From 1b8415a1c1c1b82338327eb14ac4872d753c5b73 Mon Sep 17 00:00:00 2001 From: Lars Hupfeldt Date: Tue, 5 May 2020 00:50:51 +0200 Subject: [PATCH 1/6] Introduce fips_safe_md5, see issue #7611 --- AUTHORS | 1 + CHANGES | 1 + sphinx/builders/html/__init__.py | 5 ++--- sphinx/ext/inheritance_diagram.py | 4 ++-- sphinx/util/__init__.py | 17 ++++++++++++++++- tests/test_build_html.py | 7 +++---- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/AUTHORS b/AUTHORS index dbe1082a6..4b6aa8ffb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -85,6 +85,7 @@ Other contributors, listed alphabetically, are: * Daniel Pizetta -- inheritance diagram improvements * KINEBUCHI Tomohiko -- typing Sphinx as well as docutils * Adrián Chaves (Gallaecio) -- coverage builder improvements +* Lars Hupfeldt Nielsen - OpenSSL FIPS mode md5 bug fix Many thanks for all contributions! diff --git a/CHANGES b/CHANGES index 9d7886063..eaa9a0760 100644 --- a/CHANGES +++ b/CHANGES @@ -17,6 +17,7 @@ Bugs fixed ---------- * #7567: autodoc: parametrized types are shown twice for generic types +* #7611: md5 fails when OpenSSL FIPS is enabled Testing -------- diff --git a/sphinx/builders/html/__init__.py b/sphinx/builders/html/__init__.py index 320c7feb6..982889b74 100644 --- a/sphinx/builders/html/__init__.py +++ b/sphinx/builders/html/__init__.py @@ -13,7 +13,6 @@ import posixpath import re import sys import warnings -from hashlib import md5 from os import path from typing import Any, Dict, IO, Iterable, Iterator, List, Set, Tuple @@ -38,7 +37,7 @@ from sphinx.highlighting import PygmentsBridge from sphinx.locale import _, __ from sphinx.search import js_index from sphinx.theming import HTMLThemeFactory -from sphinx.util import logging, progress_message, status_iterator +from sphinx.util import logging, progress_message, status_iterator, fips_safe_md5 from sphinx.util.docutils import is_html5_writer_available, new_document from sphinx.util.fileutil import copy_asset from sphinx.util.i18n import format_date @@ -77,7 +76,7 @@ def get_stable_hash(obj: Any) -> str: return get_stable_hash(list(obj.items())) elif isinstance(obj, (list, tuple)): obj = sorted(get_stable_hash(o) for o in obj) - return md5(str(obj).encode()).hexdigest() + return fips_safe_md5(str(obj).encode()).hexdigest() class Stylesheet(str): diff --git a/sphinx/ext/inheritance_diagram.py b/sphinx/ext/inheritance_diagram.py index db2a15b14..407fe09f5 100644 --- a/sphinx/ext/inheritance_diagram.py +++ b/sphinx/ext/inheritance_diagram.py @@ -38,7 +38,6 @@ r""" import builtins import inspect import re -from hashlib import md5 from importlib import import_module from typing import Any, Dict, Iterable, List, Tuple from typing import cast @@ -55,6 +54,7 @@ from sphinx.ext.graphviz import ( graphviz, figure_wrapper, render_dot_html, render_dot_latex, render_dot_texinfo ) +from sphinx.util import fips_safe_md5 from sphinx.util.docutils import SphinxDirective from sphinx.writers.html import HTMLTranslator from sphinx.writers.latex import LaTeXTranslator @@ -387,7 +387,7 @@ class InheritanceDiagram(SphinxDirective): def get_graph_hash(node: inheritance_diagram) -> str: encoded = (node['content'] + str(node['parts'])).encode() - return md5(encoded).hexdigest()[-10:] + return fips_safe_md5(encoded).hexdigest()[-10:] def html_visit_inheritance_diagram(self: HTMLTranslator, node: inheritance_diagram) -> None: diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index 12ae051f1..3c219cc67 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -170,6 +170,21 @@ class FilenameUniqDict(dict): self._existing = state +def fips_safe_md5(data=b'', **kwargs): + """Wrapper around hashlib.md5 + + Attempt call with 'usedforsecurity=False' if we get a ValueError, which happens when OpenSSL FIPS mode is enabled: + ValueError: error:060800A3:digital envelope routines:EVP_DigestInit_ex:disabled for fips + + See: https://github.com/sphinx-doc/sphinx/issues/7611 + """ + + try: + return md5(data, **kwargs) + except ValueError: + return md5(data, **kwargs, usedforsecurity=False) + + class DownloadFiles(dict): """A special dictionary for download files. @@ -179,7 +194,7 @@ class DownloadFiles(dict): def add_file(self, docname: str, filename: str) -> str: if filename not in self: - digest = md5(filename.encode()).hexdigest() + digest = fips_safe_md5(filename.encode()).hexdigest() dest = '%s/%s' % (digest, os.path.basename(filename)) self[filename] = (set(), dest) diff --git a/tests/test_build_html.py b/tests/test_build_html.py index ede368315..a93553361 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -10,7 +10,6 @@ import os import re -from hashlib import md5 from itertools import cycle, chain import pytest @@ -19,7 +18,7 @@ from html5lib import HTMLParser from sphinx.builders.html import validate_html_extra_path, validate_html_static_path from sphinx.errors import ConfigError from sphinx.testing.util import strip_escseq -from sphinx.util import docutils +from sphinx.util import docutils, fips_safe_md5 from sphinx.util.inventory import InventoryFile @@ -450,9 +449,9 @@ def test_html_download(app): @pytest.mark.sphinx('html', testroot='roles-download') def test_html_download_role(app, status, warning): app.build() - digest = md5(b'dummy.dat').hexdigest() + digest = fips_safe_md5(b'dummy.dat').hexdigest() assert (app.outdir / '_downloads' / digest / 'dummy.dat').exists() - digest_another = md5(b'another/dummy.dat').hexdigest() + digest_another = fips_safe_md5(b'another/dummy.dat').hexdigest() assert (app.outdir / '_downloads' / digest_another / 'dummy.dat').exists() content = (app.outdir / 'index.html').read_text() From 193d1410acf297a32b9e69b0626901790c7e1647 Mon Sep 17 00:00:00 2001 From: Lars Hupfeldt Date: Tue, 5 May 2020 02:27:33 +0200 Subject: [PATCH 2/6] Fix mypy and flake8 checks --- sphinx/util/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index 3c219cc67..3400041ac 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -173,16 +173,17 @@ class FilenameUniqDict(dict): def fips_safe_md5(data=b'', **kwargs): """Wrapper around hashlib.md5 - Attempt call with 'usedforsecurity=False' if we get a ValueError, which happens when OpenSSL FIPS mode is enabled: + Attempt call with 'usedforsecurity=False' if we get a ValueError, which happens when + OpenSSL FIPS mode is enabled: ValueError: error:060800A3:digital envelope routines:EVP_DigestInit_ex:disabled for fips See: https://github.com/sphinx-doc/sphinx/issues/7611 """ try: - return md5(data, **kwargs) + return md5(data, **kwargs) # type: ignore except ValueError: - return md5(data, **kwargs, usedforsecurity=False) + return md5(data, **kwargs, usedforsecurity=False) # type: ignore class DownloadFiles(dict): From 8bbc7b83c059e099db41dc8774c155cb1723e9aa Mon Sep 17 00:00:00 2001 From: Lars Hupfeldt Date: Tue, 5 May 2020 10:12:41 +0200 Subject: [PATCH 3/6] Rename fips_safe_md5 to md5 --- sphinx/builders/html/__init__.py | 4 ++-- sphinx/ext/inheritance_diagram.py | 4 ++-- sphinx/util/__init__.py | 10 +++++----- tests/test_build_html.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sphinx/builders/html/__init__.py b/sphinx/builders/html/__init__.py index 982889b74..463e19b7b 100644 --- a/sphinx/builders/html/__init__.py +++ b/sphinx/builders/html/__init__.py @@ -37,7 +37,7 @@ from sphinx.highlighting import PygmentsBridge from sphinx.locale import _, __ from sphinx.search import js_index from sphinx.theming import HTMLThemeFactory -from sphinx.util import logging, progress_message, status_iterator, fips_safe_md5 +from sphinx.util import logging, progress_message, status_iterator, md5 from sphinx.util.docutils import is_html5_writer_available, new_document from sphinx.util.fileutil import copy_asset from sphinx.util.i18n import format_date @@ -76,7 +76,7 @@ def get_stable_hash(obj: Any) -> str: return get_stable_hash(list(obj.items())) elif isinstance(obj, (list, tuple)): obj = sorted(get_stable_hash(o) for o in obj) - return fips_safe_md5(str(obj).encode()).hexdigest() + return md5(str(obj).encode()).hexdigest() class Stylesheet(str): diff --git a/sphinx/ext/inheritance_diagram.py b/sphinx/ext/inheritance_diagram.py index 407fe09f5..7b2383fca 100644 --- a/sphinx/ext/inheritance_diagram.py +++ b/sphinx/ext/inheritance_diagram.py @@ -54,7 +54,7 @@ from sphinx.ext.graphviz import ( graphviz, figure_wrapper, render_dot_html, render_dot_latex, render_dot_texinfo ) -from sphinx.util import fips_safe_md5 +from sphinx.util import md5 from sphinx.util.docutils import SphinxDirective from sphinx.writers.html import HTMLTranslator from sphinx.writers.latex import LaTeXTranslator @@ -387,7 +387,7 @@ class InheritanceDiagram(SphinxDirective): def get_graph_hash(node: inheritance_diagram) -> str: encoded = (node['content'] + str(node['parts'])).encode() - return fips_safe_md5(encoded).hexdigest()[-10:] + return md5(encoded).hexdigest()[-10:] def html_visit_inheritance_diagram(self: HTMLTranslator, node: inheritance_diagram) -> None: diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index 3400041ac..eeee1248e 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -10,6 +10,7 @@ import fnmatch import functools +import hashlib import os import posixpath import re @@ -21,7 +22,6 @@ import warnings from codecs import BOM_UTF8 from collections import deque from datetime import datetime -from hashlib import md5 from importlib import import_module from os import path from time import mktime, strptime @@ -170,7 +170,7 @@ class FilenameUniqDict(dict): self._existing = state -def fips_safe_md5(data=b'', **kwargs): +def md5(data=b'', **kwargs): """Wrapper around hashlib.md5 Attempt call with 'usedforsecurity=False' if we get a ValueError, which happens when @@ -181,9 +181,9 @@ def fips_safe_md5(data=b'', **kwargs): """ try: - return md5(data, **kwargs) # type: ignore + return hashlib.md5(data, **kwargs) # type: ignore except ValueError: - return md5(data, **kwargs, usedforsecurity=False) # type: ignore + return hashlib.md5(data, **kwargs, usedforsecurity=False) # type: ignore class DownloadFiles(dict): @@ -195,7 +195,7 @@ class DownloadFiles(dict): def add_file(self, docname: str, filename: str) -> str: if filename not in self: - digest = fips_safe_md5(filename.encode()).hexdigest() + digest = md5(filename.encode()).hexdigest() dest = '%s/%s' % (digest, os.path.basename(filename)) self[filename] = (set(), dest) diff --git a/tests/test_build_html.py b/tests/test_build_html.py index a93553361..b3406b74c 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -18,7 +18,7 @@ from html5lib import HTMLParser from sphinx.builders.html import validate_html_extra_path, validate_html_static_path from sphinx.errors import ConfigError from sphinx.testing.util import strip_escseq -from sphinx.util import docutils, fips_safe_md5 +from sphinx.util import docutils, md5 from sphinx.util.inventory import InventoryFile @@ -449,9 +449,9 @@ def test_html_download(app): @pytest.mark.sphinx('html', testroot='roles-download') def test_html_download_role(app, status, warning): app.build() - digest = fips_safe_md5(b'dummy.dat').hexdigest() + digest = md5(b'dummy.dat').hexdigest() assert (app.outdir / '_downloads' / digest / 'dummy.dat').exists() - digest_another = fips_safe_md5(b'another/dummy.dat').hexdigest() + digest_another = md5(b'another/dummy.dat').hexdigest() assert (app.outdir / '_downloads' / digest_another / 'dummy.dat').exists() content = (app.outdir / 'index.html').read_text() From 7d41eddd6ee4b8fb47ffa2d8fa75d69801950815 Mon Sep 17 00:00:00 2001 From: Lars Hupfeldt Date: Tue, 5 May 2020 11:26:21 +0200 Subject: [PATCH 4/6] Introduce fips safe sha1, see issue #7611 --- sphinx/ext/graphviz.py | 3 +-- sphinx/ext/imgmath.py | 3 +-- sphinx/transforms/post_transforms/images.py | 3 +-- sphinx/util/__init__.py | 14 ++++++++++++++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/sphinx/ext/graphviz.py b/sphinx/ext/graphviz.py index 27b64fbce..c21868a6f 100644 --- a/sphinx/ext/graphviz.py +++ b/sphinx/ext/graphviz.py @@ -12,7 +12,6 @@ import posixpath import re import subprocess -from hashlib import sha1 from os import path from subprocess import CalledProcessError, PIPE from typing import Any, Dict, List, Tuple @@ -25,7 +24,7 @@ import sphinx from sphinx.application import Sphinx from sphinx.errors import SphinxError from sphinx.locale import _, __ -from sphinx.util import logging +from sphinx.util import logging, sha1 from sphinx.util.docutils import SphinxDirective, SphinxTranslator from sphinx.util.fileutil import copy_asset from sphinx.util.i18n import search_image_for_language diff --git a/sphinx/ext/imgmath.py b/sphinx/ext/imgmath.py index 4297ad182..2050e470e 100644 --- a/sphinx/ext/imgmath.py +++ b/sphinx/ext/imgmath.py @@ -14,7 +14,6 @@ import shutil import subprocess import sys import tempfile -from hashlib import sha1 from os import path from subprocess import CalledProcessError, PIPE from typing import Any, Dict, List, Tuple @@ -30,7 +29,7 @@ from sphinx.config import Config from sphinx.deprecation import RemovedInSphinx40Warning, deprecated_alias from sphinx.errors import SphinxError from sphinx.locale import _, __ -from sphinx.util import logging +from sphinx.util import logging, sha1 from sphinx.util.math import get_node_equation_number, wrap_displaymath from sphinx.util.osutil import ensuredir from sphinx.util.png import read_png_depth, write_png_depth diff --git a/sphinx/transforms/post_transforms/images.py b/sphinx/transforms/post_transforms/images.py index d1b513b27..57b36b4b0 100644 --- a/sphinx/transforms/post_transforms/images.py +++ b/sphinx/transforms/post_transforms/images.py @@ -10,7 +10,6 @@ import os import re -from hashlib import sha1 from math import ceil from typing import Any, Dict, List, Tuple @@ -19,7 +18,7 @@ from docutils import nodes from sphinx.application import Sphinx from sphinx.locale import __ from sphinx.transforms import SphinxTransform -from sphinx.util import epoch_to_rfc1123, rfc1123_to_epoch +from sphinx.util import epoch_to_rfc1123, rfc1123_to_epoch, sha1 from sphinx.util import logging, requests from sphinx.util.images import guess_mimetype, get_image_extension, parse_data_uri from sphinx.util.osutil import ensuredir, movefile diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index eeee1248e..2a08821e1 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -186,6 +186,20 @@ def md5(data=b'', **kwargs): return hashlib.md5(data, **kwargs, usedforsecurity=False) # type: ignore +def sha1(data=b'', **kwargs): + """Wrapper around hashlib.sha1 + + Attempt call with 'usedforsecurity=False' if we get a ValueError + + See: https://github.com/sphinx-doc/sphinx/issues/7611 + """ + + try: + return hashlib.sha1(data, **kwargs) # type: ignore + except ValueError: + return hashlib.sha1(data, **kwargs, usedforsecurity=False) # type: ignore + + class DownloadFiles(dict): """A special dictionary for download files. From cf5660cb399e1eddd1c27d0c1380e4fd8a675a0a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 9 May 2020 17:46:53 +0900 Subject: [PATCH 5/6] Fix #7626: release package does not contain ``CODE_OF_CONDUCT`` --- CHANGES | 1 + MANIFEST.in | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index eaa9a0760..5b21a626e 100644 --- a/CHANGES +++ b/CHANGES @@ -18,6 +18,7 @@ Bugs fixed * #7567: autodoc: parametrized types are shown twice for generic types * #7611: md5 fails when OpenSSL FIPS is enabled +* #7626: release package does not contain ``CODE_OF_CONDUCT`` Testing -------- diff --git a/MANIFEST.in b/MANIFEST.in index a5699c23c..54da42895 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include LICENSE include AUTHORS include CHANGES include CHANGES.old +include CODE_OF_CONDUCT include CONTRIBUTING.rst include EXAMPLES From 1fbfb7281e6298311fde3403558d6a8a3c03372f Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 9 May 2020 14:25:29 +0900 Subject: [PATCH 6/6] Suppress arguments if all system defined TypeVars on py39 --- CHANGES | 1 + sphinx/util/typing.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index eaa9a0760..77e4189d7 100644 --- a/CHANGES +++ b/CHANGES @@ -17,6 +17,7 @@ Bugs fixed ---------- * #7567: autodoc: parametrized types are shown twice for generic types +* #7637: autodoc: system defined TypeVars are shown in Python 3.9 * #7611: md5 fails when OpenSSL FIPS is enabled Testing diff --git a/sphinx/util/typing.py b/sphinx/util/typing.py index 4644b1378..08bf7203b 100644 --- a/sphinx/util/typing.py +++ b/sphinx/util/typing.py @@ -39,6 +39,12 @@ TitleGetter = Callable[[nodes.Node], str] Inventory = Dict[str, Dict[str, Tuple[str, str, str, str]]] +def is_system_TypeVar(typ: Any) -> bool: + """Check *typ* is system defined TypeVar.""" + modname = getattr(typ, '__module__', '') + return modname == 'typing' and isinstance(typ, TypeVar) # type: ignore + + def stringify(annotation: Any) -> str: """Stringify type annotation object.""" if isinstance(annotation, str): @@ -96,7 +102,8 @@ def _stringify_py37(annotation: Any) -> str: return '%s[[%s], %s]' % (qualname, args, returns) elif str(annotation).startswith('typing.Annotated'): # for py39+ return stringify(annotation.__args__[0]) - elif getattr(annotation, '_special', False): + elif all(is_system_TypeVar(a) for a in annotation.__args__): + # Suppress arguments if all system defined TypeVars (ex. Dict[KT, VT]) return qualname else: args = ', '.join(stringify(a) for a in annotation.__args__)