From a8f7e4f5638d4b34a03b6c17bf0cc87d14977011 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 8 Jan 2021 01:25:24 +0900 Subject: [PATCH 01/30] doc: Update branch name in release-checklist --- utils/release-checklist | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/utils/release-checklist b/utils/release-checklist index 671f932d8..477ddcbbe 100644 --- a/utils/release-checklist +++ b/utils/release-checklist @@ -18,10 +18,10 @@ for stable releases * ``python utils/bump_version.py --in-develop X.Y.Zb0`` (ex. 1.5.3b0) * Check diff by ``git diff`` * ``git commit -am 'Bump version'`` -* ``git push origin X.Y --tags`` -* ``git checkout master`` -* ``git merge X.Y`` -* ``git push origin master`` +* ``git push origin X.Y.x --tags`` +* ``git checkout X.x`` +* ``git merge X.Y.x`` +* ``git push origin X.x`` * Add new version/milestone to tracker categories * Write announcement and send to sphinx-dev, sphinx-users and python-announce @@ -43,10 +43,10 @@ for first beta releases * ``python utils/bump_version.py --in-develop X.Y.0b2`` (ex. 1.6.0b2) * Check diff by ``git diff`` * ``git commit -am 'Bump version'`` -* ``git checkout -b X.Y`` -* ``git push origin X.Y --tags`` +* ``git checkout -b X.x`` +* ``git push origin X.x --tags`` * ``git checkout master`` -* ``git merge X.Y`` +* ``git merge X.x`` * ``python utils/bump_version.py --in-develop A.B.0b0`` (ex. 1.7.0b0) * Check diff by ``git diff`` * ``git commit -am 'Bump version'`` @@ -71,9 +71,9 @@ for other beta releases * ``python utils/bump_version.py --in-develop X.Y.0bM`` (ex. 1.6.0b3) * Check diff by `git diff`` * ``git commit -am 'Bump version'`` -* ``git push origin X.Y --tags`` +* ``git push origin X.x --tags`` * ``git checkout master`` -* ``git merge X.Y`` +* ``git merge X.x`` * ``git push origin master`` * Add new version/milestone to tracker categories * Write announcement and send to sphinx-dev, sphinx-users and python-announce @@ -99,9 +99,9 @@ for major releases * ``python utils/bump_version.py --in-develop X.Y.1b0`` (ex. 1.6.1b0) * Check diff by ``git diff`` * ``git commit -am 'Bump version'`` -* ``git push origin X.Y --tags`` +* ``git push origin X.x --tags`` * ``git checkout master`` -* ``git merge X.Y`` +* ``git merge X.x`` * ``git push origin master`` * open https://github.com/sphinx-doc/sphinx/settings/branches and make ``A.B`` branch *not* protected * ``git checkout A.B`` (checkout old stable) From 8a11f909a7ad53d292dddbbb1700f572e145e6f9 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 10 Jan 2021 11:21:23 +0100 Subject: [PATCH 02/30] C, fix namespace lookup for expr role --- CHANGES | 4 ++++ sphinx/domains/c.py | 8 ++++---- tests/roots/test-domain-c/ns_lookup.rst | 13 +++++++++++++ tests/test_domain_c.py | 7 +++++++ 4 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 tests/roots/test-domain-c/ns_lookup.rst diff --git a/CHANGES b/CHANGES index 4dfc95bd9..bbf710f02 100644 --- a/CHANGES +++ b/CHANGES @@ -16,6 +16,10 @@ Features added Bugs fixed ---------- +* #8655: autodoc: Failed to generate document if target module contains an + object that raises an exception on ``hasattr()`` +* C, ``expr`` role should start symbol lookup in the current scope. + Testing -------- diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 87f115c4a..7a27261cd 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -137,8 +137,7 @@ class ASTIdentifier(ASTBaseBase): reftype='identifier', reftarget=targetText, modname=None, classname=None) - key = symbol.get_lookup_key() - pnode['c:parent_key'] = key + pnode['c:parent_key'] = symbol.get_lookup_key() if self.is_anon(): pnode += nodes.strong(text="[anonymous]") else: @@ -3204,7 +3203,8 @@ class CObject(ObjectDescription): def parse_pre_v3_type_definition(self, parser: DefinitionParser) -> ASTDeclaration: return parser.parse_pre_v3_type_definition() - def describe_signature(self, signode: TextElement, ast: Any, options: Dict) -> None: + def describe_signature(self, signode: TextElement, ast: ASTDeclaration, + options: Dict) -> None: ast.describe_signature(signode, 'lastIsName', self.env, options) def run(self) -> List[Node]: @@ -3642,7 +3642,7 @@ class CExprRole(SphinxRole): location=self.get_source_info()) # see below return [self.node_type(text, text, classes=classes)], [] - parentSymbol = self.env.temp_data.get('cpp:parent_symbol', None) + parentSymbol = self.env.temp_data.get('c:parent_symbol', None) if parentSymbol is None: parentSymbol = self.env.domaindata['c']['root_symbol'] # ...most if not all of these classes should really apply to the individual references, diff --git a/tests/roots/test-domain-c/ns_lookup.rst b/tests/roots/test-domain-c/ns_lookup.rst new file mode 100644 index 000000000..87f9d68e7 --- /dev/null +++ b/tests/roots/test-domain-c/ns_lookup.rst @@ -0,0 +1,13 @@ +.. c:namespace:: ns_lookup + +.. c:var:: int i + +.. c:function:: void f(int j) + + - :c:var:`i` + - :c:var:`j` + - :c:expr:`i` + - :c:expr:`j` + +- :c:var:`i` +- :c:expr:`i` diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index 10d618712..acbaf7946 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -595,6 +595,13 @@ def test_build_function_param_target(app, warning): ] +@pytest.mark.sphinx(testroot='domain-c', confoverrides={'nitpicky': True}) +def test_build_ns_lookup(app, warning): + app.builder.build_all() + ws = filter_warnings(warning, "ns_lookup") + assert len(ws) == 0 + + def _get_obj(app, queryName): domain = app.env.get_domain('c') for name, dispname, objectType, docname, anchor, prio in domain.get_objects(): From 8f5744be5e688069b6335fea94b430c61e955db2 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 16 Jan 2021 22:46:10 +0900 Subject: [PATCH 03/30] refactor: Remove meaningless variable: attrs It would be better to use `name in node` and `node[name]` to check and access an attribute of the node instead of `node.attributes`. --- sphinx/writers/latex.py | 27 +++++++++++++-------------- sphinx/writers/texinfo.py | 7 +++---- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 1591b5056..918dda774 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -1205,7 +1205,6 @@ class LaTeXTranslator(SphinxTranslator): return isinstance(node.parent, nodes.TextElement) def visit_image(self, node: Element) -> None: - attrs = node.attributes pre = [] # type: List[str] # in reverse order post = [] # type: List[str] @@ -1215,27 +1214,27 @@ class LaTeXTranslator(SphinxTranslator): is_inline = self.is_inline(node.parent) else: is_inline = self.is_inline(node) - if 'width' in attrs: - if 'scale' in attrs: - w = self.latex_image_length(attrs['width'], attrs['scale']) + if 'width' in node: + if 'scale' in node: + w = self.latex_image_length(node['width'], node['scale']) else: - w = self.latex_image_length(attrs['width']) + w = self.latex_image_length(node['width']) if w: include_graphics_options.append('width=%s' % w) - if 'height' in attrs: - if 'scale' in attrs: - h = self.latex_image_length(attrs['height'], attrs['scale']) + if 'height' in node: + if 'scale' in node: + h = self.latex_image_length(node['height'], node['scale']) else: - h = self.latex_image_length(attrs['height']) + h = self.latex_image_length(node['height']) if h: include_graphics_options.append('height=%s' % h) - if 'scale' in attrs: + if 'scale' in node: if not include_graphics_options: # if no "width" nor "height", \sphinxincludegraphics will fit # to the available text width if oversized after rescaling. include_graphics_options.append('scale=%s' - % (float(attrs['scale']) / 100.0)) - if 'align' in attrs: + % (float(node['scale']) / 100.0)) + if 'align' in node: align_prepost = { # By default latex aligns the top of an image. (1, 'top'): ('', ''), @@ -1250,8 +1249,8 @@ class LaTeXTranslator(SphinxTranslator): (0, 'right'): ('{\\hspace*{\\fill}', '}'), } try: - pre.append(align_prepost[is_inline, attrs['align']][0]) - post.append(align_prepost[is_inline, attrs['align']][1]) + pre.append(align_prepost[is_inline, node['align']][0]) + post.append(align_prepost[is_inline, node['align']][1]) except KeyError: pass if self.in_parsed_literal: diff --git a/sphinx/writers/texinfo.py b/sphinx/writers/texinfo.py index 69c8b12a7..6518d10da 100644 --- a/sphinx/writers/texinfo.py +++ b/sphinx/writers/texinfo.py @@ -1206,11 +1206,10 @@ class TexinfoTranslator(SphinxTranslator): # ignore remote images return name, ext = path.splitext(uri) - attrs = node.attributes # width and height ignored in non-tex output - width = self.tex_image_length(attrs.get('width', '')) - height = self.tex_image_length(attrs.get('height', '')) - alt = self.escape_arg(attrs.get('alt', '')) + width = self.tex_image_length(node.get('width', '')) + height = self.tex_image_length(node.get('height', '')) + alt = self.escape_arg(node.get('alt', '')) filename = "%s-figures/%s" % (self.elements['filename'][:-5], name) # type: ignore self.body.append('\n@image{%s,%s,%s,%s,%s}\n' % (filename, width, height, alt, ext[1:])) From 425cd1af02ad5f806e25c725fa161a3464970eb2 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 17 Jan 2021 01:50:31 +0900 Subject: [PATCH 04/30] Fix #8693: autodoc: Default values for overloads are rendered as string The default values for overloaded functions are rendered as string literal unexpectedly because autodoc extracts code snippets from the source code, not actual value (ex. int, ellipsis, and so on). This introduces a simple wrapper class; `DefaultValue` to render these code snippets like actual values, not string literals. --- CHANGES | 1 + sphinx/util/inspect.py | 17 +++++++++++++++-- tests/roots/test-ext-autodoc/target/overload.py | 12 ++++++------ tests/test_ext_autodoc.py | 12 ++++++------ 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/CHANGES b/CHANGES index 9fc0c40f7..a289f5ebc 100644 --- a/CHANGES +++ b/CHANGES @@ -45,6 +45,7 @@ Bugs fixed * #8315: autodoc: Failed to resolve struct.Struct type annotation * #8652: autodoc: All variable comments in the module are ignored if the module contains invalid type comments +* #8693: autodoc: Default values for overloaded functions are rendered as string * #8306: autosummary: mocked modules are documented as empty page when using :recursive: option * #8618: html: kbd role produces incorrect HTML when compound-key separators (-, diff --git a/sphinx/util/inspect.py b/sphinx/util/inspect.py index e91e5df96..202e170c1 100644 --- a/sphinx/util/inspect.py +++ b/sphinx/util/inspect.py @@ -499,6 +499,19 @@ def is_builtin_class_method(obj: Any, attr_name: str) -> bool: return getattr(builtins, name, None) is cls +class DefaultValue: + """A simple wrapper for default value of the parameters of overload functions.""" + + def __init__(self, value: str) -> None: + self.value = value + + def __eq__(self, other: object) -> bool: + return self.value == other + + def __repr__(self) -> str: + return self.value + + def _should_unwrap(subject: Callable) -> bool: """Check the function should be unwrapped on getting signature.""" if (safe_getattr(subject, '__globals__', None) and @@ -704,7 +717,7 @@ def signature_from_ast(node: ast.FunctionDef, code: str = '') -> inspect.Signatu if defaults[i] is Parameter.empty: default = Parameter.empty else: - default = ast_unparse(defaults[i], code) + default = DefaultValue(ast_unparse(defaults[i], code)) annotation = ast_unparse(arg.annotation, code) or Parameter.empty params.append(Parameter(arg.arg, Parameter.POSITIONAL_ONLY, @@ -714,7 +727,7 @@ def signature_from_ast(node: ast.FunctionDef, code: str = '') -> inspect.Signatu if defaults[i + posonlyargs] is Parameter.empty: default = Parameter.empty else: - default = ast_unparse(defaults[i + posonlyargs], code) + default = DefaultValue(ast_unparse(defaults[i + posonlyargs], code)) annotation = ast_unparse(arg.annotation, code) or Parameter.empty params.append(Parameter(arg.arg, Parameter.POSITIONAL_OR_KEYWORD, diff --git a/tests/roots/test-ext-autodoc/target/overload.py b/tests/roots/test-ext-autodoc/target/overload.py index cc4e509f2..35d078b66 100644 --- a/tests/roots/test-ext-autodoc/target/overload.py +++ b/tests/roots/test-ext-autodoc/target/overload.py @@ -2,17 +2,17 @@ from typing import Any, overload @overload -def sum(x: int, y: int) -> int: +def sum(x: int, y: int = 0) -> int: ... @overload -def sum(x: "float", y: "float") -> "float": +def sum(x: "float", y: "float" = 0.0) -> "float": ... @overload -def sum(x: str, y: str) -> str: +def sum(x: str, y: str = ...) -> str: ... @@ -25,15 +25,15 @@ class Math: """docstring""" @overload - def sum(self, x: int, y: int) -> int: + def sum(self, x: int, y: int = 0) -> int: ... @overload - def sum(self, x: "float", y: "float") -> "float": + def sum(self, x: "float", y: "float" = 0.0) -> "float": ... @overload - def sum(self, x: str, y: str) -> str: + def sum(self, x: str, y: str = ...) -> str: ... def sum(self, x, y): diff --git a/tests/test_ext_autodoc.py b/tests/test_ext_autodoc.py index 4ac9381ae..3ff33cea2 100644 --- a/tests/test_ext_autodoc.py +++ b/tests/test_ext_autodoc.py @@ -2078,17 +2078,17 @@ def test_overload(app): ' docstring', '', '', - ' .. py:method:: Math.sum(x: int, y: int) -> int', - ' Math.sum(x: float, y: float) -> float', - ' Math.sum(x: str, y: str) -> str', + ' .. py:method:: Math.sum(x: int, y: int = 0) -> int', + ' Math.sum(x: float, y: float = 0.0) -> float', + ' Math.sum(x: str, y: str = ...) -> str', ' :module: target.overload', '', ' docstring', '', '', - '.. py:function:: sum(x: int, y: int) -> int', - ' sum(x: float, y: float) -> float', - ' sum(x: str, y: str) -> str', + '.. py:function:: sum(x: int, y: int = 0) -> int', + ' sum(x: float, y: float = 0.0) -> float', + ' sum(x: str, y: str = ...) -> str', ' :module: target.overload', '', ' docstring', From d88166e84b2b63ff5c7c6b5001fc7c536775d918 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 17 Jan 2021 01:14:28 +0900 Subject: [PATCH 05/30] Close #8514: autodoc: Default values of overloads are taken from actual implementation As a well-known idiom, mypy recommends to use ellipsis ("...") for default argument values as a elided style. This allows to write the style and helps to document it with copying the default argument values from actual implementation. Note: This does not copy the default argument value when the argument of overloaded function has its own default value. --- CHANGES | 2 ++ sphinx/ext/autodoc/__init__.py | 31 +++++++++++++++++++ .../roots/test-ext-autodoc/target/overload.py | 4 +-- tests/test_ext_autodoc.py | 4 +-- tests/test_ext_autodoc_configs.py | 4 +-- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index a289f5ebc..5a7c980b1 100644 --- a/CHANGES +++ b/CHANGES @@ -20,6 +20,8 @@ Features added * #8022: autodoc: autodata and autoattribute directives does not show right-hand value of the variable if docstring contains ``:meta hide-value:`` in info-field-list +* #8514: autodoc: Default values of overloaded functions are taken from actual + implementation if they're ellipsis * #8619: html: kbd role generates customizable HTML tags for compound keys * #8634: html: Allow to change the order of JS/CSS via ``priority`` parameter for :meth:`Sphinx.add_js_file()` and :meth:`Sphinx.add_css_file()` diff --git a/sphinx/ext/autodoc/__init__.py b/sphinx/ext/autodoc/__init__.py index bf80ef4a8..83c7d28c4 100644 --- a/sphinx/ext/autodoc/__init__.py +++ b/sphinx/ext/autodoc/__init__.py @@ -1355,8 +1355,11 @@ class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter): # typ documenter.objpath = [None] sigs.append(documenter.format_signature()) if overloaded: + actual = inspect.signature(self.object, + type_aliases=self.config.autodoc_type_aliases) __globals__ = safe_getattr(self.object, '__globals__', {}) for overload in self.analyzer.overloads.get('.'.join(self.objpath)): + overload = self.merge_default_value(actual, overload) overload = evaluate_signature(overload, __globals__, self.config.autodoc_type_aliases) @@ -1365,6 +1368,16 @@ class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter): # typ return "\n".join(sigs) + def merge_default_value(self, actual: Signature, overload: Signature) -> Signature: + """Merge default values of actual implementation to the overload variants.""" + parameters = list(overload.parameters.values()) + for i, param in enumerate(parameters): + actual_param = actual.parameters.get(param.name) + if actual_param and param.default == '...': + parameters[i] = param.replace(default=actual_param.default) + + return overload.replace(parameters=parameters) + def annotate_to_first_argument(self, func: Callable, typ: Type) -> None: """Annotate type hint to the first argument of function if needed.""" try: @@ -2117,8 +2130,16 @@ class MethodDocumenter(DocstringSignatureMixin, ClassLevelDocumenter): # type: documenter.objpath = [None] sigs.append(documenter.format_signature()) if overloaded: + if inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name): + actual = inspect.signature(self.object, bound_method=False, + type_aliases=self.config.autodoc_type_aliases) + else: + actual = inspect.signature(self.object, bound_method=True, + type_aliases=self.config.autodoc_type_aliases) + __globals__ = safe_getattr(self.object, '__globals__', {}) for overload in self.analyzer.overloads.get('.'.join(self.objpath)): + overload = self.merge_default_value(actual, overload) overload = evaluate_signature(overload, __globals__, self.config.autodoc_type_aliases) @@ -2131,6 +2152,16 @@ class MethodDocumenter(DocstringSignatureMixin, ClassLevelDocumenter): # type: return "\n".join(sigs) + def merge_default_value(self, actual: Signature, overload: Signature) -> Signature: + """Merge default values of actual implementation to the overload variants.""" + parameters = list(overload.parameters.values()) + for i, param in enumerate(parameters): + actual_param = actual.parameters.get(param.name) + if actual_param and param.default == '...': + parameters[i] = param.replace(default=actual_param.default) + + return overload.replace(parameters=parameters) + def annotate_to_first_argument(self, func: Callable, typ: Type) -> None: """Annotate type hint to the first argument of function if needed.""" try: diff --git a/tests/roots/test-ext-autodoc/target/overload.py b/tests/roots/test-ext-autodoc/target/overload.py index 35d078b66..1b395ee5b 100644 --- a/tests/roots/test-ext-autodoc/target/overload.py +++ b/tests/roots/test-ext-autodoc/target/overload.py @@ -16,7 +16,7 @@ def sum(x: str, y: str = ...) -> str: ... -def sum(x, y): +def sum(x, y=None): """docstring""" return x + y @@ -36,7 +36,7 @@ class Math: def sum(self, x: str, y: str = ...) -> str: ... - def sum(self, x, y): + def sum(self, x, y=None): """docstring""" return x + y diff --git a/tests/test_ext_autodoc.py b/tests/test_ext_autodoc.py index 3ff33cea2..d555359cf 100644 --- a/tests/test_ext_autodoc.py +++ b/tests/test_ext_autodoc.py @@ -2080,7 +2080,7 @@ def test_overload(app): '', ' .. py:method:: Math.sum(x: int, y: int = 0) -> int', ' Math.sum(x: float, y: float = 0.0) -> float', - ' Math.sum(x: str, y: str = ...) -> str', + ' Math.sum(x: str, y: str = None) -> str', ' :module: target.overload', '', ' docstring', @@ -2088,7 +2088,7 @@ def test_overload(app): '', '.. py:function:: sum(x: int, y: int = 0) -> int', ' sum(x: float, y: float = 0.0) -> float', - ' sum(x: str, y: str = ...) -> str', + ' sum(x: str, y: str = None) -> str', ' :module: target.overload', '', ' docstring', diff --git a/tests/test_ext_autodoc_configs.py b/tests/test_ext_autodoc_configs.py index 9cd5f5e32..bae684397 100644 --- a/tests/test_ext_autodoc_configs.py +++ b/tests/test_ext_autodoc_configs.py @@ -647,13 +647,13 @@ def test_autodoc_typehints_none_for_overload(app): ' docstring', '', '', - ' .. py:method:: Math.sum(x, y)', + ' .. py:method:: Math.sum(x, y=None)', ' :module: target.overload', '', ' docstring', '', '', - '.. py:function:: sum(x, y)', + '.. py:function:: sum(x, y=None)', ' :module: target.overload', '', ' docstring', From de0a4ee198cc18cf04ee4e1eb447e343d4c22b30 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 18 Jan 2021 02:19:30 +0900 Subject: [PATCH 06/30] refactor: viewcode: Use a constant for dirname --- sphinx/ext/viewcode.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py index 3459e45ca..6f0e4a84f 100644 --- a/sphinx/ext/viewcode.py +++ b/sphinx/ext/viewcode.py @@ -8,6 +8,7 @@ :license: BSD, see LICENSE for details. """ +import posixpath import traceback from typing import Any, Dict, Iterable, Iterator, Set, Tuple @@ -26,6 +27,9 @@ from sphinx.util.nodes import make_refnode logger = logging.getLogger(__name__) +OUTPUT_DIRNAME = '_modules' + + def _get_full_modname(app: Sphinx, modname: str, attribute: str) -> str: try: return get_full_modname(modname, attribute) @@ -108,7 +112,7 @@ def doctree_read(app: Sphinx, doctree: Node) -> None: # only one link per name, please continue names.add(fullname) - pagename = '_modules/' + modname.replace('.', '/') + pagename = posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/')) inline = nodes.inline('', _('[source]'), classes=['viewcode-link']) onlynode = addnodes.only(expr='html') onlynode += addnodes.pending_xref('', inline, reftype='viewcode', refdomain='std', @@ -156,7 +160,7 @@ def collect_pages(app: Sphinx) -> Iterator[Tuple[str, Dict[str, Any], str]]: continue code, tags, used, refname = entry # construct a page name for the highlighted source - pagename = '_modules/' + modname.replace('.', '/') + pagename = posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/')) # highlight the source using the builder's highlighter if env.config.highlight_language in ('python3', 'default', 'none'): lexer = env.config.highlight_language @@ -188,10 +192,10 @@ def collect_pages(app: Sphinx) -> Iterator[Tuple[str, Dict[str, Any], str]]: parent = parent.rsplit('.', 1)[0] if parent in modnames: parents.append({ - 'link': urito(pagename, '_modules/' + - parent.replace('.', '/')), + 'link': urito(pagename, + posixpath.join(OUTPUT_DIRNAME, parent.replace('.', '/'))), 'title': parent}) - parents.append({'link': urito(pagename, '_modules/index'), + parents.append({'link': urito(pagename, posixpath.join(OUTPUT_DIRNAME, 'index')), 'title': _('Module code')}) parents.reverse() # putting it all together @@ -220,7 +224,8 @@ def collect_pages(app: Sphinx) -> Iterator[Tuple[str, Dict[str, Any], str]]: html.append('') stack.append(modname + '.') html.append('
  • %s
    • \n' % ( - urito('_modules/index', '_modules/' + modname.replace('.', '/')), + urito(posixpath.join(OUTPUT_DIRNAME, 'index'), + posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/'))), modname)) html.append('' * (len(stack) - 1)) context = { @@ -229,7 +234,7 @@ def collect_pages(app: Sphinx) -> Iterator[Tuple[str, Dict[str, Any], str]]: ''.join(html)), } - yield ('_modules/index', context, 'page.html') + yield (posixpath.join(OUTPUT_DIRNAME, 'index'), context, 'page.html') def setup(app: Sphinx) -> Dict[str, Any]: From bc56384fb90122aec6f7f9388411ad15f9fddb32 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 18 Jan 2021 00:00:32 +0900 Subject: [PATCH 07/30] Fix #1112: download role creates duplicated copies `:download:` role creates duplicated copies when the document contains two or more the role for the same file, but in different form. It considers two paths are different when one contains relative path like `path/to/../file.dat`. Internally, `env.relfn2path()` does not normalize the given path in relative form. As a result, download role can't detect the same paths are given. This adds `os.path.normpath()` to `env.relfn2path()` to normalize the path. --- CHANGES | 2 ++ sphinx/environment/__init__.py | 7 ++++--- tests/test_environment.py | 5 +++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 5a7c980b1..7462381a4 100644 --- a/CHANGES +++ b/CHANGES @@ -61,6 +61,8 @@ Bugs fixed * C, C++: in general fix intersphinx and role lookup types. * #8683: :confval:`html_last_updated_fmt` does not support UTC offset (%z) * #8683: :confval:`html_last_updated_fmt` generates wrong time zone for %Z +* #1112: ``download`` role creates duplicated copies when relative path is + specified Testing -------- diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index be5e52f49..af3e2b8d5 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -10,6 +10,7 @@ import os import pickle +import posixpath import warnings from collections import defaultdict from copy import copy @@ -356,9 +357,9 @@ class BuildEnvironment: docdir = path.dirname(self.doc2path(docname or self.docname, base=None)) rel_fn = path.join(docdir, filename) - # the path.abspath() might seem redundant, but otherwise artifacts - # such as ".." will remain in the path - return rel_fn, path.abspath(path.join(self.srcdir, rel_fn)) + + return (posixpath.normpath(rel_fn), + path.normpath(path.join(self.srcdir, rel_fn))) @property def found_docs(self) -> Set[str]: diff --git a/tests/test_environment.py b/tests/test_environment.py index ccf396d9c..9791c2d5b 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -138,6 +138,11 @@ def test_env_relfn2path(app): assert relfn == '../logo.jpg' assert absfn == app.srcdir.parent / 'logo.jpg' + # relative path traversal + relfn, absfn = app.env.relfn2path('subdir/../logo.jpg', 'index') + assert relfn == 'logo.jpg' + assert absfn == app.srcdir / 'logo.jpg' + # omit docname (w/ current docname) app.env.temp_data['docname'] = 'subdir/document' relfn, absfn = app.env.relfn2path('images/logo.jpg') From 2e01c34acfd4db1ca2633aeadaca4614e29f97aa Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 18 Jan 2021 02:11:47 +0900 Subject: [PATCH 08/30] Close #8681: viewcode: Support incremental build Now viewcode supports incremental build. It generates HTML files only when the original python code is updated from last build. --- CHANGES | 1 + sphinx/ext/viewcode.py | 41 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 5a7c980b1..6316df378 100644 --- a/CHANGES +++ b/CHANGES @@ -32,6 +32,7 @@ Features added type * #6241: mathjax: Include mathjax.js only on the document using equations * #8651: std domain: cross-reference for a rubric having inline item is broken +* #8681: viewcode: Support incremental build * #8132: Add :confval:`project_copyright` as an alias of :confval:`copyright` * #207: Now :confval:`highlight_language` supports multiple languages * #2030: :rst:dir:`code-block` and :rst:dir:`literalinclude` supports automatic diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py index 3459e45ca..ff9dc6a0c 100644 --- a/sphinx/ext/viewcode.py +++ b/sphinx/ext/viewcode.py @@ -9,7 +9,8 @@ """ import traceback -from typing import Any, Dict, Iterable, Iterator, Set, Tuple +from os import path +from typing import Any, Dict, Iterable, Iterator, Optional, Set, Tuple, cast from docutils import nodes from docutils.nodes import Element, Node @@ -17,6 +18,7 @@ from docutils.nodes import Element, Node import sphinx from sphinx import addnodes from sphinx.application import Sphinx +from sphinx.builders.html import StandaloneHTMLBuilder from sphinx.environment import BuildEnvironment from sphinx.locale import _, __ from sphinx.pycode import ModuleAnalyzer @@ -138,6 +140,40 @@ def missing_reference(app: Sphinx, env: BuildEnvironment, node: Element, contnod return None +def get_module_filename(app: Sphinx, modname: str) -> Optional[str]: + """Get module filename for *modname*.""" + source_info = app.emit_firstresult('viewcode-find-source', modname) + if source_info: + return None + else: + try: + filename, source = ModuleAnalyzer.get_module_source(modname) + return filename + except Exception: + return None + + +def should_generate_module_page(app: Sphinx, modname: str) -> bool: + """Check generation of module page is needed.""" + module_filename = get_module_filename(app, modname) + if module_filename is None: + # Always (re-)generate module page when module filename is not found. + return True + + builder = cast(StandaloneHTMLBuilder, app.builder) + basename = modname.replace('.', '/') + builder.out_suffix + page_filename = path.join(app.outdir, '_modules/', basename) + + try: + if path.getmtime(module_filename) <= path.getmtime(page_filename): + # generation is not needed if the HTML page is newer than module file. + return False + except IOError: + pass + + return True + + def collect_pages(app: Sphinx) -> Iterator[Tuple[str, Dict[str, Any], str]]: env = app.builder.env if not hasattr(env, '_viewcode_modules'): @@ -154,6 +190,9 @@ def collect_pages(app: Sphinx) -> Iterator[Tuple[str, Dict[str, Any], str]]: app.verbosity, lambda x: x[0]): if not entry: continue + if not should_generate_module_page(app, modname): + continue + code, tags, used, refname = entry # construct a page name for the highlighted source pagename = '_modules/' + modname.replace('.', '/') From 84150b2106f947d310960e7951ac0bea0eb36731 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 19 Jan 2021 02:50:11 +0900 Subject: [PATCH 09/30] refactor: viewcode: Fix type annotations --- sphinx/ext/viewcode.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py index 89b97de02..a7d52a91c 100644 --- a/sphinx/ext/viewcode.py +++ b/sphinx/ext/viewcode.py @@ -11,7 +11,7 @@ import posixpath import traceback from os import path -from typing import Any, Dict, Iterable, Iterator, Optional, Set, Tuple, cast +from typing import Any, Dict, Generator, Iterable, Optional, Set, Tuple, cast from docutils import nodes from docutils.nodes import Element, Node @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) OUTPUT_DIRNAME = '_modules' -def _get_full_modname(app: Sphinx, modname: str, attribute: str) -> str: +def _get_full_modname(app: Sphinx, modname: str, attribute: str) -> Optional[str]: try: return get_full_modname(modname, attribute) except AttributeError: @@ -135,7 +135,7 @@ def env_merge_info(app: Sphinx, env: BuildEnvironment, docnames: Iterable[str], def missing_reference(app: Sphinx, env: BuildEnvironment, node: Element, contnode: Node - ) -> Node: + ) -> Optional[Node]: # resolve our "viewcode" reference nodes -- they need special treatment if node['reftype'] == 'viewcode': return make_refnode(app.builder, node['refdoc'], node['reftarget'], @@ -178,7 +178,7 @@ def should_generate_module_page(app: Sphinx, modname: str) -> bool: return True -def collect_pages(app: Sphinx) -> Iterator[Tuple[str, Dict[str, Any], str]]: +def collect_pages(app: Sphinx) -> Generator[Tuple[str, Dict[str, Any], str], None, None]: env = app.builder.env if not hasattr(env, '_viewcode_modules'): return From 7ad85cd1e0e2c020cb649c720ef230d147aff0df Mon Sep 17 00:00:00 2001 From: SolidifiedRay Date: Tue, 5 Jan 2021 12:40:25 -0800 Subject: [PATCH 10/30] Close #8573: napoleon: Add more custom section styles --- doc/usage/extensions/napoleon.rst | 26 +++++++++++++++++++++++++- sphinx/ext/napoleon/__init__.py | 7 ++++++- sphinx/ext/napoleon/docstring.py | 24 +++++++++++++++++++----- tests/test_ext_napoleon_docstring.py | 19 ++++++++++++++++++- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/doc/usage/extensions/napoleon.rst b/doc/usage/extensions/napoleon.rst index cf5b3080f..066c56e2d 100644 --- a/doc/usage/extensions/napoleon.rst +++ b/doc/usage/extensions/napoleon.rst @@ -546,4 +546,28 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: If an attribute is documented in the docstring without a type and has an annotation in the class body, that type is used. - .. versionadded:: 3.4 \ No newline at end of file + .. versionadded:: 3.4 + +.. confval:: napoleon_custom_sections + + Add a list of custom sections to include, expanding the list of parsed sections. + *Defaults to None.* + + The entries can either be strings or tuples, depending on the intention: + + * To create a custom "generic" section, just pass a string. + * To create an alias for an existing section, pass a tuple containing the + alias name and the original, in that order. + * To create a custom section that displays like the parameters or returns + section, pass a tuple containing the custom section name and a string + value, "params_style" or "returns_style". + + If an entry is just a string, it is interpreted as a header for a generic + section. If the entry is a tuple/list/indexed container, the first entry + is the name of the section, the second is the section key to emulate. If the + second entry value is "params_style" or "returns_style", the custom section + will be displayed like the parameters section or returns section. + + .. versionadded:: 1.8 + .. versionchanged:: 3.5 + Support ``params_style`` and ``returns_style`` \ No newline at end of file diff --git a/sphinx/ext/napoleon/__init__.py b/sphinx/ext/napoleon/__init__.py index 5b2715bac..4a8c2135a 100644 --- a/sphinx/ext/napoleon/__init__.py +++ b/sphinx/ext/napoleon/__init__.py @@ -253,10 +253,15 @@ class Config: * To create a custom "generic" section, just pass a string. * To create an alias for an existing section, pass a tuple containing the alias name and the original, in that order. + * To create a custom section that displays like the parameters or returns + section, pass a tuple containing the custom section name and a string + value, "params_style" or "returns_style". If an entry is just a string, it is interpreted as a header for a generic section. If the entry is a tuple/list/indexed container, the first entry - is the name of the section, the second is the section key to emulate. + is the name of the section, the second is the section key to emulate. If the + second entry value is "params_style" or "returns_style", the custom section + will be displayed like the parameters section or returns section. napoleon_attr_annotations : :obj:`bool` (Defaults to True) Use the type annotations of class attributes that are documented in the docstring diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 141be022e..b6408427a 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -549,11 +549,18 @@ class GoogleDocstring: self._sections[entry.lower()] = self._parse_custom_generic_section else: # otherwise, assume entry is container; - # [0] is new section, [1] is the section to alias. - # in the case of key mismatch, just handle as generic section. - self._sections[entry[0].lower()] = \ - self._sections.get(entry[1].lower(), - self._parse_custom_generic_section) + if entry[1] == "params_style": + self._sections[entry[0].lower()] = \ + self._parse_custom_params_style_section + elif entry[1] == "returns_style": + self._sections[entry[0].lower()] = \ + self._parse_custom_returns_style_section + else: + # [0] is new section, [1] is the section to alias. + # in the case of key mismatch, just handle as generic section. + self._sections[entry[0].lower()] = \ + self._sections.get(entry[1].lower(), + self._parse_custom_generic_section) def _parse(self) -> None: self._parsed_lines = self._consume_empty() @@ -641,6 +648,13 @@ class GoogleDocstring: # for now, no admonition for simple custom sections return self._parse_generic_section(section, False) + def _parse_custom_params_style_section(self, section: str) -> List[str]: + return self._format_fields(section, self._consume_fields()) + + def _parse_custom_returns_style_section(self, section: str) -> List[str]: + fields = self._consume_returns_section() + return self._format_fields(section, fields) + def _parse_usage_section(self, section: str) -> List[str]: header = ['.. rubric:: Usage:', ''] block = ['.. code-block:: python', ''] diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index ec5f90ac2..28dfa371b 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1072,10 +1072,27 @@ You should listen to me! Sooper Warning: Stop hitting yourself! """, """:Warns: **Stop hitting yourself!** +"""), + ("""\ +Params Style: + arg1 (int): Description of arg1 + arg2 (str): Description of arg2 + +""", """\ +:Params Style: * **arg1** (*int*) -- Description of arg1 + * **arg2** (*str*) -- Description of arg2 +"""), + ("""\ +Returns Style: + description of custom section + +""", """:Returns Style: description of custom section """)) testConfig = Config(napoleon_custom_sections=['Really Important Details', - ('Sooper Warning', 'warns')]) + ('Sooper Warning', 'warns'), + ('Params Style', 'params_style'), + ('Returns Style', 'returns_style')]) for docstring, expected in docstrings: actual = str(GoogleDocstring(docstring, testConfig)) From ef7f57325c38e67091b38a92a5c322693866a62c Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 19 Jan 2021 21:17:26 +0900 Subject: [PATCH 11/30] Update CHANGES for PR #8658 --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index d7ef44f96..78680721a 100644 --- a/CHANGES +++ b/CHANGES @@ -30,6 +30,8 @@ Features added :event:`html-page-context` event * #8649: imgconverter: Skip availability check if builder supports the image type +* #8573: napoleon: Allow to change the style of custom sections using + :confval:`napoleon_custom_styles` * #6241: mathjax: Include mathjax.js only on the document using equations * #8651: std domain: cross-reference for a rubric having inline item is broken * #8681: viewcode: Support incremental build From ab7562513b5fdd65d203f68a0712f5f96dc7b6fd Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Mon, 18 Jan 2021 22:06:45 +0100 Subject: [PATCH 12/30] C++, also hyperlink operators in expr and alias --- CHANGES | 1 + sphinx/domains/cpp.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGES b/CHANGES index 9fc0c40f7..9476056c8 100644 --- a/CHANGES +++ b/CHANGES @@ -34,6 +34,7 @@ Features added * #207: Now :confval:`highlight_language` supports multiple languages * #2030: :rst:dir:`code-block` and :rst:dir:`literalinclude` supports automatic dedent via no-argument ``:dedent:`` option +* C++, also hyperlink operator overloads in expressions and alias declarations. Bugs fixed ---------- diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py index 4d6e189a3..25e6f1421 100644 --- a/sphinx/domains/cpp.py +++ b/sphinx/domains/cpp.py @@ -1592,6 +1592,15 @@ class ASTOperator(ASTBase): identifier = str(self) if mode == 'lastIsName': signode += addnodes.desc_name(identifier, identifier) + elif mode == 'markType': + targetText = prefix + identifier + templateArgs + pnode = addnodes.pending_xref('', refdomain='cpp', + reftype='identifier', + reftarget=targetText, modname=None, + classname=None) + pnode['cpp:parent_key'] = symbol.get_lookup_key() + pnode += nodes.Text(identifier) + signode += pnode else: signode += addnodes.desc_addname(identifier, identifier) From d49f3a1c78cadd6ed5c9623b80a351b080d9405e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 20 Jan 2021 02:23:34 +0900 Subject: [PATCH 13/30] refactor: autosummary: Deprecate _simple_info() and _simple_warn() --- CHANGES | 2 ++ doc/extdev/deprecated.rst | 10 ++++++++++ sphinx/ext/autosummary/generate.py | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/CHANGES b/CHANGES index 78680721a..46038f649 100644 --- a/CHANGES +++ b/CHANGES @@ -13,6 +13,8 @@ Deprecated * ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()`` * ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter`` * ``sphinx.ext.autodoc.importer.get_module_members()`` +* ``sphinx.ext.autosummary.generate._simple_info()`` +* ``sphinx.ext.autosummary.generate._simple_warn()`` Features added -------------- diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index 0e9a8f9e3..03a307228 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -36,6 +36,16 @@ The following is a list of deprecated interfaces. - 5.0 - ``sphinx.ext.autodoc.ModuleDocumenter.get_module_members()`` + * - ``sphinx.ext.autosummary.generate._simple_info()`` + - 3.5 + - 5.0 + - :ref:`logging-api` + + * - ``sphinx.ext.autosummary.generate._simple_warn()`` + - 3.5 + - 5.0 + - :ref:`logging-api` + * - The ``follow_wrapped`` argument of ``sphinx.util.inspect.signature()`` - 3.4 - 5.0 diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index e21e1d94e..0240d2c7c 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -101,10 +101,14 @@ def setup_documenters(app: Any) -> None: def _simple_info(msg: str) -> None: + warnings.warn('_simple_info() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) print(msg) def _simple_warn(msg: str) -> None: + warnings.warn('_simple_warn() is deprecated.', + RemovedInSphinx50Warning, stacklevel=2) print('WARNING: ' + msg, file=sys.stderr) From 088d0591963f6a6f3a1a913f1fc86819d08cb983 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 20 Jan 2021 04:43:39 +0000 Subject: [PATCH 14/30] minor typing fix --- sphinx/transforms/post_transforms/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/transforms/post_transforms/__init__.py b/sphinx/transforms/post_transforms/__init__.py index 2c563540a..1c424050a 100644 --- a/sphinx/transforms/post_transforms/__init__.py +++ b/sphinx/transforms/post_transforms/__init__.py @@ -8,7 +8,7 @@ :license: BSD, see LICENSE for details. """ -from typing import Any, Dict, List, Tuple, Type, cast +from typing import Any, Dict, List, Optional, Tuple, Type, cast from docutils import nodes from docutils.nodes import Element @@ -150,7 +150,7 @@ class ReferencesResolver(SphinxPostTransform): return newnode def warn_missing_reference(self, refdoc: str, typ: str, target: str, - node: pending_xref, domain: Domain) -> None: + node: pending_xref, domain: Optional[Domain]) -> None: warn = node.get('refwarn') if self.config.nitpicky: warn = True From cead0f6ddfaa58ac4e8a365918dbad07b0b633fc Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 19 Jan 2021 00:23:25 +0900 Subject: [PATCH 15/30] linkcheck: Fix race condition that could lead to checking the availability of the same URL twice So far, linkcheck scans all of references and images from documents, and checks them parallel. As a result, some URL would be checked twice (or more) by race condition. This collects the URL via post-transforms, and removes duplicated URLs before checking availability. refs: #4303 --- CHANGES | 3 + doc/extdev/deprecated.rst | 5 ++ sphinx/builders/linkcheck.py | 86 ++++++++++++------- .../conf.py | 1 - .../index.rst | 6 -- tests/test_build_linkcheck.py | 37 -------- 6 files changed, 64 insertions(+), 74 deletions(-) delete mode 100644 tests/roots/test-linkcheck-localserver-two-links/conf.py delete mode 100644 tests/roots/test-linkcheck-localserver-two-links/index.rst diff --git a/CHANGES b/CHANGES index 78680721a..2d392372c 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,7 @@ Incompatible changes Deprecated ---------- +* ``sphinx.builders.linkcheck.node_line_or_0()`` * ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()`` * ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter`` * ``sphinx.ext.autodoc.importer.get_module_members()`` @@ -57,6 +58,8 @@ Bugs fixed + or ^) are used as keystrokes * #8629: html: A type warning for html_use_opensearch is shown twice * #8665: html theme: Could not override globaltoc_maxdepth in theme.conf +* #4304: linkcheck: Fix race condition that could lead to checking the + availability of the same URL twice * #8094: texinfo: image files on the different directory with document are not copied * #8671: :confval:`highlight_options` is not working diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index 0e9a8f9e3..b0ef9b696 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -26,6 +26,11 @@ The following is a list of deprecated interfaces. - (will be) Removed - Alternatives + * - ``sphinx.builders.linkcheck.node_line_or_0()`` + - 3.5 + - 5.0 + - ``sphinx.util.nodes.get_node_line()`` + * - ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()`` - 3.5 - 5.0 diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index cfcc182e1..ce328df33 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -14,11 +14,12 @@ import re import socket import threading import time +import warnings from datetime import datetime, timezone from email.utils import parsedate_to_datetime from html.parser import HTMLParser from os import path -from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple +from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast from urllib.parse import unquote, urlparse from docutils import nodes @@ -28,7 +29,9 @@ from requests.exceptions import HTTPError, TooManyRedirects from sphinx.application import Sphinx from sphinx.builders import Builder +from sphinx.deprecation import RemovedInSphinx40Warning from sphinx.locale import __ +from sphinx.transforms.post_transforms import SphinxPostTransform from sphinx.util import encode_uri, logging, requests from sphinx.util.console import darkgray, darkgreen, purple, red, turquoise # type: ignore from sphinx.util.nodes import get_node_line @@ -37,6 +40,10 @@ logger = logging.getLogger(__name__) uri_re = re.compile('([a-z]+:)?//') # matches to foo:// and // (a protocol relative URL) +Hyperlink = NamedTuple('Hyperlink', (('next_check', float), + ('uri', Optional[str]), + ('docname', Optional[str]), + ('lineno', Optional[int]))) RateLimit = NamedTuple('RateLimit', (('delay', float), ('next_check', float))) DEFAULT_REQUEST_HEADERS = { @@ -52,6 +59,8 @@ def node_line_or_0(node: Element) -> int: PriorityQueue items must be comparable. The line number is part of the tuple used by the PriorityQueue, keep an homogeneous type for comparison. """ + warnings.warn('node_line_or_0() is deprecated.', + RemovedInSphinx40Warning, stacklevel=2) return get_node_line(node) or 0 @@ -98,6 +107,7 @@ class CheckExternalLinksBuilder(Builder): '%(outdir)s/output.txt') def init(self) -> None: + self.hyperlinks = {} # type: Dict[str, Hyperlink] self.to_ignore = [re.compile(x) for x in self.app.config.linkcheck_ignore] self.anchors_ignore = [re.compile(x) for x in self.app.config.linkcheck_anchors_ignore] @@ -406,35 +416,7 @@ class CheckExternalLinksBuilder(Builder): return def write_doc(self, docname: str, doctree: Node) -> None: - logger.info('') - n = 0 - - # reference nodes - for refnode in doctree.traverse(nodes.reference): - if 'refuri' not in refnode: - continue - uri = refnode['refuri'] - lineno = node_line_or_0(refnode) - uri_info = (CHECK_IMMEDIATELY, uri, docname, lineno) - self.wqueue.put(uri_info, False) - n += 1 - - # image nodes - for imgnode in doctree.traverse(nodes.image): - uri = imgnode['candidates'].get('?') - if uri and '://' in uri: - lineno = node_line_or_0(imgnode) - uri_info = (CHECK_IMMEDIATELY, uri, docname, lineno) - self.wqueue.put(uri_info, False) - n += 1 - - done = 0 - while done < n: - self.process_result(self.rqueue.get()) - done += 1 - - if self.broken: - self.app.statuscode = 1 + pass def write_entry(self, what: str, docname: str, filename: str, line: int, uri: str) -> None: @@ -447,14 +429,58 @@ class CheckExternalLinksBuilder(Builder): output.write('\n') def finish(self) -> None: + logger.info('') + n = 0 + + for hyperlink in self.hyperlinks.values(): + self.wqueue.put(hyperlink, False) + n += 1 + + done = 0 + while done < n: + self.process_result(self.rqueue.get()) + done += 1 + + if self.broken: + self.app.statuscode = 1 + self.wqueue.join() # Shutdown threads. for worker in self.workers: self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False) +class HyperlinkCollector(SphinxPostTransform): + builders = ('linkcheck',) + default_priority = 800 + + def run(self, **kwargs: Any) -> None: + builder = cast(CheckExternalLinksBuilder, self.app.builder) + hyperlinks = builder.hyperlinks + + # reference nodes + for refnode in self.document.traverse(nodes.reference): + if 'refuri' not in refnode: + continue + uri = refnode['refuri'] + lineno = get_node_line(refnode) + uri_info = Hyperlink(CHECK_IMMEDIATELY, uri, self.env.docname, lineno) + if uri not in hyperlinks: + hyperlinks[uri] = uri_info + + # image nodes + for imgnode in self.document.traverse(nodes.image): + uri = imgnode['candidates'].get('?') + if uri and '://' in uri: + lineno = get_node_line(imgnode) + uri_info = Hyperlink(CHECK_IMMEDIATELY, uri, self.env.docname, lineno) + if uri not in hyperlinks: + hyperlinks[uri] = uri_info + + def setup(app: Sphinx) -> Dict[str, Any]: app.add_builder(CheckExternalLinksBuilder) + app.add_post_transform(HyperlinkCollector) app.add_config_value('linkcheck_ignore', [], None) app.add_config_value('linkcheck_auth', [], None) diff --git a/tests/roots/test-linkcheck-localserver-two-links/conf.py b/tests/roots/test-linkcheck-localserver-two-links/conf.py deleted file mode 100644 index a45d22e28..000000000 --- a/tests/roots/test-linkcheck-localserver-two-links/conf.py +++ /dev/null @@ -1 +0,0 @@ -exclude_patterns = ['_build'] diff --git a/tests/roots/test-linkcheck-localserver-two-links/index.rst b/tests/roots/test-linkcheck-localserver-two-links/index.rst deleted file mode 100644 index 4c1bcfd6a..000000000 --- a/tests/roots/test-linkcheck-localserver-two-links/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. image:: http://localhost:7777/ - :target: http://localhost:7777/ - -`weblate.org`_ - -.. _weblate.org: http://localhost:7777/ diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index 8427dfb59..55a2bf233 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -573,40 +573,3 @@ def test_limit_rate_bails_out_after_waiting_max_time(app): checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)} next_check = checker.limit_rate(FakeResponse()) assert next_check is None - - -@pytest.mark.sphinx( - 'linkcheck', testroot='linkcheck-localserver-two-links', freshenv=True, -) -def test_priorityqueue_items_are_comparable(app): - with http_server(OKHandler): - app.builder.build_all() - content = (app.outdir / 'output.json').read_text() - rows = [json.loads(x) for x in sorted(content.splitlines())] - assert rows == [ - { - 'filename': 'index.rst', - # Should not be None. - 'lineno': 0, - 'status': 'working', - 'code': 0, - 'uri': 'http://localhost:7777/', - 'info': '', - }, - { - 'filename': 'index.rst', - 'lineno': 0, - 'status': 'working', - 'code': 0, - 'uri': 'http://localhost:7777/', - 'info': '', - }, - { - 'filename': 'index.rst', - 'lineno': 4, - 'status': 'working', - 'code': 0, - 'uri': 'http://localhost:7777/', - 'info': '', - } - ] From bd103a82c9db4f8d1d4d9edf017af9937bf54a06 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 20 Jan 2021 21:14:15 +0900 Subject: [PATCH 16/30] refactor: linkcheck: Make linkcheck builder to a subclass of DummyBuilder After recent refactoring, the linkcheck builder does not do "writing". So it would be better to inherit the DummyBuilder. --- sphinx/builders/linkcheck.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index ce328df33..0ffb2b3d8 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -23,12 +23,12 @@ from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast from urllib.parse import unquote, urlparse from docutils import nodes -from docutils.nodes import Element, Node +from docutils.nodes import Element from requests import Response from requests.exceptions import HTTPError, TooManyRedirects from sphinx.application import Sphinx -from sphinx.builders import Builder +from sphinx.builders.dummy import DummyBuilder from sphinx.deprecation import RemovedInSphinx40Warning from sphinx.locale import __ from sphinx.transforms.post_transforms import SphinxPostTransform @@ -98,7 +98,7 @@ def check_anchor(response: requests.requests.Response, anchor: str) -> bool: return parser.found -class CheckExternalLinksBuilder(Builder): +class CheckExternalLinksBuilder(DummyBuilder): """ Checks for broken external links. """ @@ -406,18 +406,6 @@ class CheckExternalLinksBuilder(Builder): lineno, uri + ' to ' + info) self.write_linkstat(linkstat) - def get_target_uri(self, docname: str, typ: str = None) -> str: - return '' - - def get_outdated_docs(self) -> Set[str]: - return self.env.found_docs - - def prepare_writing(self, docnames: Set[str]) -> None: - return - - def write_doc(self, docname: str, doctree: Node) -> None: - pass - def write_entry(self, what: str, docname: str, filename: str, line: int, uri: str) -> None: with open(path.join(self.outdir, 'output.txt'), 'a') as output: From 73db152cf65d090ef433db9acd49107ffd21aba3 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 20 Jan 2021 23:21:11 +0900 Subject: [PATCH 17/30] Close #5560: napoleon_use_param also affect "other parameters" section --- CHANGES | 2 ++ sphinx/ext/napoleon/docstring.py | 8 +++++++- tests/test_ext_napoleon_docstring.py | 9 +++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index d55f3dc8c..f9844e53b 100644 --- a/CHANGES +++ b/CHANGES @@ -16,6 +16,8 @@ Incompatible changes MathJax configuration may have to set the old MathJax path or update their configuration for version 3. See :mod:`sphinx.ext.mathjax`. * #7784: i18n: The msgid for alt text of image is changed +* #5560: napoleon: :confval:`napoleon_use_param` also affect "other parameters" + section * #7996: manpage: Make a section directory on build manpage by default (see :confval:`man_make_section_directory`) * #8380: html search: search results are wrapped with ``

    `` instead of diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 755088ca5..a5a986e49 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -682,7 +682,13 @@ class GoogleDocstring: return self._parse_generic_section(_('Notes'), use_admonition) def _parse_other_parameters_section(self, section: str) -> List[str]: - return self._format_fields(_('Other Parameters'), self._consume_fields()) + if self._config.napoleon_use_param: + # Allow to declare multiple parameters at once (ex: x, y: int) + fields = self._consume_fields(multiple=True) + return self._format_docutils_params(fields) + else: + fields = self._consume_fields() + return self._format_fields(_('Other Parameters'), fields) def _parse_parameters_section(self, section: str) -> List[str]: if self._config.napoleon_use_param: diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index ec5f90ac2..4ee66aaa5 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1441,12 +1441,18 @@ Parameters ---------- param1 : :class:`MyClass ` instance +Other Parameters +---------------- +param2 : :class:`MyClass ` instance + """ config = Config(napoleon_use_param=False) actual = str(NumpyDocstring(docstring, config)) expected = """\ :Parameters: **param1** (:class:`MyClass ` instance) + +:Other Parameters: **param2** (:class:`MyClass ` instance) """ self.assertEqual(expected, actual) @@ -1455,6 +1461,9 @@ param1 : :class:`MyClass ` instance expected = """\ :param param1: :type param1: :class:`MyClass ` instance + +:param param2: +:type param2: :class:`MyClass ` instance """ self.assertEqual(expected, actual) From 59694da63a92922a902f51b2bb875b3eacf5e816 Mon Sep 17 00:00:00 2001 From: jfbu Date: Wed, 20 Jan 2021 21:15:52 +0100 Subject: [PATCH 18/30] Fix some mark-up issues in latex.rst with "Default:" lines --- doc/latex.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/latex.rst b/doc/latex.rst index 53fe9301a..35e38acb4 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -195,8 +195,8 @@ Keys that you may want to override include: "Bjornstrup". You can also set this to ``''`` to disable fncychap. Default: ``'\\usepackage[Bjarne]{fncychap}'`` for English documents, - ``'\\usepackage[Sonny]{fncychap}'`` for internationalized documents, and - ``''`` for Japanese documents. + ``'\\usepackage[Sonny]{fncychap}'`` for internationalized documents, and + ``''`` for Japanese documents. ``'preamble'`` Additional preamble content. One may move all needed macros into some file @@ -300,7 +300,7 @@ Keys that don't need to be overridden unless in special cases are: "inputenc" package inclusion. Default: ``'\\usepackage[utf8]{inputenc}'`` when using pdflatex, else - ``''`` + ``''`` .. versionchanged:: 1.4.3 Previously ``'\\usepackage[utf8]{inputenc}'`` was used for all @@ -389,7 +389,7 @@ Keys that don't need to be overridden unless in special cases are: key is ignored. Default: ``'\\usepackage{textalpha}'`` or ``''`` if ``fontenc`` does not - include the ``LGR`` option. + include the ``LGR`` option. .. versionadded:: 2.0 @@ -407,7 +407,7 @@ Keys that don't need to be overridden unless in special cases are: `. Default: ``'\\usepackage{geometry}'`` (or - ``'\\usepackage[dvipdfm]{geometry}'`` for Japanese documents) + ``'\\usepackage[dvipdfm]{geometry}'`` for Japanese documents) .. versionadded:: 1.5 @@ -784,14 +784,14 @@ macros may be significant. |warningbdcolors| The colour for the admonition frame. - Default: ``{rgb}{0,0,0}`` (black) + Default: ``{rgb}{0,0,0}`` (black) .. only:: latex |wgbdcolorslatex| The colour for the admonition frame. - Default: ``{rgb}{0,0,0}`` (black) + Default: ``{rgb}{0,0,0}`` (black) |warningbgcolors| The background colours for the respective admonitions. From 3919d7313c7380c53af2604f095a0c0ad6d98f58 Mon Sep 17 00:00:00 2001 From: jfbu Date: Thu, 21 Jan 2021 10:10:48 +0100 Subject: [PATCH 19/30] pour tester token new file: dummy --- dummy | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 dummy diff --git a/dummy b/dummy new file mode 100644 index 000000000..e69de29bb From 43ed4897534e2c3bc893d9698688525fbc44a31b Mon Sep 17 00:00:00 2001 From: jfbu Date: Thu, 21 Jan 2021 10:11:44 +0100 Subject: [PATCH 20/30] test --- dummy | 1 + 1 file changed, 1 insertion(+) diff --git a/dummy b/dummy index e69de29bb..0e46b3147 100644 --- a/dummy +++ b/dummy @@ -0,0 +1 @@ +baba From 753f5404b9af39c7d29aa0e9ecd12d4bcbea92d1 Mon Sep 17 00:00:00 2001 From: jfbu Date: Thu, 21 Jan 2021 10:13:56 +0100 Subject: [PATCH 21/30] test --- dummy | 1 + 1 file changed, 1 insertion(+) diff --git a/dummy b/dummy index 0e46b3147..293c09e23 100644 --- a/dummy +++ b/dummy @@ -1 +1,2 @@ baba +uuu From a23781a2e540ff84ee4b40990833aa69f639ac8b Mon Sep 17 00:00:00 2001 From: jfbu Date: Thu, 21 Jan 2021 10:22:14 +0100 Subject: [PATCH 22/30] Cleaning up accidental mess My apologies. I was testing authentication token, pushing master to my forked repo. But I ended up accidentally pushing to sphinx-doc/sphinx, and force pushing afterwards to clean up then was rejected. deleted: dummy --- dummy | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 dummy diff --git a/dummy b/dummy deleted file mode 100644 index 293c09e23..000000000 --- a/dummy +++ /dev/null @@ -1,2 +0,0 @@ -baba -uuu From 52fde7e7b1a304c69021f3a801f128b9f3c7e26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Freitag?= Date: Thu, 21 Jan 2021 16:39:40 +0100 Subject: [PATCH 23/30] Match linkcheck deprecation warning version with deprecated.rst Deprecated.rst states the node_line_or_0 helper will be removed in Sphinx 5.0, use a RemovedInSphinx50Warning. --- sphinx/builders/linkcheck.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index ce328df33..f405910e9 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -29,7 +29,7 @@ from requests.exceptions import HTTPError, TooManyRedirects from sphinx.application import Sphinx from sphinx.builders import Builder -from sphinx.deprecation import RemovedInSphinx40Warning +from sphinx.deprecation import RemovedInSphinx50Warning from sphinx.locale import __ from sphinx.transforms.post_transforms import SphinxPostTransform from sphinx.util import encode_uri, logging, requests @@ -60,7 +60,7 @@ def node_line_or_0(node: Element) -> int: tuple used by the PriorityQueue, keep an homogeneous type for comparison. """ warnings.warn('node_line_or_0() is deprecated.', - RemovedInSphinx40Warning, stacklevel=2) + RemovedInSphinx50Warning, stacklevel=2) return get_node_line(node) or 0 From b2bb12ad9193ce0c0047ae7bf3cde9ea10f46d8a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 22 Jan 2021 00:33:39 +0900 Subject: [PATCH 24/30] Fix #8720: viewcode: module pages are generated for epub on incremental build The module pages should be generated for epub only if enabled via configuration. But they are generated after the build for other viewcode-supported builders. This checks the current builder on generating module pages. --- CHANGES | 1 + sphinx/ext/viewcode.py | 4 ++++ tests/test_ext_viewcode.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/CHANGES b/CHANGES index bc9dfa335..f38a0dc60 100644 --- a/CHANGES +++ b/CHANGES @@ -65,6 +65,7 @@ Bugs fixed availability of the same URL twice * #8094: texinfo: image files on the different directory with document are not copied +* #8720: viewcode: module pages are generated for epub on incremental build * #8671: :confval:`highlight_options` is not working * #8341: C, fix intersphinx lookup types for names in declarations. * C, C++: in general fix intersphinx and role lookup types. diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py index a7d52a91c..c2bcee4f5 100644 --- a/sphinx/ext/viewcode.py +++ b/sphinx/ext/viewcode.py @@ -182,6 +182,10 @@ def collect_pages(app: Sphinx) -> Generator[Tuple[str, Dict[str, Any], str], Non env = app.builder.env if not hasattr(env, '_viewcode_modules'): return + if app.builder.name == "singlehtml": + return + if app.builder.name.startswith("epub") and not env.config.viewcode_enable_epub: + return highlighter = app.builder.highlighter # type: ignore urito = app.builder.get_relative_uri diff --git a/tests/test_ext_viewcode.py b/tests/test_ext_viewcode.py index 79864095b..21002966b 100644 --- a/tests/test_ext_viewcode.py +++ b/tests/test_ext_viewcode.py @@ -49,6 +49,21 @@ def test_viewcode(app, status, warning): ' """\n') in result +@pytest.mark.sphinx('epub', testroot='ext-viewcode') +def test_viewcode_epub_default(app, status, warning): + app.builder.build_all() + + assert not (app.outdir / '_modules/spam/mod1.xhtml').exists() + + +@pytest.mark.sphinx('epub', testroot='ext-viewcode', + confoverrides={'viewcode_enable_epub': True}) +def test_viewcode_epub_enabled(app, status, warning): + app.builder.build_all() + + assert (app.outdir / '_modules/spam/mod1.xhtml').exists() + + @pytest.mark.sphinx(testroot='ext-viewcode', tags=['test_linkcode']) def test_linkcode(app, status, warning): app.builder.build(['objects']) From aa5e4e2da0b47966f0a0f5779132d9e865728af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Freitag?= Date: Thu, 21 Jan 2021 17:02:11 +0100 Subject: [PATCH 25/30] Deprecate linkcheck builder {broken,good,redirected} These attributes were used to cache checked links and avoid issuing another web request to the same URI. Since 82ef497a8c88f0f6e50d84520e7276bfbf65025d, links are pre-processed to ensure uniqueness. This caching the results of checked links is no longer useful. --- CHANGES | 3 ++ doc/extdev/deprecated.rst | 15 ++++++++++ sphinx/builders/linkcheck.py | 55 +++++++++++++++++++++++++++--------- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/CHANGES b/CHANGES index bc9dfa335..1e541f8e2 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,9 @@ Incompatible changes Deprecated ---------- +* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken`` +* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good`` +* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected`` * ``sphinx.builders.linkcheck.node_line_or_0()`` * ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()`` * ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter`` diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index d7d041279..17a606a49 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -26,6 +26,21 @@ The following is a list of deprecated interfaces. - (will be) Removed - Alternatives + * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken`` + - 3.5 + - 5.0 + - N/A + + * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good`` + - 3.5 + - 5.0 + - N/A + + * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected`` + - 3.5 + - 5.0 + - N/A + * - ``sphinx.builders.linkcheck.node_line_or_0()`` - 3.5 - 5.0 diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index 0ffb2b3d8..5a32cb319 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -29,7 +29,7 @@ from requests.exceptions import HTTPError, TooManyRedirects from sphinx.application import Sphinx from sphinx.builders.dummy import DummyBuilder -from sphinx.deprecation import RemovedInSphinx40Warning +from sphinx.deprecation import RemovedInSphinx40Warning, RemovedInSphinx50Warning from sphinx.locale import __ from sphinx.transforms.post_transforms import SphinxPostTransform from sphinx.util import encode_uri, logging, requests @@ -113,9 +113,9 @@ class CheckExternalLinksBuilder(DummyBuilder): for x in self.app.config.linkcheck_anchors_ignore] self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info in self.app.config.linkcheck_auth] - self.good = set() # type: Set[str] - self.broken = {} # type: Dict[str, str] - self.redirected = {} # type: Dict[str, Tuple[str, int]] + self._good = set() # type: Set[str] + self._broken = {} # type: Dict[str, str] + self._redirected = {} # type: Dict[str, Tuple[str, int]] # set a timeout for non-responding servers socket.setdefaulttimeout(5.0) # create output file @@ -133,6 +133,33 @@ class CheckExternalLinksBuilder(DummyBuilder): thread.start() self.workers.append(thread) + @property + def good(self): + warnings.warn( + "%s.%s is deprecated." % (self.__class__.__name__, "good"), + RemovedInSphinx50Warning, + stacklevel=2, + ) + return self._good + + @property + def broken(self): + warnings.warn( + "%s.%s is deprecated." % (self.__class__.__name__, "broken"), + RemovedInSphinx50Warning, + stacklevel=2, + ) + return self._broken + + @property + def redirected(self): + warnings.warn( + "%s.%s is deprecated." % (self.__class__.__name__, "redirected"), + RemovedInSphinx50Warning, + stacklevel=2, + ) + return self._redirected + def check_thread(self) -> None: kwargs = {} if self.app.config.linkcheck_timeout: @@ -261,14 +288,14 @@ class CheckExternalLinksBuilder(DummyBuilder): if rex.match(uri): return 'ignored', '', 0 else: - self.broken[uri] = '' + self._broken[uri] = '' return 'broken', '', 0 - elif uri in self.good: + elif uri in self._good: return 'working', 'old', 0 - elif uri in self.broken: - return 'broken', self.broken[uri], 0 - elif uri in self.redirected: - return 'redirected', self.redirected[uri][0], self.redirected[uri][1] + elif uri in self._broken: + return 'broken', self._broken[uri], 0 + elif uri in self._redirected: + return 'redirected', self._redirected[uri][0], self._redirected[uri][1] for rex in self.to_ignore: if rex.match(uri): return 'ignored', '', 0 @@ -280,11 +307,11 @@ class CheckExternalLinksBuilder(DummyBuilder): break if status == "working": - self.good.add(uri) + self._good.add(uri) elif status == "broken": - self.broken[uri] = info + self._broken[uri] = info elif status == "redirected": - self.redirected[uri] = (info, code) + self._redirected[uri] = (info, code) return (status, info, code) @@ -429,7 +456,7 @@ class CheckExternalLinksBuilder(DummyBuilder): self.process_result(self.rqueue.get()) done += 1 - if self.broken: + if self._broken: self.app.statuscode = 1 self.wqueue.join() From e59365923b4ba5d4f4a61dc772c648a4fa636138 Mon Sep 17 00:00:00 2001 From: markus-oehme-pg40 <63052527+markus-oehme-pg40@users.noreply.github.com> Date: Fri, 22 Jan 2021 09:29:19 +0100 Subject: [PATCH 26/30] doc: Link to detailed extension documentation I missed that there are actually two sections explaining extensions in the docs and just saw the examples based one. This lead to me thinking "kind of lousy documentation, but that's probably par for the course". Would there have been a pointer to the detailed extension API description it would have saved me hours of guesswork. --- doc/development/index.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/development/index.rst b/doc/development/index.rst index 04918acd6..b4a7920ba 100644 --- a/doc/development/index.rst +++ b/doc/development/index.rst @@ -2,10 +2,12 @@ Extending Sphinx ================ -This guide is aimed at those wishing to develop their own extensions for -Sphinx. Sphinx possesses significant extensibility capabilities including the -ability to hook into almost every point of the build process. If you simply -wish to use Sphinx with existing extensions, refer to :doc:`/usage/index`. +This guide is aimed at giving a quick introduction for those wishing to +develop their own extensions for Sphinx. Sphinx possesses significant +extensibility capabilities including the ability to hook into almost every +point of the build process. If you simply wish to use Sphinx with existing +extensions, refer to :doc:`/usage/index`. For a more detailed discussion of +the extension interface see :doc:`/extdev/index`. .. toctree:: :maxdepth: 2 From 1679a0916da70f2ba25873a7babc0b1b06711325 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 22 Jan 2021 21:26:58 +0900 Subject: [PATCH 28/30] Fix mypy violations (with mypy-0.800) --- setup.py | 2 +- sphinx/builders/html/__init__.py | 4 ++-- sphinx/util/i18n.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 8d40de1a8..dfc80578f 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ extras_require = { 'lint': [ 'flake8>=3.5.0', 'isort', - 'mypy>=0.790', + 'mypy>=0.800', 'docutils-stubs', ], 'test': [ diff --git a/sphinx/builders/html/__init__.py b/sphinx/builders/html/__init__.py index 2c96ede32..699ab8189 100644 --- a/sphinx/builders/html/__init__.py +++ b/sphinx/builders/html/__init__.py @@ -92,7 +92,7 @@ class Stylesheet(str): filename = None # type: str def __new__(cls, filename: str, *args: str, **attributes: str) -> "Stylesheet": - self = str.__new__(cls, filename) # type: ignore + self = str.__new__(cls, filename) self.filename = filename self.attributes = attributes self.attributes.setdefault('rel', 'stylesheet') @@ -115,7 +115,7 @@ class JavaScript(str): filename = None # type: str def __new__(cls, filename: str, **attributes: str) -> "JavaScript": - self = str.__new__(cls, filename) # type: ignore + self = str.__new__(cls, filename) self.filename = filename self.attributes = attributes diff --git a/sphinx/util/i18n.py b/sphinx/util/i18n.py index e149976ef..3c1030d62 100644 --- a/sphinx/util/i18n.py +++ b/sphinx/util/i18n.py @@ -34,7 +34,7 @@ if False: logger = logging.getLogger(__name__) -LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain,charset') +LocaleFileInfoBase = namedtuple('LocaleFileInfoBase', 'base_dir,domain,charset') class CatalogInfo(LocaleFileInfoBase): From 5260143afe020676513b87709fbdd2612d2bdeb3 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 19 Jan 2021 03:18:39 +0900 Subject: [PATCH 29/30] Fix #8704: viewcode: anchors are generated in incremental build The anchors for viewcode was generated in the reading phase only if supported builder is used. It causes anchors are missing on the incremental build after the build for non supported builder. This introduces `viewcode_anchor` node to insert the anchor even if non supported builders. They will be converted to the anchor tag in the resolving phase for supported builders. Or, they will be removed for non supported builders. --- CHANGES | 2 ++ doc/extdev/deprecated.rst | 5 +++ sphinx/ext/viewcode.py | 64 ++++++++++++++++++++++++++++++-------- tests/test_ext_viewcode.py | 6 ++++ 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/CHANGES b/CHANGES index 8c7160a98..c9f572bba 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,7 @@ Incompatible changes Deprecated ---------- +* pending_xref node for viewcode extension * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken`` * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good`` * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected`` @@ -69,6 +70,7 @@ Bugs fixed * #8094: texinfo: image files on the different directory with document are not copied * #8720: viewcode: module pages are generated for epub on incremental build +* #8704: viewcode: anchors are generated in incremental build after singlehtml * #8671: :confval:`highlight_options` is not working * #8341: C, fix intersphinx lookup types for names in declarations. * C, C++: in general fix intersphinx and role lookup types. diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index 17a606a49..e8ec65dfe 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -26,6 +26,11 @@ The following is a list of deprecated interfaces. - (will be) Removed - Alternatives + * - pending_xref node for viewcode extension + - 3.5 + - 5.0 + - ``sphinx.ext.viewcode.viewcode_anchor`` + * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken`` - 3.5 - 5.0 diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py index c2bcee4f5..baf86dbbf 100644 --- a/sphinx/ext/viewcode.py +++ b/sphinx/ext/viewcode.py @@ -10,6 +10,7 @@ import posixpath import traceback +import warnings from os import path from typing import Any, Dict, Generator, Iterable, Optional, Set, Tuple, cast @@ -19,10 +20,13 @@ from docutils.nodes import Element, Node import sphinx from sphinx import addnodes from sphinx.application import Sphinx +from sphinx.builders import Builder from sphinx.builders.html import StandaloneHTMLBuilder +from sphinx.deprecation import RemovedInSphinx50Warning from sphinx.environment import BuildEnvironment from sphinx.locale import _, __ from sphinx.pycode import ModuleAnalyzer +from sphinx.transforms.post_transforms import SphinxPostTransform from sphinx.util import get_full_modname, logging, status_iterator from sphinx.util.nodes import make_refnode @@ -32,6 +36,15 @@ logger = logging.getLogger(__name__) OUTPUT_DIRNAME = '_modules' +class viewcode_anchor(Element): + """Node for viewcode anchors. + + This node will be processed in the resolving phase. + For viewcode supported builders, they will be all converted to the anchors. + For not supported builders, they will be removed. + """ + + def _get_full_modname(app: Sphinx, modname: str, attribute: str) -> Optional[str]: try: return get_full_modname(modname, attribute) @@ -50,14 +63,21 @@ def _get_full_modname(app: Sphinx, modname: str, attribute: str) -> Optional[str return None +def is_supported_builder(builder: Builder) -> bool: + if builder.format != 'html': + return False + elif builder.name == 'singlehtml': + return False + elif builder.name.startswith('epub') and not builder.config.viewcode_enable_epub: + return False + else: + return True + + def doctree_read(app: Sphinx, doctree: Node) -> None: env = app.builder.env if not hasattr(env, '_viewcode_modules'): env._viewcode_modules = {} # type: ignore - if app.builder.name == "singlehtml": - return - if app.builder.name.startswith("epub") and not env.config.viewcode_enable_epub: - return def has_tag(modname: str, fullname: str, docname: str, refname: str) -> bool: entry = env._viewcode_modules.get(modname, None) # type: ignore @@ -115,12 +135,7 @@ def doctree_read(app: Sphinx, doctree: Node) -> None: continue names.add(fullname) pagename = posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/')) - inline = nodes.inline('', _('[source]'), classes=['viewcode-link']) - onlynode = addnodes.only(expr='html') - onlynode += addnodes.pending_xref('', inline, reftype='viewcode', refdomain='std', - refexplicit=False, reftarget=pagename, - refid=fullname, refdoc=env.docname) - signode += onlynode + signode += viewcode_anchor(reftarget=pagename, refid=fullname, refdoc=env.docname) def env_merge_info(app: Sphinx, env: BuildEnvironment, docnames: Iterable[str], @@ -134,10 +149,34 @@ def env_merge_info(app: Sphinx, env: BuildEnvironment, docnames: Iterable[str], env._viewcode_modules.update(other._viewcode_modules) # type: ignore +class ViewcodeAnchorTransform(SphinxPostTransform): + """Convert or remove viewcode_anchor nodes depends on builder.""" + default_priority = 100 + + def run(self, **kwargs: Any) -> None: + if is_supported_builder(self.app.builder): + self.convert_viewcode_anchors() + else: + self.remove_viewcode_anchors() + + def convert_viewcode_anchors(self) -> None: + for node in self.document.traverse(viewcode_anchor): + anchor = nodes.inline('', _('[source]'), classes=['viewcode-link']) + refnode = make_refnode(self.app.builder, node['refdoc'], node['reftarget'], + node['refid'], anchor) + node.replace_self(refnode) + + def remove_viewcode_anchors(self) -> None: + for node in self.document.traverse(viewcode_anchor): + node.parent.remove(node) + + def missing_reference(app: Sphinx, env: BuildEnvironment, node: Element, contnode: Node ) -> Optional[Node]: # resolve our "viewcode" reference nodes -- they need special treatment if node['reftype'] == 'viewcode': + warnings.warn('viewcode extension is no longer use pending_xref node. ' + 'Please update your extension.', RemovedInSphinx50Warning) return make_refnode(app.builder, node['refdoc'], node['reftarget'], node['refid'], contnode) @@ -182,9 +221,7 @@ def collect_pages(app: Sphinx) -> Generator[Tuple[str, Dict[str, Any], str], Non env = app.builder.env if not hasattr(env, '_viewcode_modules'): return - if app.builder.name == "singlehtml": - return - if app.builder.name.startswith("epub") and not env.config.viewcode_enable_epub: + if not is_supported_builder(app.builder): return highlighter = app.builder.highlighter # type: ignore urito = app.builder.get_relative_uri @@ -292,6 +329,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: # app.add_config_value('viewcode_exclude_modules', [], 'env') app.add_event('viewcode-find-source') app.add_event('viewcode-follow-imported') + app.add_post_transform(ViewcodeAnchorTransform) return { 'version': sphinx.__display_version__, 'env_version': 1, diff --git a/tests/test_ext_viewcode.py b/tests/test_ext_viewcode.py index 21002966b..d75fb7196 100644 --- a/tests/test_ext_viewcode.py +++ b/tests/test_ext_viewcode.py @@ -55,6 +55,9 @@ def test_viewcode_epub_default(app, status, warning): assert not (app.outdir / '_modules/spam/mod1.xhtml').exists() + result = (app.outdir / 'index.xhtml').read_text() + assert result.count('href="_modules/spam/mod1.xhtml#func1"') == 0 + @pytest.mark.sphinx('epub', testroot='ext-viewcode', confoverrides={'viewcode_enable_epub': True}) @@ -63,6 +66,9 @@ def test_viewcode_epub_enabled(app, status, warning): assert (app.outdir / '_modules/spam/mod1.xhtml').exists() + result = (app.outdir / 'index.xhtml').read_text() + assert result.count('href="_modules/spam/mod1.xhtml#func1"') == 2 + @pytest.mark.sphinx(testroot='ext-viewcode', tags=['test_linkcode']) def test_linkcode(app, status, warning): From d6f41227d7dd42594f85438ec816fb3df180e0a7 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 22 Jan 2021 00:02:45 +0900 Subject: [PATCH 30/30] Fix #8714: html: kbd role with "Caps Lock" rendered incorrectly `:kbd:` role has to support keys in consist of multiple words (ex. caps lock, num lock, etc.) --- CHANGES | 1 + sphinx/builders/html/transforms.py | 27 ++++++++++++++++++++++++--- tests/test_markup.py | 7 +++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 8c7160a98..1ba8d2f40 100644 --- a/CHANGES +++ b/CHANGES @@ -63,6 +63,7 @@ Bugs fixed * #8618: html: kbd role produces incorrect HTML when compound-key separators (-, + or ^) are used as keystrokes * #8629: html: A type warning for html_use_opensearch is shown twice +* #8714: html: kbd role with "Caps Lock" rendered incorrectly * #8665: html theme: Could not override globaltoc_maxdepth in theme.conf * #4304: linkcheck: Fix race condition that could lead to checking the availability of the same URL twice diff --git a/sphinx/builders/html/transforms.py b/sphinx/builders/html/transforms.py index 29a989936..cb9af5f28 100644 --- a/sphinx/builders/html/transforms.py +++ b/sphinx/builders/html/transforms.py @@ -9,7 +9,7 @@ """ import re -from typing import Any, Dict +from typing import Any, Dict, List from docutils import nodes @@ -38,18 +38,29 @@ class KeyboardTransform(SphinxPostTransform): default_priority = 400 builders = ('html',) pattern = re.compile(r'(?<=.)(-|\+|\^|\s+)(?=.)') + multiwords_keys = (('caps', 'lock'), + ('page' 'down'), + ('page', 'up'), + ('scroll' 'lock'), + ('num', 'lock'), + ('sys' 'rq'), + ('back' 'space')) def run(self, **kwargs: Any) -> None: matcher = NodeMatcher(nodes.literal, classes=["kbd"]) for node in self.document.traverse(matcher): # type: nodes.literal parts = self.pattern.split(node[-1].astext()) - if len(parts) == 1: + if len(parts) == 1 or self.is_multiwords_key(parts): continue node['classes'].append('compound') node.pop() while parts: - key = parts.pop(0) + if self.is_multiwords_key(parts): + key = ''.join(parts[:3]) + parts[:3] = [] + else: + key = parts.pop(0) node += nodes.literal('', key, classes=["kbd"]) try: @@ -59,6 +70,16 @@ class KeyboardTransform(SphinxPostTransform): except IndexError: pass + def is_multiwords_key(self, parts: List[str]) -> bool: + if len(parts) >= 3 and parts[1].strip() == '': + name = parts[0].lower(), parts[2].lower() + if name in self.multiwords_keys: + return True + else: + return False + else: + return False + def setup(app: Sphinx) -> Dict[str, Any]: app.add_post_transform(KeyboardTransform) diff --git a/tests/test_markup.py b/tests/test_markup.py index e762dbb3b..8341b8826 100644 --- a/tests/test_markup.py +++ b/tests/test_markup.py @@ -284,6 +284,13 @@ def get_verifier(verify, verify_re): '

    -

    ', '\\sphinxkeyboard{\\sphinxupquote{\\sphinxhyphen{}}}', ), + ( + # kbd role + 'verify', + ':kbd:`Caps Lock`', + '

    Caps Lock

    ', + '\\sphinxkeyboard{\\sphinxupquote{Caps Lock}}', + ), ( # non-interpolation of dashes in option role 'verify_re',