From b150d79b4b00e6d2d8c6c33b45b955c0b134bdbb Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Wed, 13 Jun 2018 20:29:46 +0200 Subject: [PATCH 01/88] BuildDoc: link verbosity to distutils' This preserves the current behavior (using 0), but picks up distutils' `-v` and `-q` options. --- sphinx/setup_command.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sphinx/setup_command.py b/sphinx/setup_command.py index c54179227..f281ce6ef 100644 --- a/sphinx/setup_command.py +++ b/sphinx/setup_command.py @@ -105,7 +105,8 @@ class BuildDoc(Command): self.config_dir = None # type: unicode self.link_index = False self.copyright = '' - self.verbosity = 0 + # Link verbosity to distutils' (which uses 1 by default). + self.verbosity = self.distribution.verbose - 1 self.traceback = False def _guess_source_dir(self): @@ -185,7 +186,8 @@ class BuildDoc(Command): builder_target_dir, self.doctree_dir, builder, confoverrides, status_stream, freshenv=self.fresh_env, - warningiserror=self.warning_is_error) + warningiserror=self.warning_is_error, + verbosity=self.verbosity) app.build(force_all=self.all_files) if app.statuscode: raise DistutilsExecError( From 72ca2bdffc2947d818b1cd36484debe577a25dbc Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 8 Jul 2020 16:45:07 +1000 Subject: [PATCH 02/88] Add autosummary_filename_map config to avoid clashes --- doc/usage/extensions/autosummary.rst | 9 ++++++ sphinx/ext/autosummary/__init__.py | 1 + sphinx/ext/autosummary/generate.py | 12 ++++++-- tests/roots/test-autosummary/sphinx.rst | 39 ++++++++----------------- tests/test_ext_autosummary.py | 16 ++++++++++ 5 files changed, 48 insertions(+), 29 deletions(-) diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index 38d18361e..c1f59170b 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -195,6 +195,15 @@ also use these config values: .. versionadded:: 2.1 +.. confval:: autosummary_filename_map + + A dict mapping object names to filenames. This is necessary to avoid + filename conflicts where multiple objects have names that are + indistinguishable when case is ignored, on file systems where filenames + are case-insensitive. + + .. versionadded:: 3.2 + Customizing templates --------------------- diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index 0984377c5..ece16c15b 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -790,5 +790,6 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_config_value('autosummary_mock_imports', lambda config: config.autodoc_mock_imports, 'env') app.add_config_value('autosummary_imported_members', [], False, [bool]) + app.add_config_value('autosummary_filename_map', {}, 'html') return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index d908e2088..2693c9008 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -28,7 +28,7 @@ import sys import warnings from gettext import NullTranslations from os import path -from typing import Any, Callable, Dict, List, NamedTuple, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Set, Tuple, Type, Union from jinja2 import TemplateNotFound from jinja2.sandbox import SandboxedEnvironment @@ -393,6 +393,14 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, # keep track of new files new_files = [] + if app: + filename_map = app.config.autosummary_filename_map + if not isinstance(filename_map, Mapping): + raise TypeError('autosummary_filename_map should be a mapping from ' + 'strings to strings') + else: + filename_map = {} + # write for entry in sorted(set(items), key=str): if entry.path is None: @@ -418,7 +426,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, imported_members, app, entry.recursive, context, modname, qualname) - filename = os.path.join(path, name + suffix) + filename = os.path.join(path, filename_map.get(name, name) + suffix) if os.path.isfile(filename): with open(filename, encoding=encoding) as f: old_content = f.read() diff --git a/tests/roots/test-autosummary/sphinx.rst b/tests/roots/test-autosummary/sphinx.rst index fc1a35a0e..7b47b2c0e 100644 --- a/tests/roots/test-autosummary/sphinx.rst +++ b/tests/roots/test-autosummary/sphinx.rst @@ -1,31 +1,16 @@ -Autosummary test -================ +sphinx +====== -.. autosummary:: - :toctree: generated +.. automodule:: sphinx - sphinx.application.Sphinx + + + -.. currentmodule:: sphinx.application + + + -.. autoclass:: TemplateBridge - - Basic test - - .. autosummary:: - - render -- some ignored stuff goes here - render_string More ignored stuff - - Test with tildes - - .. autosummary:: - - ~TemplateBridge.render - ~TemplateBridge.render_string - - Methods: - - .. automethod:: render - - .. automethod:: render_string + + + \ No newline at end of file diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 3dd90b8ce..fe16092dd 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -386,6 +386,22 @@ def test_autosummary_recursive(app, status, warning): assert 'package.package.module' in content +@pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive', + confoverrides={'autosummary_filename_map': + {"package": "package_mangled", + "package.package": "package_package_mangled"}}) +def test_autosummary_filename_map(app, status, warning): + app.build() + + assert (app.srcdir / 'generated' / 'package_mangled.rst').exists() + assert not (app.srcdir / 'generated' / 'package.rst').exists() + assert (app.srcdir / 'generated' / 'package.module.rst').exists() + assert (app.srcdir / 'generated' / 'package.module_importfail.rst').exists() is False + assert (app.srcdir / 'generated' / 'package_package_mangled.rst').exists() + assert not (app.srcdir / 'generated' / 'package.package.rst').exists() + assert (app.srcdir / 'generated' / 'package.package.module.rst').exists() + + @pytest.mark.sphinx('latex', **default_kw) def test_autosummary_latex_table_colspec(app, status, warning): app.builder.build_all() From d4010540e551f4d34946b605aae700db660ab69e Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 8 Jul 2020 20:29:17 +1000 Subject: [PATCH 03/88] Flake8 --- sphinx/ext/autosummary/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 2693c9008..7c5a6e0a4 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -28,7 +28,7 @@ import sys import warnings from gettext import NullTranslations from os import path -from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Set, Tuple, Type, Union +from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Set, Tuple, Union from jinja2 import TemplateNotFound from jinja2.sandbox import SandboxedEnvironment From 4e0ff5cac2a47fab136955a4aab4da75b599c5c9 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 8 Jul 2020 22:36:40 +1000 Subject: [PATCH 04/88] Fix attempt with new test root --- sphinx/ext/autosummary/generate.py | 1 + .../autosummary_dummy_module.py | 41 +++++++++++++++++++ .../autosummary_importfail.py | 4 ++ .../test-ext-autosummary-filename-map/conf.py | 10 +++++ .../index.rst | 15 +++++++ tests/test_ext_autosummary.py | 18 ++++---- 6 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 tests/roots/test-ext-autosummary-filename-map/autosummary_dummy_module.py create mode 100644 tests/roots/test-ext-autosummary-filename-map/autosummary_importfail.py create mode 100644 tests/roots/test-ext-autosummary-filename-map/conf.py create mode 100644 tests/roots/test-ext-autosummary-filename-map/index.rst diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 7c5a6e0a4..4454e47fc 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -74,6 +74,7 @@ class DummyApplication: self.warningiserror = False self.config.add('autosummary_context', {}, True, None) + self.config.add('autosummary_filename_map', {}, True, None) self.config.init_values() def emit_firstresult(self, *args: Any) -> None: diff --git a/tests/roots/test-ext-autosummary-filename-map/autosummary_dummy_module.py b/tests/roots/test-ext-autosummary-filename-map/autosummary_dummy_module.py new file mode 100644 index 000000000..85981a0f8 --- /dev/null +++ b/tests/roots/test-ext-autosummary-filename-map/autosummary_dummy_module.py @@ -0,0 +1,41 @@ +from os import path # NOQA +from typing import Union + + +class Foo: + class Bar: + pass + + def __init__(self): + pass + + def bar(self): + pass + + @property + def baz(self): + pass + + +class _Baz: + pass + + +def bar(x: Union[int, str], y: int = 1) -> None: + pass + + +def _quux(): + pass + + +class Exc(Exception): + pass + + +class _Exc(Exception): + pass + + +#: a module-level attribute +qux = 2 diff --git a/tests/roots/test-ext-autosummary-filename-map/autosummary_importfail.py b/tests/roots/test-ext-autosummary-filename-map/autosummary_importfail.py new file mode 100644 index 000000000..9e3f9f195 --- /dev/null +++ b/tests/roots/test-ext-autosummary-filename-map/autosummary_importfail.py @@ -0,0 +1,4 @@ +import sys + +# Fail module import in a catastrophic way +sys.exit(1) diff --git a/tests/roots/test-ext-autosummary-filename-map/conf.py b/tests/roots/test-ext-autosummary-filename-map/conf.py new file mode 100644 index 000000000..55c769c6f --- /dev/null +++ b/tests/roots/test-ext-autosummary-filename-map/conf.py @@ -0,0 +1,10 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) + +extensions = ['sphinx.ext.autosummary'] +autosummary_generate = True + +# The suffix of source filenames. +source_suffix = '.rst' diff --git a/tests/roots/test-ext-autosummary-filename-map/index.rst b/tests/roots/test-ext-autosummary-filename-map/index.rst new file mode 100644 index 000000000..9f657bb73 --- /dev/null +++ b/tests/roots/test-ext-autosummary-filename-map/index.rst @@ -0,0 +1,15 @@ + +:autolink:`autosummary_dummy_module.Foo` + +:autolink:`autosummary_importfail` + +.. autosummary:: + :toctree: generated + :caption: An autosummary + + autosummary_dummy_module + autosummary_dummy_module.Foo + autosummary_dummy_module.Foo.Bar + autosummary_dummy_module.bar + autosummary_dummy_module.qux + autosummary_importfail diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index fe16092dd..7a35d7c90 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -386,20 +386,18 @@ def test_autosummary_recursive(app, status, warning): assert 'package.package.module' in content -@pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive', +@pytest.mark.sphinx('dummy', testroot='ext-autosummary-filename-map', confoverrides={'autosummary_filename_map': - {"package": "package_mangled", - "package.package": "package_package_mangled"}}) + {"autosummary_dummy_module": "module_mangled", + "autosummary_dummy_module.bar": "bar"}}) def test_autosummary_filename_map(app, status, warning): app.build() - assert (app.srcdir / 'generated' / 'package_mangled.rst').exists() - assert not (app.srcdir / 'generated' / 'package.rst').exists() - assert (app.srcdir / 'generated' / 'package.module.rst').exists() - assert (app.srcdir / 'generated' / 'package.module_importfail.rst').exists() is False - assert (app.srcdir / 'generated' / 'package_package_mangled.rst').exists() - assert not (app.srcdir / 'generated' / 'package.package.rst').exists() - assert (app.srcdir / 'generated' / 'package.package.module.rst').exists() + assert (app.srcdir / 'generated' / 'module_mangled.rst').exists() + assert not (app.srcdir / 'generated' / 'autosummary_dummy_module.rst').exists() + assert (app.srcdir / 'generated' / 'bar.rst').exists() + assert not (app.srcdir / 'generated' / 'autosummary_dummy_module.bar.rst').exists() + assert (app.srcdir / 'generated' / 'autosummary_dummy_module.Foo.rst').exists() @pytest.mark.sphinx('latex', **default_kw) From 52c173bdb9fa804c14c3852ae339781a50035e67 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Fri, 10 Jul 2020 11:30:06 +1000 Subject: [PATCH 05/88] Responding to comments --- tests/roots/test-autosummary/sphinx.rst | 39 +++++++++++++------ .../autosummary_importfail.py | 4 -- .../test-ext-autosummary-filename-map/conf.py | 7 ++-- .../index.rst | 1 - tests/test_ext_autosummary.py | 5 +-- 5 files changed, 32 insertions(+), 24 deletions(-) delete mode 100644 tests/roots/test-ext-autosummary-filename-map/autosummary_importfail.py diff --git a/tests/roots/test-autosummary/sphinx.rst b/tests/roots/test-autosummary/sphinx.rst index 7b47b2c0e..fc1a35a0e 100644 --- a/tests/roots/test-autosummary/sphinx.rst +++ b/tests/roots/test-autosummary/sphinx.rst @@ -1,16 +1,31 @@ -sphinx -====== +Autosummary test +================ -.. automodule:: sphinx +.. autosummary:: + :toctree: generated - - - + sphinx.application.Sphinx - - - +.. currentmodule:: sphinx.application - - - \ No newline at end of file +.. autoclass:: TemplateBridge + + Basic test + + .. autosummary:: + + render -- some ignored stuff goes here + render_string More ignored stuff + + Test with tildes + + .. autosummary:: + + ~TemplateBridge.render + ~TemplateBridge.render_string + + Methods: + + .. automethod:: render + + .. automethod:: render_string diff --git a/tests/roots/test-ext-autosummary-filename-map/autosummary_importfail.py b/tests/roots/test-ext-autosummary-filename-map/autosummary_importfail.py deleted file mode 100644 index 9e3f9f195..000000000 --- a/tests/roots/test-ext-autosummary-filename-map/autosummary_importfail.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys - -# Fail module import in a catastrophic way -sys.exit(1) diff --git a/tests/roots/test-ext-autosummary-filename-map/conf.py b/tests/roots/test-ext-autosummary-filename-map/conf.py index 55c769c6f..17e2fa445 100644 --- a/tests/roots/test-ext-autosummary-filename-map/conf.py +++ b/tests/roots/test-ext-autosummary-filename-map/conf.py @@ -5,6 +5,7 @@ sys.path.insert(0, os.path.abspath('.')) extensions = ['sphinx.ext.autosummary'] autosummary_generate = True - -# The suffix of source filenames. -source_suffix = '.rst' +autosummary_filename_map = { + "autosummary_dummy_module": "module_mangled", + "autosummary_dummy_module.bar": "bar" +} diff --git a/tests/roots/test-ext-autosummary-filename-map/index.rst b/tests/roots/test-ext-autosummary-filename-map/index.rst index 9f657bb73..ef27d9d19 100644 --- a/tests/roots/test-ext-autosummary-filename-map/index.rst +++ b/tests/roots/test-ext-autosummary-filename-map/index.rst @@ -12,4 +12,3 @@ autosummary_dummy_module.Foo.Bar autosummary_dummy_module.bar autosummary_dummy_module.qux - autosummary_importfail diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 7a35d7c90..62fe4cc56 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -386,10 +386,7 @@ def test_autosummary_recursive(app, status, warning): assert 'package.package.module' in content -@pytest.mark.sphinx('dummy', testroot='ext-autosummary-filename-map', - confoverrides={'autosummary_filename_map': - {"autosummary_dummy_module": "module_mangled", - "autosummary_dummy_module.bar": "bar"}}) +@pytest.mark.sphinx('dummy', testroot='ext-autosummary-filename-map') def test_autosummary_filename_map(app, status, warning): app.build() From 71d1b37f8ad9b470c1779d0b74a8581b1e131353 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Fri, 10 Jul 2020 11:31:27 +1000 Subject: [PATCH 06/88] Simplify dummy module --- .../autosummary_dummy_module.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/tests/roots/test-ext-autosummary-filename-map/autosummary_dummy_module.py b/tests/roots/test-ext-autosummary-filename-map/autosummary_dummy_module.py index 85981a0f8..1f57eeb25 100644 --- a/tests/roots/test-ext-autosummary-filename-map/autosummary_dummy_module.py +++ b/tests/roots/test-ext-autosummary-filename-map/autosummary_dummy_module.py @@ -17,25 +17,5 @@ class Foo: pass -class _Baz: - pass - - def bar(x: Union[int, str], y: int = 1) -> None: pass - - -def _quux(): - pass - - -class Exc(Exception): - pass - - -class _Exc(Exception): - pass - - -#: a module-level attribute -qux = 2 From aae8ce8efa64963c6c5f73f318da6762499c705e Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 15 Jul 2020 18:46:51 +1000 Subject: [PATCH 07/88] Fix toctree generation --- sphinx/ext/autosummary/__init__.py | 4 +++- sphinx/ext/autosummary/generate.py | 3 --- tests/roots/test-ext-autosummary-filename-map/index.rst | 3 +-- tests/test_ext_autosummary.py | 3 +++ 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index ece16c15b..350604387 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -252,7 +252,9 @@ class Autosummary(SphinxDirective): tree_prefix = self.options['toctree'].strip() docnames = [] excluded = Matcher(self.config.exclude_patterns) + filename_map = self.config.autosummary_filename_map for name, sig, summary, real_name in items: + real_name = filename_map.get(real_name, real_name) docname = posixpath.join(tree_prefix, real_name) docname = posixpath.normpath(posixpath.join(dirname, docname)) if docname not in self.env.found_docs: @@ -785,11 +787,11 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_role('autolink', AutoLink()) app.connect('builder-inited', process_generate_options) app.add_config_value('autosummary_context', {}, True) + app.add_config_value('autosummary_filename_map', {}, 'html') app.add_config_value('autosummary_generate', [], True, [bool]) app.add_config_value('autosummary_generate_overwrite', True, False) app.add_config_value('autosummary_mock_imports', lambda config: config.autodoc_mock_imports, 'env') app.add_config_value('autosummary_imported_members', [], False, [bool]) - app.add_config_value('autosummary_filename_map', {}, 'html') return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 4454e47fc..98e2a047a 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -396,9 +396,6 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, if app: filename_map = app.config.autosummary_filename_map - if not isinstance(filename_map, Mapping): - raise TypeError('autosummary_filename_map should be a mapping from ' - 'strings to strings') else: filename_map = {} diff --git a/tests/roots/test-ext-autosummary-filename-map/index.rst b/tests/roots/test-ext-autosummary-filename-map/index.rst index ef27d9d19..e85389395 100644 --- a/tests/roots/test-ext-autosummary-filename-map/index.rst +++ b/tests/roots/test-ext-autosummary-filename-map/index.rst @@ -9,6 +9,5 @@ autosummary_dummy_module autosummary_dummy_module.Foo - autosummary_dummy_module.Foo.Bar + autosummary_dummy_module.Foo.bar autosummary_dummy_module.bar - autosummary_dummy_module.qux diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 62fe4cc56..462a93954 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -396,6 +396,9 @@ def test_autosummary_filename_map(app, status, warning): assert not (app.srcdir / 'generated' / 'autosummary_dummy_module.bar.rst').exists() assert (app.srcdir / 'generated' / 'autosummary_dummy_module.Foo.rst').exists() + html_warnings = app._warning.getvalue() + assert html_warnings == '' + @pytest.mark.sphinx('latex', **default_kw) def test_autosummary_latex_table_colspec(app, status, warning): From 2b843dd856ab54d7e0a9769fe0e507ac7671b263 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 15 Jul 2020 20:49:05 +1000 Subject: [PATCH 08/88] Rm unused import --- sphinx/ext/autosummary/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 98e2a047a..c1b50de57 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -28,7 +28,7 @@ import sys import warnings from gettext import NullTranslations from os import path -from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Set, Tuple, Union +from typing import Any, Callable, Dict, List, NamedTuple, Set, Tuple, Union from jinja2 import TemplateNotFound from jinja2.sandbox import SandboxedEnvironment From a7bbedfbb71066614f5fc01da17caecd78da45c3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 31 Jan 2020 22:01:55 +0100 Subject: [PATCH 09/88] add a test for the parameter type conversions --- tests/test_ext_napoleon_docstring.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index f9cd40104..93a9f6773 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1976,6 +1976,37 @@ definition_after_normal_text : int actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) + def test_parameter_types(self): + docstring = """\ +Parameters +---------- +param1 : DataFrame + the data to work on +param2 : int or float or None + a parameter with different types +param3 : dict-like, optional + a optional mapping +param4 : int or float or None, optional + a optional parameter with different types +param5 : {"F", "C", "N"}, optional + a optional parameter with fixed values +""" + expected = """\ +:param param1: the data to work on +:type param1: :obj:`DataFrame` +:param param2: a parameter with different types +:type param2: :obj:`int` or :obj:`float` or :obj:`None` +:param param3: a optional mapping +:type param3: :obj:`dict-like`, optional +:param param4: a optional parameter with different types +:type param4: :obj:`int` or :obj:`float` or :obj:`None`, optional +:param param5: a optional parameter with fixed values +:type param5: {"F", "C", "N"}, optional +""" + config = Config(napoleon_use_param=True, napoleon_use_rtype=True) + actual = str(NumpyDocstring(docstring, config)) + self.assertEqual(expected, actual) + def test_keywords_with_types(self): docstring = """\ Do as you please From b1f43d2fff8f1202f59441eb29b1b80eb7496670 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 22:50:32 +0200 Subject: [PATCH 10/88] use textwrap for a normal indentation depth --- tests/test_ext_napoleon_docstring.py | 53 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 93a9f6773..e80e7e348 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1977,32 +1977,33 @@ definition_after_normal_text : int self.assertEqual(expected, actual) def test_parameter_types(self): - docstring = """\ -Parameters ----------- -param1 : DataFrame - the data to work on -param2 : int or float or None - a parameter with different types -param3 : dict-like, optional - a optional mapping -param4 : int or float or None, optional - a optional parameter with different types -param5 : {"F", "C", "N"}, optional - a optional parameter with fixed values -""" - expected = """\ -:param param1: the data to work on -:type param1: :obj:`DataFrame` -:param param2: a parameter with different types -:type param2: :obj:`int` or :obj:`float` or :obj:`None` -:param param3: a optional mapping -:type param3: :obj:`dict-like`, optional -:param param4: a optional parameter with different types -:type param4: :obj:`int` or :obj:`float` or :obj:`None`, optional -:param param5: a optional parameter with fixed values -:type param5: {"F", "C", "N"}, optional -""" + import textwrap + docstring = textwrap.dedent("""\ + Parameters + ---------- + param1 : DataFrame + the data to work on + param2 : int or float or None + a parameter with different types + param3 : dict-like, optional + a optional mapping + param4 : int or float or None, optional + a optional parameter with different types + param5 : {"F", "C", "N"}, optional + a optional parameter with fixed values + """) + expected = textwrap.dedent("""\ + :param param1: the data to work on + :type param1: :obj:`DataFrame` + :param param2: a parameter with different types + :type param2: :obj:`int` or :obj:`float` or :obj:`None` + :param param3: a optional mapping + :type param3: :obj:`dict-like`, optional + :param param4: a optional parameter with different types + :type param4: :obj:`int` or :obj:`float` or :obj:`None`, optional + :param param5: a optional parameter with fixed values + :type param5: {"F", "C", "N"}, optional + """) config = Config(napoleon_use_param=True, napoleon_use_rtype=True) actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) From bc25a3d136bb5947152347ff037da9063055002c Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 22:56:47 +0200 Subject: [PATCH 11/88] try to mark literals as such --- sphinx/ext/napoleon/docstring.py | 92 +++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 5857fcf92..935d98107 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -20,11 +20,23 @@ from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter from sphinx.locale import _ +from sphinx.util.docutils import SphinxRole +from docutils import nodes +from docutils.nodes import Node, system_message +from docutils.parsers.rst import roles + + +class LiteralText(SphinxRole): + def run(self) -> Tuple[List[Node], List[system_message]]: + return [nodes.Text(self.text, self.text, **self.options)], [] + + +roles.register_local_role("noref", LiteralText()) + if False: # For type annotation from typing import Type # for python3.5.1 - _directive_regex = re.compile(r'\.\. \S+::') _google_section_regex = re.compile(r'^(\s|\w)+:\s*$') _google_typed_arg_regex = re.compile(r'\s*(.+?)\s*\(\s*(.*[^\s]+)\s*\)') @@ -879,6 +891,84 @@ class NumpyDocstring(GoogleDocstring): self._directive_sections = ['.. index::'] super().__init__(docstring, config, app, what, name, obj, options) + def _convert_type_spec(self, _type): + def recombine_set(tokens): + def combine_set(tokens): + in_set = False + set_items = [] + + for token in tokens: + if token.startswith("{"): + in_set = True + elif token.endswith("}"): + in_set = False + set_items.append(token) + + if in_set: + set_items.append(token) + else: + if set_items: + token = "".join(set_items) + set_items = [] + yield token + + return list(combine_set(tokens)) + + def tokenize_type_spec(spec): + delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" + + split = [ + item + for item in re.split(delimiters, _type) + if item + ] + tokens = recombine_set(split) + return tokens + + def token_type(token): + if token.startswith(" ") or token.endswith(" "): + type_ = "delimiter" + elif token.startswith("{") and token.endswith("}"): + type_ = "value_set" + elif token in ("optional", "default"): + type_ = "control" + elif "instance" in token: + type_ = "literal" + elif re.match(":[^:]+:`[^`]+`", token): + type_ = "reference" + elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): + type_ = "literal" + else: + type_ = "obj" + + return type_ + + def convert_obj(obj, translations): + return translations.get(obj, ":obj:`{}`".format(obj)) + + tokens = tokenize_type_spec(_type) + types = [ + (token, token_type(token)) + for token in tokens + ] + + # TODO: make this configurable + translations = { + "sequence": ":term:`sequence`", + "dict-like": ":term:`mapping`", + } + + converters = { + "value_set": lambda x: f":noref:`{x}`", + "literal": lambda x: f":noref:`{x}`", + "obj": lambda x: convert_obj(x, translations), + "control": lambda x: f":noref:`{x}`", + "delimiter": lambda x: x, + "reference": lambda x: x, + } + + return "".join(converters.get(type_)(token) for token, type_ in types) + def _consume_field(self, parse_type: bool = True, prefer_type: bool = False ) -> Tuple[str, str, List[str]]: line = next(self._line_iter) From bafb24dd931aef3bc5488f9318664c918cad8d75 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 23:02:25 +0200 Subject: [PATCH 12/88] actually apply the type conversion --- sphinx/ext/napoleon/docstring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 935d98107..20955be12 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -978,6 +978,7 @@ class NumpyDocstring(GoogleDocstring): _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) + _type = self._convert_type_spec(_type) if prefer_type and not _type: _type, _name = _name, _type From bdea34e54edfa546e04b1d53c54fee2215743f9f Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 23:02:35 +0200 Subject: [PATCH 13/88] don't treat instance as special --- sphinx/ext/napoleon/docstring.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 20955be12..380776ba9 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -932,8 +932,6 @@ class NumpyDocstring(GoogleDocstring): type_ = "value_set" elif token in ("optional", "default"): type_ = "control" - elif "instance" in token: - type_ = "literal" elif re.match(":[^:]+:`[^`]+`", token): type_ = "reference" elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): From bd33b61d64f88b05ce65279e4825d9ab78dd9a07 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 17 May 2020 23:23:04 +0200 Subject: [PATCH 14/88] update the translations --- sphinx/ext/napoleon/docstring.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 380776ba9..abfcf2fba 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -953,7 +953,8 @@ class NumpyDocstring(GoogleDocstring): # TODO: make this configurable translations = { "sequence": ":term:`sequence`", - "dict-like": ":term:`mapping`", + "mapping": ":term:`mapping`", + "dict-like": ":term:`dict-like `", } converters = { From e9822139eea115685f622a1ce3cb77edebad44c8 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 19 May 2020 18:46:51 +0200 Subject: [PATCH 16/88] more flake8 --- sphinx/ext/napoleon/docstring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index abfcf2fba..fddfd39a5 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -15,15 +15,15 @@ import re from functools import partial from typing import Any, Callable, Dict, List, Tuple, Union +from docutils import nodes +from docutils.nodes import Node, system_message +from docutils.parsers.rst import roles + from sphinx.application import Sphinx from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter from sphinx.locale import _ - from sphinx.util.docutils import SphinxRole -from docutils import nodes -from docutils.nodes import Node, system_message -from docutils.parsers.rst import roles class LiteralText(SphinxRole): From ace933107a645ab648497e4485d87f97973de178 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 19:26:31 +0200 Subject: [PATCH 17/88] move the numpy type spec parsing function out of NumpyDocstring --- sphinx/ext/napoleon/docstring.py | 157 ++++++++++++++++--------------- 1 file changed, 79 insertions(+), 78 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index fddfd39a5..c20e8f2b7 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -792,6 +792,84 @@ class GoogleDocstring: return lines +def _parse_numpy_type_spec(_type): + def recombine_set(tokens): + def combine_set(tokens): + in_set = False + set_items = [] + + for token in tokens: + if token.startswith("{"): + in_set = True + elif token.endswith("}"): + in_set = False + set_items.append(token) + + if in_set: + set_items.append(token) + else: + if set_items: + token = "".join(set_items) + set_items = [] + yield token + + return list(combine_set(tokens)) + + def tokenize_type_spec(spec): + delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" + + split = [ + item + for item in re.split(delimiters, _type) + if item + ] + tokens = recombine_set(split) + return tokens + + def token_type(token): + if token.startswith(" ") or token.endswith(" "): + type_ = "delimiter" + elif token.startswith("{") and token.endswith("}"): + type_ = "value_set" + elif token in ("optional", "default"): + type_ = "control" + elif re.match(":[^:]+:`[^`]+`", token): + type_ = "reference" + elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): + type_ = "literal" + else: + type_ = "obj" + + return type_ + + def convert_obj(obj, translations): + return translations.get(obj, ":obj:`{}`".format(obj)) + + tokens = tokenize_type_spec(_type) + types = [ + (token, token_type(token)) + for token in tokens + ] + + # TODO: make this configurable + translations = { + "sequence": ":term:`sequence`", + "mapping": ":term:`mapping`", + "dict-like": ":term:`dict-like `", + } + + converters = { + "value_set": lambda x: f":noref:`{x}`", + "literal": lambda x: f":noref:`{x}`", + "obj": lambda x: convert_obj(x, translations), + "control": lambda x: f":noref:`{x}`", + "delimiter": lambda x: x, + "reference": lambda x: x, + } + + return "".join(converters.get(type_)(token) for token, type_ in types) + + class NumpyDocstring(GoogleDocstring): """Convert NumPy style docstrings to reStructuredText. @@ -891,83 +969,6 @@ class NumpyDocstring(GoogleDocstring): self._directive_sections = ['.. index::'] super().__init__(docstring, config, app, what, name, obj, options) - def _convert_type_spec(self, _type): - def recombine_set(tokens): - def combine_set(tokens): - in_set = False - set_items = [] - - for token in tokens: - if token.startswith("{"): - in_set = True - elif token.endswith("}"): - in_set = False - set_items.append(token) - - if in_set: - set_items.append(token) - else: - if set_items: - token = "".join(set_items) - set_items = [] - yield token - - return list(combine_set(tokens)) - - def tokenize_type_spec(spec): - delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" - - split = [ - item - for item in re.split(delimiters, _type) - if item - ] - tokens = recombine_set(split) - return tokens - - def token_type(token): - if token.startswith(" ") or token.endswith(" "): - type_ = "delimiter" - elif token.startswith("{") and token.endswith("}"): - type_ = "value_set" - elif token in ("optional", "default"): - type_ = "control" - elif re.match(":[^:]+:`[^`]+`", token): - type_ = "reference" - elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): - type_ = "literal" - else: - type_ = "obj" - - return type_ - - def convert_obj(obj, translations): - return translations.get(obj, ":obj:`{}`".format(obj)) - - tokens = tokenize_type_spec(_type) - types = [ - (token, token_type(token)) - for token in tokens - ] - - # TODO: make this configurable - translations = { - "sequence": ":term:`sequence`", - "mapping": ":term:`mapping`", - "dict-like": ":term:`dict-like `", - } - - converters = { - "value_set": lambda x: f":noref:`{x}`", - "literal": lambda x: f":noref:`{x}`", - "obj": lambda x: convert_obj(x, translations), - "control": lambda x: f":noref:`{x}`", - "delimiter": lambda x: x, - "reference": lambda x: x, - } - - return "".join(converters.get(type_)(token) for token, type_ in types) - def _consume_field(self, parse_type: bool = True, prefer_type: bool = False ) -> Tuple[str, str, List[str]]: line = next(self._line_iter) @@ -977,7 +978,7 @@ class NumpyDocstring(GoogleDocstring): _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) - _type = self._convert_type_spec(_type) + _type = _parse_numpy_type_spec(_type) if prefer_type and not _type: _type, _name = _name, _type From 8ab210f1b024c1abd534bf9b75a6013722a158d5 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 19:27:14 +0200 Subject: [PATCH 18/88] don't use the obj role if it is not necessary --- sphinx/ext/napoleon/docstring.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index c20e8f2b7..2bf8459de 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -842,8 +842,8 @@ def _parse_numpy_type_spec(_type): return type_ - def convert_obj(obj, translations): - return translations.get(obj, ":obj:`{}`".format(obj)) + def convert_obj(obj, translations, default_translation=":obj:`{}`"): + return translations.get(obj, default_translation.format(obj)) tokens = tokenize_type_spec(_type) types = [ @@ -858,10 +858,17 @@ def _parse_numpy_type_spec(_type): "dict-like": ":term:`dict-like `", } + # don't use the object role if it's not necessary + default_translation = ( + ":obj:`{}`" + if not all(type_ == "obj" for _, type_ in types) + else "{}" + ) + converters = { "value_set": lambda x: f":noref:`{x}`", "literal": lambda x: f":noref:`{x}`", - "obj": lambda x: convert_obj(x, translations), + "obj": lambda x: convert_obj(x, translations, default_translation), "control": lambda x: f":noref:`{x}`", "delimiter": lambda x: x, "reference": lambda x: x, From 25937f745a3dbaa76c5cf87a31122c3602730670 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 21:06:52 +0200 Subject: [PATCH 19/88] move tokenize_type_spec to its own function and add tests for it --- sphinx/ext/napoleon/docstring.py | 83 +++++++++++++-------- tests/test_ext_napoleon_docstring.py | 103 +++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 31 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 2bf8459de..112bb0831 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -792,40 +792,61 @@ class GoogleDocstring: return lines +def _recombine_set_tokens(tokens): + def takewhile_set(iterable): + yield "{" + + open_braces = 1 + while True: + try: + token = next(iterable) + except StopIteration: + if open_braces != 0: + raise ValueError("invalid value set: {}".format("".join(tokens))) + + break + + if token == "{": + open_braces += 1 + elif token == "}": + open_braces -= 1 + + yield token + + if open_braces == 0: + break + + def combine_set(tokens): + iterable = iter(tokens) + while True: + try: + token = next(iterable) + except StopIteration: + break + + yield "".join(takewhile_set(iterable)) if token == "{" else token + + return list(combine_set(tokens)) + + +def _tokenize_type_spec(spec): + delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" + + tokens = tuple( + item + for item in re.split(delimiters, spec) + if item + ) + return _recombine_set_tokens(tokens) + + def _parse_numpy_type_spec(_type): - def recombine_set(tokens): - def combine_set(tokens): - in_set = False - set_items = [] + raw_tokens = _tokenize_type_spec(_type) + tokens = list(_recombine_set_tokens(raw_tokens)) + return tokens - for token in tokens: - if token.startswith("{"): - in_set = True - elif token.endswith("}"): - in_set = False - set_items.append(token) - - if in_set: - set_items.append(token) - else: - if set_items: - token = "".join(set_items) - set_items = [] - yield token - - return list(combine_set(tokens)) - - def tokenize_type_spec(spec): - delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" - - split = [ - item - for item in re.split(delimiters, _type) - if item - ] - tokens = recombine_set(split) - return tokens +def _parse_numpy_type_spec2(_type): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index e80e7e348..3f5330b40 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -16,6 +16,7 @@ from unittest import TestCase, mock from sphinx.ext.napoleon import Config from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring +from sphinx.ext.napoleon.docstring import _tokenize_type_spec, _recombine_set_tokens, _parse_numpy_type_spec class NamedtupleSubclass(namedtuple('NamedtupleSubclass', ('attr1', 'attr2'))): @@ -1976,6 +1977,108 @@ definition_after_normal_text : int actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) + def test_recombine_set_tokens(self): + tokens = ( + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], + ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], + ["{", "1", ", ", "2", "}"], + ) + recombined_tokens = ( + ["{'F', 'C', 'N'}"], + ['{"F", "C", "N"}'], + ["{1, 2}"], + ) + + for input_tokens, expected in zip(tokens, recombined_tokens): + actual = _recombine_set_tokens(input_tokens) + self.assertEqual(expected, actual) + + def test_recombine_set_tokens_invalid(self): + invalid_tokens = ( + ["{", "1", ", ", "2"], + ) + + for input_tokens in invalid_tokens: + with self.assertRaisesRegex(ValueError, "invalid value set:"): + _recombine_set_tokens(input_tokens) + + + def test_tokenize_type_spec(self): + types = ( + "str", + "int or float or None", + '{"F", "C", "N"}', + "{'F', 'C', 'N'}", + ) + modifiers = ( + "optional", + "default: None", + ) + + type_tokens = ( + ["str"], + ["int", " or ", "float", " or ", "None"], + ['{"F", "C", "N"}'], + ["{'F', 'C', 'N'}"], + ) + modifier_tokens = ( + ["optional"], + ["default", ": ", "None"], + ) + + type_specs = tuple( + ", ".join([type_, modifier]) + for type_ in types + for modifier in modifiers + ) + tokens = tuple( + tokens_ + [", "] + modifier_tokens_ + for tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + for type_spec, expected in zip(type_specs, tokens): + actual = _tokenize_type_spec(type_spec) + self.assertEqual(expected, actual) + + def test_parse_numpy_type_spec(self): + types = ( + "str", + "int or float or None", + '{"F", "C", "N"}', + "{'F', 'C', 'N'}", + ) + modifiers = ( + "optional", + "default: None", + ) + + type_tokens = ( + ["str"], + ["int", " or ", "float", " or ", "None"], + ['{"F", "C", "N"}'], + ["{'F', 'C', 'N'}"], + ) + modifier_tokens = ( + ["optional"], + ["default", ": ", "None"], + ) + + type_specs = tuple( + ", ".join([type_, modifier]) + for type_ in types + for modifier in modifiers + ) + tokens = tuple( + tuple(tokens_ + [", "] + modifier_tokens_) + for tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + for type_spec, expected in zip(type_specs, tokens): + actual = _parse_numpy_type_spec(type_spec) + self.assertEqual(expected, actual) + def test_parameter_types(self): import textwrap docstring = textwrap.dedent("""\ From fc70205fb4236490a1526e7667cd695c1f91763e Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 21:29:38 +0200 Subject: [PATCH 20/88] get the type converter function to work, verified by new tests --- sphinx/ext/napoleon/docstring.py | 16 ++++----- tests/test_ext_napoleon_docstring.py | 49 ++++++++++++++++------------ 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 112bb0831..ae5ba4456 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -840,13 +840,7 @@ def _tokenize_type_spec(spec): return _recombine_set_tokens(tokens) -def _parse_numpy_type_spec(_type): - raw_tokens = _tokenize_type_spec(_type) - tokens = list(_recombine_set_tokens(raw_tokens)) - return tokens - - -def _parse_numpy_type_spec2(_type): +def _convert_numpy_type_spec(_type): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" @@ -866,7 +860,7 @@ def _parse_numpy_type_spec2(_type): def convert_obj(obj, translations, default_translation=":obj:`{}`"): return translations.get(obj, default_translation.format(obj)) - tokens = tokenize_type_spec(_type) + tokens = _tokenize_type_spec(_type) types = [ (token, token_type(token)) for token in tokens @@ -895,7 +889,9 @@ def _parse_numpy_type_spec2(_type): "reference": lambda x: x, } - return "".join(converters.get(type_)(token) for token, type_ in types) + converted = "".join(converters.get(type_)(token) for token, type_ in types) + + return converted class NumpyDocstring(GoogleDocstring): @@ -1006,7 +1002,7 @@ class NumpyDocstring(GoogleDocstring): _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) - _type = _parse_numpy_type_spec(_type) + _type = _convert_numpy_type_spec(_type) if prefer_type and not _type: _type, _name = _name, _type diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 3f5330b40..f956e4d2a 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -16,7 +16,7 @@ from unittest import TestCase, mock from sphinx.ext.napoleon import Config from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring -from sphinx.ext.napoleon.docstring import _tokenize_type_spec, _recombine_set_tokens, _parse_numpy_type_spec +from sphinx.ext.napoleon.docstring import _tokenize_type_spec, _recombine_set_tokens, _convert_numpy_type_spec class NamedtupleSubclass(namedtuple('NamedtupleSubclass', ('attr1', 'attr2'))): @@ -2041,7 +2041,7 @@ definition_after_normal_text : int actual = _tokenize_type_spec(type_spec) self.assertEqual(expected, actual) - def test_parse_numpy_type_spec(self): + def test_convert_numpy_type_spec(self): types = ( "str", "int or float or None", @@ -2049,34 +2049,43 @@ definition_after_normal_text : int "{'F', 'C', 'N'}", ) modifiers = ( + "", "optional", "default: None", ) - - type_tokens = ( - ["str"], - ["int", " or ", "float", " or ", "None"], - ['{"F", "C", "N"}'], - ["{'F', 'C', 'N'}"], - ) - modifier_tokens = ( - ["optional"], - ["default", ": ", "None"], - ) - type_specs = tuple( ", ".join([type_, modifier]) + if modifier + else type_ for type_ in types for modifier in modifiers ) - tokens = tuple( - tuple(tokens_ + [", "] + modifier_tokens_) - for tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens + + converted_types = ( + ":obj:`str`", + ":obj:`int` or :obj:`float` or :obj:`None`", + ':noref:`{"F", "C", "N"}`', + ":noref:`{'F', 'C', 'N'}`", + ) + converted_modifiers = ( + "", + ":noref:`optional`", + ":noref:`default`: :obj:`None`", + ) + converted = tuple( + ", ".join([converted_type, converted_modifier]) + if converted_modifier + else ( + type_ + if ("{" not in type_ and "or" not in type_) + else converted_type + ) + for converted_type, type_ in zip(converted_types, types) + for converted_modifier in converted_modifiers ) - for type_spec, expected in zip(type_specs, tokens): - actual = _parse_numpy_type_spec(type_spec) + for type_, expected in zip(type_specs, converted): + actual = _convert_numpy_type_spec(type_) self.assertEqual(expected, actual) def test_parameter_types(self): From 2882c3465adc37d86107cca0dcd51d383a288f82 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 22 May 2020 21:37:06 +0200 Subject: [PATCH 21/88] fix the expected parameters section to match the current status --- tests/test_ext_napoleon_docstring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index f956e4d2a..c310665d5 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2106,15 +2106,15 @@ definition_after_normal_text : int """) expected = textwrap.dedent("""\ :param param1: the data to work on - :type param1: :obj:`DataFrame` + :type param1: DataFrame :param param2: a parameter with different types :type param2: :obj:`int` or :obj:`float` or :obj:`None` :param param3: a optional mapping - :type param3: :obj:`dict-like`, optional + :type param3: :term:`dict-like `, :noref:`optional` :param param4: a optional parameter with different types - :type param4: :obj:`int` or :obj:`float` or :obj:`None`, optional + :type param4: :obj:`int` or :obj:`float` or :obj:`None`, :noref:`optional` :param param5: a optional parameter with fixed values - :type param5: {"F", "C", "N"}, optional + :type param5: :noref:`{"F", "C", "N"}`, :noref:`optional` """) config = Config(napoleon_use_param=True, napoleon_use_rtype=True) actual = str(NumpyDocstring(docstring, config)) From ad89b1f76a5a031e14022c96d678a6240061eb11 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:12:31 +0200 Subject: [PATCH 22/88] replace the hard-coded mapping of translations with a config option --- sphinx/ext/napoleon/__init__.py | 5 +++++ sphinx/ext/napoleon/docstring.py | 14 +++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/sphinx/ext/napoleon/__init__.py b/sphinx/ext/napoleon/__init__.py index 2b1818425..e3b03fd75 100644 --- a/sphinx/ext/napoleon/__init__.py +++ b/sphinx/ext/napoleon/__init__.py @@ -41,6 +41,7 @@ class Config: napoleon_use_param = True napoleon_use_rtype = True napoleon_use_keyword = True + napoleon_numpy_type_aliases = None napoleon_custom_sections = None .. _Google style: @@ -236,6 +237,9 @@ class Config: :returns: *bool* -- True if successful, False otherwise + napoleon_numpy_type_aliases : :obj:`dict` (Defaults to None) + Add a mapping of strings to string, translating types. + napoleon_custom_sections : :obj:`list` (Defaults to None) Add a list of custom sections to include, expanding the list of parsed sections. @@ -263,6 +267,7 @@ class Config: 'napoleon_use_param': (True, 'env'), 'napoleon_use_rtype': (True, 'env'), 'napoleon_use_keyword': (True, 'env'), + 'napoleon_numpy_type_aliases': (None, 'env'), 'napoleon_custom_sections': (None, 'env') } diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index ae5ba4456..63e2df6bc 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -840,7 +840,7 @@ def _tokenize_type_spec(spec): return _recombine_set_tokens(tokens) -def _convert_numpy_type_spec(_type): +def _convert_numpy_type_spec(_type, translations): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" @@ -866,13 +866,6 @@ def _convert_numpy_type_spec(_type): for token in tokens ] - # TODO: make this configurable - translations = { - "sequence": ":term:`sequence`", - "mapping": ":term:`mapping`", - "dict-like": ":term:`dict-like `", - } - # don't use the object role if it's not necessary default_translation = ( ":obj:`{}`" @@ -1002,7 +995,10 @@ class NumpyDocstring(GoogleDocstring): _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) - _type = _convert_numpy_type_spec(_type) + _type = _convert_numpy_type_spec( + _type, + translations=self._config.napoleon_numpy_type_aliases or {}, + ) if prefer_type and not _type: _type, _name = _name, _type From e1d7edac25d0195a34ab23e584a019842884a6f7 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:16:17 +0200 Subject: [PATCH 23/88] rename the configuration option --- sphinx/ext/napoleon/__init__.py | 8 ++++---- sphinx/ext/napoleon/docstring.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/napoleon/__init__.py b/sphinx/ext/napoleon/__init__.py index e3b03fd75..128fbaab5 100644 --- a/sphinx/ext/napoleon/__init__.py +++ b/sphinx/ext/napoleon/__init__.py @@ -41,7 +41,7 @@ class Config: napoleon_use_param = True napoleon_use_rtype = True napoleon_use_keyword = True - napoleon_numpy_type_aliases = None + napoleon_type_aliases = None napoleon_custom_sections = None .. _Google style: @@ -237,8 +237,8 @@ class Config: :returns: *bool* -- True if successful, False otherwise - napoleon_numpy_type_aliases : :obj:`dict` (Defaults to None) - Add a mapping of strings to string, translating types. + napoleon_type_aliases : :obj:`dict` (Defaults to None) + Add a mapping of strings to string, translating types in numpy style docstrings. napoleon_custom_sections : :obj:`list` (Defaults to None) Add a list of custom sections to include, expanding the list of parsed sections. @@ -267,7 +267,7 @@ class Config: 'napoleon_use_param': (True, 'env'), 'napoleon_use_rtype': (True, 'env'), 'napoleon_use_keyword': (True, 'env'), - 'napoleon_numpy_type_aliases': (None, 'env'), + 'napoleon_type_aliases': (None, 'env'), 'napoleon_custom_sections': (None, 'env') } diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 63e2df6bc..5d65887df 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -997,7 +997,7 @@ class NumpyDocstring(GoogleDocstring): _name = self._escape_args_and_kwargs(_name) _type = _convert_numpy_type_spec( _type, - translations=self._config.napoleon_numpy_type_aliases or {}, + translations=self._config.napoleon_type_aliases or {}, ) if prefer_type and not _type: From 27733d6f618c9f4875499ded1374e25712ebe3b3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:17:58 +0200 Subject: [PATCH 24/88] replace the custom role with markup --- sphinx/ext/napoleon/docstring.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 5d65887df..bf314adbd 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -15,24 +15,11 @@ import re from functools import partial from typing import Any, Callable, Dict, List, Tuple, Union -from docutils import nodes -from docutils.nodes import Node, system_message -from docutils.parsers.rst import roles - from sphinx.application import Sphinx from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter -from sphinx.locale import _ -from sphinx.util.docutils import SphinxRole -class LiteralText(SphinxRole): - def run(self) -> Tuple[List[Node], List[system_message]]: - return [nodes.Text(self.text, self.text, **self.options)], [] - - -roles.register_local_role("noref", LiteralText()) - if False: # For type annotation from typing import Type # for python3.5.1 @@ -874,10 +861,10 @@ def _convert_numpy_type_spec(_type, translations): ) converters = { - "value_set": lambda x: f":noref:`{x}`", - "literal": lambda x: f":noref:`{x}`", + "value_set": lambda x: f"``{x}``", + "literal": lambda x: f"``{x}``", "obj": lambda x: convert_obj(x, translations, default_translation), - "control": lambda x: f":noref:`{x}`", + "control": lambda x: f"*{x}*", "delimiter": lambda x: x, "reference": lambda x: x, } From b846db7e53bf43108c64c981dc5cec24f6b5ca25 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:20:33 +0200 Subject: [PATCH 25/88] emit a warning instead of raising an error --- sphinx/ext/napoleon/docstring.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index bf314adbd..f7eebb766 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -13,12 +13,15 @@ import inspect import re from functools import partial +import logging from typing import Any, Callable, Dict, List, Tuple, Union from sphinx.application import Sphinx from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter +from sphinx.locale import _, __ +logger = logging.Logger(__name__) if False: # For type annotation @@ -789,7 +792,7 @@ def _recombine_set_tokens(tokens): token = next(iterable) except StopIteration: if open_braces != 0: - raise ValueError("invalid value set: {}".format("".join(tokens))) + logger.warning(__("invalid value set: %r"), "".join(tokens)) break From ce60b555eeebf71ef04b2e4614e6de7d620e0191 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 19:57:20 +0200 Subject: [PATCH 26/88] properly use sphinx's logger --- sphinx/ext/napoleon/docstring.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index f7eebb766..e1e9e1a49 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -13,15 +13,15 @@ import inspect import re from functools import partial -import logging from typing import Any, Callable, Dict, List, Tuple, Union from sphinx.application import Sphinx from sphinx.config import Config as SphinxConfig from sphinx.ext.napoleon.iterators import modify_iter from sphinx.locale import _, __ +from sphinx.util import logging -logger = logging.Logger(__name__) +logger = logging.getLogger(__name__) if False: # For type annotation @@ -792,7 +792,12 @@ def _recombine_set_tokens(tokens): token = next(iterable) except StopIteration: if open_braces != 0: - logger.warning(__("invalid value set: %r"), "".join(tokens)) + location = ("", "") + logger.warning( + __("invalid value set: %r"), + "".join(tokens), + location=location, + ) break From eab49125e9236ac3ba615663dbfbadfc44c37e82 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 20:30:55 +0200 Subject: [PATCH 27/88] update the splitting regexp to handle braces in strings and escaped quotes --- sphinx/ext/napoleon/docstring.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index e1e9e1a49..2582babc5 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -825,12 +825,25 @@ def _recombine_set_tokens(tokens): def _tokenize_type_spec(spec): - delimiters = r"(\sor\s|\sof\s|:\s|,\s|[{]|[}])" - + delimiters = [ + r"\sor\s", + r"\sof\s", + r":\s", + r",\s", + ] + braces = [ + "[{]", + "[}]", + ] + quoted_strings = [ + r'"(?:[^"]|\\")*"', + r"'(?:[^']|\\')*'", + ] + tokenization_re = re.compile(f"({'|'.join(delimiters + braces + quoted_strings)})") tokens = tuple( item - for item in re.split(delimiters, spec) - if item + for item in tokenization_re.split(spec) + if item is not None and item.strip() ) return _recombine_set_tokens(tokens) From 9835f1fff80c781c63a08e974235f046657b8c9b Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:07:19 +0200 Subject: [PATCH 28/88] test that braces and quotes in strings work --- tests/test_ext_napoleon_docstring.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index c310665d5..e1c296e79 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2009,6 +2009,8 @@ definition_after_normal_text : int "int or float or None", '{"F", "C", "N"}', "{'F', 'C', 'N'}", + '"ma{icious"', + r"'with \'quotes\''", ) modifiers = ( "optional", @@ -2020,6 +2022,8 @@ definition_after_normal_text : int ["int", " or ", "float", " or ", "None"], ['{"F", "C", "N"}'], ["{'F', 'C', 'N'}"], + ['"ma{icious"'], + [r"'with \'quotes\''"], ) modifier_tokens = ( ["optional"], From 9bfbe252f190bd3b426b8ff2e3d0454f5e49fb1e Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:07:53 +0200 Subject: [PATCH 29/88] set a default so translations don't to be specified --- sphinx/ext/napoleon/docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 2582babc5..5f4a64e06 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -848,7 +848,7 @@ def _tokenize_type_spec(spec): return _recombine_set_tokens(tokens) -def _convert_numpy_type_spec(_type, translations): +def _convert_numpy_type_spec(_type, translations={}): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" From e3b7e16b0ade75e7660fe364cdd122dbc55a022d Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:08:36 +0200 Subject: [PATCH 30/88] move the regexes to top-level --- sphinx/ext/napoleon/docstring.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 5f4a64e06..dfc2605ab 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -35,11 +35,19 @@ _single_colon_regex = re.compile(r'(?\()?' r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])' r'(?(paren)\)|\.)(\s+\S|\s*$)') +_token_regex = re.compile( + r"(\sor\s|\sof\s|:\s|,\s|[{]|[}]" + r'|"(?:\\"|[^"])*"' + r"|'(?:\\'|[^'])*')" +) class GoogleDocstring: @@ -825,24 +833,9 @@ def _recombine_set_tokens(tokens): def _tokenize_type_spec(spec): - delimiters = [ - r"\sor\s", - r"\sof\s", - r":\s", - r",\s", - ] - braces = [ - "[{]", - "[}]", - ] - quoted_strings = [ - r'"(?:[^"]|\\")*"', - r"'(?:[^']|\\')*'", - ] - tokenization_re = re.compile(f"({'|'.join(delimiters + braces + quoted_strings)})") tokens = tuple( item - for item in tokenization_re.split(spec) + for item in _token_regex.split(spec) if item is not None and item.strip() ) return _recombine_set_tokens(tokens) @@ -856,7 +849,7 @@ def _convert_numpy_type_spec(_type, translations={}): type_ = "value_set" elif token in ("optional", "default"): type_ = "control" - elif re.match(":[^:]+:`[^`]+`", token): + elif _xref_regex.match(token): type_ = "reference" elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): type_ = "literal" From 20e36007fe69ff4f27552ff6ebc72c9c0c1abd1f Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:16:49 +0200 Subject: [PATCH 31/88] treat value sets as literals --- sphinx/ext/napoleon/docstring.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index dfc2605ab..b47490d0d 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -845,14 +845,17 @@ def _convert_numpy_type_spec(_type, translations={}): def token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" - elif token.startswith("{") and token.endswith("}"): - type_ = "value_set" + elif ( + token.isnumeric() + or (token.startswith("{") and token.endswith("}")) + or (token.startswith('"') and token.endswith('"')) + or (token.startswith("'") and token.endswith("'")) + ): + type_ = "literal" elif token in ("optional", "default"): type_ = "control" elif _xref_regex.match(token): type_ = "reference" - elif token.isnumeric() or (token.startswith('"') and token.endswith('"')): - type_ = "literal" else: type_ = "obj" @@ -875,7 +878,6 @@ def _convert_numpy_type_spec(_type, translations={}): ) converters = { - "value_set": lambda x: f"``{x}``", "literal": lambda x: f"``{x}``", "obj": lambda x: convert_obj(x, translations, default_translation), "control": lambda x: f"*{x}*", From dc8c7ac9f8efd9b984c406a6bea410333a7be404 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:30:14 +0200 Subject: [PATCH 32/88] update the integration test --- tests/test_ext_napoleon_docstring.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index e1c296e79..49479fa59 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2068,13 +2068,13 @@ definition_after_normal_text : int converted_types = ( ":obj:`str`", ":obj:`int` or :obj:`float` or :obj:`None`", - ':noref:`{"F", "C", "N"}`', - ":noref:`{'F', 'C', 'N'}`", + '``{"F", "C", "N"}``', + "``{'F', 'C', 'N'}``", ) converted_modifiers = ( "", - ":noref:`optional`", - ":noref:`default`: :obj:`None`", + "*optional*", + "*default*: :obj:`None`", ) converted = tuple( ", ".join([converted_type, converted_modifier]) @@ -2114,13 +2114,20 @@ definition_after_normal_text : int :param param2: a parameter with different types :type param2: :obj:`int` or :obj:`float` or :obj:`None` :param param3: a optional mapping - :type param3: :term:`dict-like `, :noref:`optional` + :type param3: :term:`dict-like `, *optional* :param param4: a optional parameter with different types - :type param4: :obj:`int` or :obj:`float` or :obj:`None`, :noref:`optional` + :type param4: :obj:`int` or :obj:`float` or :obj:`None`, *optional* :param param5: a optional parameter with fixed values - :type param5: :noref:`{"F", "C", "N"}`, :noref:`optional` + :type param5: ``{"F", "C", "N"}``, *optional* """) - config = Config(napoleon_use_param=True, napoleon_use_rtype=True) + translations = { + "dict-like": ":term:`dict-like `", + } + config = Config( + napoleon_use_param=True, + napoleon_use_rtype=True, + napoleon_type_aliases=translations, + ) actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) From af6071e5719aef6650370ffee504e0e98e1dc1a2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 21:58:44 +0200 Subject: [PATCH 33/88] expect a warning instead of an error --- tests/test_ext_napoleon_docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 49479fa59..abde56212 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1999,7 +1999,7 @@ definition_after_normal_text : int ) for input_tokens in invalid_tokens: - with self.assertRaisesRegex(ValueError, "invalid value set:"): + with self.assertWarnsRegex(UserWarning, "invalid value set:"): _recombine_set_tokens(input_tokens) From b0da0e5aefc37b635250c5f84a367d9245d1ec2d Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 22:10:24 +0200 Subject: [PATCH 34/88] remove the default for the default translation --- sphinx/ext/napoleon/docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index b47490d0d..7437f5f39 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -861,7 +861,7 @@ def _convert_numpy_type_spec(_type, translations={}): return type_ - def convert_obj(obj, translations, default_translation=":obj:`{}`"): + def convert_obj(obj, translations, default_translation): return translations.get(obj, default_translation.format(obj)) tokens = _tokenize_type_spec(_type) From 37e02512fc5e356bab491861b57576e066207dd2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 29 May 2020 22:10:39 +0200 Subject: [PATCH 35/88] make invalid value sets a literal to avoid further warnings --- sphinx/ext/napoleon/docstring.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 7437f5f39..6ce3798f9 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -852,6 +852,9 @@ def _convert_numpy_type_spec(_type, translations={}): or (token.startswith("'") and token.endswith("'")) ): type_ = "literal" + elif token.startswith("{"): + # invalid value set, make it a literal to avoid further warnings + type_ = "literal" elif token in ("optional", "default"): type_ = "control" elif _xref_regex.match(token): From 866c822e1190bfadd4fa276899d3d479c918f196 Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 4 Jun 2020 14:33:35 +0200 Subject: [PATCH 36/88] move the warnings to token_type --- sphinx/ext/napoleon/docstring.py | 81 +++++++++++++++++----------- tests/test_ext_napoleon_docstring.py | 43 ++++++++++++--- 2 files changed, 86 insertions(+), 38 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 6ce3798f9..bf4bf0d8f 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -799,14 +799,6 @@ def _recombine_set_tokens(tokens): try: token = next(iterable) except StopIteration: - if open_braces != 0: - location = ("", "") - logger.warning( - __("invalid value set: %r"), - "".join(tokens), - location=location, - ) - break if token == "{": @@ -841,35 +833,62 @@ def _tokenize_type_spec(spec): return _recombine_set_tokens(tokens) + +def _token_type(token): + if token.startswith(" ") or token.endswith(" "): + type_ = "delimiter" + elif ( + token.isnumeric() + or (token.startswith("{") and token.endswith("}")) + or (token.startswith('"') and token.endswith('"')) + or (token.startswith("'") and token.endswith("'")) + ): + type_ = "literal" + elif token.startswith("{"): + logger.warning( + __("invalid value set (missing closing brace): %s"), + token, + location=None, + ) + type_ = "literal" + elif token.endswith("}"): + logger.warning( + __("invalid value set (missing opening brace): %s"), + token, + location=None, + ) + type_ = "literal" + elif token.startswith("'") or token.startswith('"'): + logger.warning( + __("malformed string literal (missing closing quote): %s"), + token, + location=None, + ) + type_ = "literal" + elif token.endswith("'") or token.endswith('"'): + logger.warning( + __("malformed string literal (missing opening quote): %s"), + token, + location=None, + ) + type_ = "literal" + elif token in ("optional", "default"): + type_ = "control" + elif _xref_regex.match(token): + type_ = "reference" + else: + type_ = "obj" + + return type_ + + def _convert_numpy_type_spec(_type, translations={}): - def token_type(token): - if token.startswith(" ") or token.endswith(" "): - type_ = "delimiter" - elif ( - token.isnumeric() - or (token.startswith("{") and token.endswith("}")) - or (token.startswith('"') and token.endswith('"')) - or (token.startswith("'") and token.endswith("'")) - ): - type_ = "literal" - elif token.startswith("{"): - # invalid value set, make it a literal to avoid further warnings - type_ = "literal" - elif token in ("optional", "default"): - type_ = "control" - elif _xref_regex.match(token): - type_ = "reference" - else: - type_ = "obj" - - return type_ - def convert_obj(obj, translations, default_translation): return translations.get(obj, default_translation.format(obj)) tokens = _tokenize_type_spec(_type) types = [ - (token, token_type(token)) + (token, _token_type(token)) for token in tokens ] diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index abde56212..1f9af8c2a 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -16,7 +16,12 @@ from unittest import TestCase, mock from sphinx.ext.napoleon import Config from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring -from sphinx.ext.napoleon.docstring import _tokenize_type_spec, _recombine_set_tokens, _convert_numpy_type_spec +from sphinx.ext.napoleon.docstring import ( + _tokenize_type_spec, + _recombine_set_tokens, + _convert_numpy_type_spec, + _token_type +) class NamedtupleSubclass(namedtuple('NamedtupleSubclass', ('attr1', 'attr2'))): @@ -1993,15 +1998,39 @@ definition_after_normal_text : int actual = _recombine_set_tokens(input_tokens) self.assertEqual(expected, actual) - def test_recombine_set_tokens_invalid(self): - invalid_tokens = ( - ["{", "1", ", ", "2"], + def test_token_type(self): + tokens = ( + ("1", "literal"), + ("'string'", "literal"), + ('"another_string"', "literal"), + ("{1, 2}", "literal"), + ("{'va{ue', 'set'}", "literal"), + ("optional", "control"), + ("default", "control"), + (", ", "delimiter"), + (" of ", "delimiter"), + (" or ", "delimiter"), + (": ", "delimiter"), + ("True", "obj"), + ("None", "obj"), + ("name", "obj"), + (":py:class:`Enum`", "reference"), ) - for input_tokens in invalid_tokens: - with self.assertWarnsRegex(UserWarning, "invalid value set:"): - _recombine_set_tokens(input_tokens) + for token, expected in tokens: + actual = _token_type(token) + self.assertEqual(expected, actual) + def test_token_type_invalid(self): + tokens = ( + "{1, 2", + "1, 2}", + "'abc", + "def'", + ) + for token in tokens: + # TODO: check for the warning + _token_type(token) def test_tokenize_type_spec(self): types = ( From d177e589993cd7401f6ff7b37ee576dc657e86ce Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 4 Jun 2020 21:06:04 +0200 Subject: [PATCH 37/88] reimplement the value set combination function using collections.deque --- sphinx/ext/napoleon/docstring.py | 51 +++++++++++----- tests/test_ext_napoleon_docstring.py | 87 ++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 38 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index bf4bf0d8f..9bd714f60 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -10,6 +10,7 @@ :license: BSD, see LICENSE for details. """ +import collections import inspect import re from functools import partial @@ -790,17 +791,34 @@ class GoogleDocstring: return lines -def _recombine_set_tokens(tokens): - def takewhile_set(iterable): - yield "{" +def _recombine_sets(tokens): + tokens = collections.deque(tokens) + keywords = ("optional", "default") - open_braces = 1 + def takewhile_set(tokens): + open_braces = 0 + previous_token = None + print("combining set:", tokens) while True: try: - token = next(iterable) - except StopIteration: + token = tokens.popleft() + except IndexError: break + if token == ", ": + previous_token = token + continue + + if token in keywords: + tokens.appendleft(token) + if previous_token is not None: + tokens.appendleft(previous_token) + break + + if previous_token is not None: + yield previous_token + previous_token = None + if token == "{": open_braces += 1 elif token == "}": @@ -812,26 +830,28 @@ def _recombine_set_tokens(tokens): break def combine_set(tokens): - iterable = iter(tokens) while True: try: - token = next(iterable) - except StopIteration: + token = tokens.popleft() + except IndexError: break - yield "".join(takewhile_set(iterable)) if token == "{" else token + if token == "{": + tokens.appendleft("{") + yield "".join(takewhile_set(tokens)) + else: + yield token return list(combine_set(tokens)) def _tokenize_type_spec(spec): - tokens = tuple( + tokens = list( item for item in _token_regex.split(spec) if item is not None and item.strip() ) - return _recombine_set_tokens(tokens) - + return tokens def _token_type(token): @@ -842,7 +862,7 @@ def _token_type(token): or (token.startswith("{") and token.endswith("}")) or (token.startswith('"') and token.endswith('"')) or (token.startswith("'") and token.endswith("'")) - ): + ): type_ = "literal" elif token.startswith("{"): logger.warning( @@ -887,9 +907,10 @@ def _convert_numpy_type_spec(_type, translations={}): return translations.get(obj, default_translation.format(obj)) tokens = _tokenize_type_spec(_type) + combined_tokens = _recombine_sets(tokens) types = [ (token, _token_type(token)) - for token in tokens + for token in combined_tokens ] # don't use the object role if it's not necessary diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 1f9af8c2a..13d4db8b8 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -18,7 +18,7 @@ from sphinx.ext.napoleon import Config from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring from sphinx.ext.napoleon.docstring import ( _tokenize_type_spec, - _recombine_set_tokens, + _recombine_sets, _convert_numpy_type_spec, _token_type ) @@ -1068,7 +1068,7 @@ Methods: .. method:: func(i, j) :noindex: - + description """ config = Config() @@ -1982,22 +1982,6 @@ definition_after_normal_text : int actual = str(NumpyDocstring(docstring, config)) self.assertEqual(expected, actual) - def test_recombine_set_tokens(self): - tokens = ( - ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], - ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], - ["{", "1", ", ", "2", "}"], - ) - recombined_tokens = ( - ["{'F', 'C', 'N'}"], - ['{"F", "C", "N"}'], - ["{1, 2}"], - ) - - for input_tokens, expected in zip(tokens, recombined_tokens): - actual = _recombine_set_tokens(input_tokens) - self.assertEqual(expected, actual) - def test_token_type(self): tokens = ( ("1", "literal"), @@ -2042,6 +2026,7 @@ definition_after_normal_text : int r"'with \'quotes\''", ) modifiers = ( + "", "optional", "default: None", ) @@ -2049,23 +2034,24 @@ definition_after_normal_text : int type_tokens = ( ["str"], ["int", " or ", "float", " or ", "None"], - ['{"F", "C", "N"}'], - ["{'F', 'C', 'N'}"], + ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], ['"ma{icious"'], [r"'with \'quotes\''"], ) modifier_tokens = ( + [], ["optional"], ["default", ": ", "None"], ) - + type_specs = tuple( - ", ".join([type_, modifier]) + ", ".join([type_, modifier]) if modifier else type_ for type_ in types for modifier in modifiers ) tokens = tuple( - tokens_ + [", "] + modifier_tokens_ + tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) for tokens_ in type_tokens for modifier_tokens_ in modifier_tokens ) @@ -2074,8 +2060,63 @@ definition_after_normal_text : int actual = _tokenize_type_spec(type_spec) self.assertEqual(expected, actual) + def test_recombine_sets(self): + type_tokens = ( + ["{", "1", ", ", "2", "}"], + ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], + ) + modifier_tokens = ( + [], + ["optional"], + ["default", ": ", "None"], + ) + tokens = tuple( + type_tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) + for type_tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + combined_tokens = tuple( + ["".join(type_tokens_)] + ([", "] + modifier_tokens_ if modifier_tokens_ else []) + for type_tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + for tokens_, expected in zip(tokens, combined_tokens): + actual = _recombine_sets(tokens_) + self.assertEqual(expected, actual) + + def test_recombine_sets_invalid(self): + type_tokens = ( + ["{", "1", ", ", "2"], + ['"F"', ", ", '"C"', ", ", '"N"', "}"], + ) + modifier_tokens = ( + [], + ["optional"], + ["default", ": ", "None"], + ) + tokens = tuple( + type_tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) + for type_tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + combined_tokens = tuple( + (["".join(type_tokens_)] if "{" in type_tokens_ else type_tokens_) + + ([", "] + modifier_tokens_ if modifier_tokens_ else []) + for type_tokens_ in type_tokens + for modifier_tokens_ in modifier_tokens + ) + + for tokens_, expected in zip(tokens, combined_tokens): + actual = _recombine_sets(tokens_) + self.assertEqual(expected, actual) + def test_convert_numpy_type_spec(self): types = ( + "", "str", "int or float or None", '{"F", "C", "N"}', From 1140f7b26d40d676c1e0300015ee93eb980892af Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 4 Jun 2020 21:07:21 +0200 Subject: [PATCH 38/88] also check type specs without actual types --- tests/test_ext_napoleon_docstring.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 13d4db8b8..c3f917081 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2128,14 +2128,13 @@ definition_after_normal_text : int "default: None", ) type_specs = tuple( - ", ".join([type_, modifier]) - if modifier - else type_ + ", ".join(part for part in (type_, modifier) if part) for type_ in types for modifier in modifiers ) converted_types = ( + "", ":obj:`str`", ":obj:`int` or :obj:`float` or :obj:`None`", '``{"F", "C", "N"}``', @@ -2147,7 +2146,7 @@ definition_after_normal_text : int "*default*: :obj:`None`", ) converted = tuple( - ", ".join([converted_type, converted_modifier]) + ", ".join(part for part in (converted_type, converted_modifier) if part) if converted_modifier else ( type_ From 26855f92d820bbef310d45a6dee934ee640fae7a Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 4 Jun 2020 21:07:59 +0200 Subject: [PATCH 39/88] also test invalid string tokens --- tests/test_ext_napoleon_docstring.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index c3f917081..8457edad8 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2008,14 +2008,18 @@ definition_after_normal_text : int def test_token_type_invalid(self): tokens = ( "{1, 2", - "1, 2}", + "}", "'abc", "def'", + '"ghi', + 'jkl"', ) for token in tokens: # TODO: check for the warning _token_type(token) + assert False + def test_tokenize_type_spec(self): types = ( "str", From 4d0b4f2931c0b8780277a06d0c2eb054dc8f978f Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 7 Jun 2020 15:51:53 +0200 Subject: [PATCH 40/88] add back the trailing whitespace --- tests/test_ext_napoleon_docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 8457edad8..b3df079f2 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1068,7 +1068,7 @@ Methods: .. method:: func(i, j) :noindex: - + description """ config = Config() From fedceb25ff9098714ad15cbca4d0e49f244ac2b0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 7 Jun 2020 19:01:29 +0200 Subject: [PATCH 41/88] move the binary operator "or" to before the newline --- sphinx/ext/napoleon/docstring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 9bd714f60..a50a695ee 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -858,10 +858,10 @@ def _token_type(token): if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" elif ( - token.isnumeric() - or (token.startswith("{") and token.endswith("}")) - or (token.startswith('"') and token.endswith('"')) - or (token.startswith("'") and token.endswith("'")) + token.isnumeric() or + (token.startswith("{") and token.endswith("}")) or + (token.startswith('"') and token.endswith('"')) or + (token.startswith("'") and token.endswith("'")) ): type_ = "literal" elif token.startswith("{"): From 7d8aaf2c0337b68af5af04b138175ffb161951e0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 7 Jul 2020 22:56:01 +0200 Subject: [PATCH 42/88] remove a debug print --- sphinx/ext/napoleon/docstring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index a50a695ee..ac67e06f0 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -798,7 +798,6 @@ def _recombine_sets(tokens): def takewhile_set(tokens): open_braces = 0 previous_token = None - print("combining set:", tokens) while True: try: token = tokens.popleft() From f4817be7a9daa4843da24a3fe754ec06120f5527 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 7 Jul 2020 22:57:37 +0200 Subject: [PATCH 43/88] use the format method instead of f-strings --- sphinx/ext/napoleon/docstring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index ac67e06f0..fd16c95ff 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -920,9 +920,9 @@ def _convert_numpy_type_spec(_type, translations={}): ) converters = { - "literal": lambda x: f"``{x}``", + "literal": lambda x: "``{x}``".format(x=x), "obj": lambda x: convert_obj(x, translations, default_translation), - "control": lambda x: f"*{x}*", + "control": lambda x: "*{x}*".format(x=x), "delimiter": lambda x: x, "reference": lambda x: x, } From 804df88e8d84d7ae5f111b6b4d0f55bf12e547c6 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 20:56:22 +0200 Subject: [PATCH 44/88] use :class: as default role and only fall back to :obj: for singletons --- sphinx/ext/napoleon/docstring.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index fd16c95ff..7f6f21d32 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -903,6 +903,10 @@ def _token_type(token): def _convert_numpy_type_spec(_type, translations={}): def convert_obj(obj, translations, default_translation): + # use :class: (the default) only if obj is not a standard singleton (None, True, False) + if obj in (None, True, False) and default_translation == ":class:`{}`": + default_translation = ":obj:`{}`" + return translations.get(obj, default_translation.format(obj)) tokens = _tokenize_type_spec(_type) @@ -914,7 +918,7 @@ def _convert_numpy_type_spec(_type, translations={}): # don't use the object role if it's not necessary default_translation = ( - ":obj:`{}`" + ":class:`{}`" if not all(type_ == "obj" for _, type_ in types) else "{}" ) From f30c0cb9f674baa0048b9d3d6a8cad85807c52c0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:13:28 +0200 Subject: [PATCH 45/88] rewrite the invalid token_type test to check the warnings --- tests/test_ext_napoleon_docstring.py | 54 +++++++++++++++++++--------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index b3df079f2..5c0941398 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -9,7 +9,9 @@ :license: BSD, see LICENSE for details. """ +import re from collections import namedtuple +from contextlib import contextmanager from inspect import cleandoc from textwrap import dedent from unittest import TestCase, mock @@ -1068,7 +1070,7 @@ Methods: .. method:: func(i, j) :noindex: - + description """ config = Config() @@ -2005,21 +2007,6 @@ definition_after_normal_text : int actual = _token_type(token) self.assertEqual(expected, actual) - def test_token_type_invalid(self): - tokens = ( - "{1, 2", - "}", - "'abc", - "def'", - '"ghi', - 'jkl"', - ) - for token in tokens: - # TODO: check for the warning - _token_type(token) - - assert False - def test_tokenize_type_spec(self): types = ( "str", @@ -2219,3 +2206,38 @@ Do as you please :kwtype gotham_is_yours: None """ self.assertEqual(expected, actual) + +@contextmanager +def warns(warning, match): + match_re = re.compile(match) + try: + yield warning + finally: + raw_warnings = warning.getvalue() + warnings = [w for w in raw_warnings.split("\n") if w.strip()] + + assert len(warnings) == 1 and all(match_re.match(w) for w in warnings) + warning.truncate(0) + + +class TestNumpyDocstring: + def test_token_type_invalid(self, warning): + tokens = ( + "{1, 2", + "}", + "'abc", + "def'", + '"ghi', + 'jkl"', + ) + errors = ( + r".+: invalid value set \(missing closing brace\):", + r".+: invalid value set \(missing opening brace\):", + r".+: malformed string literal \(missing closing quote\):", + r".+: malformed string literal \(missing opening quote\):", + r".+: malformed string literal \(missing closing quote\):", + r".+: malformed string literal \(missing opening quote\):", + ) + for token, error in zip(tokens, errors): + with warns(warning, match=error): + _token_type(token) From 4fc22cd0c40290e52a6ee5820ac54d1db309496d Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:14:13 +0200 Subject: [PATCH 46/88] use the dedent function imported at module-level --- tests/test_ext_napoleon_docstring.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 5c0941398..7a273b26a 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2153,8 +2153,7 @@ definition_after_normal_text : int self.assertEqual(expected, actual) def test_parameter_types(self): - import textwrap - docstring = textwrap.dedent("""\ + docstring = dedent("""\ Parameters ---------- param1 : DataFrame @@ -2168,17 +2167,17 @@ definition_after_normal_text : int param5 : {"F", "C", "N"}, optional a optional parameter with fixed values """) - expected = textwrap.dedent("""\ - :param param1: the data to work on - :type param1: DataFrame - :param param2: a parameter with different types - :type param2: :obj:`int` or :obj:`float` or :obj:`None` - :param param3: a optional mapping - :type param3: :term:`dict-like `, *optional* - :param param4: a optional parameter with different types - :type param4: :obj:`int` or :obj:`float` or :obj:`None`, *optional* - :param param5: a optional parameter with fixed values - :type param5: ``{"F", "C", "N"}``, *optional* + expected = dedent("""\ + :param param1: the data to work on + :type param1: DataFrame + :param param2: a parameter with different types + :type param2: :obj:`int` or :obj:`float` or :obj:`None` + :param param3: a optional mapping + :type param3: :term:`dict-like `, *optional* + :param param4: a optional parameter with different types + :type param4: :obj:`int` or :obj:`float` or :obj:`None`, *optional* + :param param5: a optional parameter with fixed values + :type param5: ``{"F", "C", "N"}``, *optional* """) translations = { "dict-like": ":term:`dict-like `", From fc43f494acfefbd81273bc69591e9707b725fd1a Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:16:04 +0200 Subject: [PATCH 47/88] add back the trailing whitespace --- tests/test_ext_napoleon_docstring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 7a273b26a..7aa7b2ea2 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -1070,7 +1070,7 @@ Methods: .. method:: func(i, j) :noindex: - + description """ config = Config() From 2b981b6abdb998f5363e947b6b84089d47f6ee52 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:19:26 +0200 Subject: [PATCH 48/88] make sure singletons actually use :obj: --- sphinx/ext/napoleon/docstring.py | 2 +- tests/test_ext_napoleon_docstring.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 7f6f21d32..48daa8ada 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -904,7 +904,7 @@ def _token_type(token): def _convert_numpy_type_spec(_type, translations={}): def convert_obj(obj, translations, default_translation): # use :class: (the default) only if obj is not a standard singleton (None, True, False) - if obj in (None, True, False) and default_translation == ":class:`{}`": + if obj in ("None", "True", "False") and default_translation == ":class:`{}`": default_translation = ":obj:`{}`" return translations.get(obj, default_translation.format(obj)) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 7aa7b2ea2..18eddb72a 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2126,8 +2126,8 @@ definition_after_normal_text : int converted_types = ( "", - ":obj:`str`", - ":obj:`int` or :obj:`float` or :obj:`None`", + ":class:`str`", + ":class:`int` or :class:`float` or :obj:`None`", '``{"F", "C", "N"}``', "``{'F', 'C', 'N'}``", ) @@ -2171,11 +2171,11 @@ definition_after_normal_text : int :param param1: the data to work on :type param1: DataFrame :param param2: a parameter with different types - :type param2: :obj:`int` or :obj:`float` or :obj:`None` + :type param2: :class:`int` or :class:`float` or :obj:`None` :param param3: a optional mapping :type param3: :term:`dict-like `, *optional* :param param4: a optional parameter with different types - :type param4: :obj:`int` or :obj:`float` or :obj:`None`, *optional* + :type param4: :class:`int` or :class:`float` or :obj:`None`, *optional* :param param5: a optional parameter with fixed values :type param5: ``{"F", "C", "N"}``, *optional* """) From 922054ed6fe0c3bf705c6988491606a3cdf66e81 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sun, 12 Jul 2020 23:26:12 +0200 Subject: [PATCH 49/88] replace .format with %-style string interpolation --- sphinx/ext/napoleon/docstring.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 48daa8ada..b0546ed37 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -904,10 +904,10 @@ def _token_type(token): def _convert_numpy_type_spec(_type, translations={}): def convert_obj(obj, translations, default_translation): # use :class: (the default) only if obj is not a standard singleton (None, True, False) - if obj in ("None", "True", "False") and default_translation == ":class:`{}`": - default_translation = ":obj:`{}`" + if obj in ("None", "True", "False") and default_translation == ":class:`%s`": + default_translation = ":obj:`%s`" - return translations.get(obj, default_translation.format(obj)) + return translations.get(obj, default_translation % obj) tokens = _tokenize_type_spec(_type) combined_tokens = _recombine_sets(tokens) @@ -918,15 +918,15 @@ def _convert_numpy_type_spec(_type, translations={}): # don't use the object role if it's not necessary default_translation = ( - ":class:`{}`" + ":class:`%s`" if not all(type_ == "obj" for _, type_ in types) - else "{}" + else "%s" ) converters = { - "literal": lambda x: "``{x}``".format(x=x), + "literal": lambda x: "``%s``" % x, "obj": lambda x: convert_obj(x, translations, default_translation), - "control": lambda x: "*{x}*".format(x=x), + "control": lambda x: "*%s*" % x, "delimiter": lambda x: x, "reference": lambda x: x, } From 660b818636bcd5b29ef6b23d565f82ac7de99266 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 13 Jul 2020 00:06:07 +0200 Subject: [PATCH 50/88] add type hints and location information --- sphinx/ext/napoleon/docstring.py | 35 ++++++++++++++++++---------- tests/test_ext_napoleon_docstring.py | 10 ++++---- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index b0546ed37..46cad9553 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -791,8 +791,8 @@ class GoogleDocstring: return lines -def _recombine_sets(tokens): - tokens = collections.deque(tokens) +def _recombine_set_tokens(tokens: List[str]) -> List[str]: + token_queue = collections.deque(tokens) keywords = ("optional", "default") def takewhile_set(tokens): @@ -841,10 +841,10 @@ def _recombine_sets(tokens): else: yield token - return list(combine_set(tokens)) + return list(combine_set(token_queue)) -def _tokenize_type_spec(spec): +def _tokenize_type_spec(spec: str) -> List[str]: tokens = list( item for item in _token_regex.split(spec) @@ -853,7 +853,7 @@ def _tokenize_type_spec(spec): return tokens -def _token_type(token): +def _token_type(token: str, location: str = None) -> str: if token.startswith(" ") or token.endswith(" "): type_ = "delimiter" elif ( @@ -867,28 +867,28 @@ def _token_type(token): logger.warning( __("invalid value set (missing closing brace): %s"), token, - location=None, + location=location, ) type_ = "literal" elif token.endswith("}"): logger.warning( __("invalid value set (missing opening brace): %s"), token, - location=None, + location=location, ) type_ = "literal" elif token.startswith("'") or token.startswith('"'): logger.warning( __("malformed string literal (missing closing quote): %s"), token, - location=None, + location=location, ) type_ = "literal" elif token.endswith("'") or token.endswith('"'): logger.warning( __("malformed string literal (missing opening quote): %s"), token, - location=None, + location=location, ) type_ = "literal" elif token in ("optional", "default"): @@ -901,7 +901,7 @@ def _token_type(token): return type_ -def _convert_numpy_type_spec(_type, translations={}): +def _convert_numpy_type_spec(_type: str, location: str = None, translations: dict = {}) -> str: def convert_obj(obj, translations, default_translation): # use :class: (the default) only if obj is not a standard singleton (None, True, False) if obj in ("None", "True", "False") and default_translation == ":class:`%s`": @@ -910,9 +910,9 @@ def _convert_numpy_type_spec(_type, translations={}): return translations.get(obj, default_translation % obj) tokens = _tokenize_type_spec(_type) - combined_tokens = _recombine_sets(tokens) + combined_tokens = _recombine_set_tokens(tokens) types = [ - (token, _token_type(token)) + (token, _token_type(token, location)) for token in combined_tokens ] @@ -1035,6 +1035,16 @@ class NumpyDocstring(GoogleDocstring): self._directive_sections = ['.. index::'] super().__init__(docstring, config, app, what, name, obj, options) + def _get_location(self) -> str: + filepath = inspect.getfile(self._obj) if self._obj is not None else "" + name = self._name + line = "" + + if filepath is None and name is None: + return None + + return ":".join([filepath, "docstring of %s" % name, line]) + def _consume_field(self, parse_type: bool = True, prefer_type: bool = False ) -> Tuple[str, str, List[str]]: line = next(self._line_iter) @@ -1046,6 +1056,7 @@ class NumpyDocstring(GoogleDocstring): _name = self._escape_args_and_kwargs(_name) _type = _convert_numpy_type_spec( _type, + location=self._get_location(), translations=self._config.napoleon_type_aliases or {}, ) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 18eddb72a..f7767065a 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -20,7 +20,7 @@ from sphinx.ext.napoleon import Config from sphinx.ext.napoleon.docstring import GoogleDocstring, NumpyDocstring from sphinx.ext.napoleon.docstring import ( _tokenize_type_spec, - _recombine_sets, + _recombine_set_tokens, _convert_numpy_type_spec, _token_type ) @@ -2051,7 +2051,7 @@ definition_after_normal_text : int actual = _tokenize_type_spec(type_spec) self.assertEqual(expected, actual) - def test_recombine_sets(self): + def test_recombine_set_tokens(self): type_tokens = ( ["{", "1", ", ", "2", "}"], ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], @@ -2075,10 +2075,10 @@ definition_after_normal_text : int ) for tokens_, expected in zip(tokens, combined_tokens): - actual = _recombine_sets(tokens_) + actual = _recombine_set_tokens(tokens_) self.assertEqual(expected, actual) - def test_recombine_sets_invalid(self): + def test_recombine_set_tokens_invalid(self): type_tokens = ( ["{", "1", ", ", "2"], ['"F"', ", ", '"C"', ", ", '"N"', "}"], @@ -2102,7 +2102,7 @@ definition_after_normal_text : int ) for tokens_, expected in zip(tokens, combined_tokens): - actual = _recombine_sets(tokens_) + actual = _recombine_set_tokens(tokens_) self.assertEqual(expected, actual) def test_convert_numpy_type_spec(self): From cc8baf60ec93f0ba429f21aa40f7c12ba8ce3e71 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 13 Jul 2020 14:25:01 +0200 Subject: [PATCH 51/88] only transform the types if napoleon_use_param is true --- sphinx/ext/napoleon/docstring.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 46cad9553..3a4ee02e6 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -1054,11 +1054,12 @@ class NumpyDocstring(GoogleDocstring): _name, _type = line, '' _name, _type = _name.strip(), _type.strip() _name = self._escape_args_and_kwargs(_name) - _type = _convert_numpy_type_spec( - _type, - location=self._get_location(), - translations=self._config.napoleon_type_aliases or {}, - ) + if self._config.napoleon_use_param: + _type = _convert_numpy_type_spec( + _type, + location=self._get_location(), + translations=self._config.napoleon_type_aliases or {}, + ) if prefer_type and not _type: _type, _name = _name, _type From 274d9fe4f98d98c604c0e9bf0a4b49ceb62d523b Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 14 Jul 2020 23:00:07 +0200 Subject: [PATCH 52/88] don't try to generate test cases in code --- tests/test_ext_napoleon_docstring.py | 134 +++++++-------------------- 1 file changed, 34 insertions(+), 100 deletions(-) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index f7767065a..2b1ac93aa 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2008,70 +2008,39 @@ definition_after_normal_text : int self.assertEqual(expected, actual) def test_tokenize_type_spec(self): - types = ( + specs = ( "str", - "int or float or None", + "int or float or None, optional", '{"F", "C", "N"}', - "{'F', 'C', 'N'}", + "{'F', 'C', 'N'}, default: 'F'", '"ma{icious"', r"'with \'quotes\''", ) - modifiers = ( - "", - "optional", - "default: None", - ) - type_tokens = ( + tokens = ( ["str"], - ["int", " or ", "float", " or ", "None"], + ["int", " or ", "float", " or ", "None", ", ", "optional"], ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], - ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "'F'"], ['"ma{icious"'], [r"'with \'quotes\''"], ) - modifier_tokens = ( - [], - ["optional"], - ["default", ": ", "None"], - ) - type_specs = tuple( - ", ".join([type_, modifier]) if modifier else type_ - for type_ in types - for modifier in modifiers - ) - tokens = tuple( - tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens - ) - - for type_spec, expected in zip(type_specs, tokens): - actual = _tokenize_type_spec(type_spec) + for spec, expected in zip(specs, tokens): + actual = _tokenize_type_spec(spec) self.assertEqual(expected, actual) def test_recombine_set_tokens(self): - type_tokens = ( + tokens = ( ["{", "1", ", ", "2", "}"], - ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], - ["{", "'F'", ", ", "'C'", ", ", "'N'", "}"], - ) - modifier_tokens = ( - [], - ["optional"], - ["default", ": ", "None"], - ) - tokens = tuple( - type_tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for type_tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens + ["{", '"F"', ", ", '"C"', ", ", '"N"', "}", ", ", "optional"], + ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "None"], ) - combined_tokens = tuple( - ["".join(type_tokens_)] + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for type_tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens + combined_tokens = ( + ["{1, 2}"], + ['{"F", "C", "N"}', ", ", "optional"], + ["{'F', 'C', 'N'}", ", ", "default", ": ", "None"], ) for tokens_, expected in zip(tokens, combined_tokens): @@ -2079,26 +2048,15 @@ definition_after_normal_text : int self.assertEqual(expected, actual) def test_recombine_set_tokens_invalid(self): - type_tokens = ( + tokens = ( ["{", "1", ", ", "2"], - ['"F"', ", ", '"C"', ", ", '"N"', "}"], + ['"F"', ", ", '"C"', ", ", '"N"', "}", ", ", "optional"], + ["{", "1", ", ", "2", ", ", "default", ": ", "None"], ) - modifier_tokens = ( - [], - ["optional"], - ["default", ": ", "None"], - ) - tokens = tuple( - type_tokens_ + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for type_tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens - ) - - combined_tokens = tuple( - (["".join(type_tokens_)] if "{" in type_tokens_ else type_tokens_) - + ([", "] + modifier_tokens_ if modifier_tokens_ else []) - for type_tokens_ in type_tokens - for modifier_tokens_ in modifier_tokens + combined_tokens = ( + ["{1, 2"], + ['"F"', ", ", '"C"', ", ", '"N"', "}", ", ", "optional"], + ["{1, 2", ", ", "default", ": ", "None"], ) for tokens_, expected in zip(tokens, combined_tokens): @@ -2106,50 +2064,26 @@ definition_after_normal_text : int self.assertEqual(expected, actual) def test_convert_numpy_type_spec(self): - types = ( - "", - "str", - "int or float or None", - '{"F", "C", "N"}', - "{'F', 'C', 'N'}", - ) - modifiers = ( + specs = ( "", "optional", - "default: None", - ) - type_specs = tuple( - ", ".join(part for part in (type_, modifier) if part) - for type_ in types - for modifier in modifiers + "str, optional", + "int or float or None, default: None", + '{"F", "C", "N"}', + "{'F', 'C', 'N'}, default: 'N'", ) - converted_types = ( - "", - ":class:`str`", - ":class:`int` or :class:`float` or :obj:`None`", - '``{"F", "C", "N"}``', - "``{'F', 'C', 'N'}``", - ) - converted_modifiers = ( + converted = ( "", "*optional*", - "*default*: :obj:`None`", - ) - converted = tuple( - ", ".join(part for part in (converted_type, converted_modifier) if part) - if converted_modifier - else ( - type_ - if ("{" not in type_ and "or" not in type_) - else converted_type - ) - for converted_type, type_ in zip(converted_types, types) - for converted_modifier in converted_modifiers + ":class:`str`, *optional*", + ":class:`int` or :class:`float` or :obj:`None`, *default*: :obj:`None`", + '``{"F", "C", "N"}``', + "``{'F', 'C', 'N'}``, *default*: ``'N'``", ) - for type_, expected in zip(type_specs, converted): - actual = _convert_numpy_type_spec(type_) + for spec, expected in zip(specs, converted): + actual = _convert_numpy_type_spec(spec) self.assertEqual(expected, actual) def test_parameter_types(self): From b3b7cbbd3836ecbd3ff053884aa09b92f823e9d6 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 19 Jul 2020 15:47:42 +0900 Subject: [PATCH 53/88] Fix #7983: autodoc: Generator type annotation is wrongly rendered in py36 This adds a special handler (if-branch) for Generator type to stringify them correctly. So far, they have been considered as a kind of Callable. --- CHANGES | 1 + sphinx/util/typing.py | 4 +++- tests/test_util_typing.py | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index bc932a7d4..e71df5a75 100644 --- a/CHANGES +++ b/CHANGES @@ -46,6 +46,7 @@ Bugs fixed * #7901: autodoc: type annotations for overloaded functions are not resolved * #904: autodoc: An instance attribute cause a crash of autofunction directive * #1362: autodoc: ``private-members`` option does not work for class attributes +* #7983: autodoc: Generator type annotation is wrongly rendered in py36 * #7839: autosummary: cannot handle umlauts in function names * #7865: autosummary: Failed to extract summary line when abbreviations found * #7866: autosummary: Failed to extract correct summary line when docstring diff --git a/sphinx/util/typing.py b/sphinx/util/typing.py index 86f9c6e5c..d71ca1b2d 100644 --- a/sphinx/util/typing.py +++ b/sphinx/util/typing.py @@ -10,7 +10,7 @@ import sys import typing -from typing import Any, Callable, Dict, List, Tuple, TypeVar, Union +from typing import Any, Callable, Dict, Generator, List, Tuple, TypeVar, Union from docutils import nodes from docutils.parsers.rst.states import Inliner @@ -164,6 +164,8 @@ def _stringify_py36(annotation: Any) -> str: # for Python 3.5.2+ if annotation.__args__ is None or len(annotation.__args__) <= 2: # type: ignore # NOQA params = annotation.__args__ # type: ignore + elif annotation.__origin__ == Generator: # type: ignore + params = annotation.__args__ # type: ignore else: # typing.Callable args = ', '.join(stringify(arg) for arg in annotation.__args__[:-1]) # type: ignore diff --git a/tests/test_util_typing.py b/tests/test_util_typing.py index 41d2a19c2..932fdbfc0 100644 --- a/tests/test_util_typing.py +++ b/tests/test_util_typing.py @@ -10,7 +10,9 @@ import sys from numbers import Integral -from typing import Any, Dict, List, TypeVar, Union, Callable, Tuple, Optional, Generic +from typing import ( + Any, Dict, Generator, List, TypeVar, Union, Callable, Tuple, Optional, Generic +) import pytest @@ -48,6 +50,7 @@ def test_stringify_type_hints_containers(): assert stringify(Tuple[str, ...]) == "Tuple[str, ...]" assert stringify(List[Dict[str, Tuple]]) == "List[Dict[str, Tuple]]" assert stringify(MyList[Tuple[int, int]]) == "test_util_typing.MyList[Tuple[int, int]]" + assert stringify(Generator[None, None, None]) == "Generator[None, None, None]" @pytest.mark.skipif(sys.version_info < (3, 9), reason='python 3.9+ is required.') From f95ba21f4a4dd1cade2f554b1d3afa7b322be1e4 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 9 Feb 2020 20:22:22 +0900 Subject: [PATCH 54/88] Close #5208: linkcheck: Support checks for local links --- CHANGES | 1 + sphinx/builders/linkcheck.py | 17 +++++++++++++++-- tests/roots/test-linkcheck/links.txt | 2 ++ tests/test_build_linkcheck.py | 13 ++++++++----- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index bc932a7d4..7af591708 100644 --- a/CHANGES +++ b/CHANGES @@ -27,6 +27,7 @@ Features added * #7902: html theme: Add a new option :confval:`globaltoc_maxdepth` to control the behavior of globaltoc in sidebar * #7840: i18n: Optimize the dependencies check on bootstrap +* #5208: linkcheck: Support checks for local links * #7052: add ``:noindexentry:`` to the Python, C, C++, and Javascript domains. Update the documentation to better reflect the relationship between this option and the ``:noindex:`` option. diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index dd5317087..ef8f9d902 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -35,6 +35,8 @@ from sphinx.util.requests import is_ssl_error logger = logging.getLogger(__name__) +uri_re = re.compile('[a-z]+://') + DEFAULT_REQUEST_HEADERS = { 'Accept': 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8', @@ -210,10 +212,21 @@ class CheckExternalLinksBuilder(Builder): def check() -> Tuple[str, str, int]: # check for various conditions without bothering the network - if len(uri) == 0 or uri.startswith(('#', 'mailto:', 'ftp:')): + if len(uri) == 0 or uri.startswith(('#', 'mailto:')): return 'unchecked', '', 0 elif not uri.startswith(('http:', 'https:')): - return 'local', '', 0 + if uri_re.match(uri): + # non supported URI schemes (ex. ftp) + return 'unchecked', '', 0 + else: + if path.exists(path.join(self.srcdir, uri)): + return 'working', '', 0 + else: + for rex in self.to_ignore: + if rex.match(uri): + return 'ignored', '', 0 + else: + return 'broken', '', 0 elif uri in self.good: return 'working', 'old', 0 elif uri in self.broken: diff --git a/tests/roots/test-linkcheck/links.txt b/tests/roots/test-linkcheck/links.txt index fa8f11e4c..90759ee63 100644 --- a/tests/roots/test-linkcheck/links.txt +++ b/tests/roots/test-linkcheck/links.txt @@ -11,6 +11,8 @@ Some additional anchors to exercise ignore code * `Example Bar invalid `_ * `Example anchor invalid `_ * `Complete nonsense `_ +* `Example valid local file `_ +* `Example invalid local file `_ .. image:: https://www.google.com/image.png .. figure:: https://www.google.com/image2.png diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index d1fec550f..7d85f10c5 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -30,7 +30,9 @@ def test_defaults(app, status, warning): # images should fail assert "Not Found for url: https://www.google.com/image.png" in content assert "Not Found for url: https://www.google.com/image2.png" in content - assert len(content.splitlines()) == 5 + # looking for local file should fail + assert "[broken] path/to/notfound" in content + assert len(content.splitlines()) == 6 @pytest.mark.sphinx('linkcheck', testroot='linkcheck', freshenv=True) @@ -47,8 +49,8 @@ def test_defaults_json(app, status, warning): "info"]: assert attr in row - assert len(content.splitlines()) == 8 - assert len(rows) == 8 + assert len(content.splitlines()) == 10 + assert len(rows) == 10 # the output order of the rows is not stable # due to possible variance in network latency rowsby = {row["uri"]:row for row in rows} @@ -69,7 +71,7 @@ def test_defaults_json(app, status, warning): assert dnerow['uri'] == 'https://localhost:7777/doesnotexist' assert rowsby['https://www.google.com/image2.png'] == { 'filename': 'links.txt', - 'lineno': 16, + 'lineno': 18, 'status': 'broken', 'code': 0, 'uri': 'https://www.google.com/image2.png', @@ -92,7 +94,8 @@ def test_defaults_json(app, status, warning): 'https://localhost:7777/doesnotexist', 'http://www.sphinx-doc.org/en/1.7/intro.html#', 'https://www.google.com/image.png', - 'https://www.google.com/image2.png'] + 'https://www.google.com/image2.png', + 'path/to/notfound'] }) def test_anchors_ignored(app, status, warning): app.builder.build_all() From 9b425606e7be92c048e1b08b41562c296fadc56c Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 21 Jul 2020 12:26:28 +0200 Subject: [PATCH 55/88] support pandas-style default spec by postprocessing tokens --- sphinx/ext/napoleon/docstring.py | 11 +++++++++-- tests/test_ext_napoleon_docstring.py | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 3a4ee02e6..ec3664f76 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -845,10 +845,17 @@ def _recombine_set_tokens(tokens: List[str]) -> List[str]: def _tokenize_type_spec(spec: str) -> List[str]: + def postprocess(item): + if item.startswith("default"): + return [item[:7], item[7:]] + else: + return [item] + tokens = list( item - for item in _token_regex.split(spec) - if item is not None and item.strip() + for raw_token in _token_regex.split(spec) + for item in postprocess(raw_token) + if item ) return tokens diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 2b1ac93aa..64fce5aa4 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2013,6 +2013,7 @@ definition_after_normal_text : int "int or float or None, optional", '{"F", "C", "N"}', "{'F', 'C', 'N'}, default: 'F'", + "{'F', 'C', 'N or C'}, default 'F'", '"ma{icious"', r"'with \'quotes\''", ) @@ -2022,6 +2023,7 @@ definition_after_normal_text : int ["int", " or ", "float", " or ", "None", ", ", "optional"], ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "'F'"], + ["{", "'F'", ", ", "'C'", ", ", "'N or C'", "}", ", ", "default", " ", "'F'"], ['"ma{icious"'], [r"'with \'quotes\''"], ) From ae35f81d3d8c7579fc81eaa11f71ba3a17e27722 Mon Sep 17 00:00:00 2001 From: Keewis Date: Tue, 21 Jul 2020 12:42:57 +0200 Subject: [PATCH 56/88] allow mapping to a long name --- sphinx/ext/napoleon/docstring.py | 9 +++++++-- tests/test_ext_napoleon_docstring.py | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index ec3664f76..3fc24fa24 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -910,11 +910,16 @@ def _token_type(token: str, location: str = None) -> str: def _convert_numpy_type_spec(_type: str, location: str = None, translations: dict = {}) -> str: def convert_obj(obj, translations, default_translation): + translation = translations.get(obj, obj) + # use :class: (the default) only if obj is not a standard singleton (None, True, False) - if obj in ("None", "True", "False") and default_translation == ":class:`%s`": + if translation in ("None", "True", "False") and default_translation == ":class:`%s`": default_translation = ":obj:`%s`" - return translations.get(obj, default_translation % obj) + if _xref_regex.match(translation) is None: + translation = default_translation % translation + + return translation tokens = _tokenize_type_spec(_type) combined_tokens = _recombine_set_tokens(tokens) diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py index 64fce5aa4..56812d193 100644 --- a/tests/test_ext_napoleon_docstring.py +++ b/tests/test_ext_napoleon_docstring.py @@ -2066,6 +2066,10 @@ definition_after_normal_text : int self.assertEqual(expected, actual) def test_convert_numpy_type_spec(self): + translations = { + "DataFrame": "pandas.DataFrame", + } + specs = ( "", "optional", @@ -2073,6 +2077,7 @@ definition_after_normal_text : int "int or float or None, default: None", '{"F", "C", "N"}', "{'F', 'C', 'N'}, default: 'N'", + "DataFrame, optional", ) converted = ( @@ -2082,10 +2087,11 @@ definition_after_normal_text : int ":class:`int` or :class:`float` or :obj:`None`, *default*: :obj:`None`", '``{"F", "C", "N"}``', "``{'F', 'C', 'N'}``, *default*: ``'N'``", + ":class:`pandas.DataFrame`, *optional*", ) for spec, expected in zip(specs, converted): - actual = _convert_numpy_type_spec(spec) + actual = _convert_numpy_type_spec(spec, translations=translations) self.assertEqual(expected, actual) def test_parameter_types(self): From 4512d2757b0d1a589ceefa6d2df5144d52e7c406 Mon Sep 17 00:00:00 2001 From: Joris Beckers Date: Wed, 22 Jul 2020 12:00:46 +0200 Subject: [PATCH 57/88] Allow searching for things like serials, phone numbers, etc --- sphinx/search/__init__.py | 2 +- sphinx/themes/basic/static/searchtools.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sphinx/search/__init__.py b/sphinx/search/__init__.py index b531145f4..4534dd333 100644 --- a/sphinx/search/__init__.py +++ b/sphinx/search/__init__.py @@ -117,7 +117,7 @@ var Stemmer = function() { len(word) == 0 or not ( ((len(word) < 3) and (12353 < ord(word[0]) < 12436)) or (ord(word[0]) < 256 and ( - len(word) < 3 or word in self.stopwords or word.isdigit() + len(word) < 3 or word in self.stopwords )))) diff --git a/sphinx/themes/basic/static/searchtools.js b/sphinx/themes/basic/static/searchtools.js index ab5649965..970d0d975 100644 --- a/sphinx/themes/basic/static/searchtools.js +++ b/sphinx/themes/basic/static/searchtools.js @@ -166,8 +166,7 @@ var Search = { objectterms.push(tmp[i].toLowerCase()); } - if ($u.indexOf(stopwords, tmp[i].toLowerCase()) != -1 || tmp[i].match(/^\d+$/) || - tmp[i] === "") { + if ($u.indexOf(stopwords, tmp[i].toLowerCase()) != -1 || tmp[i] === "") { // skip this "word" continue; } From 144f18a5284f1df792bd1ba217d6d35a7a20d8dc Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 23 Jul 2020 17:27:57 +1000 Subject: [PATCH 58/88] Remove irrelevant test RST --- tests/roots/test-ext-autosummary-filename-map/index.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/roots/test-ext-autosummary-filename-map/index.rst b/tests/roots/test-ext-autosummary-filename-map/index.rst index e85389395..57d902b6a 100644 --- a/tests/roots/test-ext-autosummary-filename-map/index.rst +++ b/tests/roots/test-ext-autosummary-filename-map/index.rst @@ -1,8 +1,4 @@ -:autolink:`autosummary_dummy_module.Foo` - -:autolink:`autosummary_importfail` - .. autosummary:: :toctree: generated :caption: An autosummary From 15d93d2c763c920883da6f8fd9c0874a82cff50a Mon Sep 17 00:00:00 2001 From: Fabio Utzig Date: Thu, 23 Jul 2020 12:27:16 -0300 Subject: [PATCH 59/88] domains: c: add support for named variadic argument Add support for gcc extensions which allow variadic arguments to be named: https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html A named variadic argument takes the form "..." with spaces around allowed and must be the last argument. Currently it generates a warning "Invalid C declaration: Expected identifier, ')', or ',' in macro parameter list." Signed-off-by: Fabio Utzig --- sphinx/domains/c.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 642fee55e..240fe30e5 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -1200,13 +1200,17 @@ class ASTTypeWithInit(ASTBase): class ASTMacroParameter(ASTBase): - def __init__(self, arg: ASTNestedName, ellipsis: bool = False) -> None: + def __init__(self, arg: ASTNestedName, ellipsis: bool = False, + variadic: bool = False) -> None: self.arg = arg self.ellipsis = ellipsis + self.variadic = variadic def _stringify(self, transform: StringifyTransform) -> str: if self.ellipsis: return '...' + elif self.variadic: + return transform(self.arg) + '...' else: return transform(self.arg) @@ -1215,6 +1219,9 @@ class ASTMacroParameter(ASTBase): verify_description_mode(mode) if self.ellipsis: signode += nodes.Text('...') + elif self.variadic: + name = str(self) + signode += nodes.emphasis(name, name) else: self.arg.describe_signature(signode, mode, env, symbol=symbol) @@ -2915,9 +2922,14 @@ class DefinitionParser(BaseParser): if not self.match(identifier_re): self.fail("Expected identifier in macro parameters.") nn = ASTNestedName([ASTIdentifier(self.matched_text)], rooted=False) - arg = ASTMacroParameter(nn) - args.append(arg) self.skip_ws() + if self.skip_string_and_ws('...'): + args.append(ASTMacroParameter(nn, False, True)) + self.skip_ws() + if not self.skip_string(')'): + self.fail('Expected ")" after "..." in macro parameters.') + break + args.append(ASTMacroParameter(nn)) if self.skip_string_and_ws(','): continue elif self.skip_string_and_ws(')'): From 3d0818f1cb7de3cad84e7c1d750c71e53bc736b1 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 24 Jul 2020 01:23:08 +0900 Subject: [PATCH 60/88] Update CHANGES for PR #7927 --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index f0dde1d25..f9ef5ec9e 100644 --- a/CHANGES +++ b/CHANGES @@ -17,6 +17,8 @@ Features added -------------- * #2076: autodoc: Allow overriding of exclude-members in skip-member function +* #2024: autosummary: Add :confval:`autosummary_filename_map` to avoid conflict + of filenames between two object with different case * #7849: html: Add :confval:`html_codeblock_linenos_style` to change the style of line numbers for code-blocks * #7853: C and C++, support parameterized GNU style attributes. From 875346307f8a10892328800122ebed7fb9657562 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 24 Jul 2020 02:12:47 +0900 Subject: [PATCH 61/88] linkcheck: Fix a protocol relative URL is considered as a local file Since #7985, a protocol relative URL (URL starts with "//") is considered as a local file incorrectly. This makes it to a "unchecked" URL. refs: #7985 --- sphinx/builders/linkcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index ef8f9d902..9b54afc7c 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -35,7 +35,7 @@ from sphinx.util.requests import is_ssl_error logger = logging.getLogger(__name__) -uri_re = re.compile('[a-z]+://') +uri_re = re.compile('([a-z]+:)?//') # matches to foo:// and // (a protocol relative URL) DEFAULT_REQUEST_HEADERS = { From d177956365a39b7045f7ef69049c5c2e0dc06829 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 24 Jul 2020 13:36:38 +0900 Subject: [PATCH 62/88] Fix a mypy violation --- sphinx/setup_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/setup_command.py b/sphinx/setup_command.py index 3622456c3..29a9dace7 100644 --- a/sphinx/setup_command.py +++ b/sphinx/setup_command.py @@ -106,7 +106,7 @@ class BuildDoc(Command): self.link_index = False self.copyright = '' # Link verbosity to distutils' (which uses 1 by default). - self.verbosity = self.distribution.verbose - 1 + self.verbosity = self.distribution.verbose - 1 # type: ignore self.traceback = False self.nitpicky = False self.keep_going = False From 0b57727233cec67c27a6a30d60ae7b44f1d05f3c Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 24 Jul 2020 13:35:28 +0900 Subject: [PATCH 63/88] Update CHANGES for PR #5090 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index f9ef5ec9e..4ae5479c4 100644 --- a/CHANGES +++ b/CHANGES @@ -30,6 +30,7 @@ Features added the behavior of globaltoc in sidebar * #7840: i18n: Optimize the dependencies check on bootstrap * #5208: linkcheck: Support checks for local links +* #5090: setuptools: Link verbosity to distutils' -v and -q option * #7052: add ``:noindexentry:`` to the Python, C, C++, and Javascript domains. Update the documentation to better reflect the relationship between this option and the ``:noindex:`` option. From 0c4edbdb382d95bf898cce50ecf3b95174f27f91 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 24 Jul 2020 23:09:54 +0900 Subject: [PATCH 64/88] doclinter: Allow a very long literal string --- utils/doclinter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/doclinter.py b/utils/doclinter.py index 52b2fe892..bb11decaf 100644 --- a/utils/doclinter.py +++ b/utils/doclinter.py @@ -50,6 +50,9 @@ def lint(path: str) -> int: if re.match(r'^\s*\.\. ', line): # ignore directives and hyperlink targets pass + elif re.match(r'^\s*``[^`]+``$', line): + # ignore a very long literal string + pass else: print('%s:%d: the line is too long (%d > %d).' % (path, i + 1, len(line), MAX_LINE_LENGTH)) From 920048466c16123ccd1267299dad7742f6b41d9e Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:22:05 +0200 Subject: [PATCH 65/88] don't provide a empty line number --- sphinx/ext/napoleon/docstring.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index 3fc24fa24..e610f0427 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -1050,12 +1050,11 @@ class NumpyDocstring(GoogleDocstring): def _get_location(self) -> str: filepath = inspect.getfile(self._obj) if self._obj is not None else "" name = self._name - line = "" if filepath is None and name is None: return None - return ":".join([filepath, "docstring of %s" % name, line]) + return ":".join([filepath, "docstring of %s" % name]) def _consume_field(self, parse_type: bool = True, prefer_type: bool = False ) -> Tuple[str, str, List[str]]: From 530793d997606fdd9131b03ca93cf066f3df67b7 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:37:32 +0200 Subject: [PATCH 66/88] update the link to the official docstring guide --- doc/usage/extensions/napoleon.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/usage/extensions/napoleon.rst b/doc/usage/extensions/napoleon.rst index 76c423dc0..fcfe36460 100644 --- a/doc/usage/extensions/napoleon.rst +++ b/doc/usage/extensions/napoleon.rst @@ -278,7 +278,7 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: .. _Google style: https://google.github.io/styleguide/pyguide.html .. _NumPy style: - https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt + https://numpydoc.readthedocs.io/en/latest/format.html#docstring-standard .. confval:: napoleon_google_docstring From 8feb5f9ac97648ba0bd0229a8539fad4f89590c8 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:38:10 +0200 Subject: [PATCH 67/88] mention that the type aliases only work with napoleon_use_param --- sphinx/ext/napoleon/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/ext/napoleon/__init__.py b/sphinx/ext/napoleon/__init__.py index 128fbaab5..6d7406ead 100644 --- a/sphinx/ext/napoleon/__init__.py +++ b/sphinx/ext/napoleon/__init__.py @@ -238,7 +238,8 @@ class Config: :returns: *bool* -- True if successful, False otherwise napoleon_type_aliases : :obj:`dict` (Defaults to None) - Add a mapping of strings to string, translating types in numpy style docstrings. + Add a mapping of strings to string, translating types in numpy + style docstrings. Only works when ``napoleon_use_param = True``. napoleon_custom_sections : :obj:`list` (Defaults to None) Add a list of custom sections to include, expanding the list of parsed sections. From 6ae1c601b93683a8b501aebc2382cdce73a2add3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:48:30 +0200 Subject: [PATCH 68/88] add a section about napoleon_type_aliases to the documentation --- doc/usage/extensions/napoleon.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/doc/usage/extensions/napoleon.rst b/doc/usage/extensions/napoleon.rst index fcfe36460..438c33950 100644 --- a/doc/usage/extensions/napoleon.rst +++ b/doc/usage/extensions/napoleon.rst @@ -274,6 +274,7 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: napoleon_use_ivar = False napoleon_use_param = True napoleon_use_rtype = True + napoleon_type_aliases = None .. _Google style: https://google.github.io/styleguide/pyguide.html @@ -435,7 +436,7 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: :param arg1: Description of `arg1` :type arg1: str :param arg2: Description of `arg2`, defaults to 0 - :type arg2: int, optional + :type arg2: :class:`int`, *optional* **If False**:: @@ -480,3 +481,31 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: **If False**:: :returns: *bool* -- True if successful, False otherwise + +.. confval:: napoleon_type_aliases + + A mapping to translate type names to other names or references. Works + only when ``napoleon_use_param = True``. *Defaults to None.* + + With:: + + napoleon_type_aliases = { + "CustomType": "mypackage.CustomType", + "dict-like": ":term:`dict-like `", + } + + This `NumPy style`_ snippet:: + + Parameters + ---------- + arg1 : CustomType + Description of `arg1` + arg2 : dict-like + Description of `arg2` + + becomes:: + + :param arg1: Description of `arg1` + :type arg1: mypackage.CustomType + :param arg2: Description of `arg2` + :type arg2: :term:`dict-like ` From 864dd0b610ff245352e3a1b51d716264cd255fc9 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 25 Jul 2020 11:54:02 +0200 Subject: [PATCH 69/88] add a comment about default not being a official keyword --- sphinx/ext/napoleon/docstring.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py index e610f0427..95fb1e538 100644 --- a/sphinx/ext/napoleon/docstring.py +++ b/sphinx/ext/napoleon/docstring.py @@ -899,6 +899,8 @@ def _token_type(token: str, location: str = None) -> str: ) type_ = "literal" elif token in ("optional", "default"): + # default is not a official keyword (yet) but supported by the + # reference implementation (numpydoc) and widely used type_ = "control" elif _xref_regex.match(token): type_ = "reference" From 778e297577c5301b75de7859609f2bb490d27192 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 25 Jul 2020 21:45:13 +0900 Subject: [PATCH 70/88] Update CHANGES for PR #7690 --- CHANGES | 2 ++ doc/usage/extensions/napoleon.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGES b/CHANGES index 4ae5479c4..b82e17656 100644 --- a/CHANGES +++ b/CHANGES @@ -23,6 +23,8 @@ Features added of line numbers for code-blocks * #7853: C and C++, support parameterized GNU style attributes. * #7888: napoleon: Add aliases Warn and Raise. +* #7690: napoleon: parse type strings and make them hyperlinks as possible. The + conversion rule can be updated via :confval:`napoleon_type_aliases` * C, added :rst:dir:`c:alias` directive for inserting copies of existing declarations. * #7745: html: inventory is broken if the docname contains a space diff --git a/doc/usage/extensions/napoleon.rst b/doc/usage/extensions/napoleon.rst index 438c33950..b16577e2d 100644 --- a/doc/usage/extensions/napoleon.rst +++ b/doc/usage/extensions/napoleon.rst @@ -509,3 +509,5 @@ sure that "sphinx.ext.napoleon" is enabled in `conf.py`:: :type arg1: mypackage.CustomType :param arg2: Description of `arg2` :type arg2: :term:`dict-like ` + + .. versionadded:: 3.2 From 70aa913bb24aa756872f4eafa378f8a1911b0d9b Mon Sep 17 00:00:00 2001 From: Yoshiki Shibukawa Date: Thu, 23 Jul 2020 11:43:41 +0900 Subject: [PATCH 71/88] fix 6863: add description literalinclude option (start/end have same string) --- doc/usage/restructuredtext/directives.rst | 40 +++++++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/doc/usage/restructuredtext/directives.rst b/doc/usage/restructuredtext/directives.rst index 7b9bd2f80..fcdbc3f16 100644 --- a/doc/usage/restructuredtext/directives.rst +++ b/doc/usage/restructuredtext/directives.rst @@ -656,9 +656,43 @@ __ http://pygments.org/docs/lexers string are included. The ``start-at`` and ``end-at`` options behave in a similar way, but the lines containing the matched string are included. - With lines selected using ``start-after`` it is still possible to use - ``lines``, the first allowed line having by convention the line number - ``1``. + ``start-after``/``start-at`` and ``end-before``/``end-at`` can have same string. + ``start-after``/``start-at`` filter lines before the line that contains + option string (``start-at`` will keep the line). Then ``end-before``/``end-at`` + filter lines after the line that contains option string (``end-at`` will keep + the line and ``end-before`` skip the first line). + + .. note:: + + If you want to select only ``[second-section]`` of ini file like the + following, you can use ``:start-at: [second-section]`` and + ``:end-before: [third-section]``: + + .. code-block:: ini + + [first-section] + + var_in_first=true + + [second-section] + + var_in_second=true + + [third-section] + + var_in_third=true + + Useful cases of these option is working with tag comments. + ``:start-after: [initialized]`` and ``:end-before: [initialized]`` options + keep lines between comments: + + .. code-block:: py + + if __name__ == "__main__": + # [initialize] + app.start(":8000") + # [initialize] + When lines have been selected in any of the ways described above, the line numbers in ``emphasize-lines`` refer to those selected lines, counted From 71a16f54aa643c3c0bee586716ecff56dd3eac80 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 26 Jul 2020 01:19:58 +0200 Subject: [PATCH 72/88] Fix typo --- sphinx/testing/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/testing/fixtures.py b/sphinx/testing/fixtures.py index eec3b4208..9197014bf 100644 --- a/sphinx/testing/fixtures.py +++ b/sphinx/testing/fixtures.py @@ -79,7 +79,7 @@ def app_params(request: Any, test_params: Dict, shared_result: SharedResult, if test_params['shared_result']: if 'srcdir' in kwargs: - raise pytest.Exception('You can not spcify shared_result and ' + raise pytest.Exception('You can not specify shared_result and ' 'srcdir in same time.') kwargs['srcdir'] = test_params['shared_result'] restore = shared_result.restore(test_params['shared_result']) From ff4330abe370895e64d6ea42a58ae8538ec8c087 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 26 Jul 2020 12:27:43 +0900 Subject: [PATCH 73/88] Update CHANGES for PR #7992 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index b82e17656..fd101f482 100644 --- a/CHANGES +++ b/CHANGES @@ -28,6 +28,7 @@ Features added * C, added :rst:dir:`c:alias` directive for inserting copies of existing declarations. * #7745: html: inventory is broken if the docname contains a space +* #7991: html search: Allow searching for numbers * #7902: html theme: Add a new option :confval:`globaltoc_maxdepth` to control the behavior of globaltoc in sidebar * #7840: i18n: Optimize the dependencies check on bootstrap From 09ca58d0dd2eee4e871da593791fe1d3af628e03 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 26 Jul 2020 13:49:50 +0900 Subject: [PATCH 74/88] Fix #7768: i18n: Wrong root element is passed to figure_language_filename The root element should be a user specified path; a relative path from current document or absolute path based on source directory. But an absolute path is passed instead. --- CHANGES | 2 ++ sphinx/environment/collectors/asset.py | 28 +++++++++++++++----------- sphinx/util/i18n.py | 4 ++-- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/CHANGES b/CHANGES index fd101f482..6f208ba49 100644 --- a/CHANGES +++ b/CHANGES @@ -70,6 +70,8 @@ Bugs fixed * #7928: py domain: failed to resolve a type annotation for the attribute * #7968: i18n: The content of ``math`` directive is interpreted as reST on translation +* #7768: i18n: The ``root`` element for :confval:`figure_language_filename` is + not a path that user specifies in the document * #7869: :rst:role:`abbr` role without an explanation will show the explanation from the previous abbr role * C and C++, removed ``noindex`` directive option as it did diff --git a/sphinx/environment/collectors/asset.py b/sphinx/environment/collectors/asset.py index 06a0d5198..3da2a6e4b 100644 --- a/sphinx/environment/collectors/asset.py +++ b/sphinx/environment/collectors/asset.py @@ -58,17 +58,13 @@ class ImageCollector(EnvironmentCollector): elif imguri.find('://') != -1: candidates['?'] = imguri continue - rel_imgpath, full_imgpath = app.env.relfn2path(imguri, docname) - if app.config.language: - # substitute figures (ex. foo.png -> foo.en.png) - i18n_full_imgpath = search_image_for_language(full_imgpath, app.env) - if i18n_full_imgpath != full_imgpath: - full_imgpath = i18n_full_imgpath - rel_imgpath = relative_path(path.join(app.srcdir, 'dummy'), - i18n_full_imgpath) - # set imgpath as default URI - node['uri'] = rel_imgpath - if rel_imgpath.endswith(os.extsep + '*'): + + if imguri.endswith(os.extsep + '*'): + # Update `node['uri']` to a relative path from srcdir + # from a relative path from current document. + rel_imgpath, full_imgpath = app.env.relfn2path(imguri, docname) + node['uri'] = rel_imgpath + if app.config.language: # Search language-specific figures at first i18n_imguri = get_image_filename_for_language(imguri, app.env) @@ -77,7 +73,15 @@ class ImageCollector(EnvironmentCollector): self.collect_candidates(app.env, full_imgpath, candidates, node) else: - candidates['*'] = rel_imgpath + if app.config.language: + # substitute imguri by figure_language_filename + # (ex. foo.png -> foo.en.png) + imguri = search_image_for_language(imguri, app.env) + + # Update `node['uri']` to a relative path from srcdir + # from a relative path from current document. + node['uri'], _ = app.env.relfn2path(imguri, docname) + candidates['*'] = node['uri'] # map image paths to unique image names (so that they can be put # into a single directory) diff --git a/sphinx/util/i18n.py b/sphinx/util/i18n.py index 499f2316f..b8839d8b0 100644 --- a/sphinx/util/i18n.py +++ b/sphinx/util/i18n.py @@ -320,8 +320,8 @@ def search_image_for_language(filename: str, env: "BuildEnvironment") -> str: return filename translated = get_image_filename_for_language(filename, env) - dirname = path.dirname(env.docname) - if path.exists(path.join(env.srcdir, dirname, translated)): + _, abspath = env.relfn2path(translated) + if path.exists(abspath): return translated else: return filename From 8c08abadeed0a5d88eb92fade2cebce7c2753791 Mon Sep 17 00:00:00 2001 From: Fabio Utzig Date: Mon, 27 Jul 2020 10:11:39 -0300 Subject: [PATCH 75/88] Fix typo Signed-off-by: Fabio Utzig --- sphinx/util/docutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/util/docutils.py b/sphinx/util/docutils.py index ce299d27a..3ba7813b6 100644 --- a/sphinx/util/docutils.py +++ b/sphinx/util/docutils.py @@ -459,7 +459,7 @@ class SphinxTranslator(nodes.NodeVisitor): The priority of visitor method is: 1. ``self.visit_{node_class}()`` - 2. ``self.visit_{supre_node_class}()`` + 2. ``self.visit_{super_node_class}()`` 3. ``self.unknown_visit()`` """ for node_class in node.__class__.__mro__: From 1b3bf4e3198bdbb59747c2b79ebbe23536059a93 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Mon, 27 Jul 2020 19:35:10 +0200 Subject: [PATCH 76/88] C, changelog and test for named variadic args --- CHANGES | 1 + sphinx/domains/c.py | 2 ++ tests/test_domain_c.py | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/CHANGES b/CHANGES index f0dde1d25..52eb2c23c 100644 --- a/CHANGES +++ b/CHANGES @@ -37,6 +37,7 @@ Features added The warnings printed from this functionality can be suppressed by setting :confval:`c_warn_on_allowed_pre_v3`` to ``True``. The functionality is immediately deprecated. +* #7999: C, add support for named variadic macro arguments. Bugs fixed ---------- diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 240fe30e5..65786b5de 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -2922,6 +2922,8 @@ class DefinitionParser(BaseParser): if not self.match(identifier_re): self.fail("Expected identifier in macro parameters.") nn = ASTNestedName([ASTIdentifier(self.matched_text)], rooted=False) + # Allow named variadic args: + # https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html self.skip_ws() if self.skip_string_and_ws('...'): args.append(ASTMacroParameter(nn, False, True)) diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index efef104f9..71bf251e9 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -296,6 +296,10 @@ def test_macro_definitions(): check('macro', 'M(arg, ...)', {1: 'M'}) check('macro', 'M(arg1, arg2, ...)', {1: 'M'}) check('macro', 'M(arg1, arg2, arg3, ...)', {1: 'M'}) + # GNU extension + check('macro', 'M(arg1, arg2, arg3...)', {1: 'M'}) + with pytest.raises(DefinitionError): + check('macro', 'M(arg1, arg2..., arg3)', {1: 'M'}) def test_member_definitions(): From d65c61f9e701ba5a1102ca7c884f5ef24c02a340 Mon Sep 17 00:00:00 2001 From: Fabio Utzig Date: Wed, 22 Jul 2020 14:03:15 -0300 Subject: [PATCH 77/88] Add compat id for cmdoption directive Add a compatibility id for the option directive, which avoids breaking old html links to anchors. Signed-off-by: Fabio Utzig --- sphinx/domains/std.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py index 016f84ebc..7eaaa531d 100644 --- a/sphinx/domains/std.py +++ b/sphinx/domains/std.py @@ -223,6 +223,11 @@ class Cmdoption(ObjectDescription): node_id = make_id(self.env, self.state.document, prefix, optname) signode['ids'].append(node_id) + old_node_id = self.make_old_id(prefix, optname) + if old_node_id not in self.state.document.ids and \ + old_node_id not in signode['ids']: + signode['ids'].append(old_node_id) + self.state.document.note_explicit_target(signode) domain = cast(StandardDomain, self.env.get_domain('std')) @@ -239,6 +244,14 @@ class Cmdoption(ObjectDescription): entry = '; '.join([descr, option]) self.indexnode['entries'].append(('pair', entry, signode['ids'][0], '', None)) + def make_old_id(self, prefix: str, optname: str) -> str: + """Generate old styled node_id for cmdoption. + + .. note:: Old Styled node_id was used until Sphinx-3.0. + This will be removed in Sphinx-5.0. + """ + return nodes.make_id(prefix + '-' + optname) + class Program(SphinxDirective): """ From cefdb566ebd29518080d103b656d0ce856f68f7d Mon Sep 17 00:00:00 2001 From: Julien Schueller Date: Tue, 28 Jul 2020 11:44:43 +0200 Subject: [PATCH 78/88] Fix circular import in addnodes.py Closes #8016 --- sphinx/addnodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/addnodes.py b/sphinx/addnodes.py index 84b0c1427..33503bb08 100644 --- a/sphinx/addnodes.py +++ b/sphinx/addnodes.py @@ -15,7 +15,6 @@ from docutils import nodes from docutils.nodes import Element, Node from sphinx.deprecation import RemovedInSphinx40Warning -from sphinx.util import docutils if False: # For type annotation @@ -34,6 +33,7 @@ class document(nodes.document): def set_id(self, node: Element, msgnode: Element = None, suggested_prefix: str = '') -> str: + from sphinx.util import docutils if docutils.__version_info__ >= (0, 16): ret = super().set_id(node, msgnode, suggested_prefix) # type: ignore else: From 60b105d6aa15fd0681c40965e261847b61f3ddd9 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Thu, 2 Jul 2020 14:22:19 -0700 Subject: [PATCH 79/88] several documentation improvements to theming, templating, and extension development --- doc/contents.rst | 1 - doc/development/tutorials/builders.rst | 32 ++++ doc/development/tutorials/index.rst | 14 ++ doc/development/tutorials/overview.rst | 32 ++++ .../tutorials/theming-dev.rst} | 147 +++++++++++++++++- doc/extdev/index.rst | 74 ++++----- doc/usage/restructuredtext/field-lists.rst | 31 +++- doc/usage/theming.rst | 9 +- 8 files changed, 283 insertions(+), 57 deletions(-) create mode 100644 doc/development/tutorials/builders.rst create mode 100644 doc/development/tutorials/overview.rst rename doc/{theming.rst => development/tutorials/theming-dev.rst} (52%) diff --git a/doc/contents.rst b/doc/contents.rst index 17a3d4b54..eb6946292 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -10,7 +10,6 @@ Sphinx documentation contents development/index man/index - theming templating latex extdev/index diff --git a/doc/development/tutorials/builders.rst b/doc/development/tutorials/builders.rst new file mode 100644 index 000000000..c745113c5 --- /dev/null +++ b/doc/development/tutorials/builders.rst @@ -0,0 +1,32 @@ +Configuring builders +==================== + +Discover builders by entry point +-------------------------------- + +.. versionadded:: 1.6 + +:term:`Builder` extensions can be discovered by means of `entry points`_ so +that they do not have to be listed in the :confval:`extensions` configuration +value. + +Builder extensions should define an entry point in the ``sphinx.builders`` +group. The name of the entry point needs to match your builder's +:attr:`~.Builder.name` attribute, which is the name passed to the +:option:`sphinx-build -b` option. The entry point value should equal the +dotted name of the extension module. Here is an example of how an entry point +for 'mybuilder' can be defined in the extension's ``setup.py``:: + + setup( + # ... + entry_points={ + 'sphinx.builders': [ + 'mybuilder = my.extension.module', + ], + } + ) + +Note that it is still necessary to register the builder using +:meth:`~.Sphinx.add_builder` in the extension's :func:`setup` function. + +.. _entry points: https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins diff --git a/doc/development/tutorials/index.rst b/doc/development/tutorials/index.rst index a79e6a8b6..9acfc6c76 100644 --- a/doc/development/tutorials/index.rst +++ b/doc/development/tutorials/index.rst @@ -1,8 +1,16 @@ +.. _extension-tutorials-index: + Extension tutorials =================== Refer to the following tutorials to get started with extension development. +.. toctree:: + :caption: General extension tutorials + + overview + builders + .. toctree:: :caption: Directive tutorials :maxdepth: 1 @@ -10,3 +18,9 @@ Refer to the following tutorials to get started with extension development. helloworld todo recipe + +.. toctree:: + :caption: Theming + :maxdepth: 1 + + theming-dev diff --git a/doc/development/tutorials/overview.rst b/doc/development/tutorials/overview.rst new file mode 100644 index 000000000..8ffb8df42 --- /dev/null +++ b/doc/development/tutorials/overview.rst @@ -0,0 +1,32 @@ +Developing extensions overview +============================== + +This page contains general information about developing Sphinx extensions. + +Make an extension depend on another extension +--------------------------------------------- + +Sometimes your extension depends on the functionality of another +Sphinx extension. Most Sphinx extensions are activated in a +project's :file:`conf.py` file, but this is not available to you as an +extension developer. + +.. module:: sphinx.application + :noindex: + +To ensure that another extension is activated as a part of your own extension, +use the :meth:`Sphinx.setup_extension` method. This will +activate another extension at run-time, ensuring that you have access to its +functionality. + +For example, the following code activates the "recommonmark" extension: + +.. code-block:: python + + def setup(app): + app.setup_extension("recommonmark") + +.. note:: + + Since your extension will depend on another, make sure to include + it as a part of your extension's installation requirements. diff --git a/doc/theming.rst b/doc/development/tutorials/theming-dev.rst similarity index 52% rename from doc/theming.rst rename to doc/development/tutorials/theming-dev.rst index 6a154affd..4b396f020 100644 --- a/doc/theming.rst +++ b/doc/development/tutorials/theming-dev.rst @@ -1,7 +1,5 @@ -.. highlight:: python - -HTML theming support -==================== +HTML theme development +====================== .. versionadded:: 0.6 @@ -20,6 +18,11 @@ the theme's look and feel. Themes are meant to be project-unaware, so they can be used for different projects without change. +.. note:: + + See :ref:`dev-extensions` for more information that may + be helpful in developing themes. + Creating themes --------------- @@ -125,7 +128,7 @@ If your theme package contains two or more themes, please call Templating ---------- -The :doc:`guide to templating ` is helpful if you want to write your +The :doc:`guide to templating ` is helpful if you want to write your own templates. What is important to keep in mind is the order in which Sphinx searches for templates: @@ -138,6 +141,9 @@ name as an explicit directory: ``{% extends "basic/layout.html" %}``. From a user ``templates_path`` template, you can still use the "exclamation mark" syntax as described in the templating document. + +.. _theming-static-templates: + Static templates ~~~~~~~~~~~~~~~~ @@ -154,6 +160,137 @@ templating to put the color options into the stylesheet. When a documentation is built with the classic theme, the output directory will contain a ``_static/classic.css`` file where all template tags have been processed. + +Use custom page metadata in HTML templates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Any key / value pairs in :doc:`field lists ` +that are placed *before* the page's title will be available to the Jinja template when +building the page within the :data:`meta` attribute. For example, if a page had the +following text before its first title: + +.. code-block:: rst + + :mykey: My value + + My first title + -------------- + +Then it could be accessed within a Jinja template like so: + +.. code-block:: jinja + + {%- if meta is mapping %} + {{ meta.get("mykey") }} + {%- endif %} + +Note the check that ``meta`` is a dictionary ("mapping" in Jinja +terminology) to ensure that using it in this way is valid. + + +Defining custom template functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes it is useful to define your own function in Python that you wish to +then use in a template. For example, if you'd like to insert a template value +with logic that depends on the user's configuration in the project, or if you'd +like to include non-trivial checks and provide friendly error messages for +incorrect configuration in the template. + +To define your own template function, you'll need to define two functions +inside your module: + +* A **page context event handler** (or **registration**) function. This is + connected to the :class:`.Sphinx` application via an event callback. +* A **template function** that you will use in your Jinja template. + +First, define the registration function, which accepts the arguments for +:event:`html-page-context`. + +Within the registration function, define the template function that you'd like to use +within Jinja. The template function should return a string or Python objects (lists, +dictionaries) with strings inside that Jinja uses in the templating process + +.. note:: + + The template function will have access to all of the variables that + are passed to the registration function. + +At the end of the registration function, add the template function to the +Sphinx application's context with ``context['template_func'] = template_func``. + +Finally, in your extension's ``setup()`` function, add your registration +function as a callback for :event:`html-page-context`. + +.. code-block:: python + + # The registration function + def setup_my_func(app, pagename, templatename, context, doctree): + # The template function + def my_func(mystring): + return "Your string is %s" % mystring + # Add it to the page's context + context['my_func'] = my_func + # Your extension's setup function + def setup(app): + app.connect("html-page-context", setup_my_func) + +Now, you will have access to this function in jinja like so: + +.. code-block:: jinja + +
+ {{ my_func("some string") }} +
+ + +Inject javsacript based on user configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If your extension makes use of JavaScript, it can be useful to allow users +to control its behavior using their Sphinx configuration. However, this can +be difficult to do if your JavaScript comes in the form of a static library +(which will not be built with Jinja). + +There are two ways to inject variables into the JavaScript space based on user +configuration. + +First, you may append ``_t`` to the end of any static files included with your +extension. This will cause Sphinx to process these files with the templating +engine, allowing you to embed variables and control behavior. See +:ref:`theming-static-templates` for more information. + +Second, you may use the :meth:`Sphinx.add_js_file` method without pointing it +to a file. Normally, this method is used to insert a new JavaScript file +into your site. However, if you do *not* pass a file path, but instead pass +a string to the "body" argument, then this text will be inserted as JavaScript +into your site's head. This allows you to insert variables into your project's +javascript from Python. + +For example, the following code will read in a user-configured value and then +insert this value as a JavaScript variable, which your extension's JavaScript +code may use: + +.. code-block:: python + + # This function reads in a variable and inserts it into JavaScript + def add_js_variable(app): + # This is a configuration that you've specified for users in `conf.py` + js_variable = app.config['my_javascript_variable'] + js_text = "var my_variable = '%s';" % js_variable + app.add_js_file(None, body=js_text) + # We connect this function to the step after the builder is initialized + def setup(app): + # Tell Sphinx about this configuration variable + app.add_config_value('my_javascript_variable') + # Run the function after the builder is initialized + app.connect('builder-inited', add_js_variable) + +As a result, in your theme you can use code that depends on the presence of +this variable. Users can control the variable's value by defining it in their +:file:`conf.py` file. + + .. [1] It is not an executable Python file, as opposed to :file:`conf.py`, because that would pose an unnecessary security risk if themes are shared. diff --git a/doc/extdev/index.rst b/doc/extdev/index.rst index 266da52b7..ad04951f3 100644 --- a/doc/extdev/index.rst +++ b/doc/extdev/index.rst @@ -3,54 +3,41 @@ Developing extensions for Sphinx ================================ -Since many projects will need special features in their documentation, Sphinx is -designed to be extensible on several levels. +Since many projects will need special features in their documentation, Sphinx +is designed to be extensible on several levels. -This is what you can do in an extension: First, you can add new -:term:`builder`\s to support new output formats or actions on the parsed -documents. Then, it is possible to register custom reStructuredText roles and -directives, extending the markup. And finally, there are so-called "hook -points" at strategic places throughout the build process, where an extension can -register a hook and run specialized code. +Here are a few things you can do in an extension: -An extension is simply a Python module. When an extension is loaded, Sphinx -imports this module and executes its ``setup()`` function, which in turn -notifies Sphinx of everything the extension offers -- see the extension tutorial -for examples. +* Add new :term:`builder`\s to support new output formats or actions on the + parsed documents. +* Register custom reStructuredText roles and directives, extending the markup + using the :doc:`markupapi`. +* Add custom code to so-called "hook points" at strategic places throughout the + build process, allowing you to register a hook and run specialized code. + For example, see the :ref:`events`. -The configuration file itself can be treated as an extension if it contains a -``setup()`` function. All other extensions to load must be listed in the -:confval:`extensions` configuration value. +An extension is simply a Python module with a ``setup()`` function. A user +activates the extension by placing the extension's module name +(or a sub-module) in their :confval:`extensions` configuration value. -Discovery of builders by entry point ------------------------------------- +When :program:`sphinx-build` is executed, Sphinx will attempt to import each +module that is listed, and execute ``yourmodule.setup(app)``. This +function is used to prepare the extension (e.g., by executing Python code), +linking resources that Sphinx uses in the build process (like CSS or HTML +files), and notifying Sphinx of everything the extension offers (such +as directive or role definitions). The ``app`` argument is an instance of +:class:`.Sphinx` and gives you control over most aspects of the Sphinx build. -.. versionadded:: 1.6 +.. note:: -:term:`builder` extensions can be discovered by means of `entry points`_ so -that they do not have to be listed in the :confval:`extensions` configuration -value. + The configuration file itself can be treated as an extension if it + contains a ``setup()`` function. All other extensions to load must be + listed in the :confval:`extensions` configuration value. -Builder extensions should define an entry point in the ``sphinx.builders`` -group. The name of the entry point needs to match your builder's -:attr:`~.Builder.name` attribute, which is the name passed to the -:option:`sphinx-build -b` option. The entry point value should equal the -dotted name of the extension module. Here is an example of how an entry point -for 'mybuilder' can be defined in the extension's ``setup.py``:: - - setup( - # ... - entry_points={ - 'sphinx.builders': [ - 'mybuilder = my.extension.module', - ], - } - ) - -Note that it is still necessary to register the builder using -:meth:`~.Sphinx.add_builder` in the extension's :func:`setup` function. - -.. _entry points: https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins +The rest of this page describes some high-level aspects of developing +extensions and various parts of Sphinx's behavior that you can control. +For some examples of how extensions can be built and used to control different +parts of Sphinx, see the :ref:`extension-tutorials-index`. .. _important-objects: @@ -192,6 +179,11 @@ as metadata of the extension. Metadata keys currently recognized are: APIs used for writing extensions -------------------------------- +These sections provide a more complete description of the tools at your +disposal when developing Sphinx extensions. Some are core to Sphinx +(such as the :doc:`appapi`) while others trigger specific behavior +(such as the :doc:`i18n`) + .. toctree:: :maxdepth: 2 diff --git a/doc/usage/restructuredtext/field-lists.rst b/doc/usage/restructuredtext/field-lists.rst index 28b3cfe1b..d6d90caf7 100644 --- a/doc/usage/restructuredtext/field-lists.rst +++ b/doc/usage/restructuredtext/field-lists.rst @@ -9,7 +9,17 @@ fields marked up like this:: :fieldname: Field content -Sphinx provides custom behavior for bibliographic fields compared to docutils. +Field lists are :duref:`originally defined in docutils ` +to show information about a page (such as the document author or date of +publication). + +.. note:: + + The values of field lists will be parsed as + strings. You cannot use Python collections such as lists or dictionaries. + +Sphinx treats field lists slightly differently, as explained +below. .. _metadata: @@ -17,11 +27,20 @@ File-wide metadata ------------------ A field list near the top of a file is normally parsed by docutils as the -*docinfo* which is generally used to record the author, date of publication and -other metadata. However, in Sphinx, a field list preceding any other markup is -moved from the *docinfo* to the Sphinx environment as document metadata and is -not displayed in the output; a field list appearing after the document title -will be part of the *docinfo* as normal and will be displayed in the output. +*docinfo* and shown on the page. However, in Sphinx, a field list preceding +any other markup is moved from the *docinfo* to the Sphinx environment as +document metadata, and is not displayed in the output. + +.. note:: + + A field list appearing after the document title *will* be part of the + *docinfo* as normal and will be displayed in the output. + + +Special metadata fields +----------------------- + +Sphinx provides custom behavior for bibliographic fields compared to docutils. At the moment, these metadata fields are recognized: diff --git a/doc/usage/theming.rst b/doc/usage/theming.rst index 5474e9620..9a5d46fbf 100644 --- a/doc/usage/theming.rst +++ b/doc/usage/theming.rst @@ -2,8 +2,8 @@ .. _html-themes: -HTML -==== +HTML Theming +============ Sphinx provides a number of builders for HTML and HTML-based formats. @@ -21,7 +21,7 @@ Themes .. note:: This section provides information about using pre-existing HTML themes. If - you wish to create your own theme, refer to :doc:`/theming`. + you wish to create your own theme, refer to :doc:`/development/tutorials/theming-dev`. Sphinx supports changing the appearance of its HTML output via *themes*. A theme is a collection of HTML templates, stylesheet(s) and other static files. @@ -80,7 +80,7 @@ zipfile-based theme:: html_theme = "dotted" For more information on the design of themes, including information about -writing your own themes, refer to :doc:`/theming`. +writing your own themes, refer to :doc:`/development/tutorials/theming-dev`. .. _builtin-themes: @@ -363,6 +363,7 @@ sphinx-themes.org__. .. versionchanged:: 1.4 **sphinx_rtd_theme** has become optional. + .. __: https://pypi.org/search/?q=&o=&c=Framework+%3A%3A+Sphinx+%3A%3A+Theme .. __: https://github.com/search?utf8=%E2%9C%93&q=sphinx+theme&type= .. __: https://sphinx-themes.org/ From 729dcd0e033b0e458bdff6ba2ba778001dc4540c Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Thu, 2 Jul 2020 14:25:04 -0700 Subject: [PATCH 80/88] flake8 --- doc/development/tutorials/theming-dev.rst | 6 +++--- doc/usage/theming.rst | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/development/tutorials/theming-dev.rst b/doc/development/tutorials/theming-dev.rst index 4b396f020..ad6b73a6c 100644 --- a/doc/development/tutorials/theming-dev.rst +++ b/doc/development/tutorials/theming-dev.rst @@ -165,9 +165,9 @@ Use custom page metadata in HTML templates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Any key / value pairs in :doc:`field lists ` -that are placed *before* the page's title will be available to the Jinja template when -building the page within the :data:`meta` attribute. For example, if a page had the -following text before its first title: +that are placed *before* the page's title will be available to the Jinja +template when building the page within the :data:`meta` attribute. For example, +if a page had the following text before its first title: .. code-block:: rst diff --git a/doc/usage/theming.rst b/doc/usage/theming.rst index 9a5d46fbf..0c088e8c9 100644 --- a/doc/usage/theming.rst +++ b/doc/usage/theming.rst @@ -21,7 +21,8 @@ Themes .. note:: This section provides information about using pre-existing HTML themes. If - you wish to create your own theme, refer to :doc:`/development/tutorials/theming-dev`. + you wish to create your own theme, refer to + :doc:`/development/tutorials/theming-dev`. Sphinx supports changing the appearance of its HTML output via *themes*. A theme is a collection of HTML templates, stylesheet(s) and other static files. From 924befa07e792739d4878e0121373ea4fa0ca9ca Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Mon, 20 Jul 2020 11:03:33 -0700 Subject: [PATCH 81/88] Update doc/development/tutorials/builders.rst --- doc/development/tutorials/builders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/development/tutorials/builders.rst b/doc/development/tutorials/builders.rst index c745113c5..b2e43a2ae 100644 --- a/doc/development/tutorials/builders.rst +++ b/doc/development/tutorials/builders.rst @@ -6,7 +6,7 @@ Discover builders by entry point .. versionadded:: 1.6 -:term:`Builder` extensions can be discovered by means of `entry points`_ so +:term:`builder` extensions can be discovered by means of `entry points`_ so that they do not have to be listed in the :confval:`extensions` configuration value. From c7085162d133bc6e1647b063995ddfb9f909b2dc Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Wed, 22 Jul 2020 16:30:49 -0700 Subject: [PATCH 82/88] rearranging a few docs and small updates --- doc/development/{tutorials => }/builders.rst | 20 +++++++------ doc/development/index.rst | 7 +++++ .../theming-dev.rst => theming.rst} | 28 ++++++++++++++++--- doc/development/tutorials/index.rst | 7 ----- doc/development/tutorials/overview.rst | 2 +- doc/usage/theming.rst | 4 +-- 6 files changed, 45 insertions(+), 23 deletions(-) rename doc/development/{tutorials => }/builders.rst (79%) rename doc/development/{tutorials/theming-dev.rst => theming.rst} (95%) diff --git a/doc/development/tutorials/builders.rst b/doc/development/builders.rst similarity index 79% rename from doc/development/tutorials/builders.rst rename to doc/development/builders.rst index b2e43a2ae..bb6777023 100644 --- a/doc/development/tutorials/builders.rst +++ b/doc/development/builders.rst @@ -15,16 +15,18 @@ group. The name of the entry point needs to match your builder's :attr:`~.Builder.name` attribute, which is the name passed to the :option:`sphinx-build -b` option. The entry point value should equal the dotted name of the extension module. Here is an example of how an entry point -for 'mybuilder' can be defined in the extension's ``setup.py``:: +for 'mybuilder' can be defined in the extension's ``setup.py`` - setup( - # ... - entry_points={ - 'sphinx.builders': [ - 'mybuilder = my.extension.module', - ], - } - ) +.. code-block:: python + + setup( + # ... + entry_points={ + 'sphinx.builders': [ + 'mybuilder = my.extension.module', + ], + } + ) Note that it is still necessary to register the builder using :meth:`~.Sphinx.add_builder` in the extension's :func:`setup` function. diff --git a/doc/development/index.rst b/doc/development/index.rst index 6a3406a20..2025b3ec7 100644 --- a/doc/development/index.rst +++ b/doc/development/index.rst @@ -11,3 +11,10 @@ wish to use Sphinx with existing extensions, refer to :doc:`/usage/index`. :maxdepth: 2 tutorials/index + builders + +.. toctree:: + :caption: Theming + :maxdepth: 2 + + theming diff --git a/doc/development/tutorials/theming-dev.rst b/doc/development/theming.rst similarity index 95% rename from doc/development/tutorials/theming-dev.rst rename to doc/development/theming.rst index ad6b73a6c..ea993e679 100644 --- a/doc/development/tutorials/theming-dev.rst +++ b/doc/development/theming.rst @@ -231,6 +231,7 @@ function as a callback for :event:`html-page-context`. return "Your string is %s" % mystring # Add it to the page's context context['my_func'] = my_func + # Your extension's setup function def setup(app): app.connect("html-page-context", setup_my_func) @@ -244,7 +245,7 @@ Now, you will have access to this function in jinja like so: -Inject javsacript based on user configuration +Inject JavaScript based on user configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If your extension makes use of JavaScript, it can be useful to allow users @@ -257,15 +258,34 @@ configuration. First, you may append ``_t`` to the end of any static files included with your extension. This will cause Sphinx to process these files with the templating -engine, allowing you to embed variables and control behavior. See -:ref:`theming-static-templates` for more information. +engine, allowing you to embed variables and control behavior. + +For example, the following JavaScript structure: + +.. code-block:: bash + + mymodule/ + ├── _static + │   └── myjsfile.js_t + └── mymodule.py + +Will result in the following static file placed in your HTML's build output: + +.. code-block:: bash + + _build/ + └── html + └── _static +    └── myjsfile.js + +See :ref:`theming-static-templates` for more information. Second, you may use the :meth:`Sphinx.add_js_file` method without pointing it to a file. Normally, this method is used to insert a new JavaScript file into your site. However, if you do *not* pass a file path, but instead pass a string to the "body" argument, then this text will be inserted as JavaScript into your site's head. This allows you to insert variables into your project's -javascript from Python. +JavaScript from Python. For example, the following code will read in a user-configured value and then insert this value as a JavaScript variable, which your extension's JavaScript diff --git a/doc/development/tutorials/index.rst b/doc/development/tutorials/index.rst index 9acfc6c76..3b625090d 100644 --- a/doc/development/tutorials/index.rst +++ b/doc/development/tutorials/index.rst @@ -9,7 +9,6 @@ Refer to the following tutorials to get started with extension development. :caption: General extension tutorials overview - builders .. toctree:: :caption: Directive tutorials @@ -18,9 +17,3 @@ Refer to the following tutorials to get started with extension development. helloworld todo recipe - -.. toctree:: - :caption: Theming - :maxdepth: 1 - - theming-dev diff --git a/doc/development/tutorials/overview.rst b/doc/development/tutorials/overview.rst index 8ffb8df42..ad474999a 100644 --- a/doc/development/tutorials/overview.rst +++ b/doc/development/tutorials/overview.rst @@ -19,7 +19,7 @@ use the :meth:`Sphinx.setup_extension` method. This will activate another extension at run-time, ensuring that you have access to its functionality. -For example, the following code activates the "recommonmark" extension: +For example, the following code activates the ``recommonmark`` extension: .. code-block:: python diff --git a/doc/usage/theming.rst b/doc/usage/theming.rst index 0c088e8c9..e5362b9f0 100644 --- a/doc/usage/theming.rst +++ b/doc/usage/theming.rst @@ -22,7 +22,7 @@ Themes This section provides information about using pre-existing HTML themes. If you wish to create your own theme, refer to - :doc:`/development/tutorials/theming-dev`. + :doc:`/development/theming`. Sphinx supports changing the appearance of its HTML output via *themes*. A theme is a collection of HTML templates, stylesheet(s) and other static files. @@ -81,7 +81,7 @@ zipfile-based theme:: html_theme = "dotted" For more information on the design of themes, including information about -writing your own themes, refer to :doc:`/development/tutorials/theming-dev`. +writing your own themes, refer to :doc:`/development/theming`. .. _builtin-themes: From 84f7dffd995fa36f5380e4b79a681c62fd3a0296 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Thu, 23 Jul 2020 17:39:20 -0700 Subject: [PATCH 83/88] comments --- doc/development/theming.rst | 20 ++++++++++++++++++++ doc/usage/restructuredtext/field-lists.rst | 7 ++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/doc/development/theming.rst b/doc/development/theming.rst index ea993e679..5de10158a 100644 --- a/doc/development/theming.rst +++ b/doc/development/theming.rst @@ -245,6 +245,26 @@ Now, you will have access to this function in jinja like so: +Add your own static files to the build assets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are packaging your own build assets with an extension +(e.g., a CSS or JavaScript file), you need to ensure that they are placed +in the ``_static/`` folder of HTML outputs. To do so, you may copy them directly +into a build's ``_static/`` folder at build time, generally via an event hook. +Here is some sample code to accomplish this: + +.. code-block:: python + + def copy_custom_files(app, exc): + if app.builder.format == 'html' and not exc: + staticdir = path.join(app.builder.outdir, '_static') + copy_asset_file('path/to/myextension/_static/myjsfile.js', staticdir) + + def setup(app): + app.connect('builder-inited', copy_custom_files) + + Inject JavaScript based on user configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/usage/restructuredtext/field-lists.rst b/doc/usage/restructuredtext/field-lists.rst index d6d90caf7..5fc897d62 100644 --- a/doc/usage/restructuredtext/field-lists.rst +++ b/doc/usage/restructuredtext/field-lists.rst @@ -9,17 +9,14 @@ fields marked up like this:: :fieldname: Field content -Field lists are :duref:`originally defined in docutils ` -to show information about a page (such as the document author or date of -publication). +Sphinx extends standard docutils behavior for field lists and adds some extra +functionality that is covered in this section. .. note:: The values of field lists will be parsed as strings. You cannot use Python collections such as lists or dictionaries. -Sphinx treats field lists slightly differently, as explained -below. .. _metadata: From b8ee3ac9a45420ca4fbd56fd1975647fdff8d650 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Tue, 28 Jul 2020 17:59:55 -0700 Subject: [PATCH 84/88] moving extension dev overview --- doc/development/index.rst | 1 + doc/development/{tutorials => }/overview.rst | 0 doc/development/tutorials/index.rst | 4 ---- 3 files changed, 1 insertion(+), 4 deletions(-) rename doc/development/{tutorials => }/overview.rst (100%) diff --git a/doc/development/index.rst b/doc/development/index.rst index 2025b3ec7..04918acd6 100644 --- a/doc/development/index.rst +++ b/doc/development/index.rst @@ -10,6 +10,7 @@ wish to use Sphinx with existing extensions, refer to :doc:`/usage/index`. .. toctree:: :maxdepth: 2 + overview tutorials/index builders diff --git a/doc/development/tutorials/overview.rst b/doc/development/overview.rst similarity index 100% rename from doc/development/tutorials/overview.rst rename to doc/development/overview.rst diff --git a/doc/development/tutorials/index.rst b/doc/development/tutorials/index.rst index 3b625090d..be126b3ca 100644 --- a/doc/development/tutorials/index.rst +++ b/doc/development/tutorials/index.rst @@ -5,10 +5,6 @@ Extension tutorials Refer to the following tutorials to get started with extension development. -.. toctree:: - :caption: General extension tutorials - - overview .. toctree:: :caption: Directive tutorials From 376d4fe4c65dadc8215959b9860e608c1c02f316 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 29 Jul 2020 23:30:09 +0900 Subject: [PATCH 85/88] Update CHANGES for PR #8017 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index bd1e17d8a..b2a771f73 100644 --- a/CHANGES +++ b/CHANGES @@ -77,6 +77,7 @@ Bugs fixed nothing. * #7619: Duplicated node IDs are generated if node has multiple IDs * #2050: Symbols sections are appeared twice in the index page +* #8017: Fix circular import in sphinx.addnodes Testing -------- From bb65ea7930205ff54986136cd519c32a83407087 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 30 Jul 2020 00:37:44 +0900 Subject: [PATCH 86/88] Update CHANGES for PR #7994 --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index b2a771f73..ff9236469 100644 --- a/CHANGES +++ b/CHANGES @@ -69,6 +69,8 @@ Bugs fixed * #7691: linkcheck: HEAD requests are not used for checking * #4888: i18n: Failed to add an explicit title to ``:ref:`` role on translation * #7928: py domain: failed to resolve a type annotation for the attribute +* #7994: std domain: option directive does not generate old node_id compatible + with 2.x or older * #7968: i18n: The content of ``math`` directive is interpreted as reST on translation * #7869: :rst:role:`abbr` role without an explanation will show the explanation From a99675bf78eec8024792da6448e89ea534c59b5a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 30 Jul 2020 01:41:18 +0900 Subject: [PATCH 87/88] Fix #8008: py domain: failed to parse a type annotation containing ellipsis Fix _parse_annotation() does not support a type annotation having ellipsis. --- CHANGES | 1 + sphinx/domains/python.py | 14 ++++++++++++++ tests/test_domain_py.py | 14 ++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/CHANGES b/CHANGES index ff9236469..4da2823e9 100644 --- a/CHANGES +++ b/CHANGES @@ -69,6 +69,7 @@ Bugs fixed * #7691: linkcheck: HEAD requests are not used for checking * #4888: i18n: Failed to add an explicit title to ``:ref:`` role on translation * #7928: py domain: failed to resolve a type annotation for the attribute +* #8008: py domain: failed to parse a type annotation containing ellipsis * #7994: std domain: option directive does not generate old node_id compatible with 2.x or older * #7968: i18n: The content of ``math`` directive is interpreted as reST on diff --git a/sphinx/domains/python.py b/sphinx/domains/python.py index fb167828f..f4bc58b69 100644 --- a/sphinx/domains/python.py +++ b/sphinx/domains/python.py @@ -11,6 +11,7 @@ import builtins import inspect import re +import sys import typing import warnings from inspect import Parameter @@ -134,6 +135,19 @@ def _parse_annotation(annotation: str, env: BuildEnvironment = None) -> List[Nod return result else: + if sys.version_info >= (3, 6): + if isinstance(node, ast.Constant): + if node.value is Ellipsis: + return [addnodes.desc_sig_punctuation('', "...")] + else: + return [nodes.Text(node.value)] + + if sys.version_info < (3, 8): + if isinstance(node, ast.Ellipsis): + return [addnodes.desc_sig_punctuation('', "...")] + elif isinstance(node, ast.NameConstant): + return [nodes.Text(node.value)] + raise SyntaxError # unsupported syntax if env is None: diff --git a/tests/test_domain_py.py b/tests/test_domain_py.py index ccf539b6d..b98f37912 100644 --- a/tests/test_domain_py.py +++ b/tests/test_domain_py.py @@ -262,6 +262,14 @@ def test_parse_annotation(app): [desc_sig_punctuation, ")"], [desc_sig_punctuation, "]"])) + doctree = _parse_annotation("Tuple[int, ...]", app.env) + assert_node(doctree, ([pending_xref, "Tuple"], + [desc_sig_punctuation, "["], + [pending_xref, "int"], + [desc_sig_punctuation, ", "], + [desc_sig_punctuation, "..."], + [desc_sig_punctuation, "]"])) + doctree = _parse_annotation("Callable[[int, int], int]", app.env) assert_node(doctree, ([pending_xref, "Callable"], [desc_sig_punctuation, "["], @@ -274,6 +282,12 @@ def test_parse_annotation(app): [pending_xref, "int"], [desc_sig_punctuation, "]"])) + doctree = _parse_annotation("List[None]", app.env) + assert_node(doctree, ([pending_xref, "List"], + [desc_sig_punctuation, "["], + [pending_xref, "None"], + [desc_sig_punctuation, "]"])) + # None type makes an object-reference (not a class reference) doctree = _parse_annotation("None", app.env) assert_node(doctree, ([pending_xref, "None"],)) From 01ff500b34589a8af5ccb3fee430d0a83695abef Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 1 Aug 2020 13:06:09 +0900 Subject: [PATCH 88/88] Fix #8023: evaluate_signature() does not work properly in python3.10 Since 3.10, ForwardRef._evalute() takes an additional argument `recursive_guard`. As a result, sphinx.util.inspect:evaluate_signature() does not work properly. This adds a simple wrapper evalute_forwardref() to allow evaluating ForwardRefs in py3.10. --- sphinx/util/inspect.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sphinx/util/inspect.py b/sphinx/util/inspect.py index ade2be924..4bb320939 100644 --- a/sphinx/util/inspect.py +++ b/sphinx/util/inspect.py @@ -497,19 +497,26 @@ def signature(subject: Callable, bound_method: bool = False, follow_wrapped: boo def evaluate_signature(sig: inspect.Signature, globalns: Dict = None, localns: Dict = None ) -> inspect.Signature: """Evaluate unresolved type annotations in a signature object.""" + def evaluate_forwardref(ref: ForwardRef, globalns: Dict, localns: Dict) -> Any: + """Evaluate a forward reference.""" + if sys.version_info > (3, 10): + return ref._evaluate(globalns, localns, frozenset()) + else: + return ref._evaluate(globalns, localns) + def evaluate(annotation: Any, globalns: Dict, localns: Dict) -> Any: """Evaluate unresolved type annotation.""" try: if isinstance(annotation, str): ref = ForwardRef(annotation, True) - annotation = ref._evaluate(globalns, localns) + annotation = evaluate_forwardref(ref, globalns, localns) if isinstance(annotation, ForwardRef): - annotation = annotation._evaluate(globalns, localns) + annotation = evaluate_forwardref(ref, globalns, localns) elif isinstance(annotation, str): # might be a ForwardRef'ed annotation in overloaded functions ref = ForwardRef(annotation, True) - annotation = ref._evaluate(globalns, localns) + annotation = evaluate_forwardref(ref, globalns, localns) except (NameError, TypeError): # failed to evaluate type. skipped. pass