From 555e675947975dcc1b3b55028cc7e435a653f523 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 8 Feb 2019 16:10:43 +0100 Subject: [PATCH 01/27] [autosummary] add modules and packages for modules --- doc/usage/extensions/autosummary.rst | 13 ++++++++++ sphinx/ext/autosummary/generate.py | 25 +++++++++++++++++++ .../templates/autosummary/module.rst | 24 ++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index 24e0d1855..d19ff0554 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -236,6 +236,19 @@ The following variables available in the templates: List containing names of "public" attributes in the class. Only available for classes. +.. data:: modules + + List containing names of "public" modules in the package. Only available for + modules that are packages. + + .. versionadded:: 2.0.0 + +.. data:: packages + + List containing names of "public" sub-packages in the package. Only available for + modules that are packages. + + .. versionadded:: 2.0.0 Additionally, the following filters are available diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 83918e998..a10eaaf4d 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -23,6 +23,7 @@ import os import pydoc import re import sys +import pkgutil from jinja2 import FileSystemLoader, TemplateNotFound from jinja2.sandbox import SandboxedEnvironment @@ -188,6 +189,29 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', if x in include_public or not x.startswith('_')] return public, items + def get_package_members(out_dict, include_public=[], imported=True): + # https://docs.python.org/3/reference/import.html#packages + # ... any module that contains a __path__ attribute is + # considered a package + if not hasattr(obj, '__path__'): + return + out_dict['modules'], out_dict['all_modules'] = [], [] + out_dict['packages'], out_dict['all_packages'] = [], [] + for _, modname, ispkg in pkgutil.iter_modules(obj.__path__): + fullname = name+'.'+modname + try: + import_by_name(fullname) + except ImportError as e: + warn('[autosummary] failed to import %s: %s' % (fullname, e)) + continue + if ispkg: + kpublic, kall = 'packages', 'all_packages' + else: + kpublic, kall = 'modules', 'all_modules' + out_dict[kall].append(fullname) + if include_public or not modname.startswith('_'): + out_dict[kpublic].append(fullname) + ns = {} # type: Dict[str, Any] if doc.objtype == 'module': @@ -198,6 +222,7 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', get_members(obj, 'class', imported=imported_members) ns['exceptions'], ns['all_exceptions'] = \ get_members(obj, 'exception', imported=imported_members) + get_package_members(ns, imported=imported_members) elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ diff --git a/sphinx/ext/autosummary/templates/autosummary/module.rst b/sphinx/ext/autosummary/templates/autosummary/module.rst index 6ec89e05e..049498021 100644 --- a/sphinx/ext/autosummary/templates/autosummary/module.rst +++ b/sphinx/ext/autosummary/templates/autosummary/module.rst @@ -34,3 +34,27 @@ {%- endfor %} {% endif %} {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: modules + +.. autosummary:: + :toctree: modules +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} + +{% block packages %} +{% if packages %} +.. rubric:: packages + +.. autosummary:: + :toctree: packages +{% for item in packages %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} From 4365c8436f6ae463331a7edf694f02bfa8ff3ae5 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 8 Feb 2019 16:11:29 +0100 Subject: [PATCH 02/27] [autosummary] tests for autosummary modules and packages --- .../test-ext-autosummary-package/conf.py | 10 ++++ .../test-ext-autosummary-package/index.rst | 9 +++ .../package/__init__.py | 0 .../package/module.py | 13 ++++ .../package/module_importfail.py | 4 ++ .../package/subpackage/__init__.py | 0 .../package/subpackage/module.py | 13 ++++ .../package/subpackage/module_importfail.py | 4 ++ tests/test_ext_autosummary.py | 60 ++++++++++++++++++- 9 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/roots/test-ext-autosummary-package/conf.py create mode 100644 tests/roots/test-ext-autosummary-package/index.rst create mode 100644 tests/roots/test-ext-autosummary-package/package/__init__.py create mode 100644 tests/roots/test-ext-autosummary-package/package/module.py create mode 100644 tests/roots/test-ext-autosummary-package/package/module_importfail.py create mode 100644 tests/roots/test-ext-autosummary-package/package/subpackage/__init__.py create mode 100644 tests/roots/test-ext-autosummary-package/package/subpackage/module.py create mode 100644 tests/roots/test-ext-autosummary-package/package/subpackage/module_importfail.py diff --git a/tests/roots/test-ext-autosummary-package/conf.py b/tests/roots/test-ext-autosummary-package/conf.py new file mode 100644 index 000000000..55c769c6f --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/conf.py @@ -0,0 +1,10 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) + +extensions = ['sphinx.ext.autosummary'] +autosummary_generate = True + +# The suffix of source filenames. +source_suffix = '.rst' diff --git a/tests/roots/test-ext-autosummary-package/index.rst b/tests/roots/test-ext-autosummary-package/index.rst new file mode 100644 index 000000000..d5292607c --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/index.rst @@ -0,0 +1,9 @@ +API Reference +============= + +.. rubric:: Packages + +.. autosummary:: + :toctree: generated + + package diff --git a/tests/roots/test-ext-autosummary-package/package/__init__.py b/tests/roots/test-ext-autosummary-package/package/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-ext-autosummary-package/package/module.py b/tests/roots/test-ext-autosummary-package/package/module.py new file mode 100644 index 000000000..5506d0bc9 --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/package/module.py @@ -0,0 +1,13 @@ +from os import * # NOQA + + +class Foo: + def __init__(self): + pass + + def bar(self): + pass + + @property + def baz(self): + pass diff --git a/tests/roots/test-ext-autosummary-package/package/module_importfail.py b/tests/roots/test-ext-autosummary-package/package/module_importfail.py new file mode 100644 index 000000000..9e3f9f195 --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/package/module_importfail.py @@ -0,0 +1,4 @@ +import sys + +# Fail module import in a catastrophic way +sys.exit(1) diff --git a/tests/roots/test-ext-autosummary-package/package/subpackage/__init__.py b/tests/roots/test-ext-autosummary-package/package/subpackage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-ext-autosummary-package/package/subpackage/module.py b/tests/roots/test-ext-autosummary-package/package/subpackage/module.py new file mode 100644 index 000000000..5506d0bc9 --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/package/subpackage/module.py @@ -0,0 +1,13 @@ +from os import * # NOQA + + +class Foo: + def __init__(self): + pass + + def bar(self): + pass + + @property + def baz(self): + pass diff --git a/tests/roots/test-ext-autosummary-package/package/subpackage/module_importfail.py b/tests/roots/test-ext-autosummary-package/package/subpackage/module_importfail.py new file mode 100644 index 000000000..9e3f9f195 --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/package/subpackage/module_importfail.py @@ -0,0 +1,4 @@ +import sys + +# Fail module import in a catastrophic way +sys.exit(1) diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index e1cbaf274..d0dee312d 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -9,6 +9,7 @@ """ from io import StringIO +import os import pytest @@ -177,7 +178,6 @@ def test_escaping(app, status, warning): @pytest.mark.sphinx('dummy', testroot='ext-autosummary') def test_autosummary_generate(app, status, warning): app.builder.build_all() - module = (app.srcdir / 'generated' / 'autosummary_dummy_module.rst').text() assert (' .. autosummary::\n' ' \n' @@ -197,6 +197,64 @@ def test_autosummary_generate(app, status, warning): ' \n' in Foo) +@pytest.mark.sphinx('dummy', testroot='ext-autosummary-package') +def test_autosummary_generate_package(app, status, warning): + app.builder.build_all() + + # Check generated .rst files + root = app.srcdir / 'generated' + n = len(root)+1 + generated = set() + for path, subdirs, files in os.walk(str(root)): + base = path[n:] + for name in files: + generated.add(os.path.join(base, name)) + for name in subdirs: + generated.add(os.path.join(base, name)) + expected = set(['package.rst', 'modules', 'packages', + os.path.join('modules', 'package.module.rst'), + os.path.join('packages', 'package.subpackage.rst'), + os.path.join('packages', 'modules'), + os.path.join('packages', 'modules', 'package.subpackage.module.rst'), + ]) + assert generated == expected + + content = (root / 'package.rst').text() + assert '.. automodule:: package' in content + assert '.. rubric:: modules' in content + assert ('.. autosummary::\n' + ' :toctree: modules\n' + '\n' + ' package.module\n' in content) + assert '.. rubric:: packages' in content + assert ('.. autosummary::\n' + ' :toctree: packages\n' + '\n' + ' package.subpackage\n' in content) + + content = (root / 'packages' / 'package.subpackage.rst').text() + assert '.. automodule:: package.subpackage' in content + assert '.. rubric:: modules' in content + assert ('.. autosummary::\n' + ' :toctree: modules\n' + '\n' + ' package.subpackage.module\n' in content) + + modules = [('modules', 'package.module.rst'), + ('packages', 'modules', 'package.subpackage.module.rst')] + for module in modules: + path = root + for sub in module: + path /= sub + content = path.text() + assert '.. automodule:: '+module[-1][:-4] in content + assert ' .. rubric:: Classes' in content + assert (' .. autosummary::\n' + ' \n' + ' Foo\n' + ' \n' in content) + + @pytest.mark.sphinx('latex', **default_kw) def test_autosummary_latex_table_colspec(app, status, warning): app.builder.build_all() From 79a928212ba2631e65c55885ff5c4a9b6686b611 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 8 Feb 2019 16:28:50 +0100 Subject: [PATCH 03/27] [autosummary] import pkgutil before sys --- sphinx/ext/autosummary/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index a10eaaf4d..54770e3d9 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -22,8 +22,8 @@ import locale import os import pydoc import re -import sys import pkgutil +import sys from jinja2 import FileSystemLoader, TemplateNotFound from jinja2.sandbox import SandboxedEnvironment From 93ba872dd54b8122fa3fbf811615c2cb2e257606 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 8 Feb 2019 16:54:32 +0100 Subject: [PATCH 04/27] [autosummary] import pkgutil first --- sphinx/ext/autosummary/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 54770e3d9..f346ba0da 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -17,12 +17,12 @@ :license: BSD, see LICENSE for details. """ +import pkgutil import argparse import locale import os import pydoc import re -import pkgutil import sys from jinja2 import FileSystemLoader, TemplateNotFound From 667e62e4e2c76bd6793f506faf5b67e0c37425f4 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 8 Feb 2019 17:33:48 +0100 Subject: [PATCH 05/27] [autosummary] fix some flake8 issues --- sphinx/ext/autosummary/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index f346ba0da..039aa89f9 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -17,10 +17,10 @@ :license: BSD, see LICENSE for details. """ -import pkgutil import argparse import locale import os +import pkgutil import pydoc import re import sys @@ -198,7 +198,7 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', out_dict['modules'], out_dict['all_modules'] = [], [] out_dict['packages'], out_dict['all_packages'] = [], [] for _, modname, ispkg in pkgutil.iter_modules(obj.__path__): - fullname = name+'.'+modname + fullname = name + '.' + modname try: import_by_name(fullname) except ImportError as e: From b0eedaadbf302922f5f009f6890392ed96609cc5 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sat, 23 Feb 2019 13:45:48 +0100 Subject: [PATCH 06/27] [autosummary] do not skip an existing source when it is a package --- sphinx/ext/autosummary/generate.py | 40 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 039aa89f9..826fe5040 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -152,10 +152,15 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', warn('[autosummary] failed to import %r: %s' % (name, e)) continue + # https://docs.python.org/3/reference/import.html#packages + # ... any module that contains a __path__ attribute is + # considered a package ... + ispackage = hasattr(obj, '__path__') + fn = os.path.join(path, name + suffix) - # skip it if it exists - if os.path.isfile(fn): + # skip it if it exists and is not a package + if os.path.isfile(fn) and not ispackage: continue new_files.append(fn) @@ -189,14 +194,10 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', if x in include_public or not x.startswith('_')] return public, items - def get_package_members(out_dict, include_public=[], imported=True): - # https://docs.python.org/3/reference/import.html#packages - # ... any module that contains a __path__ attribute is - # considered a package - if not hasattr(obj, '__path__'): - return - out_dict['modules'], out_dict['all_modules'] = [], [] - out_dict['packages'], out_dict['all_packages'] = [], [] + def get_package_members(obj, typ, include_public=[]): + # type: (Any, bool, List[str]) -> Tuple[List[str], List[str]] + items = [] # type: List[str] + pkg_required = typ == 'package' for _, modname, ispkg in pkgutil.iter_modules(obj.__path__): fullname = name + '.' + modname try: @@ -204,13 +205,11 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', except ImportError as e: warn('[autosummary] failed to import %s: %s' % (fullname, e)) continue - if ispkg: - kpublic, kall = 'packages', 'all_packages' - else: - kpublic, kall = 'modules', 'all_modules' - out_dict[kall].append(fullname) - if include_public or not modname.startswith('_'): - out_dict[kpublic].append(fullname) + if ispkg == pkg_required: + items.append(fullname) + public = [x for x in items + if x in include_public or not x.split('.')[-1].startswith('_')] + return public, items ns = {} # type: Dict[str, Any] @@ -222,7 +221,12 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', get_members(obj, 'class', imported=imported_members) ns['exceptions'], ns['all_exceptions'] = \ get_members(obj, 'exception', imported=imported_members) - get_package_members(ns, imported=imported_members) + if ispackage: + ns['modules'], ns['all_modules'] = \ + get_package_members(obj, 'module') + ns['packages'], ns['all_packages'] = \ + get_package_members(obj, 'package') + print(ns['modules'], ns['all_modules']) elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ From 170aa5ab33b6857771e79597c8608c6a1978c4b2 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 24 Feb 2019 11:59:11 +0100 Subject: [PATCH 07/27] [autosummary] add recursion limit config parameter --- sphinx/ext/autosummary/__init__.py | 3 +++ sphinx/ext/autosummary/generate.py | 35 +++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index a18ce2a6d..72b875d1b 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -702,9 +702,11 @@ def process_generate_options(app): 'But your source_suffix does not contain .rst. Skipped.')) return + recursion_limit = app.config.autosummary_recursion_limit generate_autosummary_docs(genfiles, builder=app.builder, warn=logger.warning, info=logger.info, suffix=suffix, base_path=app.srcdir, + recursion_limit=recursion_limit, app=app) @@ -729,4 +731,5 @@ def setup(app): app.connect('doctree-read', process_autosummary_toc) app.connect('builder-inited', process_generate_options) app.add_config_value('autosummary_generate', [], True, [bool]) + app.add_config_value('autosummary_recursion_limit', 0, 0) return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 826fe5040..27974eeac 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -94,9 +94,9 @@ def _underline(title, line='='): def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', warn=_simple_warn, info=_simple_info, base_path=None, builder=None, template_dir=None, - imported_members=False, app=None): + imported_members=False, recursion_limit=0, + app=None): # type: (List[str], str, str, Callable, Callable, str, Builder, str, bool, Any) -> None - showed_sources = list(sorted(sources)) if len(showed_sources) > 20: showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:] @@ -156,12 +156,28 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', # ... any module that contains a __path__ attribute is # considered a package ... ispackage = hasattr(obj, '__path__') + if ispackage: + depth = len(name.split('.')) - 1 + recurse_package = depth < recursion_limit or recursion_limit < 0 + if recurse_package: + info(__('[autosummary] add modules/packages from %s') % repr(name)) + else: + recurse_package = False fn = os.path.join(path, name + suffix) - # skip it if it exists and is not a package - if os.path.isfile(fn) and not ispackage: - continue + # Skip it if it exists and not a package. + if os.path.isfile(fn): + if ispackage and recursion_limit != 0: + # Overwrite the file because submodules/packages + # could have been added/removed from the package + # or the recursion limit might have changed. + # Warning: this file could have been created by the user + # in which case it is lost. + warn('[autosummary] overwriting docs of package %s' % (repr(name))) + os.remove(fn) + else: + continue new_files.append(fn) @@ -221,12 +237,11 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', get_members(obj, 'class', imported=imported_members) ns['exceptions'], ns['all_exceptions'] = \ get_members(obj, 'exception', imported=imported_members) - if ispackage: + if recurse_package: ns['modules'], ns['all_modules'] = \ get_package_members(obj, 'module') ns['packages'], ns['all_packages'] = \ get_package_members(obj, 'package') - print(ns['modules'], ns['all_modules']) elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ @@ -261,7 +276,10 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', generate_autosummary_docs(new_files, output_dir=output_dir, suffix=suffix, warn=warn, info=info, base_path=base_path, builder=builder, - template_dir=template_dir, app=app) + template_dir=template_dir, + imported_members=imported_members, + recursion_limit=recursion_limit, + app=app) # -- Finding documented entries in files --------------------------------------- @@ -442,6 +460,7 @@ def main(argv=sys.argv[1:]): '.' + args.suffix, template_dir=args.templates, imported_members=args.imported_members, + recursion_limit=args.recursion_limit, app=app) From 902adaff02da7d1c855e720f7cc81972b0899e1f Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 24 Feb 2019 11:59:48 +0100 Subject: [PATCH 08/27] [main] allow for docstring extraction --- sphinx/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/__main__.py b/sphinx/__main__.py index 22ea029b0..074957509 100644 --- a/sphinx/__main__.py +++ b/sphinx/__main__.py @@ -12,4 +12,5 @@ import sys from sphinx.cmd.build import main -sys.exit(main(sys.argv[1:])) +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From cb51216cac6cb2007313cd74d091595853ddd2f8 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 24 Feb 2019 12:12:33 +0100 Subject: [PATCH 09/27] [autosummary] modify docs --- doc/usage/extensions/autosummary.rst | 47 +++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index d19ff0554..c9d04dda8 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -13,7 +13,7 @@ those output e.g. by Epydoc and other API doc generation tools. This is especially useful when your docstrings are long and detailed, and putting each one of them on a separate page makes them easier to read. -The :mod:`sphinx.ext.autosummary` extension does this in two parts: +The :mod:`sphinx.ext.autosummary` extension does this in three parts: 1. There is an :rst:dir:`autosummary` directive for generating summary listings that contain links to the documented items, and short summary blurbs @@ -25,6 +25,10 @@ The :mod:`sphinx.ext.autosummary` extension does this in two parts: These files by default contain only the corresponding :mod:`sphinx.ext.autodoc` directive, but can be customized with templates. +3. Optionally, the new :confval:`autosummary_recursion_limit` config value can be + used to generate "stub" files recursively for packages. Warning: this will overwrite + existing files for packages. + .. rst:directive:: autosummary Insert a table that contains links to documented items, and a short summary @@ -32,7 +36,8 @@ The :mod:`sphinx.ext.autosummary` extension does this in two parts: The :rst:dir:`autosummary` directive can also optionally serve as a :rst:dir:`toctree` entry for the included items. Optionally, stub - ``.rst`` files for these items can also be automatically generated. + ``.rst`` files for these items can also be automatically generated + when :confval:`autosummary_generate` is `True`. For example, :: @@ -54,6 +59,27 @@ The :mod:`sphinx.ext.autosummary` extension does this in two parts: .. currentmodule:: sphinx.ext.autosummary + When :confval:`autosummary_recursion_limit != 0`, the :rst:dir:`autosummary` + directive recursively generates stub ``.rst`` files for packages. + + For example, :: + + .. currentmodule:: sphinx + + .. autosummary:: + + autosummary + + produces a table like this: + + .. currentmodule:: sphinx.ext + + .. autosummary:: + + autosummary + + .. currentmodule:: sphinx.ext.autosummary + Autosummary preprocesses the docstrings and signatures with the same :event:`autodoc-process-docstring` and :event:`autodoc-process-signature` hooks as :mod:`~sphinx.ext.autodoc`. @@ -136,13 +162,24 @@ also use this new config value: .. confval:: autosummary_generate Boolean indicating whether to scan all found documents for autosummary - directives, and to generate stub pages for each. + directives, and to generate stub pages for each. It is disabled by default. Can also be a list of documents for which stub pages should be generated. The new files will be placed in the directories specified in the ``:toctree:`` options of the directives. +Packages can be explored recursively when generating stub pages. + +.. confval:: autosummary_recursion_limit + + Integer that determines the maximal recursion depth. + + - ``autosummary_recursion_limit = -1``: no recursion limit + - ``autosummary_recursion_limit = 0``: disable recursion (default) + - ``autosummary_recursion_limit > 0``: limit recursion depth + + .. versionadded:: 2.0 Customizing templates --------------------- @@ -241,14 +278,14 @@ The following variables available in the templates: List containing names of "public" modules in the package. Only available for modules that are packages. - .. versionadded:: 2.0.0 + .. versionadded:: 2.0 .. data:: packages List containing names of "public" sub-packages in the package. Only available for modules that are packages. - .. versionadded:: 2.0.0 + .. versionadded:: 2.0 Additionally, the following filters are available From 2a7d6081d9d1a2c3129a9b8eaeb01166c0b1a5e3 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 24 Feb 2019 12:13:04 +0100 Subject: [PATCH 10/27] [autosummary] test package recursion --- .../{subpackage => package}/__init__.py | 0 .../package/{subpackage => package}/module.py | 0 .../module_importfail.py | 0 .../package/package/package/__init__.py | 0 .../package/package/package/module.py | 13 ++ .../package/package/module_importfail.py | 4 + .../package/package/package/__init__.py | 0 .../package/package/package/package/module.py | 13 ++ .../package/package/module_importfail.py | 4 + tests/test_ext_autosummary.py | 179 +++++++++++++----- 10 files changed, 163 insertions(+), 50 deletions(-) rename tests/roots/test-ext-autosummary-package/package/{subpackage => package}/__init__.py (100%) rename tests/roots/test-ext-autosummary-package/package/{subpackage => package}/module.py (100%) rename tests/roots/test-ext-autosummary-package/package/{subpackage => package}/module_importfail.py (100%) create mode 100644 tests/roots/test-ext-autosummary-package/package/package/package/__init__.py create mode 100644 tests/roots/test-ext-autosummary-package/package/package/package/module.py create mode 100644 tests/roots/test-ext-autosummary-package/package/package/package/module_importfail.py create mode 100644 tests/roots/test-ext-autosummary-package/package/package/package/package/__init__.py create mode 100644 tests/roots/test-ext-autosummary-package/package/package/package/package/module.py create mode 100644 tests/roots/test-ext-autosummary-package/package/package/package/package/module_importfail.py diff --git a/tests/roots/test-ext-autosummary-package/package/subpackage/__init__.py b/tests/roots/test-ext-autosummary-package/package/package/__init__.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/subpackage/__init__.py rename to tests/roots/test-ext-autosummary-package/package/package/__init__.py diff --git a/tests/roots/test-ext-autosummary-package/package/subpackage/module.py b/tests/roots/test-ext-autosummary-package/package/package/module.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/subpackage/module.py rename to tests/roots/test-ext-autosummary-package/package/package/module.py diff --git a/tests/roots/test-ext-autosummary-package/package/subpackage/module_importfail.py b/tests/roots/test-ext-autosummary-package/package/package/module_importfail.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/subpackage/module_importfail.py rename to tests/roots/test-ext-autosummary-package/package/package/module_importfail.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/__init__.py b/tests/roots/test-ext-autosummary-package/package/package/package/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/module.py b/tests/roots/test-ext-autosummary-package/package/package/package/module.py new file mode 100644 index 000000000..5506d0bc9 --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/package/package/package/module.py @@ -0,0 +1,13 @@ +from os import * # NOQA + + +class Foo: + def __init__(self): + pass + + def bar(self): + pass + + @property + def baz(self): + pass diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-package/package/package/package/module_importfail.py new file mode 100644 index 000000000..9e3f9f195 --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/package/package/package/module_importfail.py @@ -0,0 +1,4 @@ +import sys + +# Fail module import in a catastrophic way +sys.exit(1) diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/package/__init__.py b/tests/roots/test-ext-autosummary-package/package/package/package/package/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/package/module.py b/tests/roots/test-ext-autosummary-package/package/package/package/package/module.py new file mode 100644 index 000000000..5506d0bc9 --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/package/package/package/package/module.py @@ -0,0 +1,13 @@ +from os import * # NOQA + + +class Foo: + def __init__(self): + pass + + def bar(self): + pass + + @property + def baz(self): + pass diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-package/package/package/package/package/module_importfail.py new file mode 100644 index 000000000..9e3f9f195 --- /dev/null +++ b/tests/roots/test-ext-autosummary-package/package/package/package/package/module_importfail.py @@ -0,0 +1,4 @@ +import sys + +# Fail module import in a catastrophic way +sys.exit(1) diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index d0dee312d..216321010 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -101,6 +101,7 @@ def test_get_items_summary(make_app, app_params): import sphinx.ext.autosummary.generate args, kwargs = app_params app = make_app(*args, **kwargs) + sphinx.ext.autosummary.generate.setup_documenters(app) # monkey-patch Autosummary.get_items so we can easily get access to it's # results.. @@ -197,62 +198,140 @@ def test_autosummary_generate(app, status, warning): ' \n' in Foo) -@pytest.mark.sphinx('dummy', testroot='ext-autosummary-package') -def test_autosummary_generate_package(app, status, warning): +def _assert_autosummary_generate_package(app): app.builder.build_all() - # Check generated .rst files - root = app.srcdir / 'generated' - n = len(root)+1 - generated = set() - for path, subdirs, files in os.walk(str(root)): - base = path[n:] - for name in files: - generated.add(os.path.join(base, name)) - for name in subdirs: - generated.add(os.path.join(base, name)) - expected = set(['package.rst', 'modules', 'packages', - os.path.join('modules', 'package.module.rst'), - os.path.join('packages', 'package.subpackage.rst'), - os.path.join('packages', 'modules'), - os.path.join('packages', 'modules', 'package.subpackage.module.rst'), - ]) - assert generated == expected + # Prepare recursion + max_depth = 3 + recursion_limit = app.config.autosummary_recursion_limit + assert recursion_limit <= max_depth + + unlimited = recursion_limit < 0 - content = (root / 'package.rst').text() - assert '.. automodule:: package' in content - assert '.. rubric:: modules' in content - assert ('.. autosummary::\n' - ' :toctree: modules\n' - '\n' - ' package.module\n' in content) - assert '.. rubric:: packages' in content - assert ('.. autosummary::\n' - ' :toctree: packages\n' - '\n' - ' package.subpackage\n' in content) + # All packages, modules and classes have the same name + package_name = 'package' + module_name = 'module' + class_name = 'Foo' + extension = 'rst' + nsuffix = len(extension)+1 - content = (root / 'packages' / 'package.subpackage.rst').text() - assert '.. automodule:: package.subpackage' in content - assert '.. rubric:: modules' in content - assert ('.. autosummary::\n' - ' :toctree: modules\n' - '\n' - ' package.subpackage.module\n' in content) + # Expected module.rst template formatting + package_rubic_name = 'packages' + module_rubic_name = 'modules' + package_rubic = '.. rubric:: ' + package_rubic_name + module_rubic = '.. rubric:: ' + module_rubic_name + package_summary = '.. autosummary::\n'\ + ' :toctree: {}\n'\ + '\n'\ + ' {{}}.{}\n'.format(package_rubic_name, package_name) + module_summary = '.. autosummary::\n'\ + ' :toctree: {}\n'\ + '\n'\ + ' {{}}.{}\n'.format(module_rubic_name, module_name) + class_summary = ' .. autosummary::\n'\ + ' \n'\ + ' {}\n'\ + ' \n'.format(class_name) - modules = [('modules', 'package.module.rst'), - ('packages', 'modules', 'package.subpackage.module.rst')] - for module in modules: - path = root - for sub in module: - path /= sub - content = path.text() - assert '.. automodule:: '+module[-1][:-4] in content + def assert_package_content(file): + pkgroot = file.basename()[:-nsuffix] + content = file.text() + expected = ['.. automodule:: ' + pkgroot] + unexpected = [] + if has_modules: + lst = expected + else: + lst = unexpected + lst.append(module_rubic) + lst.append(module_summary.format(pkgroot)) + if has_packages: + lst = expected + else: + lst = unexpected + lst.append(package_rubic) + lst.append(package_summary.format(pkgroot)) + for text in expected: + assert text in content + for text in unexpected: + assert text not in content + + def assert_module_content(file): + modname = file.basename()[:-nsuffix] + content = file.text() + assert '.. automodule:: '+modname in content assert ' .. rubric:: Classes' in content - assert (' .. autosummary::\n' - ' \n' - ' Foo\n' - ' \n' in content) + assert class_summary in content + + root = app.srcdir / 'generated' + depth = 0 + pkgroot = '' + packages = [] + modules = [] + directories = [] + while True: + limit_reached = (depth == recursion_limit) and not unlimited + last_package = depth == max_depth + has_packages = not limit_reached and not last_package + has_modules = not limit_reached + + if pkgroot: + pkgroot = '.'.join((pkgroot, package_name)) + else: + pkgroot = package_name + package = root / '.'.join((pkgroot, extension)) + packages.append(package) + assert_package_content(package) + + if limit_reached: + break + + directories.append(root / module_rubic_name) + module = root / module_rubic_name / \ + '.'.join((pkgroot, module_name, extension)) + modules.append(module) + assert_module_content(module) + + if last_package: + # The last package contains a module but no subpackage + break + + directories.append(root / package_rubic_name) + root /= package_rubic_name + depth += 1 + + # Check generated files and directories + generated = [] + root = app.srcdir / 'generated' + for dir, subdirs, files in os.walk(str(root)): + root = root / dir + for name in files: + generated.append(root / name) + for name in subdirs: + generated.append(root / name) + expected = directories + packages + modules + assert set(generated) == set(expected) + + +@pytest.mark.sphinx('dummy', testroot='ext-autosummary-package') +def test_autosummary_generate_package(make_app, app_params): + import sphinx.ext.autosummary + import sphinx.ext.autosummary.generate + logger = sphinx.ext.autosummary.logger + args, kwargs = app_params + confoverrides = {'autosummary_recursion_limit': 0} + kwargs['confoverrides'] = confoverrides + # Try different recursion limits + # TODO: going from higher to lower limit + # currently fails because generated files + # are not deleted + for limit in 0, 1, 2, 3, -1: + logger.info('autosummary_recursion_limit = {}'.format(limit)) + confoverrides['autosummary_recursion_limit'] = limit + app = make_app(*args, **kwargs) + _assert_autosummary_generate_package(app) + # Cleanup + # root = app.srcdir / 'generated' + # root.rmtree() @pytest.mark.sphinx('latex', **default_kw) From 3ef575cbca7ac46a677be3b0201ddf1d2e1b9e41 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 24 Feb 2019 12:41:47 +0100 Subject: [PATCH 11/27] [autosummary] small flake8/mypy bugs --- sphinx/ext/autosummary/__init__.py | 4 ++-- sphinx/ext/autosummary/generate.py | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index 1381b4eff..39b00ce9d 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -704,7 +704,7 @@ def process_generate_options(app): return recursion_limit = app.config.autosummary_recursion_limit - + with mock(app.config.autosummary_mock_imports): generate_autosummary_docs(genfiles, builder=app.builder, warn=logger.warning, info=logger.info, @@ -734,7 +734,7 @@ def setup(app): app.connect('doctree-read', process_autosummary_toc) app.connect('builder-inited', process_generate_options) app.add_config_value('autosummary_generate', [], True, [bool]) - app.add_config_value('autosummary_recursion_limit', 0, 0) + app.add_config_value('autosummary_recursion_limit', 0, True, [int]) app.add_config_value('autosummary_mock_imports', lambda config: config.autodoc_mock_imports, 'env') return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 27974eeac..e1bdfdef8 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -91,12 +91,19 @@ def _underline(title, line='='): # -- Generating output --------------------------------------------------------- -def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', - warn=_simple_warn, info=_simple_info, - base_path=None, builder=None, template_dir=None, - imported_members=False, recursion_limit=0, - app=None): - # type: (List[str], str, str, Callable, Callable, str, Builder, str, bool, Any) -> None +def generate_autosummary_docs(sources, # type: List[str] + output_dir=None, # type: str + suffix='.rst', # type: str + warn=_simple_warn, # type: Callable + info=_simple_info, # type: Callable + base_path=None, # type: str + builder=None, # type: Builder + template_dir=None, # type: str + imported_members=False, # type: bool + recursion_limit=0, # type: int + app=None # type: Any + ): + # type: (...) -> None showed_sources = list(sorted(sources)) if len(showed_sources) > 20: showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:] @@ -211,7 +218,7 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', return public, items def get_package_members(obj, typ, include_public=[]): - # type: (Any, bool, List[str]) -> Tuple[List[str], List[str]] + # type: (Any, str, List[str]) -> Tuple[List[str], List[str]] items = [] # type: List[str] pkg_required = typ == 'package' for _, modname, ispkg in pkgutil.iter_modules(obj.__path__): From 6e0b0517c06a9f5c5db7d12890e8f699c10d8514 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 7 Apr 2019 11:43:22 +0200 Subject: [PATCH 12/27] [autosummary] address code review --- doc/usage/extensions/autosummary.rst | 46 +++++-------------- sphinx/__main__.py | 3 +- sphinx/ext/autosummary/__init__.py | 7 ++- sphinx/ext/autosummary/generate.py | 20 ++++---- .../test-ext-autosummary-package/conf.py | 3 -- tests/test_ext_autosummary.py | 18 ++++---- 6 files changed, 35 insertions(+), 62 deletions(-) diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index 891fa54ab..3d373908a 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -25,9 +25,9 @@ The :mod:`sphinx.ext.autosummary` extension does this in three parts: These files by default contain only the corresponding :mod:`sphinx.ext.autodoc` directive, but can be customized with templates. -3. Optionally, the new :confval:`autosummary_recursion_limit` config value can be - used to generate "stub" files recursively for packages. Warning: this will overwrite - existing files for packages. +3. Optionally, :confval:`autosummary_depth_limit` config value can be + used to generate "stub" files of modules and sub-packages for packages + under the :rst:dir:`autosummary` directive. .. rst:directive:: autosummary @@ -59,27 +59,6 @@ The :mod:`sphinx.ext.autosummary` extension does this in three parts: .. currentmodule:: sphinx.ext.autosummary - When :confval:`autosummary_recursion_limit != 0`, the :rst:dir:`autosummary` - directive recursively generates stub ``.rst`` files for packages. - - For example, :: - - .. currentmodule:: sphinx - - .. autosummary:: - - autosummary - - produces a table like this: - - .. currentmodule:: sphinx.ext - - .. autosummary:: - - autosummary - - .. currentmodule:: sphinx.ext.autosummary - Autosummary preprocesses the docstrings and signatures with the same :event:`autodoc-process-docstring` and :event:`autodoc-process-signature` hooks as :mod:`~sphinx.ext.autodoc`. @@ -169,17 +148,16 @@ also use these config values: The new files will be placed in the directories specified in the ``:toctree:`` options of the directives. -Packages can be explored recursively when generating stub pages. +.. confval:: autosummary_depth_limit -.. confval:: autosummary_recursion_limit - - Integer that determines the maximal recursion depth. + Integer that determines the maximal depth (starting from root) when adding + modules and sub-packages of packages. - - ``autosummary_recursion_limit = -1``: no recursion limit - - ``autosummary_recursion_limit = 0``: disable recursion (default) - - ``autosummary_recursion_limit > 0``: limit recursion depth + - ``autosummary_depth_limit = -1``: no limit + - ``autosummary_depth_limit = 0``: disable adding modules and sub-packages (default) + - ``autosummary_depth_limit > 0``: limited depth starting from root - .. versionadded:: 2.0 + .. versionadded:: 2.1 .. confval:: autosummary_mock_imports @@ -284,14 +262,14 @@ The following variables available in the templates: List containing names of "public" modules in the package. Only available for modules that are packages. - .. versionadded:: 2.0 + .. versionadded:: 2.1 .. data:: packages List containing names of "public" sub-packages in the package. Only available for modules that are packages. - .. versionadded:: 2.0 + .. versionadded:: 2.1 Additionally, the following filters are available diff --git a/sphinx/__main__.py b/sphinx/__main__.py index 074957509..22ea029b0 100644 --- a/sphinx/__main__.py +++ b/sphinx/__main__.py @@ -12,5 +12,4 @@ import sys from sphinx.cmd.build import main -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) +sys.exit(main(sys.argv[1:])) diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index af1e50a0a..6f4159aab 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -733,14 +733,13 @@ def process_generate_options(app): 'But your source_suffix does not contain .rst. Skipped.')) return - recursion_limit = app.config.autosummary_recursion_limit + depth_limit = app.config.autosummary_depth_limit with mock(app.config.autosummary_mock_imports): generate_autosummary_docs(genfiles, builder=app.builder, warn=logger.warning, info=logger.info, suffix=suffix, base_path=app.srcdir, - recursion_limit=recursion_limit, - app=app) + app=app, depth_limit=depth_limit) def setup(app): @@ -764,7 +763,7 @@ def setup(app): app.connect('doctree-read', process_autosummary_toc) app.connect('builder-inited', process_generate_options) app.add_config_value('autosummary_generate', [], True, [bool]) - app.add_config_value('autosummary_recursion_limit', 0, True, [int]) + app.add_config_value('autosummary_depth_limit', 0, 'env', [int]) app.add_config_value('autosummary_mock_imports', lambda config: config.autodoc_mock_imports, 'env') return {'version': sphinx.__display_version__, 'parallel_read_safe': True} diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 13033692a..d17ab869f 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -98,8 +98,8 @@ def generate_autosummary_docs(sources, # type: List[str] builder=None, # type: Builder template_dir=None, # type: str imported_members=False, # type: bool - recursion_limit=0, # type: int - app=None # type: Any + app=None, # type: Any + depth_limit=0, # type: int ): # type: (...) -> None showed_sources = list(sorted(sources)) @@ -163,20 +163,20 @@ def generate_autosummary_docs(sources, # type: List[str] ispackage = hasattr(obj, '__path__') if ispackage: depth = len(name.split('.')) - 1 - recurse_package = depth < recursion_limit or recursion_limit < 0 - if recurse_package: + add_package_children = depth < depth_limit or depth_limit < 0 + if add_package_children: info(__('[autosummary] add modules/packages from %s') % repr(name)) else: - recurse_package = False + add_package_children = False fn = os.path.join(path, name + suffix) # Skip it if it exists and not a package. if os.path.isfile(fn): - if ispackage and recursion_limit != 0: + if ispackage and depth_limit != 0: # Overwrite the file because submodules/packages # could have been added/removed from the package - # or the recursion limit might have changed. + # or the depth limit might have changed. # Warning: this file could have been created by the user # in which case it is lost. warn('[autosummary] overwriting docs of package %s' % (repr(name))) @@ -242,7 +242,7 @@ def generate_autosummary_docs(sources, # type: List[str] get_members(obj, 'class', imported=imported_members) ns['exceptions'], ns['all_exceptions'] = \ get_members(obj, 'exception', imported=imported_members) - if recurse_package: + if add_package_children: ns['modules'], ns['all_modules'] = \ get_package_members(obj, 'module') ns['packages'], ns['all_packages'] = \ @@ -283,7 +283,7 @@ def generate_autosummary_docs(sources, # type: List[str] base_path=base_path, builder=builder, template_dir=template_dir, imported_members=imported_members, - recursion_limit=recursion_limit, + depth_limit=depth_limit, app=app) @@ -465,7 +465,7 @@ def main(argv=sys.argv[1:]): '.' + args.suffix, template_dir=args.templates, imported_members=args.imported_members, - recursion_limit=args.recursion_limit, + depth_limit=args.depth_limit, app=app) diff --git a/tests/roots/test-ext-autosummary-package/conf.py b/tests/roots/test-ext-autosummary-package/conf.py index 55c769c6f..1c0d02202 100644 --- a/tests/roots/test-ext-autosummary-package/conf.py +++ b/tests/roots/test-ext-autosummary-package/conf.py @@ -5,6 +5,3 @@ sys.path.insert(0, os.path.abspath('.')) extensions = ['sphinx.ext.autosummary'] autosummary_generate = True - -# The suffix of source filenames. -source_suffix = '.rst' diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 0d1282691..1da5b73a4 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -202,12 +202,12 @@ def test_autosummary_generate(app, status, warning): def _assert_autosummary_generate_package(app): app.builder.build_all() - # Prepare recursion + # Prepare package exploration max_depth = 3 - recursion_limit = app.config.autosummary_recursion_limit - assert recursion_limit <= max_depth + depth_limit = app.config.autosummary_depth_limit + assert depth_limit <= max_depth - unlimited = recursion_limit < 0 + unlimited = depth_limit < 0 # All packages, modules and classes have the same name package_name = 'package' @@ -270,7 +270,7 @@ def _assert_autosummary_generate_package(app): modules = [] directories = [] while True: - limit_reached = (depth == recursion_limit) and not unlimited + limit_reached = (depth == depth_limit) and not unlimited last_package = depth == max_depth has_packages = not limit_reached and not last_package has_modules = not limit_reached @@ -319,15 +319,15 @@ def test_autosummary_generate_package(make_app, app_params): import sphinx.ext.autosummary.generate logger = sphinx.ext.autosummary.logger args, kwargs = app_params - confoverrides = {'autosummary_recursion_limit': 0} + confoverrides = {'autosummary_depth_limit': 0} kwargs['confoverrides'] = confoverrides - # Try different recursion limits + # Try different depth limits # TODO: going from higher to lower limit # currently fails because generated files # are not deleted for limit in 0, 1, 2, 3, -1: - logger.info('autosummary_recursion_limit = {}'.format(limit)) - confoverrides['autosummary_recursion_limit'] = limit + logger.info('autosummary_depth_limit = {}'.format(limit)) + confoverrides['autosummary_depth_limit'] = limit app = make_app(*args, **kwargs) _assert_autosummary_generate_package(app) # Cleanup From 4fb58b2146dd6daea2dd326b577a7dd1ee6f43e2 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sat, 1 Jun 2019 13:08:23 +0200 Subject: [PATCH 13/27] [autosummary] doclint violation --- doc/usage/extensions/autosummary.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index 8e43f8a7b..0bdc4e640 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -154,7 +154,7 @@ also use these config values: modules and sub-packages of packages. - ``autosummary_depth_limit = -1``: no limit - - ``autosummary_depth_limit = 0``: disable adding modules and sub-packages (default) + - ``autosummary_depth_limit = 0``: disable adding sub-packages (default) - ``autosummary_depth_limit > 0``: limited depth starting from root .. versionadded:: 2.1 @@ -276,8 +276,8 @@ The following variables available in the templates: .. data:: packages - List containing names of "public" sub-packages in the package. Only available for - modules that are packages. + List containing names of "public" sub-packages in the package. Only available + for modules that are packages. .. versionadded:: 2.1 From ec8656bd27e24697a5ce4aa8a0ddebd705768f26 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 2 Jun 2019 16:26:40 +0200 Subject: [PATCH 14/27] [autosummary] remove recursion limit and module/package separation --- doc/usage/extensions/autosummary.rst | 23 +--- sphinx/ext/autosummary/__init__.py | 6 +- sphinx/ext/autosummary/generate.py | 34 ++---- .../templates/autosummary/module.rst | 12 -- .../conf.py | 0 .../index.rst | 0 .../package/__init__.py | 0 .../package/module.py | 0 .../package/module_importfail.py | 0 .../package/package/__init__.py | 0 .../package/package/module.py | 0 .../package/package/module_importfail.py | 0 .../package/package/package/__init__.py | 0 .../package/package/package/module.py | 0 .../package/package/module_importfail.py | 0 .../package/package/package/__init__.py | 0 .../package/package/package/package/module.py | 0 .../package/package/module_importfail.py | 0 tests/test_ext_autosummary.py | 111 +++++++----------- 19 files changed, 63 insertions(+), 123 deletions(-) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/conf.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/index.rst (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/__init__.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/module.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/module_importfail.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/__init__.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/module.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/module_importfail.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/package/__init__.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/package/module.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/package/module_importfail.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/package/package/__init__.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/package/package/module.py (100%) rename tests/roots/{test-ext-autosummary-package => test-ext-autosummary-recursive}/package/package/package/package/module_importfail.py (100%) diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index 0bdc4e640..316ffcb9d 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -25,7 +25,7 @@ The :mod:`sphinx.ext.autosummary` extension does this in three parts: These files by default contain only the corresponding :mod:`sphinx.ext.autodoc` directive, but can be customized with templates. -3. Optionally, :confval:`autosummary_depth_limit` config value can be +3. Optionally, the :confval:`autosummary_recursive` config value can be used to generate "stub" files of modules and sub-packages for packages under the :rst:dir:`autosummary` directive. @@ -148,16 +148,12 @@ also use these config values: The new files will be placed in the directories specified in the ``:toctree:`` options of the directives. -.. confval:: autosummary_depth_limit +.. confval:: autosummary_recursive - Integer that determines the maximal depth (starting from root) when adding - modules and sub-packages of packages. - - - ``autosummary_depth_limit = -1``: no limit - - ``autosummary_depth_limit = 0``: disable adding sub-packages (default) - - ``autosummary_depth_limit > 0``: limited depth starting from root + Boolean that determines whether to add modules and sub-packages + recursively. It is disabled by default. - .. versionadded:: 2.1 + .. versionadded:: 2.2 .. confval:: autosummary_mock_imports @@ -272,14 +268,7 @@ The following variables available in the templates: List containing names of "public" modules in the package. Only available for modules that are packages. - .. versionadded:: 2.1 - -.. data:: packages - - List containing names of "public" sub-packages in the package. Only available - for modules that are packages. - - .. versionadded:: 2.1 + .. versionadded:: 2.2 Additionally, the following filters are available diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index cf768dc4f..87ae90386 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -751,14 +751,14 @@ def process_generate_options(app): 'But your source_suffix does not contain .rst. Skipped.')) return - depth_limit = app.config.autosummary_depth_limit imported_members = app.config.autosummary_imported_members + recursive = app.config.autosummary_recursive with mock(app.config.autosummary_mock_imports): generate_autosummary_docs(genfiles, builder=app.builder, warn=logger.warning, info=logger.info, suffix=suffix, base_path=app.srcdir, app=app, imported_members=imported_members, - depth_limit=depth_limit) + recursive=recursive) def setup(app): @@ -782,7 +782,7 @@ def setup(app): app.connect('doctree-read', process_autosummary_toc) app.connect('builder-inited', process_generate_options) app.add_config_value('autosummary_generate', [], True, [bool]) - app.add_config_value('autosummary_depth_limit', 0, 'env', [int]) + app.add_config_value('autosummary_recursive', False, 'env', [bool]) app.add_config_value('autosummary_mock_imports', lambda config: config.autodoc_mock_imports, 'env') app.add_config_value('autosummary_imported_members', [], False, [bool]) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 129c43f65..5fcb1fe96 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -136,8 +136,8 @@ def generate_autosummary_docs(sources, # type: List[str] builder=None, # type: Builder template_dir=None, # type: str imported_members=False, # type: bool + recursive=False, # type: bool app=None, # type: Any - depth_limit=0, # type: int ): # type: (...) -> None showed_sources = list(sorted(sources)) @@ -180,22 +180,16 @@ def generate_autosummary_docs(sources, # type: List[str] # ... any module that contains a __path__ attribute is # considered a package ... ispackage = hasattr(obj, '__path__') - if ispackage: - depth = len(name.split('.')) - 1 - add_package_children = depth < depth_limit or depth_limit < 0 - if add_package_children: - info(__('[autosummary] add modules/packages from %s') % repr(name)) - else: - add_package_children = False + if ispackage and recursive: + info(__('[autosummary] add modules/packages from %s') % repr(name)) fn = os.path.join(path, name + suffix) # Skip it if it exists and not a package. if os.path.isfile(fn): - if ispackage and depth_limit != 0: - # Overwrite the file because submodules/packages - # could have been added/removed from the package - # or the depth limit might have changed. + if ispackage and recursive: + # Overwrite the file because modules/subpackages + # could have been added/removed from the package. # Warning: this file could have been created by the user # in which case it is lost. warn('[autosummary] overwriting docs of package %s' % (repr(name))) @@ -230,10 +224,9 @@ def generate_autosummary_docs(sources, # type: List[str] if x in include_public or not x.startswith('_')] return public, items - def get_package_members(obj, typ, include_public=[]): + def get_modules(obj, include_public=[]): # type: (Any, str, List[str]) -> Tuple[List[str], List[str]] items = [] # type: List[str] - pkg_required = typ == 'package' for _, modname, ispkg in pkgutil.iter_modules(obj.__path__): fullname = name + '.' + modname try: @@ -241,8 +234,7 @@ def generate_autosummary_docs(sources, # type: List[str] except ImportError as e: warn('[autosummary] failed to import %s: %s' % (fullname, e)) continue - if ispkg == pkg_required: - items.append(fullname) + items.append(fullname) public = [x for x in items if x in include_public or not x.split('.')[-1].startswith('_')] return public, items @@ -257,11 +249,9 @@ def generate_autosummary_docs(sources, # type: List[str] get_members(obj, {'class'}, imported=imported_members) ns['exceptions'], ns['all_exceptions'] = \ get_members(obj, {'exception'}, imported=imported_members) - if add_package_children: + if ispackage and recursive: ns['modules'], ns['all_modules'] = \ - get_package_members(obj, 'module') - ns['packages'], ns['all_packages'] = \ - get_package_members(obj, 'package') + get_modules(obj) elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ @@ -298,7 +288,7 @@ def generate_autosummary_docs(sources, # type: List[str] base_path=base_path, builder=builder, template_dir=template_dir, imported_members=imported_members, - depth_limit=depth_limit, + recursive=recursive, app=app) @@ -480,7 +470,7 @@ def main(argv=sys.argv[1:]): '.' + args.suffix, template_dir=args.templates, imported_members=args.imported_members, - depth_limit=args.depth_limit, + recursive=args.recursive, app=app) diff --git a/sphinx/ext/autosummary/templates/autosummary/module.rst b/sphinx/ext/autosummary/templates/autosummary/module.rst index 049498021..b9644ebca 100644 --- a/sphinx/ext/autosummary/templates/autosummary/module.rst +++ b/sphinx/ext/autosummary/templates/autosummary/module.rst @@ -46,15 +46,3 @@ {%- endfor %} {% endif %} {% endblock %} - -{% block packages %} -{% if packages %} -.. rubric:: packages - -.. autosummary:: - :toctree: packages -{% for item in packages %} - {{ item }} -{%- endfor %} -{% endif %} -{% endblock %} diff --git a/tests/roots/test-ext-autosummary-package/conf.py b/tests/roots/test-ext-autosummary-recursive/conf.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/conf.py rename to tests/roots/test-ext-autosummary-recursive/conf.py diff --git a/tests/roots/test-ext-autosummary-package/index.rst b/tests/roots/test-ext-autosummary-recursive/index.rst similarity index 100% rename from tests/roots/test-ext-autosummary-package/index.rst rename to tests/roots/test-ext-autosummary-recursive/index.rst diff --git a/tests/roots/test-ext-autosummary-package/package/__init__.py b/tests/roots/test-ext-autosummary-recursive/package/__init__.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/__init__.py rename to tests/roots/test-ext-autosummary-recursive/package/__init__.py diff --git a/tests/roots/test-ext-autosummary-package/package/module.py b/tests/roots/test-ext-autosummary-recursive/package/module.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/module.py rename to tests/roots/test-ext-autosummary-recursive/package/module.py diff --git a/tests/roots/test-ext-autosummary-package/package/module_importfail.py b/tests/roots/test-ext-autosummary-recursive/package/module_importfail.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/module_importfail.py rename to tests/roots/test-ext-autosummary-recursive/package/module_importfail.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/__init__.py b/tests/roots/test-ext-autosummary-recursive/package/package/__init__.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/__init__.py rename to tests/roots/test-ext-autosummary-recursive/package/package/__init__.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/module.py b/tests/roots/test-ext-autosummary-recursive/package/package/module.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/module.py rename to tests/roots/test-ext-autosummary-recursive/package/package/module.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/module_importfail.py rename to tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/__init__.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/__init__.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/package/__init__.py rename to tests/roots/test-ext-autosummary-recursive/package/package/package/__init__.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/module.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/module.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/package/module.py rename to tests/roots/test-ext-autosummary-recursive/package/package/package/module.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/package/module_importfail.py rename to tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/package/__init__.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/package/__init__.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/package/package/__init__.py rename to tests/roots/test-ext-autosummary-recursive/package/package/package/package/__init__.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/package/module.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/package/module.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/package/package/module.py rename to tests/roots/test-ext-autosummary-recursive/package/package/package/package/module.py diff --git a/tests/roots/test-ext-autosummary-package/package/package/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/package/module_importfail.py similarity index 100% rename from tests/roots/test-ext-autosummary-package/package/package/package/package/module_importfail.py rename to tests/roots/test-ext-autosummary-recursive/package/package/package/package/module_importfail.py diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index c58fdc0e1..bdfea0633 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -224,16 +224,9 @@ def test_autosummary_generate(app, status, warning): ' \n' in Foo) -def _assert_autosummary_generate_package(app): +def _assert_autosummary_recursive(app): app.builder.build_all() - # Prepare package exploration - max_depth = 3 - depth_limit = app.config.autosummary_depth_limit - assert depth_limit <= max_depth - - unlimited = depth_limit < 0 - # All packages, modules and classes have the same name package_name = 'package' module_name = 'module' @@ -242,18 +235,21 @@ def _assert_autosummary_generate_package(app): nsuffix = len(extension)+1 # Expected module.rst template formatting - package_rubic_name = 'packages' + recursive = app.config.autosummary_recursive module_rubic_name = 'modules' - package_rubic = '.. rubric:: ' + package_rubic_name module_rubic = '.. rubric:: ' + module_rubic_name - package_summary = '.. autosummary::\n'\ - ' :toctree: {}\n'\ - '\n'\ - ' {{}}.{}\n'.format(package_rubic_name, package_name) module_summary = '.. autosummary::\n'\ ' :toctree: {}\n'\ '\n'\ - ' {{}}.{}\n'.format(module_rubic_name, module_name) + ' {{}}.{}\n'\ + ' {{}}.{}\n'.format(module_rubic_name, + module_name, + package_name) + last_module_summary = '.. autosummary::\n'\ + ' :toctree: {}\n'\ + '\n'\ + ' {{}}.{}\n'.format(module_rubic_name, + module_name) class_summary = ' .. autosummary::\n'\ ' \n'\ ' {}\n'\ @@ -264,18 +260,15 @@ def _assert_autosummary_generate_package(app): content = file.text() expected = ['.. automodule:: ' + pkgroot] unexpected = [] - if has_modules: + if recursive: lst = expected else: lst = unexpected lst.append(module_rubic) - lst.append(module_summary.format(pkgroot)) - if has_packages: - lst = expected + if depth == max_depth: + lst.append(last_module_summary.format(pkgroot)) else: - lst = unexpected - lst.append(package_rubic) - lst.append(package_summary.format(pkgroot)) + lst.append(module_summary.format(pkgroot, pkgroot)) for text in expected: assert text in content for text in unexpected: @@ -289,75 +282,55 @@ def _assert_autosummary_generate_package(app): assert class_summary in content root = app.srcdir / 'generated' - depth = 0 pkgroot = '' - packages = [] - modules = [] - directories = [] - while True: - limit_reached = (depth == depth_limit) and not unlimited - last_package = depth == max_depth - has_packages = not limit_reached and not last_package - has_modules = not limit_reached - + expected_paths = [] + if app.config.autosummary_recursive: + max_depth = 3 + else: + max_depth = 0 + for depth in range(max_depth+1): if pkgroot: pkgroot = '.'.join((pkgroot, package_name)) else: pkgroot = package_name package = root / '.'.join((pkgroot, extension)) - packages.append(package) + expected_paths.append(package) assert_package_content(package) - - if limit_reached: - break - - directories.append(root / module_rubic_name) - module = root / module_rubic_name / \ - '.'.join((pkgroot, module_name, extension)) - modules.append(module) - assert_module_content(module) - - if last_package: - # The last package contains a module but no subpackage - break - - directories.append(root / package_rubic_name) - root /= package_rubic_name - depth += 1 + if recursive: + module = root / module_rubic_name / \ + '.'.join((pkgroot, module_name, extension)) + expected_paths.append(root / module_rubic_name) + expected_paths.append(module) + assert_module_content(module) + root /= module_rubic_name # Check generated files and directories - generated = [] + generated_paths = [] root = app.srcdir / 'generated' for dir, subdirs, files in os.walk(str(root)): root = root / dir for name in files: - generated.append(root / name) + generated_paths.append(root / name) for name in subdirs: - generated.append(root / name) - expected = directories + packages + modules - assert set(generated) == set(expected) + generated_paths.append(root / name) + assert set(generated_paths) == set(expected_paths) -@pytest.mark.sphinx('dummy', testroot='ext-autosummary-package') -def test_autosummary_generate_package(make_app, app_params): +@pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive') +def test_autosummary_recursive(make_app, app_params): import sphinx.ext.autosummary import sphinx.ext.autosummary.generate logger = sphinx.ext.autosummary.logger args, kwargs = app_params - confoverrides = {'autosummary_depth_limit': 0} + confoverrides = {'autosummary_recursive': False} kwargs['confoverrides'] = confoverrides - # Try different depth limits - # TODO: going from higher to lower limit - # currently fails because generated files - # are not deleted - for limit in 0, 1, 2, 3, -1: - logger.info('autosummary_depth_limit = {}'.format(limit)) - confoverrides['autosummary_depth_limit'] = limit + # Remarks: non-recursive after recursive doesn't work + # recursively generated files are not deleted + for recursive in False, True: + logger.info('autosummary_recursive = {}'.format(recursive)) + confoverrides['autosummary_recursive'] = recursive app = make_app(*args, **kwargs) - _assert_autosummary_generate_package(app) - # Cleanup - # root = app.srcdir / 'generated' - # root.rmtree() + _assert_autosummary_recursive(app) @pytest.mark.sphinx('latex', **default_kw) From 063f2d066b02fe9c36d278a699d4e354c12698fb Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 2 Jun 2019 16:39:57 +0200 Subject: [PATCH 15/27] [autosummary] forgot to adapt types --- sphinx/ext/autosummary/generate.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 5fcb1fe96..44c2d96b8 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -225,7 +225,7 @@ def generate_autosummary_docs(sources, # type: List[str] return public, items def get_modules(obj, include_public=[]): - # type: (Any, str, List[str]) -> Tuple[List[str], List[str]] + # type: (Any, List[str]) -> Tuple[List[str], List[str]] items = [] # type: List[str] for _, modname, ispkg in pkgutil.iter_modules(obj.__path__): fullname = name + '.' + modname @@ -250,8 +250,7 @@ def generate_autosummary_docs(sources, # type: List[str] ns['exceptions'], ns['all_exceptions'] = \ get_members(obj, {'exception'}, imported=imported_members) if ispackage and recursive: - ns['modules'], ns['all_modules'] = \ - get_modules(obj) + ns['modules'], ns['all_modules'] = get_modules(obj) elif doc.objtype == 'class': ns['members'] = dir(obj) ns['inherited_members'] = \ From 4e3046b01ac3333291255cec6ec3f1d7481e92cb Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 8 Jul 2019 08:43:18 +0200 Subject: [PATCH 16/27] remove redundant import check --- sphinx/ext/autosummary/generate.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 042d2e225..23fa7f2fa 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -153,15 +153,10 @@ def generate_autosummary_content(name: str, obj: Any, parent: Any, if x in include_public or not x.startswith('_')] return public, items - def get_modules(obj: Any, include_public: List[str] = [])\ - -> Tuple[List[str], List[str]]: + def get_modules(obj: Any, include_public: List[str] = []) -> Tuple[List[str], List[str]]: items = [] # type: List[str] for _, modname, ispkg in pkgutil.iter_modules(obj.__path__): fullname = name + '.' + modname - try: - import_by_name(fullname) - except ImportError: - continue items.append(fullname) public = [x for x in items if x in include_public or @@ -215,7 +210,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, info: Callable = None, base_path: str = None, builder: Builder = None, template_dir: str = None, imported_members: bool = False, app: Any = None, - overwrite: bool = True, recursive: bool=True) -> None: + overwrite: bool = True, recursive: bool = True) -> None: if info: warnings.warn('info argument for generate_autosummary_docs() is deprecated.', RemovedInSphinx40Warning) From 11d9a97a34f68c15f62a74f157444d4f7678291b Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 8 Jul 2019 09:06:52 +0200 Subject: [PATCH 17/27] refactor autosummary recursion tests --- tests/test_ext_autosummary.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index c59dec316..5454b4e73 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -260,6 +260,7 @@ def test_autosummary_generate_overwrite2(app_params, make_app): confoverrides={'autosummary_recursive': True}) def test_autosummary_recursive_enabled(app, status, warning): app.build() + toctree = 'modules' # see module.rst template # Top-level package generated = app.srcdir / 'generated' @@ -269,7 +270,7 @@ def test_autosummary_recursive_enabled(app, status, warning): assert 'package.package' in content # Recursively generate modules of top-level package - generated /= 'modules' + generated /= toctree assert (generated / 'package.module.rst').exists() assert (generated / 'package.package.rst').exists() content = (generated / 'package.package.rst').text() @@ -277,18 +278,19 @@ def test_autosummary_recursive_enabled(app, status, warning): assert 'package.package.package' in content # Recursively generate modules of sub-package - generated /= 'modules' + generated /= toctree assert (generated / 'package.package.module.rst').exists() assert (generated / 'package.package.package.rst').exists() content = (generated / 'package.package.package.rst').text() assert 'package.package.package.module' in content assert 'package.package.package.package' not in content - # Last sub-package has no modules - generated /= 'modules' + # Last sub-package has no sub-packages + generated /= toctree assert (generated / 'package.package.package.module.rst').exists() assert not (generated / 'package.package.package.package.rst').exists() - assert not (generated / 'modules').exists() + if toctree: + assert not (generated / toctree).exists() @pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive', @@ -296,6 +298,7 @@ def test_autosummary_recursive_enabled(app, status, warning): confoverrides={'autosummary_recursive': False}) def test_autosummary_recursive_disabled(app, status, warning): app.build() + toctree = 'modules' # see module.rst template # Modules should not be generated generated = app.srcdir / 'generated' @@ -303,7 +306,11 @@ def test_autosummary_recursive_disabled(app, status, warning): content = (generated / 'package.rst').text() assert 'package.module' not in content assert 'package.package' not in content - assert not (generated / 'modules').exists() + + # Recursively generate modules of top-level package (should be missing) + generated /= toctree + assert not (generated / 'package.module.rst').exists() + assert not (generated / 'package.package.rst').exists() @pytest.mark.sphinx('latex', **default_kw) From 55083d5e03b8669764e34d3cc99ed409f01c4d71 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 23 Feb 2020 00:45:30 +0900 Subject: [PATCH 18/27] refactor: autosummary: Define AutosummaryEntry as a return type of find_autosummary_*() --- sphinx/ext/autosummary/generate.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 23fa7f2fa..74bd90bbe 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -25,7 +25,7 @@ import pydoc import re import sys import warnings -from typing import Any, Callable, Dict, List, Set, Tuple, Type +from typing import Any, Callable, Dict, List, NamedTuple, Set, Tuple, Type from jinja2 import BaseLoader, FileSystemLoader, TemplateNotFound from jinja2.sandbox import SandboxedEnvironment @@ -58,6 +58,11 @@ class DummyApplication: self.verbosity = 0 +AutosummaryEntry = NamedTuple('AutosummaryEntry', [('name', str), + ('path', str), + ('template', str)]) + + def setup_documenters(app: Any) -> None: from sphinx.ext.autodoc import ( ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter, @@ -246,22 +251,22 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, new_files = [] # write - for name, path, template_name in sorted(set(items), key=str): - if path is None: + for entry in sorted(set(items), key=str): + if entry.path is None: # The corresponding autosummary:: directive did not have # a :toctree: option continue - path = output_dir or os.path.abspath(path) + path = output_dir or os.path.abspath(entry.path) ensuredir(path) try: - name, obj, parent, mod_name = import_by_name(name) + name, obj, parent, mod_name = import_by_name(entry.name) except ImportError as e: _warn('[autosummary] failed to import %r: %s' % (name, e)) continue - content = generate_autosummary_content(name, obj, parent, template, template_name, + content = generate_autosummary_content(name, obj, parent, template, entry.template, imported_members, app, recursive) filename = os.path.join(path, name + suffix) @@ -291,12 +296,12 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, # -- Finding documented entries in files --------------------------------------- -def find_autosummary_in_files(filenames: List[str]) -> List[Tuple[str, str, str]]: +def find_autosummary_in_files(filenames: List[str]) -> List[AutosummaryEntry]: """Find out what items are documented in source/*.rst. See `find_autosummary_in_lines`. """ - documented = [] # type: List[Tuple[str, str, str]] + documented = [] # type: List[AutosummaryEntry] for filename in filenames: with open(filename, encoding='utf-8', errors='ignore') as f: lines = f.read().splitlines() @@ -305,7 +310,7 @@ def find_autosummary_in_files(filenames: List[str]) -> List[Tuple[str, str, str] def find_autosummary_in_docstring(name: str, module: Any = None, filename: str = None - ) -> List[Tuple[str, str, str]]: + ) -> List[AutosummaryEntry]: """Find out what items are documented in the given object's docstring. See `find_autosummary_in_lines`. @@ -325,7 +330,7 @@ def find_autosummary_in_docstring(name: str, module: Any = None, filename: str = def find_autosummary_in_lines(lines: List[str], module: Any = None, filename: str = None - ) -> List[Tuple[str, str, str]]: + ) -> List[AutosummaryEntry]: """Find out what items appear in autosummary:: directives in the given lines. @@ -345,7 +350,7 @@ def find_autosummary_in_lines(lines: List[str], module: Any = None, filename: st toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$') template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$') - documented = [] # type: List[Tuple[str, str, str]] + documented = [] # type: List[AutosummaryEntry] toctree = None # type: str template = None @@ -379,7 +384,7 @@ def find_autosummary_in_lines(lines: List[str], module: Any = None, filename: st if current_module and \ not name.startswith(current_module + '.'): name = "%s.%s" % (current_module, name) - documented.append((name, toctree, template)) + documented.append(AutosummaryEntry(name, toctree, template)) continue if not line.strip() or line.startswith(base_indent + " "): From 665458561fb28c882f8e66c569e6586ada8a5f8b Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 23 Feb 2020 01:41:31 +0900 Subject: [PATCH 19/27] fix --- tests/test_ext_autosummary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 5454b4e73..7b59c8dd5 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -20,7 +20,7 @@ from sphinx import addnodes from sphinx.ext.autosummary import ( autosummary_table, autosummary_toc, mangle_signature, import_by_name, extract_summary ) -from sphinx.ext.autosummary.generate import generate_autosummary_docs +from sphinx.ext.autosummary.generate import AutosummaryEntry, generate_autosummary_docs from sphinx.testing.util import assert_node, etree_parse from sphinx.util.docutils import new_document @@ -382,7 +382,7 @@ def test_autosummary_imported_members(app, status, warning): @pytest.mark.sphinx(testroot='ext-autodoc') def test_generate_autosummary_docs_property(app): with patch('sphinx.ext.autosummary.generate.find_autosummary_in_files') as mock: - mock.return_value = [('target.methods.Base.prop', 'prop', None)] + mock.return_value = [AutosummaryEntry('target.methods.Base.prop', 'prop', None)] generate_autosummary_docs([], output_dir=app.srcdir, builder=app.builder, app=app) content = (app.srcdir / 'target.methods.Base.prop.rst').text() From 476b73b6ca6dd3e64a557495e3f8137e702f77dd Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 23 Feb 2020 01:51:43 +0900 Subject: [PATCH 20/27] autosummary: Add :recursive: option to autosummary directive --- doc/usage/extensions/autosummary.rst | 14 +++++++++++++ sphinx/ext/autosummary/__init__.py | 1 + sphinx/ext/autosummary/generate.py | 20 ++++++++++++++++--- .../templates/autosummary/module.rst | 1 + .../test-ext-autosummary-recursive/index.rst | 6 ++++++ .../package2/__init__.py | 0 .../package2/module.py | 13 ++++++++++++ tests/test_ext_autosummary.py | 7 ++++++- 8 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 tests/roots/test-ext-autosummary-recursive/package2/__init__.py create mode 100644 tests/roots/test-ext-autosummary-recursive/package2/module.py diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index 59620b4a8..3f4a52322 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -104,6 +104,20 @@ The :mod:`sphinx.ext.autosummary` extension does this in three parts: .. versionadded:: 1.0 + * You can specify the ``recursive`` option to generate documents for + modules and sub-packages recursively. It defaults to disabled. + For example, :: + + .. autosummary:: + :recursive: + + sphinx.environment.BuildEnvironment + + It is needed to enable :confval:`autosummary_recursive` also to + use this option. + + .. versionadded:: 3.1 + :program:`sphinx-autogen` -- generate autodoc stub pages -------------------------------------------------------- diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index d32cfa5af..72eb5afcf 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -224,6 +224,7 @@ class Autosummary(SphinxDirective): option_spec = { 'toctree': directives.unchanged, 'nosignatures': directives.flag, + 'recursive': directives.flag, 'template': directives.unchanged, } diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 74bd90bbe..2b4a19971 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -60,7 +60,8 @@ class DummyApplication: AutosummaryEntry = NamedTuple('AutosummaryEntry', [('name', str), ('path', str), - ('template', str)]) + ('template', str), + ('recursive', bool)]) def setup_documenters(app: Any) -> None: @@ -266,8 +267,13 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, _warn('[autosummary] failed to import %r: %s' % (name, e)) continue + if entry.recursive and not recursive: + _warn('[autosummary] :resursive: option found. But ignored. ' + 'Please read document for autosummary_recursive option') + content = generate_autosummary_content(name, obj, parent, template, entry.template, - imported_members, app, recursive) + imported_members, app, + recursive and entry.recursive) filename = os.path.join(path, name + suffix) if os.path.isfile(filename): @@ -347,11 +353,13 @@ def find_autosummary_in_lines(lines: List[str], module: Any = None, filename: st module_re = re.compile( r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$') autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?') + recursive_arg_re = re.compile(r'^\s+:recursive:\s*$') toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$') template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$') documented = [] # type: List[AutosummaryEntry] + recursive = False toctree = None # type: str template = None current_module = module @@ -360,6 +368,11 @@ def find_autosummary_in_lines(lines: List[str], module: Any = None, filename: st for line in lines: if in_autosummary: + m = recursive_arg_re.match(line) + if m: + recursive = True + continue + m = toctree_arg_re.match(line) if m: toctree = m.group(1) @@ -384,7 +397,7 @@ def find_autosummary_in_lines(lines: List[str], module: Any = None, filename: st if current_module and \ not name.startswith(current_module + '.'): name = "%s.%s" % (current_module, name) - documented.append(AutosummaryEntry(name, toctree, template)) + documented.append(AutosummaryEntry(name, toctree, template, recursive)) continue if not line.strip() or line.startswith(base_indent + " "): @@ -396,6 +409,7 @@ def find_autosummary_in_lines(lines: List[str], module: Any = None, filename: st if m: in_autosummary = True base_indent = m.group(1) + recursive = False toctree = None template = None continue diff --git a/sphinx/ext/autosummary/templates/autosummary/module.rst b/sphinx/ext/autosummary/templates/autosummary/module.rst index 141ab779b..6b8b0392d 100644 --- a/sphinx/ext/autosummary/templates/autosummary/module.rst +++ b/sphinx/ext/autosummary/templates/autosummary/module.rst @@ -41,6 +41,7 @@ .. autosummary:: :toctree: modules + :recursive: {% for item in modules %} {{ item }} {%- endfor %} diff --git a/tests/roots/test-ext-autosummary-recursive/index.rst b/tests/roots/test-ext-autosummary-recursive/index.rst index d5292607c..5855bfa71 100644 --- a/tests/roots/test-ext-autosummary-recursive/index.rst +++ b/tests/roots/test-ext-autosummary-recursive/index.rst @@ -5,5 +5,11 @@ API Reference .. autosummary:: :toctree: generated + :recursive: package + +.. autosummary:: + :toctree: generated + + package2 diff --git a/tests/roots/test-ext-autosummary-recursive/package2/__init__.py b/tests/roots/test-ext-autosummary-recursive/package2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-ext-autosummary-recursive/package2/module.py b/tests/roots/test-ext-autosummary-recursive/package2/module.py new file mode 100644 index 000000000..5506d0bc9 --- /dev/null +++ b/tests/roots/test-ext-autosummary-recursive/package2/module.py @@ -0,0 +1,13 @@ +from os import * # NOQA + + +class Foo: + def __init__(self): + pass + + def bar(self): + pass + + @property + def baz(self): + pass diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 7b59c8dd5..de439248e 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -292,6 +292,11 @@ def test_autosummary_recursive_enabled(app, status, warning): if toctree: assert not (generated / toctree).exists() + # autosummary without :recursive: option + generated = app.srcdir / 'generated' + assert (generated / 'package2.rst').exists() + assert not (generated / 'package2.module.rst').exists() + @pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive', srcdir='ext-autosummary-recursive-disabled', @@ -382,7 +387,7 @@ def test_autosummary_imported_members(app, status, warning): @pytest.mark.sphinx(testroot='ext-autodoc') def test_generate_autosummary_docs_property(app): with patch('sphinx.ext.autosummary.generate.find_autosummary_in_files') as mock: - mock.return_value = [AutosummaryEntry('target.methods.Base.prop', 'prop', None)] + mock.return_value = [AutosummaryEntry('target.methods.Base.prop', 'prop', None, False)] generate_autosummary_docs([], output_dir=app.srcdir, builder=app.builder, app=app) content = (app.srcdir / 'target.methods.Base.prop.rst').text() From 7671bcc23bbcacc0c2dd0e927477f6e6d11b93eb Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 21 Apr 2020 23:22:05 +0900 Subject: [PATCH 21/27] Remove autosummary_recursive configuration Now autosummary directive has :recursive: option to enable the recursive feature individually. So the configuration is no longer needed. --- doc/usage/extensions/autosummary.rst | 16 +--------------- sphinx/ext/autosummary/__init__.py | 4 +--- sphinx/ext/autosummary/generate.py | 12 +++--------- tests/test_ext_autosummary.py | 26 ++------------------------ 4 files changed, 7 insertions(+), 51 deletions(-) diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index 1a1b945d3..a1fb9f405 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -13,7 +13,7 @@ those output e.g. by Epydoc and other API doc generation tools. This is especially useful when your docstrings are long and detailed, and putting each one of them on a separate page makes them easier to read. -The :mod:`sphinx.ext.autosummary` extension does this in three parts: +The :mod:`sphinx.ext.autosummary` extension does this in two parts: 1. There is an :rst:dir:`autosummary` directive for generating summary listings that contain links to the documented items, and short summary blurbs @@ -25,10 +25,6 @@ The :mod:`sphinx.ext.autosummary` extension does this in three parts: These files by default contain only the corresponding :mod:`sphinx.ext.autodoc` directive, but can be customized with templates. -3. Optionally, the :confval:`autosummary_recursive` config value can be - used to generate "stub" files of modules and sub-packages for packages - under the :rst:dir:`autosummary` directive. - .. rst:directive:: autosummary Insert a table that contains links to documented items, and a short summary @@ -119,9 +115,6 @@ The :mod:`sphinx.ext.autosummary` extension does this in three parts: sphinx.environment.BuildEnvironment - It is needed to enable :confval:`autosummary_recursive` also to - use this option. - .. versionadded:: 3.1 @@ -180,13 +173,6 @@ also use these config values: .. versionadded:: 3.0 -.. confval:: autosummary_recursive - - Boolean that determines whether to add modules and sub-packages - recursively. It is disabled by default. - - .. versionadded:: 3.0 - .. confval:: autosummary_mock_imports This value contains a list of modules to be mocked up. See diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index c9427d8ce..23527d5c8 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -755,8 +755,7 @@ def process_generate_options(app: Sphinx) -> None: generate_autosummary_docs(genfiles, builder=app.builder, suffix=suffix, base_path=app.srcdir, app=app, imported_members=imported_members, - overwrite=app.config.autosummary_generate_overwrite, - recursive=app.config.autosummary_recursive) + overwrite=app.config.autosummary_generate_overwrite) def setup(app: Sphinx) -> Dict[str, Any]: @@ -779,7 +778,6 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.connect('builder-inited', process_generate_options) app.add_config_value('autosummary_generate', [], True, [bool]) app.add_config_value('autosummary_generate_overwrite', True, False) - app.add_config_value('autosummary_recursive', False, True) app.add_config_value('autosummary_mock_imports', lambda config: config.autodoc_mock_imports, 'env') app.add_config_value('autosummary_imported_members', [], False, [bool]) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index 6616cc0fe..f5e65eb44 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -252,7 +252,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, info: Callable = None, base_path: str = None, builder: Builder = None, template_dir: str = None, imported_members: bool = False, app: Any = None, - overwrite: bool = True, recursive: bool = True) -> None: + overwrite: bool = True) -> None: if info: warnings.warn('info argument for generate_autosummary_docs() is deprecated.', RemovedInSphinx40Warning) @@ -303,13 +303,8 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, _warn(__('[autosummary] failed to import %r: %s') % (entry.name, e)) continue - if entry.recursive and not recursive: - _warn('[autosummary] :resursive: option found. But ignored. ' - 'Please read document for autosummary_recursive option') - content = generate_autosummary_content(name, obj, parent, template, entry.template, - imported_members, app, - recursive and entry.recursive) + imported_members, app, entry.recursive) filename = os.path.join(path, name + suffix) if os.path.isfile(filename): @@ -334,7 +329,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None, base_path=base_path, builder=builder, template_dir=template_dir, imported_members=imported_members, app=app, - overwrite=overwrite, recursive=recursive) + overwrite=overwrite) # -- Finding documented entries in files --------------------------------------- @@ -525,7 +520,6 @@ def main(argv: List[str] = sys.argv[1:]) -> None: '.' + args.suffix, template_dir=args.templates, imported_members=args.imported_members, - recursive=args.recursive, app=app) diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 41758003b..880fae078 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -262,10 +262,8 @@ def test_autosummary_generate_overwrite2(app_params, make_app): assert 'autosummary_dummy_module.rst' not in app._warning.getvalue() -@pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive', - srcdir='ext-autosummary-recursive-enabled', - confoverrides={'autosummary_recursive': True}) -def test_autosummary_recursive_enabled(app, status, warning): +@pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive') +def test_autosummary_recursive(app, status, warning): app.build() toctree = 'modules' # see module.rst template @@ -305,26 +303,6 @@ def test_autosummary_recursive_enabled(app, status, warning): assert not (generated / 'package2.module.rst').exists() -@pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive', - srcdir='ext-autosummary-recursive-disabled', - confoverrides={'autosummary_recursive': False}) -def test_autosummary_recursive_disabled(app, status, warning): - app.build() - toctree = 'modules' # see module.rst template - - # Modules should not be generated - generated = app.srcdir / 'generated' - assert (generated / 'package.rst').exists() - content = (generated / 'package.rst').text() - assert 'package.module' not in content - assert 'package.package' not in content - - # Recursively generate modules of top-level package (should be missing) - generated /= toctree - assert not (generated / 'package.module.rst').exists() - assert not (generated / 'package.package.rst').exists() - - @pytest.mark.sphinx('latex', **default_kw) def test_autosummary_latex_table_colspec(app, status, warning): app.builder.build_all() From 6e28675727d1f6e6852b47e49d11708b9bd165ea Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 21 Apr 2020 23:24:52 +0900 Subject: [PATCH 22/27] doc: Fix version --- doc/usage/extensions/autosummary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/usage/extensions/autosummary.rst b/doc/usage/extensions/autosummary.rst index a1fb9f405..5915b30cd 100644 --- a/doc/usage/extensions/autosummary.rst +++ b/doc/usage/extensions/autosummary.rst @@ -286,7 +286,7 @@ The following variables available in the templates: List containing names of "public" modules in the package. Only available for modules that are packages. - .. versionadded:: 2.2 + .. versionadded:: 3.1 Additionally, the following filters are available From b09978a5d67d65bcc22a4b29a97e5ad66799c196 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 21 Apr 2020 23:38:00 +0900 Subject: [PATCH 23/27] refactor: autosummary: Remove unused parameter --- sphinx/ext/autosummary/generate.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py index f5e65eb44..0449837ff 100644 --- a/sphinx/ext/autosummary/generate.py +++ b/sphinx/ext/autosummary/generate.py @@ -195,14 +195,12 @@ def generate_autosummary_content(name: str, obj: Any, parent: Any, public.append(name) return public, items - def get_modules(obj: Any, include_public: List[str] = []) -> Tuple[List[str], List[str]]: + def get_modules(obj: Any) -> Tuple[List[str], List[str]]: items = [] # type: List[str] for _, modname, ispkg in pkgutil.iter_modules(obj.__path__): fullname = name + '.' + modname items.append(fullname) - public = [x for x in items - if x in include_public or - not x.split('.')[-1].startswith('_')] + public = [x for x in items if not x.split('.')[-1].startswith('_')] return public, items ns = {} # type: Dict[str, Any] From ef1362886ff113b58326c5c150058c2e5d155a8a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 21 Apr 2020 23:39:45 +0900 Subject: [PATCH 24/27] refactor: Not to use deprecated helper: Path.text() --- tests/test_ext_autosummary.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 880fae078..3bcbe56bc 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -270,7 +270,7 @@ def test_autosummary_recursive(app, status, warning): # Top-level package generated = app.srcdir / 'generated' assert (generated / 'package.rst').exists() - content = (generated / 'package.rst').text() + content = (generated / 'package.rst').read_text() assert 'package.module' in content assert 'package.package' in content @@ -278,7 +278,7 @@ def test_autosummary_recursive(app, status, warning): generated /= toctree assert (generated / 'package.module.rst').exists() assert (generated / 'package.package.rst').exists() - content = (generated / 'package.package.rst').text() + content = (generated / 'package.package.rst').read_text() assert 'package.package.module' in content assert 'package.package.package' in content @@ -286,7 +286,7 @@ def test_autosummary_recursive(app, status, warning): generated /= toctree assert (generated / 'package.package.module.rst').exists() assert (generated / 'package.package.package.rst').exists() - content = (generated / 'package.package.package.rst').text() + content = (generated / 'package.package.package.rst').read_text() assert 'package.package.package.module' in content assert 'package.package.package.package' not in content From cc6ba63f5e9497503c3bd48bc50fc3dddff1e63f Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 22 Apr 2020 00:23:37 +0900 Subject: [PATCH 25/27] test: Remove meaningless testcases --- .../package/package/module_importfail.py | 4 ---- .../package/package/package/__init__.py | 0 .../package/package/package/module.py | 13 ------------- .../package/package/package/module_importfail.py | 4 ---- tests/test_ext_autosummary.py | 14 ++------------ 5 files changed, 2 insertions(+), 33 deletions(-) delete mode 100644 tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py delete mode 100644 tests/roots/test-ext-autosummary-recursive/package/package/package/__init__.py delete mode 100644 tests/roots/test-ext-autosummary-recursive/package/package/package/module.py delete mode 100644 tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py diff --git a/tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py deleted file mode 100644 index 9e3f9f195..000000000 --- a/tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys - -# Fail module import in a catastrophic way -sys.exit(1) diff --git a/tests/roots/test-ext-autosummary-recursive/package/package/package/__init__.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/roots/test-ext-autosummary-recursive/package/package/package/module.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/module.py deleted file mode 100644 index 5506d0bc9..000000000 --- a/tests/roots/test-ext-autosummary-recursive/package/package/package/module.py +++ /dev/null @@ -1,13 +0,0 @@ -from os import * # NOQA - - -class Foo: - def __init__(self): - pass - - def bar(self): - pass - - @property - def baz(self): - pass diff --git a/tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py deleted file mode 100644 index 9e3f9f195..000000000 --- a/tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys - -# Fail module import in a catastrophic way -sys.exit(1) diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 3bcbe56bc..0890a1e5d 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -273,29 +273,19 @@ def test_autosummary_recursive(app, status, warning): content = (generated / 'package.rst').read_text() assert 'package.module' in content assert 'package.package' in content + assert 'package.module_importfail' in content # Recursively generate modules of top-level package generated /= toctree assert (generated / 'package.module.rst').exists() + assert (generated / 'package.module_importfail.rst').exists() is False assert (generated / 'package.package.rst').exists() content = (generated / 'package.package.rst').read_text() assert 'package.package.module' in content - assert 'package.package.package' in content # Recursively generate modules of sub-package generated /= toctree assert (generated / 'package.package.module.rst').exists() - assert (generated / 'package.package.package.rst').exists() - content = (generated / 'package.package.package.rst').read_text() - assert 'package.package.package.module' in content - assert 'package.package.package.package' not in content - - # Last sub-package has no sub-packages - generated /= toctree - assert (generated / 'package.package.package.module.rst').exists() - assert not (generated / 'package.package.package.package.rst').exists() - if toctree: - assert not (generated / toctree).exists() # autosummary without :recursive: option generated = app.srcdir / 'generated' From 4a7934ba1a4f992b0bd44d45ce37f9af2987c6d5 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 22 Apr 2020 00:27:59 +0900 Subject: [PATCH 26/27] autosummary: Make recursively generated stub-files flatten --- .../templates/autosummary/module.rst | 2 +- tests/test_ext_autosummary.py | 34 ++++++++----------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/sphinx/ext/autosummary/templates/autosummary/module.rst b/sphinx/ext/autosummary/templates/autosummary/module.rst index f8cd0deb6..5b70d5c40 100644 --- a/sphinx/ext/autosummary/templates/autosummary/module.rst +++ b/sphinx/ext/autosummary/templates/autosummary/module.rst @@ -40,7 +40,7 @@ .. rubric:: Modules .. autosummary:: - :toctree: modules + :toctree: :recursive: {% for item in modules %} {{ item }} diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 0890a1e5d..59f52bd9f 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -265,33 +265,27 @@ def test_autosummary_generate_overwrite2(app_params, make_app): @pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive') def test_autosummary_recursive(app, status, warning): app.build() - toctree = 'modules' # see module.rst template - # Top-level package - generated = app.srcdir / 'generated' - assert (generated / 'package.rst').exists() - content = (generated / 'package.rst').read_text() + # autosummary having :recursive: option + assert (app.srcdir / 'generated' / 'package.rst').exists() + assert (app.srcdir / 'generated' / 'package.module.rst').exists() + assert (app.srcdir / 'generated' / 'package.module_importfail.rst').exists() is False + assert (app.srcdir / 'generated' / 'package.package.rst').exists() + assert (app.srcdir / 'generated' / 'package.package.module.rst').exists() + + # autosummary not having :recursive: option + assert (app.srcdir / 'generated' / 'package2.rst').exists() + assert (app.srcdir / 'generated' / 'package2.module.rst').exists() is False + + # Check content of recursively generated stub-files + content = (app.srcdir / 'generated' / 'package.rst').read_text() assert 'package.module' in content assert 'package.package' in content assert 'package.module_importfail' in content - # Recursively generate modules of top-level package - generated /= toctree - assert (generated / 'package.module.rst').exists() - assert (generated / 'package.module_importfail.rst').exists() is False - assert (generated / 'package.package.rst').exists() - content = (generated / 'package.package.rst').read_text() + content = (app.srcdir / 'generated' / 'package.package.rst').read_text() assert 'package.package.module' in content - # Recursively generate modules of sub-package - generated /= toctree - assert (generated / 'package.package.module.rst').exists() - - # autosummary without :recursive: option - generated = app.srcdir / 'generated' - assert (generated / 'package2.rst').exists() - assert not (generated / 'package2.module.rst').exists() - @pytest.mark.sphinx('latex', **default_kw) def test_autosummary_latex_table_colspec(app, status, warning): From 72c57ff1317fd5e44f386327df7fb3685991e48d Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 22 Apr 2020 01:29:48 +0900 Subject: [PATCH 27/27] test: Fix flake8 violation --- tests/test_ext_autosummary.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py index 59f52bd9f..58fbb0967 100644 --- a/tests/test_ext_autosummary.py +++ b/tests/test_ext_autosummary.py @@ -10,7 +10,6 @@ import sys from io import StringIO -import os from unittest.mock import Mock, patch import pytest