Merge pull request #7527 from tk0miya/6040_autosummary_recursive

recursive autosummary feature

doc/usage/extensions/autosummary.rst

@@ -32,7 +32,8 @@ The :mod:`sphinx.ext.autosummary` extension does this in two parts:
 
    The :rst:dir:`autosummary` directive can also optionally serve as a
    :rst:dir:`toctree` entry for the included items. Optionally, stub
-   ``.rst`` files for these items can also be automatically generated.
+   ``.rst`` files for these items can also be automatically generated
+   when :confval:`autosummary_generate` is `True`.
 
    For example, ::
 
@@ -105,6 +106,17 @@ The :mod:`sphinx.ext.autosummary` extension does this in two parts:
 
      .. versionadded:: 1.0
 
+   * You can specify the ``recursive`` option to generate documents for
+     modules and sub-packages recursively.  It defaults to disabled.
+     For example, ::
+
+         .. autosummary::
+            :recursive:
+
+            sphinx.environment.BuildEnvironment
+
+     .. versionadded:: 3.1
+
 
 :program:`sphinx-autogen` -- generate autodoc stub pages
 --------------------------------------------------------
@@ -142,7 +154,7 @@ also use these config values:
 .. confval:: autosummary_generate
 
    Boolean indicating whether to scan all found documents for autosummary
-   directives, and to generate stub pages for each.
+   directives, and to generate stub pages for each. It is disabled by default.
 
    Can also be a list of documents for which stub pages should be generated.
 
@@ -269,6 +281,12 @@ The following variables available in the templates:
    List containing names of "public" attributes in the class.  Only available
    for classes.
 
+.. data:: modules
+
+   List containing names of "public" modules in the package.  Only available for
+   modules that are packages.
+
+   .. versionadded:: 3.1
 
 Additionally, the following filters are available
 

sphinx/ext/autosummary/__init__.py

@@ -231,6 +231,7 @@ class Autosummary(SphinxDirective):
         'caption': directives.unchanged_required,
         'toctree': directives.unchanged,
         'nosignatures': directives.flag,
+        'recursive': directives.flag,
         'template': directives.unchanged,
     }
 

sphinx/ext/autosummary/generate.py

@@ -20,6 +20,7 @@
 import argparse
 import locale
 import os
+import pkgutil
 import pydoc
 import re
 import sys
@@ -68,7 +69,8 @@ class DummyApplication:
 
 AutosummaryEntry = NamedTuple('AutosummaryEntry', [('name', str),
                                                    ('path', str),
-                                                   ('template', str)])
+                                                   ('template', str),
+                                                   ('recursive', bool)])
 
 
 def setup_documenters(app: Any) -> None:
@@ -147,7 +149,8 @@ class AutosummaryRenderer:
 
 def generate_autosummary_content(name: str, obj: Any, parent: Any,
                                  template: AutosummaryRenderer, template_name: str,
-                                 imported_members: bool, app: Any) -> str:
+                                 imported_members: bool, app: Any,
+                                 recursive: bool) -> str:
     doc = get_documenter(app, obj, parent)
 
     if template_name is None:
@@ -192,6 +195,14 @@ def generate_autosummary_content(name: str, obj: Any, parent: Any,
                             public.append(name)
         return public, items
 
+    def get_modules(obj: Any) -> Tuple[List[str], List[str]]:
+        items = []  # type: List[str]
+        for _, modname, ispkg in pkgutil.iter_modules(obj.__path__):
+            fullname = name + '.' + modname
+            items.append(fullname)
+        public = [x for x in items if not x.split('.')[-1].startswith('_')]
+        return public, items
+
     ns = {}  # type: Dict[str, Any]
 
     if doc.objtype == 'module':
@@ -202,6 +213,9 @@ def generate_autosummary_content(name: str, obj: Any, parent: Any,
             get_members(obj, {'class'}, imported=imported_members)
         ns['exceptions'], ns['all_exceptions'] = \
             get_members(obj, {'exception'}, imported=imported_members)
+        ispackage = hasattr(obj, '__path__')
+        if ispackage and recursive:
+            ns['modules'], ns['all_modules'] = get_modules(obj)
     elif doc.objtype == 'class':
         ns['members'] = dir(obj)
         ns['inherited_members'] = \
@@ -288,7 +302,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None,
             continue
 
         content = generate_autosummary_content(name, obj, parent, template, entry.template,
-                                               imported_members, app)
+                                               imported_members, app, entry.recursive)
 
         filename = os.path.join(path, name + suffix)
         if os.path.isfile(filename):
@@ -373,11 +387,13 @@ def find_autosummary_in_lines(lines: List[str], module: str = None, filename: st
     module_re = re.compile(
         r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$')
     autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?')
+    recursive_arg_re = re.compile(r'^\s+:recursive:\s*$')
     toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$')
     template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$')
 
     documented = []  # type: List[AutosummaryEntry]
 
+    recursive = False
     toctree = None  # type: str
     template = None
     current_module = module
@@ -386,6 +402,11 @@ def find_autosummary_in_lines(lines: List[str], module: str = None, filename: st
 
     for line in lines:
         if in_autosummary:
+            m = recursive_arg_re.match(line)
+            if m:
+                recursive = True
+                continue
+
             m = toctree_arg_re.match(line)
             if m:
                 toctree = m.group(1)
@@ -410,7 +431,7 @@ def find_autosummary_in_lines(lines: List[str], module: str = None, filename: st
                 if current_module and \
                    not name.startswith(current_module + '.'):
                     name = "%s.%s" % (current_module, name)
-                documented.append(AutosummaryEntry(name, toctree, template))
+                documented.append(AutosummaryEntry(name, toctree, template, recursive))
                 continue
 
             if not line.strip() or line.startswith(base_indent + " "):
@@ -422,6 +443,7 @@ def find_autosummary_in_lines(lines: List[str], module: str = None, filename: st
         if m:
             in_autosummary = True
             base_indent = m.group(1)
+            recursive = False
             toctree = None
             template = None
             continue

sphinx/ext/autosummary/templates/autosummary/module.rst

@@ -34,3 +34,16 @@
    {%- endfor %}
    {% endif %}
    {% endblock %}
+
+{% block modules %}
+{% if modules %}
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :recursive:
+{% for item in modules %}
+   {{ item }}
+{%- endfor %}
+{% endif %}
+{% endblock %}

tests/roots/test-ext-autosummary-recursive/conf.py

@@ -0,0 +1,7 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('.'))
+
+extensions = ['sphinx.ext.autosummary']
+autosummary_generate = True

tests/roots/test-ext-autosummary-recursive/index.rst

@@ -0,0 +1,15 @@
+API Reference
+=============
+
+.. rubric:: Packages
+
+.. autosummary::
+   :toctree: generated
+   :recursive:
+
+    package
+
+.. autosummary::
+   :toctree: generated
+
+    package2

tests/roots/test-ext-autosummary-recursive/package/module.py

@@ -0,0 +1,13 @@
+from os import *  # NOQA
+
+
+class Foo:
+    def __init__(self):
+        pass
+
+    def bar(self):
+        pass
+
+    @property
+    def baz(self):
+        pass

tests/roots/test-ext-autosummary-recursive/package/module_importfail.py

@@ -0,0 +1,4 @@
+import sys
+
+# Fail module import in a catastrophic way
+sys.exit(1)

tests/roots/test-ext-autosummary-recursive/package/package/module.py

@@ -0,0 +1,13 @@
+from os import *  # NOQA
+
+
+class Foo:
+    def __init__(self):
+        pass
+
+    def bar(self):
+        pass
+
+    @property
+    def baz(self):
+        pass

tests/roots/test-ext-autosummary-recursive/package2/module.py

@@ -0,0 +1,13 @@
+from os import *  # NOQA
+
+
+class Foo:
+    def __init__(self):
+        pass
+
+    def bar(self):
+        pass
+
+    @property
+    def baz(self):
+        pass

tests/test_ext_autosummary.py

@@ -261,6 +261,31 @@ def test_autosummary_generate_overwrite2(app_params, make_app):
     assert 'autosummary_dummy_module.rst' not in app._warning.getvalue()
 
 
+@pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive')
+def test_autosummary_recursive(app, status, warning):
+    app.build()
+
+    # autosummary having :recursive: option
+    assert (app.srcdir / 'generated' / 'package.rst').exists()
+    assert (app.srcdir / 'generated' / 'package.module.rst').exists()
+    assert (app.srcdir / 'generated' / 'package.module_importfail.rst').exists() is False
+    assert (app.srcdir / 'generated' / 'package.package.rst').exists()
+    assert (app.srcdir / 'generated' / 'package.package.module.rst').exists()
+
+    # autosummary not having :recursive: option
+    assert (app.srcdir / 'generated' / 'package2.rst').exists()
+    assert (app.srcdir / 'generated' / 'package2.module.rst').exists() is False
+
+    # Check content of recursively generated stub-files
+    content = (app.srcdir / 'generated' / 'package.rst').read_text()
+    assert 'package.module' in content
+    assert 'package.package' in content
+    assert 'package.module_importfail' in content
+
+    content = (app.srcdir / 'generated' / 'package.package.rst').read_text()
+    assert 'package.package.module' in content
+
+
 @pytest.mark.sphinx('latex', **default_kw)
 def test_autosummary_latex_table_colspec(app, status, warning):
     app.builder.build_all()
@@ -330,7 +355,7 @@ def test_autosummary_imported_members(app, status, warning):
 @pytest.mark.sphinx(testroot='ext-autodoc')
 def test_generate_autosummary_docs_property(app):
     with patch('sphinx.ext.autosummary.generate.find_autosummary_in_files') as mock:
-        mock.return_value = [AutosummaryEntry('target.methods.Base.prop', 'prop', None)]
+        mock.return_value = [AutosummaryEntry('target.methods.Base.prop', 'prop', None, False)]
         generate_autosummary_docs([], output_dir=app.srcdir, builder=app.builder, app=app)
 
     content = (app.srcdir / 'target.methods.Base.prop.rst').read_text()