Add `sphinx.ext.apidoc` extension

A common use-case is that users simply want to point Sphinx
towards a Python module, and have it generate documentation
automatically.

This is not possible currently, without a "pre-build" step
of running the `sphinx-autogen` CLI.

This PR adds `sphinx.ext.apidoc` as a sphinx extension,
to incorporate the source file generation into the sphinx build.

Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com>

doc/usage/extensions/apidoc.rst

@@ -0,0 +1,126 @@
+.. _ext-apidoc:
+
+:mod:`sphinx.ext.apidoc` -- Generate API documentation from Python packages
+===========================================================================
+
+.. py:module:: sphinx.ext.apidoc
+   :synopsis: Generate API documentation from Python modules
+
+.. index:: pair: automatic; documentation
+.. index:: pair: generation; documentation
+.. index:: pair: generate; documentation
+
+.. versionadded:: 8.2
+
+.. role:: code-py(code)
+   :language: Python
+
+:mod:`sphinx.ext.apidoc` is a tool for automatic generation
+of Sphinx sources from Python packages.
+It provides the :program:`sphinx-apidoc` command-line tool as an extension,
+allowing it to be run during the Sphinx build process.
+
+The extension writes generated source files to a provided directory,
+which are then read by Sphinx using the :mod:`sphinx.ext.autodoc` extension.
+
+.. warning::
+
+   :mod:`sphinx.ext.apidoc` generates source files that
+   use :mod:`sphinx.ext.autodoc` to document all found modules.
+   If any modules have side effects on import,
+   these will be executed by ``autodoc`` when :program:`sphinx-build` is run.
+
+   If you document scripts (as opposed to library modules),
+   make sure their main routine is protected by
+   an ``if __name__ == '__main__'`` condition.
+
+
+Configuration
+-------------
+
+The apidoc extension uses the following configuration values:
+
+.. confval:: apidoc_modules
+   :no-index:
+   :type: :code-py:`Sequence[dict[str, Any]]`
+   :default: :code-py:`()`
+
+   A list or sequence of dictionaries describing modules to document.
+
+   For example:
+
+   .. code-block:: python
+
+      apidoc_modules = [
+          {'destination': 'source/', 'path': 'path/to/module'},
+          {
+              'destination': 'source/',
+              'path': 'path/to/another_module',
+              'exclude_patterns': ['**/test*'],
+              'maxdepth': 4,
+              'followlinks': False,
+              'separatemodules': False,
+              'includeprivate': False,
+              'noheadings': False,
+              'modulefirst': False,
+              'implicit_namespaces': False,
+              'automodule_options': {
+                  'members', 'show-inheritance', 'undoc-members'
+              },
+          },
+      ]
+
+
+   Valid keys are:
+
+   :code-py:`'destination'`
+     The output directory for generated files (**required**).
+     This must be relative to the source directory,
+     and will be created if it does not exist.
+
+   :code-py:`'path'`
+     The path to the module to document (**required**).
+     This must be absolute or relative to the configuration directory.
+
+   :code-py:`'exclude_patterns'`
+     A sequence of patterns to exclude from generation.
+     These may be literal paths or :py:mod:`fnmatch`-style patterns.
+     Defaults to :code-py:`()`.
+
+   :code-py:`'maxdepth'`
+     The maximum depth of submodules to show in the generated table of contents.
+     Defaults to :code-py:`4`.
+
+   :code-py:`'followlinks'`
+     Follow symbolic links.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'separatemodules'`
+     Put documentation for each module on an individual page.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'includeprivate'`
+     Generate documentation for '_private' modules with leading underscores.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'noheadings'`
+     Do not create headings for the modules/packages.
+     Useful when source docstrings already contain headings.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'modulefirst'`
+     Place module documentation before submodule documentation.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'implicit_namespaces'`
+     By default sphinx-apidoc processes sys.path searching for modules only.
+     Python 3.3 introduced :pep:`420` implicit namespaces that allow module path
+     structures such as ``foo/bar/module.py`` or ``foo/bar/baz/__init__.py``
+     (notice that ``bar`` and ``foo`` are namespaces, not modules).
+
+     Interpret module paths using :pep:`420` implicit namespaces.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'automodule_options'`
+     Options to pass to generated :rst:dir:`automodule` directives.
+     Defaults to :code-py:`{'members', 'show-inheritance', 'undoc-members'}`.

doc/usage/extensions/index.rst

@@ -21,6 +21,7 @@ These extensions are built in and can be activated by respective entries in the
 .. toctree::
    :maxdepth: 1
 
+   apidoc
    autodoc
    autosectionlabel
    autosummary

sphinx/ext/apidoc/__init__.py

@@ -13,9 +13,26 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+import sphinx
 from sphinx.ext.apidoc._cli import main
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
-__all__: Sequence[str] = ('main',)
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+__all__: Sequence[str] = 'main', 'setup'
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    from sphinx.ext.apidoc._extension import run_apidoc
+
+    # Require autodoc
+    app.setup_extension('sphinx.ext.autodoc')
+    app.add_config_value('apidoc_modules', (), 'env', types=frozenset((list, tuple)))
+    app.connect('builder-inited', run_apidoc)
+    return {
+        'version': sphinx.__display_version__,
+        'parallel_read_safe': True,
+    }

sphinx/ext/apidoc/_extension.py

@@ -0,0 +1,224 @@
+"""Sphinx extension for auto-generating API documentation."""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from sphinx.ext.apidoc._generate import create_modules_toc_file, recurse_tree
+from sphinx.ext.apidoc._shared import LOGGER, ApidocOptions, _remove_old_files
+from sphinx.locale import __
+from sphinx.util.console import bold
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sphinx.application import Sphinx
+
+_BOOL_KEYS = frozenset({
+    'followlinks',
+    'separatemodules',
+    'includeprivate',
+    'noheadings',
+    'modulefirst',
+    'implicit_namespaces',
+})
+_ALLOWED_KEYS = _BOOL_KEYS | frozenset({
+    'path',
+    'destination',
+    'exclude_patterns',
+    'automodule_options',
+    'maxdepth',
+})
+
+
+def run_apidoc(app: Sphinx) -> None:
+    """Run the apidoc extension."""
+    apidoc_modules: Sequence[dict[str, Any]] = app.config.apidoc_modules
+    srcdir: Path = app.srcdir
+    confdir: Path = app.confdir
+
+    LOGGER.info(bold(__('Running apidoc')))
+
+    module_options: dict[str, Any]
+    for i, module_options in enumerate(apidoc_modules):
+        _run_apidoc_module(i, options=module_options, srcdir=srcdir, confdir=confdir)
+
+
+def _run_apidoc_module(
+    i: int, *, options: dict[str, Any], srcdir: Path, confdir: Path
+) -> None:
+    args = _parse_module_options(i, options=options, srcdir=srcdir, confdir=confdir)
+    if args is None:
+        return
+
+    exclude_patterns_compiled: list[re.Pattern[str]] = [
+        re.compile(fnmatch.translate(exclude)) for exclude in args.exclude_pattern
+    ]
+
+    written_files, modules = recurse_tree(
+        args.module_path, exclude_patterns_compiled, args, args.templatedir
+    )
+    if args.tocfile:
+        written_files.append(
+            create_modules_toc_file(modules, args, args.tocfile, args.templatedir)
+        )
+    if args.remove_old:
+        _remove_old_files(written_files, args.destdir, args.suffix)
+
+
+def _parse_module_options(
+    i: int, *, options: dict[str, Any], srcdir: Path, confdir: Path
+) -> ApidocOptions | None:
+    if not isinstance(options, dict):
+        LOGGER.warning(__('apidoc_modules item %i must be a dict'), i, type='apidoc')
+        return None
+
+    # module path should be absolute or relative to the conf directory
+    try:
+        path = Path(os.fspath(options['path']))
+    except KeyError:
+        LOGGER.warning(
+            __("apidoc_modules item %i must have a 'path' key"), i, type='apidoc'
+        )
+        return None
+    except TypeError:
+        LOGGER.warning(
+            __("apidoc_modules item %i 'path' must be a string"), i, type='apidoc'
+        )
+        return None
+    module_path = confdir / path
+    if not module_path.is_dir():
+        LOGGER.warning(
+            __("apidoc_modules item %i 'path' is not an existing folder: %s"),
+            i,
+            module_path,
+            type='apidoc',
+        )
+        return None
+
+    # destination path should be relative to the source directory
+    try:
+        destination = Path(os.fspath(options['destination']))
+    except KeyError:
+        LOGGER.warning(
+            __("apidoc_modules item %i must have a 'destination' key"),
+            i,
+            type='apidoc',
+        )
+        return None
+    except TypeError:
+        LOGGER.warning(
+            __("apidoc_modules item %i 'destination' must be a string"),
+            i,
+            type='apidoc',
+        )
+        return None
+    if destination.is_absolute():
+        LOGGER.warning(
+            __("apidoc_modules item %i 'destination' should be a relative path"),
+            i,
+            type='apidoc',
+        )
+        return None
+    dest_path = srcdir / destination
+    try:
+        dest_path.mkdir(parents=True, exist_ok=True)
+    except OSError as exc:
+        LOGGER.warning(
+            __('apidoc_modules item %i cannot create destination directory: %s'),
+            i,
+            exc.strerror,
+            type='apidoc',
+        )
+        return None
+
+    # exclude patterns should be absolute or relative to the conf directory
+    exclude_patterns: list[str] = [
+        str(confdir / pattern)
+        for pattern in _check_list_of_strings(i, options, key='exclude_patterns')
+    ]
+
+    # TODO template_dir
+
+    maxdepth = 4
+    if 'maxdepth' in options:
+        if not isinstance(options['maxdepth'], int):
+            LOGGER.warning(
+                __("apidoc_modules item %i '%s' must be an int"),
+                i,
+                'maxdepth',
+                type='apidoc',
+            )
+        else:
+            maxdepth = options['maxdepth']
+
+    extra_options = {}
+    for key in sorted(_BOOL_KEYS):
+        if key not in options:
+            continue
+        if not isinstance(options[key], bool):
+            LOGGER.warning(
+                __("apidoc_modules item %i '%s' must be a boolean"),
+                i,
+                key,
+                type='apidoc',
+            )
+            continue
+        extra_options[key] = options[key]
+
+    if _options := _check_list_of_strings(i, options, key='automodule_options'):
+        automodule_options = set(_options)
+    else:
+        # TODO per-module automodule_options
+        automodule_options = {'members', 'undoc-members', 'show-inheritance'}
+
+    if diff := set(options) - _ALLOWED_KEYS:
+        LOGGER.warning(
+            __('apidoc_modules item %i has unexpected keys: %s'),
+            i,
+            ', '.join(sorted(diff)),
+            type='apidoc',
+        )
+
+    return ApidocOptions(
+        destdir=dest_path,
+        module_path=module_path,
+        exclude_pattern=exclude_patterns,
+        automodule_options=automodule_options,
+        maxdepth=maxdepth,
+        quiet=True,
+        **extra_options,
+    )
+
+
+def _check_list_of_strings(
+    index: int, options: dict[str, Any], *, key: str
+) -> list[str]:
+    """Check that a key's value is a list of strings in the options.
+
+    :returns: the value of the key, or the empty list if invalid.
+    """
+    if key not in options:
+        return []
+    if not isinstance(options[key], list | tuple | set | frozenset):
+        LOGGER.warning(
+            __("apidoc_modules item %i '%s' must be a sequence"),
+            index,
+            key,
+            type='apidoc',
+        )
+        return []
+    for item in options[key]:
+        if not isinstance(item, str):
+            LOGGER.warning(
+                __("apidoc_modules item %i '%s' must contain strings"),
+                index,
+                key,
+                type='apidoc',
+            )
+            return []
+    return options[key]

sphinx/ext/apidoc/_generate.py

@@ -19,6 +19,8 @@ if TYPE_CHECKING:
 
     from sphinx.ext.apidoc._shared import ApidocOptions
 
+    from sphinx.ext.apidoc._shared import ApidocOptions
+
 
 # automodule options
 if 'SPHINX_APIDOC_OPTIONS' in os.environ:

sphinx/ext/apidoc/_shared.py

@@ -35,15 +35,12 @@ def _remove_old_files(
 class ApidocOptions:
     """Options for apidoc."""
 
-    module_path: Path
     destdir: Path
+    module_path: Path
 
     exclude_pattern: Sequence[str] = ()
-    quiet: bool = False
     maxdepth: int = 4
-    force: bool = False
     followlinks: bool = False
-    dryrun: bool = False
     separatemodules: bool = False
     includeprivate: bool = False
     tocfile: str = 'modules'
@@ -53,7 +50,11 @@ class ApidocOptions:
     automodule_options: set[str] = dataclasses.field(default_factory=set)
     suffix: str = 'rst'
 
-    remove_old: bool = False
+    remove_old: bool = True
+
+    quiet: bool = False
+    dryrun: bool = False
+    force: bool = True
 
     # --full only
     full: bool = False
@@ -62,5 +63,5 @@ class ApidocOptions:
     author: str | None = None
     version: str | None = None
     release: str | None = None
-    extensions: Sequence[str] | None = None
+    extensions: list[str] | None = None
     templatedir: str | None = None

tests/roots/test-ext-apidoc/conf.py

@@ -0,0 +1,22 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path.cwd().resolve() / 'src'))
+
+extensions = ['sphinx.ext.apidoc']
+
+apidoc_modules = [
+    {
+        'path': 'src',
+        'destination': 'generated',
+        'exclude_patterns': ['src/exclude_package.py'],
+        'automodule_options': ['members', 'undoc-members'],
+        'maxdepth': 3,
+        'followlinks': False,
+        'separatemodules': True,
+        'includeprivate': True,
+        'noheadings': False,
+        'modulefirst': True,
+        'implicit_namespaces': False,
+    }
+]

tests/roots/test-ext-apidoc/index.rst

@@ -0,0 +1,6 @@
+Heading
+=======
+
+.. toctree::
+
+    generated/modules

tests/roots/test-ext-apidoc/src/exclude_package.py

@@ -0,0 +1 @@
+"""A module that should be excluded."""

tests/roots/test-ext-apidoc/src/my_package.py

@@ -0,0 +1,6 @@
+"""An example module."""
+
+
+def example_function(a: str) -> str:
+    """An example function."""
+    return a

tests/test_extensions/test_ext_apidoc.py

@@ -13,6 +13,8 @@ from sphinx.ext.apidoc._cli import main as apidoc_main
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from sphinx.testing.util import SphinxTestApp
+
 _apidoc = namedtuple('_apidoc', 'coderoot,outdir')  # NoQA: PYI024
 
 
@@ -771,3 +773,26 @@ def test_remove_old_files(tmp_path: Path):
     apidoc_main(['--remove-old', '-o', str(gen_dir), str(module_dir)])
     assert set(gen_dir.iterdir()) == {gen_dir / 'modules.rst', gen_dir / 'example.rst'}
     assert (gen_dir / 'example.rst').stat().st_mtime_ns == example_mtime
+
+
+@pytest.mark.sphinx(testroot='ext-apidoc')
+def test_sphinx_extension(app: SphinxTestApp):
+    """Test running apidoc as an extension."""
+    app.build()
+    assert app.warning.getvalue() == ''
+
+    assert set((app.srcdir / 'generated').iterdir()) == {
+        app.srcdir / 'generated' / 'modules.rst',
+        app.srcdir / 'generated' / 'my_package.rst',
+    }
+    assert 'show-inheritance' not in (
+        app.srcdir / 'generated' / 'my_package.rst'
+    ).read_text(encoding='utf8')
+    assert (app.outdir / 'generated' / 'my_package.html').is_file()
+
+    # test a re-build
+    app.build()
+    assert app.warning.getvalue() == ''
+
+    # TODO check nothing got re-built
+    # TODO test that old files are removed