Merge pull request #7538 from tk0miya/7535_autogen_crashes_on_inherited_template

Fix #7535: sphinx-autogen: crashes when custom template uses inheritance

CHANGES

@@ -12,8 +12,12 @@ Incompatible changes
 Deprecated
 ----------
 
+* The ``template_dir`` argument of ``sphinx.ext.autosummary.generate.
+  AutosummaryRenderer``
 * The ``module`` argument of ``sphinx.ext.autosummary.generate.
   find_autosummary_in_docstring()``
+* The ``template_dir`` argument of ``sphinx.ext.autosummary.generate.
+  generate_autosummary_docs()``
 
 Features added
 --------------
@@ -30,6 +34,7 @@ Features added
   caption to the toctree
 * #248, #6040: autosummary: Add ``:recursive:`` option to autosummary directive
   to generate stub files recursively
+* #7535: sphinx-autogen: crashes when custom template uses inheritance
 * #7481: html theme: Add right margin to footnote/citation labels
 * #7482: html theme: CSS spacing for code blocks with captions and line numbers
 * #7443: html theme: Add new options :confval:`globaltoc_collapse` and

doc/extdev/deprecated.rst

@@ -26,12 +26,24 @@ The following is a list of deprecated interfaces.
      - (will be) Removed
      - Alternatives
 
+   * - The ``template_dir`` argument of
+       ``sphinx.ext.autosummary.generate.AutosummaryRenderer``
+     - 3.1
+     - 5.0
+     - N/A
+
    * - The ``module`` argument of
        ``sphinx.ext.autosummary.generate.find_autosummary_in_docstring()``
      - 3.0
      - 5.0
      - N/A
 
+   * - The ``template_dir`` argument of
+       ``sphinx.ext.autosummary.generate.generate_autosummary_docs()``
+     - 3.1
+     - 5.0
+     - N/A
+
    * - ``desc_signature['first']``
      -
      - 3.0

sphinx/ext/autosummary/generate.py

@@ -25,25 +25,27 @@ import pydoc
 import re
 import sys
 import warnings
+from os import path
 from typing import Any, Callable, Dict, List, NamedTuple, Set, Tuple
 
-from jinja2 import BaseLoader, FileSystemLoader, TemplateNotFound
+from jinja2 import TemplateNotFound
 from jinja2.sandbox import SandboxedEnvironment
 
 import sphinx.locale
 from sphinx import __display_version__
 from sphinx import package_dir
 from sphinx.builders import Builder
+from sphinx.config import Config
 from sphinx.deprecation import RemovedInSphinx40Warning, RemovedInSphinx50Warning
 from sphinx.ext.autodoc import Documenter
 from sphinx.ext.autosummary import import_by_name, get_documenter
-from sphinx.jinja2glue import BuiltinTemplateLoader
 from sphinx.locale import __
 from sphinx.registry import SphinxComponentRegistry
 from sphinx.util import logging
 from sphinx.util import rst
 from sphinx.util.inspect import safe_getattr
 from sphinx.util.osutil import ensuredir
+from sphinx.util.template import SphinxTemplateLoader
 
 if False:
     # For type annotation
@@ -59,6 +61,7 @@ class DummyApplication:
     def __init__(self) -> None:
         self.registry = SphinxComponentRegistry()
         self.messagelog = []  # type: List[str]
+        self.translator = None
         self.verbosity = 0
         self._warncount = 0
         self.warningiserror = False
@@ -67,6 +70,21 @@ class DummyApplication:
         pass
 
 
+class DummyBuilder:
+    """Dummy builder class for sphinx-autogen command."""
+
+    def __init__(self, app: DummyApplication, template_path: str) -> None:
+        if template_path:
+            templates_path = [path.abspath(template_path)]
+        else:
+            templates_path = []
+
+        self.app = app
+        self.srcdir = "/"
+        self.config = Config(overrides={'templates_path': templates_path})
+        self.config.init_values()
+
+
 AutosummaryEntry = NamedTuple('AutosummaryEntry', [('name', str),
                                                    ('path', str),
                                                    ('template', str),
@@ -109,17 +127,15 @@ def _underline(title: str, line: str = '=') -> str:
 class AutosummaryRenderer:
     """A helper class for rendering."""
 
-    def __init__(self, builder: Builder, template_dir: str) -> None:
+    def __init__(self, builder: Builder, template_dir: str = None) -> None:
-        loader = None  # type: BaseLoader
+        if template_dir:
-        template_dirs = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')]
+            warnings.warn('template_dir argument for AutosummaryRenderer is deprecated.',
-        if builder is None:
+                          RemovedInSphinx50Warning)
-            if template_dir:
+
-                template_dirs.insert(0, template_dir)
+        system_templates_path = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')]
-            loader = FileSystemLoader(template_dirs)
+        loader = SphinxTemplateLoader(builder.srcdir,
-        else:
+                                      builder.config.templates_path,
-            # allow the user to override the templates
+                                      system_templates_path)
-            loader = BuiltinTemplateLoader()
-            loader.init(builder, dirs=template_dirs)
 
         self.env = SandboxedEnvironment(loader=loader)
         self.env.filters['escape'] = rst.escape
@@ -265,6 +281,10 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None,
     else:
         _warn = logger.warning
 
+    if template_dir:
+        warnings.warn('template_dir argument for generate_autosummary_docs() is deprecated.',
+                      RemovedInSphinx50Warning)
+
     showed_sources = list(sorted(sources))
     if len(showed_sources) > 20:
         showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:]
@@ -277,7 +297,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None,
     if base_path is not None:
         sources = [os.path.join(base_path, filename) for filename in sources]
 
-    template = AutosummaryRenderer(builder, template_dir)
+    template = AutosummaryRenderer(builder)
 
     # read
     items = find_autosummary_in_files(sources)
@@ -325,7 +345,6 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None,
         generate_autosummary_docs(new_files, output_dir=output_dir,
                                   suffix=suffix, warn=warn, info=info,
                                   base_path=base_path, builder=builder,
-                                  template_dir=template_dir,
                                   imported_members=imported_members, app=app,
                                   overwrite=overwrite)
 
@@ -514,9 +533,9 @@ def main(argv: List[str] = sys.argv[1:]) -> None:
     logging.setup(app, sys.stdout, sys.stderr)  # type: ignore
     setup_documenters(app)
     args = get_parser().parse_args(argv)
+    builder = DummyBuilder(app, args.templates)
     generate_autosummary_docs(args.source_file, args.output_dir,
-                              '.' + args.suffix,
+                              '.' + args.suffix, builder=builder,  # type: ignore
-                              template_dir=args.templates,
                               imported_members=args.imported_members,
                               app=app)
 

sphinx/util/template.py

@@ -10,8 +10,11 @@
 
 import os
 from functools import partial
-from typing import Dict, List, Union
+from os import path
+from typing import Callable, Dict, List, Tuple, Union
 
+from jinja2 import TemplateNotFound
+from jinja2.environment import Environment
 from jinja2.loaders import BaseLoader
 from jinja2.sandbox import SandboxedEnvironment
 
@@ -94,3 +97,36 @@ class ReSTRenderer(SphinxRenderer):
         self.env.filters['e'] = rst.escape
         self.env.filters['escape'] = rst.escape
         self.env.filters['heading'] = rst.heading
+
+
+class SphinxTemplateLoader(BaseLoader):
+    """A loader supporting template inheritance"""
+
+    def __init__(self, confdir: str, templates_paths: List[str],
+                 system_templates_paths: List[str]) -> None:
+        self.loaders = []
+        self.sysloaders = []
+
+        for templates_path in templates_paths:
+            loader = SphinxFileSystemLoader(path.join(confdir, templates_path))
+            self.loaders.append(loader)
+
+        for templates_path in system_templates_paths:
+            loader = SphinxFileSystemLoader(templates_path)
+            self.loaders.append(loader)
+            self.sysloaders.append(loader)
+
+    def get_source(self, environment: Environment, template: str) -> Tuple[str, str, Callable]:
+        if template.startswith('!'):
+            # search a template from ``system_templates_paths``
+            loaders = self.sysloaders
+            template = template[1:]
+        else:
+            loaders = self.loaders
+
+        for loader in loaders:
+            try:
+                return loader.get_source(environment, template)
+            except TemplateNotFound:
+                pass
+        raise TemplateNotFound(template)