Merge branch '3.x' into 8172_napoleon_redos

CHANGES

@@ -10,17 +10,31 @@ Incompatible changes
 Deprecated
 ----------
 
+* ``sphinx.builders.latex.LaTeXBuilder.usepackages``
+* ``sphinx.builders.latex.LaTeXBuilder.usepackages_afger_hyperref``
+
 Features added
 --------------
 
+* #8100: html: Show a better error message for failures on copying
+  html_static_files
+* #8141: C: added a ``maxdepth`` option to :rst:dir:`c:alias` to insert
+  nested declarations.
+* #8081: LaTeX: Allow to add LaTeX package via ``app.add_latex_package()`` until
+  just before writing .tex file
+
 Bugs fixed
 ----------
 
 * #8085: i18n: Add support for having single text domain
 * #8143: autodoc: AttributeError is raised when False value is passed to
   autodoc_default_options
+* #8103: autodoc: functools.cached_property is not considered as a property
+* #8190: autodoc: parsing error is raised if some extension replaces docstring
+  by string not ending with blank lines
 * #8192: napoleon: description is disappeared when it contains inline literals
 * #8172: napoleon: Potential of regex denial of service in google style docs
+* #8169: LaTeX: pxjahyper loaded even when latex_engine is not platex
 * #8093: The highlight warning has wrong location in some builders (LaTeX,
   singlehtml and so on)
 

EXAMPLES

@@ -330,6 +330,7 @@ Documentation using a custom theme or integrated in a website
 * `Lasso <http://lassoguide.com/>`__
 * `Mako <http://docs.makotemplates.org/>`__
 * `MirrorBrain <http://mirrorbrain.org/docs/>`__
+* `Mitiq <https://mitiq.readthedocs.io/>`__
 * `MongoDB <https://docs.mongodb.com/>`__
 * `Music21 <https://web.mit.edu/music21/doc/>`__
 * `MyHDL <http://docs.myhdl.org/>`__

doc/extdev/deprecated.rst

@@ -26,6 +26,16 @@ The following is a list of deprecated interfaces.
      - (will be) Removed
      - Alternatives
 
+   * - ``sphinx.builders.latex.LaTeXBuilder.usepackages``
+     - 3.3
+     - 5.0
+     - N/A
+
+   * - ``sphinx.builders.latex.LaTeXBuilder.usepackages_afger_hyperref``
+     - 3.3
+     - 5.0
+     - N/A
+
    * - ``sphinx.ext.autodoc.members_set_option()``
      - 3.2
      - 5.0

doc/usage/extensions/autodoc.rst

@@ -229,7 +229,7 @@ inserting them into the page source under a suitable :rst:dir:`py:module`,
 
      .. versionchanged:: 3.0
 
-        It takes an anchestor class name as an argument.
+        It takes an ancestor class name as an argument.
 
    * It's possible to override the signature for explicitly documented callable
      objects (functions, methods, classes) with the regular syntax that will

doc/usage/restructuredtext/domains.rst

@@ -744,6 +744,18 @@ The following directive can be used for this purpose.
 
    .. versionadded:: 3.2
 
+
+   .. rubric:: Options
+
+   .. rst:directive:option:: maxdepth: int
+
+      Insert nested declarations as well, up to the total depth given.
+      Use 0 for infinite depth and 1 for just the mentioned declaration.
+      Defaults to 1.
+
+      .. versionadded:: 3.3
+
+
 .. c:namespace-pop::
 
 

sphinx/builders/html/__init__.py

@@ -751,18 +751,27 @@ class StandaloneHTMLBuilder(Builder):
                 copyfile(jsfile, path.join(self.outdir, '_static', '_stemmer.js'))
 
     def copy_theme_static_files(self, context: Dict) -> None:
+        def onerror(filename: str, error: Exception) -> None:
+            logger.warning(__('Failed to copy a file in html_static_file: %s: %r'),
+                           filename, error)
+
         if self.theme:
             for entry in self.theme.get_theme_dirs()[::-1]:
                 copy_asset(path.join(entry, 'static'),
                            path.join(self.outdir, '_static'),
-                           excluded=DOTFILES, context=context, renderer=self.templates)
+                           excluded=DOTFILES, context=context,
+                           renderer=self.templates, onerror=onerror)
 
     def copy_html_static_files(self, context: Dict) -> None:
+        def onerror(filename: str, error: Exception) -> None:
+            logger.warning(__('Failed to copy a file in html_static_file: %s: %r'),
+                           filename, error)
+
         excluded = Matcher(self.config.exclude_patterns + ["**/.*"])
         for entry in self.config.html_static_path:
             copy_asset(path.join(self.confdir, entry),
                        path.join(self.outdir, '_static'),
-                       excluded, context=context, renderer=self.templates)
+                       excluded, context=context, renderer=self.templates, onerror=onerror)
 
     def copy_html_logo(self) -> None:
         if self.config.html_logo:

sphinx/builders/latex/__init__.py

@@ -24,7 +24,7 @@ from sphinx.builders.latex.constants import ADDITIONAL_SETTINGS, DEFAULT_SETTING
 from sphinx.builders.latex.theming import Theme, ThemeFactory
 from sphinx.builders.latex.util import ExtBabel
 from sphinx.config import Config, ENUM
-from sphinx.deprecation import RemovedInSphinx40Warning
+from sphinx.deprecation import RemovedInSphinx40Warning, RemovedInSphinx50Warning
 from sphinx.environment.adapters.asset import ImageAdapter
 from sphinx.errors import NoUri, SphinxError
 from sphinx.locale import _, __
@@ -128,8 +128,6 @@ class LaTeXBuilder(Builder):
         self.docnames = []          # type: Iterable[str]
         self.document_data = []     # type: List[Tuple[str, str, str, str, str, bool]]
         self.themes = ThemeFactory(self.app)
-        self.usepackages = self.app.registry.latex_packages
-        self.usepackages_after_hyperref = self.app.registry.latex_packages_after_hyperref
         texescape.init()
 
         self.init_context()
@@ -179,10 +177,6 @@ class LaTeXBuilder(Builder):
             key = (self.config.latex_engine, self.config.language[:2])
             self.context.update(ADDITIONAL_SETTINGS.get(key, {}))
 
-        # Apply extension settings to context
-        self.context['packages'] = self.usepackages
-        self.context['packages_after_hyperref'] = self.usepackages_after_hyperref
-
         # Apply user settings to context
         self.context.update(self.config.latex_elements)
         self.context['release'] = self.config.release
@@ -203,6 +197,13 @@ class LaTeXBuilder(Builder):
             # Show the release label only if release value exists
             self.context.setdefault('releasename', _('Release'))
 
+    def update_context(self) -> None:
+        """Update template variables for .tex file just before writing."""
+        # Apply extension settings to context
+        registry = self.app.registry
+        self.context['packages'] = registry.latex_packages
+        self.context['packages_after_hyperref'] = registry.latex_packages_after_hyperref
+
     def init_babel(self) -> None:
         self.babel = ExtBabel(self.config.language, not self.context['babel'])
         if self.config.language and not self.babel.is_supported_language():
@@ -290,6 +291,7 @@ class LaTeXBuilder(Builder):
                 doctree['tocdepth'] = tocdepth
                 self.post_process_images(doctree)
                 self.update_doc_context(title, author, theme)
+                self.update_context()
 
             with progress_message(__("writing")):
                 docsettings._author = author
@@ -448,6 +450,18 @@ class LaTeXBuilder(Builder):
         filename = path.join(package_dir, 'templates', 'latex', 'sphinxmessages.sty_t')
         copy_asset_file(filename, self.outdir, context=context, renderer=LaTeXRenderer())
 
+    @property
+    def usepackages(self) -> List[Tuple[str, str]]:
+        warnings.warn('LaTeXBuilder.usepackages is deprecated.',
+                      RemovedInSphinx50Warning, stacklevel=2)
+        return self.app.registry.latex_packages
+
+    @property
+    def usepackages_after_hyperref(self) -> List[Tuple[str, str]]:
+        warnings.warn('LaTeXBuilder.usepackages_after_hyperref is deprecated.',
+                      RemovedInSphinx50Warning, stacklevel=2)
+        return self.app.registry.latex_packages_after_hyperref
+
 
 def patch_settings(settings: Any) -> Any:
     """Make settings object to show deprecation messages."""
@@ -505,7 +519,7 @@ def validate_latex_theme_options(app: Sphinx, config: Config) -> None:
 
 def install_pakcages_for_ja(app: Sphinx) -> None:
     """Install packages for Japanese."""
-    if app.config.language == 'ja':
+    if app.config.language == 'ja' and app.config.latex_engine in ('platex', 'uplatex'):
         app.add_latex_package('pxjahyper', after_hyperref=True)
 
 

sphinx/domains/c.py

@@ -136,8 +136,8 @@ class ASTIdentifier(ASTBaseBase):
                                           reftype='identifier',
                                           reftarget=targetText, modname=None,
                                           classname=None)
-            # key = symbol.get_lookup_key()
+            key = symbol.get_lookup_key()
-            # pnode['c:parent_key'] = key
+            pnode['c:parent_key'] = key
             if self.is_anon():
                 pnode += nodes.strong(text="[anonymous]")
             else:
@@ -1562,6 +1562,11 @@ class Symbol:
             for s in sChild.get_all_symbols():
                 yield s
 
+    @property
+    def children(self) -> Iterator["Symbol"]:
+        for c in self._children:
+            yield c
+
     @property
     def children_recurse_anon(self) -> Iterator["Symbol"]:
         for c in self._children:
@@ -3408,10 +3413,13 @@ class CNamespacePopObject(SphinxDirective):
 
 
 class AliasNode(nodes.Element):
-    def __init__(self, sig: str, env: "BuildEnvironment" = None,
+    def __init__(self, sig: str, maxdepth: int, document: Any, env: "BuildEnvironment" = None,
                  parentKey: LookupKey = None) -> None:
         super().__init__()
         self.sig = sig
+        self.maxdepth = maxdepth
+        assert maxdepth >= 0
+        self.document = document
         if env is not None:
             if 'c:parent_symbol' not in env.temp_data:
                 root = env.domaindata['c']['root_symbol']
@@ -3428,6 +3436,37 @@ class AliasNode(nodes.Element):
 class AliasTransform(SphinxTransform):
     default_priority = ReferencesResolver.default_priority - 1
 
+    def _render_symbol(self, s: Symbol, maxdepth: int, document: Any) -> List[Node]:
+        nodes = []  # type: List[Node]
+        options = dict()  # type: ignore
+        signode = addnodes.desc_signature('', '')
+        nodes.append(signode)
+        s.declaration.describe_signature(signode, 'markName', self.env, options)
+        if maxdepth == 0:
+            recurse = True
+        elif maxdepth == 1:
+            recurse = False
+        else:
+            maxdepth -= 1
+            recurse = True
+        if recurse:
+            content = addnodes.desc_content()
+            desc = addnodes.desc()
+            content.append(desc)
+            desc.document = document
+            desc['domain'] = 'c'
+            # 'desctype' is a backwards compatible attribute
+            desc['objtype'] = desc['desctype'] = 'alias'
+            desc['noindex'] = True
+
+            for sChild in s.children:
+                childNodes = self._render_symbol(sChild, maxdepth, document)
+                desc.extend(childNodes)
+
+            if len(desc.children) != 0:
+                nodes.append(content)
+        return nodes
+
     def apply(self, **kwargs: Any) -> None:
         for node in self.document.traverse(AliasNode):
             sig = node.sig
@@ -3468,17 +3507,16 @@ class AliasTransform(SphinxTransform):
                 logger.warning("Could not find C declaration for alias '%s'." % name,
                                location=node)
                 node.replace_self(signode)
-            else:
+                continue
-                nodes = []
+
-                options = dict()  # type: ignore
+            nodes = self._render_symbol(s, maxdepth=node.maxdepth, document=node.document)
-                signode = addnodes.desc_signature(sig, '')
+            node.replace_self(nodes)
-                nodes.append(signode)
-                s.declaration.describe_signature(signode, 'markName', self.env, options)
-                node.replace_self(nodes)
 
 
 class CAliasObject(ObjectDescription):
-    option_spec = {}  # type: Dict
+    option_spec = {
+        'maxdepth': directives.nonnegative_int
+    }  # type: Dict
 
     def run(self) -> List[Node]:
         if ':' in self.name:
@@ -3494,16 +3532,10 @@ class CAliasObject(ObjectDescription):
         node['noindex'] = True
 
         self.names = []  # type: List[str]
+        maxdepth = self.options.get('maxdepth', 1)
         signatures = self.get_signatures()
         for i, sig in enumerate(signatures):
-            node.append(AliasNode(sig, env=self.env))
+            node.append(AliasNode(sig, maxdepth, self.state.document, env=self.env))
-
-        contentnode = addnodes.desc_content()
-        node.append(contentnode)
-        self.before_content()
-        self.state.nested_parse(self.content, self.content_offset, contentnode)
-        self.env.temp_data['object'] = None
-        self.after_content()
         return [node]
 
 

sphinx/ext/autodoc/__init__.py

@@ -535,6 +535,11 @@ class Documenter:
                 self.env.app.emit('autodoc-process-docstring',
                                   self.objtype, self.fullname, self.object,
                                   self.options, docstringlines)
+
+                if docstringlines and docstringlines[-1] != '':
+                    # append a blank line to the end of the docstring
+                    docstringlines.append('')
+
             yield from docstringlines
 
     def get_sourcename(self) -> str:

sphinx/ext/inheritance_diagram.py

@@ -66,6 +66,10 @@ module_sig_re = re.compile(r'''^(?:([\w.]*)\.)?  # module names
                            ''', re.VERBOSE)
 
 
+py_builtins = [obj for obj in vars(builtins).values()
+               if inspect.isclass(obj)]
+
+
 def try_import(objname: str) -> Any:
     """Import a object or module using *name* and *currentmodule*.
     *name* should be a relative name from *currentmodule* or
@@ -178,7 +182,6 @@ class InheritanceGraph:
         traverse to. Multiple names can be specified separated by comma.
         """
         all_classes = {}
-        py_builtins = vars(builtins).values()
 
         def recurse(cls: Any) -> None:
             if not show_builtins and cls in py_builtins:

sphinx/testing/util.py

@@ -21,7 +21,6 @@ from docutils.nodes import Node
 from docutils.parsers.rst import directives, roles
 
 from sphinx import application, locale
-from sphinx.builders.latex import LaTeXBuilder
 from sphinx.deprecation import RemovedInSphinx40Warning
 from sphinx.pycode import ModuleAnalyzer
 from sphinx.testing.path import path
@@ -141,7 +140,6 @@ class SphinxTestApp(application.Sphinx):
 
     def cleanup(self, doctrees: bool = False) -> None:
         ModuleAnalyzer.cache.clear()
-        LaTeXBuilder.usepackages = []
         locale.translators.clear()
         sys.path[:] = self._saved_path
         sys.modules.pop('autodoc_fodder', None)

sphinx/util/fileutil.py

@@ -10,7 +10,7 @@
 
 import os
 import posixpath
-from typing import Dict
+from typing import Callable, Dict
 
 from docutils.utils import relative_path
 
@@ -56,7 +56,8 @@ def copy_asset_file(source: str, destination: str,
 
 
 def copy_asset(source: str, destination: str, excluded: PathMatcher = lambda path: False,
-               context: Dict = None, renderer: "BaseRenderer" = None) -> None:
+               context: Dict = None, renderer: "BaseRenderer" = None,
+               onerror: Callable[[str, Exception], None] = None) -> None:
     """Copy asset files to destination recursively.
 
     On copying, it expands the template variables if context argument is given and
@@ -67,6 +68,7 @@ def copy_asset(source: str, destination: str, excluded: PathMatcher = lambda pat
     :param excluded: The matcher to determine the given path should be copied or not
     :param context: The template variables.  If not given, template files are simply copied
     :param renderer: The template engine.  If not given, SphinxRenderer is used by default
+    :param onerror: The error handler.
     """
     if not os.path.exists(source):
         return
@@ -90,6 +92,12 @@ def copy_asset(source: str, destination: str, excluded: PathMatcher = lambda pat
 
         for filename in files:
             if not excluded(posixpath.join(reldir, filename)):
-                copy_asset_file(posixpath.join(root, filename),
+                try:
-                                posixpath.join(destination, reldir),
+                    copy_asset_file(posixpath.join(root, filename),
-                                context, renderer)
+                                    posixpath.join(destination, reldir),
+                                    context, renderer)
+                except Exception as exc:
+                    if onerror:
+                        onerror(posixpath.join(root, filename), exc)
+                    else:
+                        raise

sphinx/util/inspect.py

@@ -304,6 +304,11 @@ def iscoroutinefunction(obj: Any) -> bool:
 
 def isproperty(obj: Any) -> bool:
     """Check if the object is property."""
+    if sys.version_info > (3, 8):
+        from functools import cached_property  # cached_property is available since py3.8
+        if isinstance(obj, cached_property):
+            return True
+
     return isinstance(obj, property)
 
 

tests/roots/test-ext-autodoc/target/cached_property.py

@@ -0,0 +1,7 @@
+from functools import cached_property
+
+
+class Foo:
+    @cached_property
+    def prop(self) -> int:
+        return 1

tests/test_ext_autodoc.py

@@ -881,6 +881,26 @@ def test_autodoc_descriptor(app):
     ]
 
 
+@pytest.mark.skipif(sys.version_info < (3, 8),
+                    reason='cached_property is available since python3.8.')
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autodoc_cached_property(app):
+    options = {"members": None,
+               "undoc-members": True}
+    actual = do_autodoc(app, 'class', 'target.cached_property.Foo', options)
+    assert list(actual) == [
+        '',
+        '.. py:class:: Foo()',
+        '   :module: target.cached_property',
+        '',
+        '',
+        '   .. py:method:: Foo.prop',
+        '      :module: target.cached_property',
+        '      :property:',
+        '',
+    ]
+
+
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
 def test_autodoc_member_order(app):
     # case member-order='bysource'

tests/test_ext_autodoc_events.py

@@ -28,7 +28,8 @@ def test_process_docstring(app):
         '.. py:function:: func()',
         '   :module: target.process_docstring',
         '',
-        '   my docstring'
+        '   my docstring',
+        '',
     ]