Merge branch '3.x'

.gitignore

@@ -9,6 +9,7 @@
 .mypy_cache/
 .pytest_cache/
 .ropeproject/
+.vscode/
 TAGS
 .tags
 .tox/

CHANGES

@@ -43,18 +43,40 @@ Incompatible changes
 Deprecated
 ----------
 
+* The first argument for sphinx.ext.autosummary.generate.AutosummaryRenderer has
+  been changed to Sphinx object
+* ``sphinx.ext.autosummary.generate.AutosummaryRenderer`` takes an object type
+  as an argument
+* The ``template_dir`` argument of ``sphinx.ext.autosummary.generate.
+  AutosummaryRenderer``
 * The ``module`` argument of ``sphinx.ext.autosummary.generate.
   find_autosummary_in_docstring()``
+* The ``builder`` argument of ``sphinx.ext.autosummary.generate.
+  generate_autosummary_docs()``
+* The ``template_dir`` argument of ``sphinx.ext.autosummary.generate.
+  generate_autosummary_docs()``
+* ``sphinx.ext.autosummary.generate.AutosummaryRenderer.exists()``
 
 Features added
 --------------
 
 * LaTeX: Make the ``toplevel_sectioning`` setting optional in LaTeX theme
+* LaTeX: Allow to override papersize and pointsize from LaTeX themes
+* LaTeX: Add :confval:`latex_theme_options` to override theme options
 * #7410: Allow to suppress "circular toctree references detected" warnings using
   :confval:`suppress_warnings`
 * C, added scope control directives, :rst:dir:`c:namespace`,
   :rst:dir:`c:namespace-push`, and :rst:dir:`c:namespace-pop`.
+* #2044: autodoc: Suppress default value for instance attributes
+* #7473: autodoc: consider a member public if docstring contains
+  ``:meta public:`` in info-field-list
 * #7466: autosummary: headings in generated documents are not translated
+* #7490: autosummary: Add ``:caption:`` option to autosummary directive to set a
+  caption to the toctree
+* #248, #6040: autosummary: Add ``:recursive:`` option to autosummary directive
+  to generate stub files recursively
+* #7535: sphinx-autogen: crashes when custom template uses inheritance
+* #7536: sphinx-autogen: crashes when template uses i18n feature
 * #7481: html theme: Add right margin to footnote/citation labels
 * #7482: html theme: CSS spacing for code blocks with captions and line numbers
 * #7443: html theme: Add new options :confval:`globaltoc_collapse` and
@@ -63,10 +85,17 @@ Features added
 * #7484: html theme: Avoid clashes between sidebar and other blocks
 * #7476: html theme: Relbar breadcrumb should contain current page
 * #7506: html theme: A canonical URL is not escaped
+* #7533: html theme: Avoid whitespace at the beginning of genindex.html
+* #7541: html theme: Add a "clearer" at the end of the "body"
+* #7542: html theme: Make admonition/topic/sidebar scrollable
+* C and C++: allow semicolon in the end of declarations.
+* C++, parse parameterized noexcept specifiers.
 
 Bugs fixed
 ----------
 
+* #6703: autodoc: incremental build does not work for imported objects
+
 Testing
 --------
 

doc/_themes/sphinx13/static/sphinx13.css

@@ -127,6 +127,7 @@ div.sphinxsidebar {
     float: right;
     font-size: 1em;
     text-align: left;
+    max-height: 0px;
 }
 
 div.sphinxsidebar .logo {

doc/extdev/deprecated.rst

@@ -56,12 +56,48 @@ The following is a list of deprecated interfaces.
      - 6.0
      - ``docutils.utils.smartyquotes``
 
+   * - The first argument for
+       ``sphinx.ext.autosummary.generate.AutosummaryRenderer`` has been changed
+       to Sphinx object
+     - 3.1
+     - 5.0
+     - N/A
+
+   * - ``sphinx.ext.autosummary.generate.AutosummaryRenderer`` takes an object
+       type as an argument
+     - 3.1
+     - 5.0
+     - N/A
+
+   * - The ``template_dir`` argument of
+       ``sphinx.ext.autosummary.generate.AutosummaryRenderer``
+     - 3.1
+     - 5.0
+     - N/A
+
    * - The ``module`` argument of
        ``sphinx.ext.autosummary.generate.find_autosummary_in_docstring()``
      - 3.0
      - 5.0
      - N/A
 
+   * - The ``builder`` argument of
+       ``sphinx.ext.autosummary.generate.generate_autosummary_docs()``
+     - 3.1
+     - 5.0
+     - N/A
+
+   * - The ``template_dir`` argument of
+       ``sphinx.ext.autosummary.generate.generate_autosummary_docs()``
+     - 3.1
+     - 5.0
+     - N/A
+
+   * - ``sphinx.ext.autosummary.generate.AutosummaryRenderer.exists()``
+     - 3.1
+     - 5.0
+     - N/A
+
    * - ``desc_signature['first']``
      -
      - 3.0

doc/usage/advanced/intl.rst

@@ -90,9 +90,9 @@ section describe an easy way to translate with *sphinx-intl*.
       locale_dirs = ['locale/']   # path is example but recommended.
       gettext_compact = False     # optional.
 
-   This case-study assumes that :confval:`locale_dirs` is set to ``locale/`` and
+   This case-study assumes that BUILDDIR is set to ``_build``,
-   :confval:`gettext_compact` is set to ``False`` (the Sphinx document is
+   :confval:`locale_dirs` is set to ``locale/`` and :confval:`gettext_compact`
-   already configured as such).
+   is set to ``False`` (the Sphinx document is already configured as such).
 
 #. Extract translatable messages into pot files.
 

doc/usage/configuration.rst

@@ -2117,6 +2117,13 @@ These options influence LaTeX output.
 
    .. versionadded:: 3.0
 
+.. confval:: latex_theme_options
+
+   A dictionary of options that influence the look and feel of the selected
+   theme.
+
+   .. versionadded:: 3.1
+
 .. confval:: latex_theme_path
 
    A list of paths that contain custom LaTeX themes as subdirectories.  Relative

doc/usage/extensions/autodoc.rst

@@ -154,6 +154,21 @@ inserting them into the page source under a suitable :rst:dir:`py:module`,
 
      .. versionadded:: 3.0
 
+   * autodoc considers a member public if its docstring contains
+     ``:meta public:`` in its :ref:`info-field-lists`, even if it starts with
+     an underscore.
+     For example:
+
+     .. code-block:: rst
+
+        def _my_function(my_arg, my_other_arg):
+            """blah blah blah
+
+            :meta public:
+            """
+
+     .. versionadded:: 3.1
+
    * Python "special" members (that is, those named like ``__special__``) will
      be included if the ``special-members`` flag option is given::
 

doc/usage/extensions/autosummary.rst

@@ -32,7 +32,8 @@ The :mod:`sphinx.ext.autosummary` extension does this in two parts:
 
    The :rst:dir:`autosummary` directive can also optionally serve as a
    :rst:dir:`toctree` entry for the included items. Optionally, stub
-   ``.rst`` files for these items can also be automatically generated.
+   ``.rst`` files for these items can also be automatically generated
+   when :confval:`autosummary_generate` is `True`.
 
    For example, ::
 
@@ -76,6 +77,12 @@ The :mod:`sphinx.ext.autosummary` extension does this in two parts:
      directory. If no argument is given, output is placed in the same directory
      as the file that contains the directive.
 
+     You can also use ``caption`` option to give a caption to the toctree.
+
+     .. versionadded:: 3.1
+
+        caption option added.
+
    * If you don't want the :rst:dir:`autosummary` to show function signatures in
      the listing, include the ``nosignatures`` option::
 
@@ -99,6 +106,17 @@ The :mod:`sphinx.ext.autosummary` extension does this in two parts:
 
      .. versionadded:: 1.0
 
+   * You can specify the ``recursive`` option to generate documents for
+     modules and sub-packages recursively.  It defaults to disabled.
+     For example, ::
+
+         .. autosummary::
+            :recursive:
+
+            sphinx.environment.BuildEnvironment
+
+     .. versionadded:: 3.1
+
 
 :program:`sphinx-autogen` -- generate autodoc stub pages
 --------------------------------------------------------
@@ -136,7 +154,7 @@ also use these config values:
 .. confval:: autosummary_generate
 
    Boolean indicating whether to scan all found documents for autosummary
-   directives, and to generate stub pages for each.
+   directives, and to generate stub pages for each. It is disabled by default.
 
    Can also be a list of documents for which stub pages should be generated.
 
@@ -263,6 +281,12 @@ The following variables available in the templates:
    List containing names of "public" attributes in the class.  Only available
    for classes.
 
+.. data:: modules
+
+   List containing names of "public" modules in the package.  Only available for
+   modules that are packages.
+
+   .. versionadded:: 3.1
 
 Additionally, the following filters are available
 

sphinx/builders/latex/__init__.py

@@ -314,6 +314,8 @@ class LaTeXBuilder(Builder):
         self.context['title'] = title
         self.context['author'] = author
         self.context['docclass'] = theme.docclass
+        self.context['papersize'] = theme.papersize
+        self.context['pointsize'] = theme.pointsize
         self.context['wrapperclass'] = theme.wrapperclass
 
     def assemble_doctree(self, indexfile: str, toctree_only: bool, appendices: List[str]) -> nodes.document:  # NOQA
@@ -491,6 +493,14 @@ def validate_config_values(app: Sphinx, config: Config) -> None:
             config.latex_elements.pop(key)
 
 
+def validate_latex_theme_options(app: Sphinx, config: Config) -> None:
+    for key in list(config.latex_theme_options):
+        if key not in Theme.UPDATABLE_KEYS:
+            msg = __("Unknown theme option: latex_theme_options[%r], ignored.")
+            logger.warning(msg % (key,))
+            config.latex_theme_options.pop(key)
+
+
 def default_latex_engine(config: Config) -> str:
     """ Better default latex_engine settings for specific languages. """
     if config.language == 'ja':
@@ -537,6 +547,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
 
     app.add_builder(LaTeXBuilder)
     app.connect('config-inited', validate_config_values, priority=800)
+    app.connect('config-inited', validate_latex_theme_options, priority=800)
 
     app.add_config_value('latex_engine', default_latex_engine, None,
                          ENUM('pdflatex', 'xelatex', 'lualatex', 'platex', 'uplatex'))
@@ -553,6 +564,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_config_value('latex_elements', {}, None)
     app.add_config_value('latex_additional_files', [], None)
     app.add_config_value('latex_theme', 'manual', None, [str])
+    app.add_config_value('latex_theme_options', {}, None)
     app.add_config_value('latex_theme_path', [], None, [list])
 
     app.add_config_value('latex_docclass', default_latex_docclass, None)

sphinx/builders/latex/constants.py

@@ -69,8 +69,8 @@ LUALATEX_DEFAULT_FONTPKG = XELATEX_DEFAULT_FONTPKG
 
 DEFAULT_SETTINGS = {
     'latex_engine':    'pdflatex',
-    'papersize':       'letterpaper',
+    'papersize':       '',
-    'pointsize':       '10pt',
+    'pointsize':       '',
     'pxunit':          '.75bp',
     'classoptions':    '',
     'extraclassoptions': '',

sphinx/builders/latex/theming.py

@@ -24,50 +24,59 @@ logger = logging.getLogger(__name__)
 class Theme:
     """A set of LaTeX configurations."""
 
+    LATEX_ELEMENTS_KEYS = ['papersize', 'pointsize']
+    UPDATABLE_KEYS = ['papersize', 'pointsize']
+
     def __init__(self, name: str) -> None:
         self.name = name
         self.docclass = name
         self.wrapperclass = name
+        self.papersize = 'letterpaper'
+        self.pointsize = '10pt'
         self.toplevel_sectioning = 'chapter'
 
+    def update(self, config: Config) -> None:
+        """Override theme settings by user's configuration."""
+        for key in self.LATEX_ELEMENTS_KEYS:
+            if config.latex_elements.get(key):
+                value = config.latex_elements[key]
+                setattr(self, key, value)
+
+        for key in self.UPDATABLE_KEYS:
+            if key in config.latex_theme_options:
+                value = config.latex_theme_options[key]
+                setattr(self, key, value)
+
 
 class BuiltInTheme(Theme):
     """A built-in LaTeX theme."""
 
     def __init__(self, name: str, config: Config) -> None:
-        # Note: Don't call supermethod here.
+        super().__init__(name)
-        self.name = name
-        self.latex_docclass = config.latex_docclass  # type: Dict[str, str]
 
-    @property
+        if name == 'howto':
-    def docclass(self) -> str:  # type: ignore
+            self.docclass = config.latex_docclass.get('howto', 'article')
-        if self.name == 'howto':
-            return self.latex_docclass.get('howto', 'article')
         else:
-            return self.latex_docclass.get('manual', 'report')
+            self.docclass = config.latex_docclass.get('manual', 'report')
 
-    @property
+        if name in ('manual', 'howto'):
-    def wrapperclass(self) -> str:  # type: ignore
+            self.wrapperclass = 'sphinx' + name
-        if self.name in ('manual', 'howto'):
-            return 'sphinx' + self.name
         else:
-            return self.name
+            self.wrapperclass = name
 
-    @property
-    def toplevel_sectioning(self) -> str:  # type: ignore
         # we assume LaTeX class provides \chapter command except in case
         # of non-Japanese 'howto' case
-        if self.name == 'howto' and not self.docclass.startswith('j'):
+        if name == 'howto' and not self.docclass.startswith('j'):
-            return 'section'
+            self.toplevel_sectioning = 'section'
         else:
-            return 'chapter'
+            self.toplevel_sectioning = 'chapter'
 
 
 class UserTheme(Theme):
     """A user defined LaTeX theme."""
 
     REQUIRED_CONFIG_KEYS = ['docclass', 'wrapperclass']
-    OPTIONAL_CONFIG_KEYS = ['toplevel_sectioning']
+    OPTIONAL_CONFIG_KEYS = ['papersize', 'pointsize', 'toplevel_sectioning']
 
     def __init__(self, name: str, filename: str) -> None:
         super().__init__(name)
@@ -97,6 +106,7 @@ class ThemeFactory:
     def __init__(self, app: Sphinx) -> None:
         self.themes = {}  # type: Dict[str, Theme]
         self.theme_paths = [path.join(app.srcdir, p) for p in app.config.latex_theme_path]
+        self.config = app.config
         self.load_builtin_themes(app.config)
 
     def load_builtin_themes(self, config: Config) -> None:
@@ -107,13 +117,14 @@ class ThemeFactory:
     def get(self, name: str) -> Theme:
         """Get a theme for given *name*."""
         if name in self.themes:
-            return self.themes[name]
+            theme = self.themes[name]
         else:
             theme = self.find_user_theme(name)
-            if theme:
+            if not theme:
-                return theme
+                theme = Theme(name)
-            else:
+
-                return Theme(name)
+        theme.update(self.config)
+        return theme
 
     def find_user_theme(self, name: str) -> Theme:
         """Find a theme named as *name* from latex_theme_path."""

sphinx/builders/latex/util.py

@@ -8,6 +8,8 @@
     :license: BSD, see LICENSE for details.
 """
 
+from typing import Optional
+
 from docutils.writers.latex2e import Babel
 
 
@@ -40,7 +42,7 @@ class ExtBabel(Babel):
             self.supported = False
             return 'english'  # fallback to english
 
-    def get_mainlanguage_options(self) -> str:
+    def get_mainlanguage_options(self) -> Optional[str]:
         """Return options for polyglossia's ``\\setmainlanguage``."""
         if self.use_polyglossia is False:
             return None

sphinx/deprecation.py

@@ -76,6 +76,6 @@ class DeprecatedDict(dict):
         warnings.warn(self.message, self.warning, stacklevel=2)
         return super().get(key, default)
 
-    def update(self, other: Dict = None) -> None:  # type: ignore
+    def update(self, other: Dict) -> None:  # type: ignore
         warnings.warn(self.message, self.warning, stacklevel=2)
         super().update(other)

sphinx/domains/c.py

@@ -1272,10 +1272,12 @@ class ASTEnumerator(ASTBase):
 
 
 class ASTDeclaration(ASTBaseBase):
-    def __init__(self, objectType: str, directiveType: str, declaration: Any) -> None:
+    def __init__(self, objectType: str, directiveType: str, declaration: Any,
+                 semicolon: bool = False) -> None:
         self.objectType = objectType
         self.directiveType = directiveType
         self.declaration = declaration
+        self.semicolon = semicolon
 
         self.symbol = None  # type: Symbol
         # set by CObject._add_enumerator_to_parent
@@ -1304,7 +1306,10 @@ class ASTDeclaration(ASTBaseBase):
         return self.get_id(_max_id, True)
 
     def _stringify(self, transform: StringifyTransform) -> str:
-        return transform(self.declaration)
+        res = transform(self.declaration)
+        if self.semicolon:
+            res += ';'
+        return res
 
     def describe_signature(self, signode: TextElement, mode: str,
                            env: "BuildEnvironment", options: Dict) -> None:
@@ -1340,6 +1345,8 @@ class ASTDeclaration(ASTBaseBase):
         else:
             assert False
         self.declaration.describe_signature(mainDeclNode, mode, env, self.symbol)
+        if self.semicolon:
+            mainDeclNode += nodes.Text(';')
 
 
 class SymbolLookupResult:
@@ -2742,7 +2749,7 @@ class DefinitionParser(BaseParser):
                 declSpecs = self._parse_decl_specs(outer=outer, typed=False)
                 decl = self._parse_declarator(named=True, paramMode=outer,
                                               typed=False)
-                self.assert_end()
+                self.assert_end(allowSemicolon=True)
             except DefinitionError as exUntyped:
                 desc = "If just a name"
                 prevErrors.append((exUntyped, desc))
@@ -2875,7 +2882,12 @@ class DefinitionParser(BaseParser):
             declaration = self._parse_type(named=True, outer='type')
         else:
             assert False
-        return ASTDeclaration(objectType, directiveType, declaration)
+        if objectType != 'macro':
+            self.skip_ws()
+            semicolon = self.skip_string(';')
+        else:
+            semicolon = False
+        return ASTDeclaration(objectType, directiveType, declaration, semicolon)
 
     def parse_namespace_object(self) -> ASTNestedName:
         return self._parse_nested_name()

sphinx/domains/cpp.py

@@ -10,7 +10,7 @@
 
 import re
 from typing import (
-    Any, Callable, Dict, Generator, Iterator, List, Tuple, Type, TypeVar, Union
+    Any, Callable, Dict, Generator, Iterator, List, Tuple, Type, TypeVar, Union, Optional
 )
 
 from docutils import nodes
@@ -109,7 +109,8 @@ T = TypeVar('T')
         simple-declaration ->
             attribute-specifier-seq[opt] decl-specifier-seq[opt]
                 init-declarator-list[opt] ;
-        # Drop the semi-colon. For now: drop the attributes (TODO).
+        # Make the semicolon optional.
+        # For now: drop the attributes (TODO).
         # Use at most 1 init-declarator.
         -> decl-specifier-seq init-declarator
         -> decl-specifier-seq declarator initializer
@@ -1266,7 +1267,7 @@ class ASTNoexceptExpr(ASTExpression):
         self.expr = expr
 
     def _stringify(self, transform: StringifyTransform) -> str:
-        return "noexcept(" + transform(self.expr) + ")"
+        return 'noexcept(' + transform(self.expr) + ')'
 
     def get_id(self, version: int) -> str:
         return 'nx' + self.expr.get_id(version)
@@ -1812,10 +1813,28 @@ class ASTFunctionParameter(ASTBase):
             self.arg.describe_signature(signode, mode, env, symbol=symbol)
 
 
+class ASTNoexceptSpec(ASTBase):
+    def __init__(self, expr: Optional[ASTExpression]):
+        self.expr = expr
+
+    def _stringify(self, transform: StringifyTransform) -> str:
+        if self.expr:
+            return 'noexcept(' + transform(self.expr) + ')'
+        return 'noexcept'
+
+    def describe_signature(self, signode: TextElement, mode: str,
+                           env: "BuildEnvironment", symbol: "Symbol") -> None:
+        signode += addnodes.desc_annotation('noexcept', 'noexcept')
+        if self.expr:
+            signode.append(nodes.Text('('))
+            self.expr.describe_signature(signode, mode, env, symbol)
+            signode.append(nodes.Text(')'))
+
+
 class ASTParametersQualifiers(ASTBase):
-    def __init__(self, args: List[ASTFunctionParameter],
+    def __init__(self, args: List[ASTFunctionParameter], volatile: bool, const: bool,
-                 volatile: bool, const: bool, refQual: str,
+                 refQual: str, exceptionSpec: ASTNoexceptSpec, override: bool, final: bool,
-                 exceptionSpec: str, override: bool, final: bool, initializer: str) -> None:
+                 initializer: str) -> None:
         self.args = args
         self.volatile = volatile
         self.const = const
@@ -1874,7 +1893,7 @@ class ASTParametersQualifiers(ASTBase):
             res.append(self.refQual)
         if self.exceptionSpec:
             res.append(' ')
-            res.append(str(self.exceptionSpec))
+            res.append(transform(self.exceptionSpec))
         if self.final:
             res.append(' final')
         if self.override:
@@ -1911,7 +1930,8 @@ class ASTParametersQualifiers(ASTBase):
         if self.refQual:
             _add_text(signode, self.refQual)
         if self.exceptionSpec:
-            _add_anno(signode, str(self.exceptionSpec))
+            signode += nodes.Text(' ')
+            self.exceptionSpec.describe_signature(signode, mode, env, symbol)
         if self.final:
             _add_anno(signode, 'final')
         if self.override:
@@ -3465,12 +3485,14 @@ class ASTTemplateDeclarationPrefix(ASTBase):
 
 class ASTDeclaration(ASTBase):
     def __init__(self, objectType: str, directiveType: str, visibility: str,
-                 templatePrefix: ASTTemplateDeclarationPrefix, declaration: Any) -> None:
+                 templatePrefix: ASTTemplateDeclarationPrefix, declaration: Any,
+                 semicolon: bool = False) -> None:
         self.objectType = objectType
         self.directiveType = directiveType
         self.visibility = visibility
         self.templatePrefix = templatePrefix
         self.declaration = declaration
+        self.semicolon = semicolon
 
         self.symbol = None  # type: Symbol
         # set by CPPObject._add_enumerator_to_parent
@@ -3483,7 +3505,7 @@ class ASTDeclaration(ASTBase):
             templatePrefixClone = None
         return ASTDeclaration(self.objectType, self.directiveType,
                               self.visibility, templatePrefixClone,
-                              self.declaration.clone())
+                              self.declaration.clone(), self.semicolon)
 
     @property
     def name(self) -> ASTNestedName:
@@ -3525,6 +3547,8 @@ class ASTDeclaration(ASTBase):
         if self.templatePrefix:
             res.append(transform(self.templatePrefix))
         res.append(transform(self.declaration))
+        if self.semicolon:
+            res.append(';')
         return ''.join(res)
 
     def describe_signature(self, signode: desc_signature, mode: str,
@@ -3578,6 +3602,8 @@ class ASTDeclaration(ASTBase):
         else:
             assert False
         self.declaration.describe_signature(mainDeclNode, mode, env, self.symbol)
+        if self.semicolon:
+            mainDeclNode += nodes.Text(';')
 
 
 class ASTNamespace(ASTBase):
@@ -5498,11 +5524,14 @@ class DefinitionParser(BaseParser):
         initializer = None
         self.skip_ws()
         if self.skip_string('noexcept'):
-            exceptionSpec = 'noexcept'
+            if self.skip_string_and_ws('('):
-            self.skip_ws()
+                expr = self._parse_constant_expression(False)
-            if self.skip_string('('):
+                self.skip_ws()
-                self.fail('Parameterised "noexcept" not yet implemented.')
+                if not self.skip_string(')'):
-
+                    self.fail("Expecting ')' to end 'noexcept'.")
+                exceptionSpec = ASTNoexceptSpec(expr)
+            else:
+                exceptionSpec = ASTNoexceptSpec(None)
         self.skip_ws()
         override = self.skip_word_and_ws('override')
         final = self.skip_word_and_ws('final')
@@ -5875,7 +5904,7 @@ class DefinitionParser(BaseParser):
                 declSpecs = self._parse_decl_specs(outer=outer, typed=False)
                 decl = self._parse_declarator(named=True, paramMode=outer,
                                               typed=False)
-                self.assert_end()
+                self.assert_end(allowSemicolon=True)
             except DefinitionError as exUntyped:
                 if outer == 'type':
                     desc = "If just a name"
@@ -6286,8 +6315,10 @@ class DefinitionParser(BaseParser):
                                                           templatePrefix,
                                                           fullSpecShorthand=False,
                                                           isMember=objectType == 'member')
+        self.skip_ws()
+        semicolon = self.skip_string(';')
         return ASTDeclaration(objectType, directiveType, visibility,
-                              templatePrefix, declaration)
+                              templatePrefix, declaration, semicolon)
 
     def parse_namespace_object(self) -> ASTNamespace:
         templatePrefix = self._parse_template_declaration_prefix(objectType="namespace")

sphinx/ext/autodoc/__init__.py

@@ -65,6 +65,7 @@ def identity(x: Any) -> Any:
 
 
 ALL = object()
+UNINITIALIZED_ATTR = object()
 INSTANCEATTR = object()
 SLOTSATTR = object()
 
@@ -573,6 +574,9 @@ class Documenter:
             if 'private' in metadata:
                 # consider a member private if docstring has "private" metadata
                 isprivate = True
+            elif 'public' in metadata:
+                # consider a member public if docstring has "public" metadata
+                isprivate = False
             else:
                 isprivate = membername.startswith('_')
 
@@ -727,7 +731,8 @@ class Documenter:
         # where the attribute documentation would actually be found in.
         # This is used for situations where you have a module that collects the
         # functions and classes of internal submodules.
-        self.real_modname = real_modname or self.get_real_modname()  # type: str
+        guess_modname = self.get_real_modname()
+        self.real_modname = real_modname or guess_modname
 
         # try to also get a source code analyzer for attribute docs
         try:
@@ -745,6 +750,14 @@ class Documenter:
         else:
             self.directive.filename_set.add(self.analyzer.srcname)
 
+        if self.real_modname != guess_modname:
+            # Add module to dependency list if target object is defined in other module.
+            try:
+                analyzer = ModuleAnalyzer.for_module(guess_modname)
+                self.directive.filename_set.add(analyzer.srcname)
+            except PycodeError:
+                pass
+
         # check __module__ of object (for members not given explicitly)
         if check_module:
             if not self.check_module():
@@ -1347,8 +1360,11 @@ class DataDocumenter(ModuleLevelDocumenter):
                                   sourcename)
 
             try:
-                objrepr = object_description(self.object)
+                if self.object is UNINITIALIZED_ATTR:
-                self.add_line('   :value: ' + objrepr, sourcename)
+                    pass
+                else:
+                    objrepr = object_description(self.object)
+                    self.add_line('   :value: ' + objrepr, sourcename)
             except ValueError:
                 pass
         elif self.options.annotation is SUPPRESS:
@@ -1389,6 +1405,7 @@ class DataDeclarationDocumenter(DataDocumenter):
         """Never import anything."""
         # disguise as a data
         self.objtype = 'data'
+        self.object = UNINITIALIZED_ATTR
         try:
             # import module to obtain type annotation
             self.parent = importlib.import_module(self.modname)
@@ -1599,8 +1616,11 @@ class AttributeDocumenter(DocstringStripSignatureMixin, ClassLevelDocumenter):
                                       sourcename)
 
                 try:
-                    objrepr = object_description(self.object)
+                    if self.object is INSTANCEATTR:
-                    self.add_line('   :value: ' + objrepr, sourcename)
+                        pass
+                    else:
+                        objrepr = object_description(self.object)
+                        self.add_line('   :value: ' + objrepr, sourcename)
                 except ValueError:
                     pass
         elif self.options.annotation is SUPPRESS:
@@ -1671,6 +1691,7 @@ class InstanceAttributeDocumenter(AttributeDocumenter):
         """Never import anything."""
         # disguise as an attribute
         self.objtype = 'attribute'
+        self.object = INSTANCEATTR
         self._datadescriptor = False
         return True
 

sphinx/ext/autosummary/__init__.py

@@ -224,8 +224,10 @@ class Autosummary(SphinxDirective):
     final_argument_whitespace = False
     has_content = True
     option_spec = {
+        'caption': directives.unchanged_required,
         'toctree': directives.unchanged,
         'nosignatures': directives.flag,
+        'recursive': directives.flag,
         'template': directives.unchanged,
     }
 
@@ -266,9 +268,14 @@ class Autosummary(SphinxDirective):
                 tocnode['entries'] = [(None, docn) for docn in docnames]
                 tocnode['maxdepth'] = -1
                 tocnode['glob'] = None
+                tocnode['caption'] = self.options.get('caption')
 
                 nodes.append(autosummary_toc('', '', tocnode))
 
+        if 'toctree' not in self.options and 'caption' in self.options:
+            logger.warning(__('A captioned autosummary requires :toctree: option. ignored.'),
+                           location=nodes[-1])
+
         return nodes
 
     def get_items(self, names: List[str]) -> List[Tuple[str, str, str, str]]:
@@ -741,8 +748,7 @@ def process_generate_options(app: Sphinx) -> None:
 
     imported_members = app.config.autosummary_imported_members
     with mock(app.config.autosummary_mock_imports):
-        generate_autosummary_docs(genfiles, builder=app.builder,
+        generate_autosummary_docs(genfiles, suffix=suffix, base_path=app.srcdir,
-                                  suffix=suffix, base_path=app.srcdir,
                                   app=app, imported_members=imported_members,
                                   overwrite=app.config.autosummary_generate_overwrite)
 

sphinx/ext/autosummary/generate.py

@@ -20,29 +20,34 @@
 import argparse
 import locale
 import os
+import pkgutil
 import pydoc
 import re
 import sys
 import warnings
-from typing import Any, Callable, Dict, List, NamedTuple, Set, Tuple, Type
+from gettext import NullTranslations
+from os import path
+from typing import Any, Callable, Dict, List, NamedTuple, Set, Tuple, Type, Union
 
-from jinja2 import BaseLoader, FileSystemLoader, TemplateNotFound
+from jinja2 import TemplateNotFound
 from jinja2.sandbox import SandboxedEnvironment
 
 import sphinx.locale
 from sphinx import __display_version__
 from sphinx import package_dir
+from sphinx.application import Sphinx
 from sphinx.builders import Builder
+from sphinx.config import Config
 from sphinx.deprecation import RemovedInSphinx40Warning, RemovedInSphinx50Warning
 from sphinx.ext.autodoc import Documenter
 from sphinx.ext.autosummary import import_by_name, get_documenter
-from sphinx.jinja2glue import BuiltinTemplateLoader
 from sphinx.locale import __
 from sphinx.registry import SphinxComponentRegistry
 from sphinx.util import logging
 from sphinx.util import rst
 from sphinx.util.inspect import safe_getattr
 from sphinx.util.osutil import ensuredir
+from sphinx.util.template import SphinxTemplateLoader
 
 
 logger = logging.getLogger(__name__)
@@ -51,20 +56,26 @@ logger = logging.getLogger(__name__)
 class DummyApplication:
     """Dummy Application class for sphinx-autogen command."""
 
-    def __init__(self) -> None:
+    def __init__(self, translator: NullTranslations) -> None:
+        self.config = Config()
         self.registry = SphinxComponentRegistry()
         self.messagelog = []  # type: List[str]
+        self.srcdir = "/"
+        self.translator = translator
         self.verbosity = 0
         self._warncount = 0
         self.warningiserror = False
 
+        self.config.init_values()
+
     def emit_firstresult(self, *args: Any) -> None:
         pass
 
 
 AutosummaryEntry = NamedTuple('AutosummaryEntry', [('name', str),
                                                    ('path', str),
-                                                   ('template', str)])
+                                                   ('template', str),
+                                                   ('recursive', bool)])
 
 
 def setup_documenters(app: Any) -> None:
@@ -103,39 +114,59 @@ def _underline(title: str, line: str = '=') -> str:
 class AutosummaryRenderer:
     """A helper class for rendering."""
 
-    def __init__(self, builder: Builder, template_dir: str) -> None:
+    def __init__(self, app: Union[Builder, Sphinx], template_dir: str = None) -> None:
-        loader = None  # type: BaseLoader
+        if isinstance(app, Builder):
-        template_dirs = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')]
+            warnings.warn('The first argument for AutosummaryRenderer has been '
-        if builder is None:
+                          'changed to Sphinx object',
-            if template_dir:
+                          RemovedInSphinx50Warning, stacklevel=2)
-                template_dirs.insert(0, template_dir)
+        if template_dir:
-            loader = FileSystemLoader(template_dirs)
+            warnings.warn('template_dir argument for AutosummaryRenderer is deprecated.',
-        else:
+                          RemovedInSphinx50Warning)
-            # allow the user to override the templates
+
-            loader = BuiltinTemplateLoader()
+        system_templates_path = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')]
-            loader.init(builder, dirs=template_dirs)
+        loader = SphinxTemplateLoader(app.srcdir, app.config.templates_path,
+                                      system_templates_path)
 
         self.env = SandboxedEnvironment(loader=loader)
         self.env.filters['escape'] = rst.escape
         self.env.filters['e'] = rst.escape
         self.env.filters['underline'] = _underline
 
-        if builder:
+        if isinstance(app, (Sphinx, DummyApplication)):
-            if builder.app.translator:
+            if app.translator:
                 self.env.add_extension("jinja2.ext.i18n")
-                self.env.install_gettext_translations(builder.app.translator)  # type: ignore
+                self.env.install_gettext_translations(app.translator)  # type: ignore
+        elif isinstance(app, Builder):
+            if app.app.translator:
+                self.env.add_extension("jinja2.ext.i18n")
+                self.env.install_gettext_translations(app.app.translator)  # type: ignore
 
     def exists(self, template_name: str) -> bool:
         """Check if template file exists."""
+        warnings.warn('AutosummaryRenderer.exists() is deprecated.',
+                      RemovedInSphinx50Warning, stacklevel=2)
         try:
             self.env.get_template(template_name)
             return True
         except TemplateNotFound:
             return False
 
-    def render(self, template_name: str, context: Dict) -> str:
+    def render(self, objtype: str, context: Dict) -> str:
         """Render a template file."""
-        return self.env.get_template(template_name).render(context)
+        if objtype.endswith('.rst'):
+            # old styled: template_name is given
+            warnings.warn('AutosummaryRenderer.render() takes an object type as an argument.',
+                          RemovedInSphinx50Warning, stacklevel=2)
+            return self.env.get_template(objtype).render(context)
+        else:
+            # objtype is given
+            try:
+                template = self.env.get_template('autosummary/%s.rst' % objtype)
+            except TemplateNotFound:
+                # fallback to base.rst
+                template = self.env.get_template('autosummary/base.rst')
+
+            return template.render(context)
 
 
 # -- Generating output ---------------------------------------------------------
@@ -143,14 +174,10 @@ class AutosummaryRenderer:
 
 def generate_autosummary_content(name: str, obj: Any, parent: Any,
                                  template: AutosummaryRenderer, template_name: str,
-                                 imported_members: bool, app: Any) -> str:
+                                 imported_members: bool, app: Any,
+                                 recursive: bool) -> str:
     doc = get_documenter(app, obj, parent)
 
-    if template_name is None:
-        template_name = 'autosummary/%s.rst' % doc.objtype
-        if not template.exists(template_name):
-            template_name = 'autosummary/base.rst'
-
     def skip_member(obj: Any, name: str, objtype: str) -> bool:
         try:
             return app.emit_firstresult('autodoc-skip-member', objtype, name,
@@ -188,6 +215,14 @@ def generate_autosummary_content(name: str, obj: Any, parent: Any,
                             public.append(name)
         return public, items
 
+    def get_modules(obj: Any) -> Tuple[List[str], List[str]]:
+        items = []  # type: List[str]
+        for _, modname, ispkg in pkgutil.iter_modules(obj.__path__):
+            fullname = name + '.' + modname
+            items.append(fullname)
+        public = [x for x in items if not x.split('.')[-1].startswith('_')]
+        return public, items
+
     ns = {}  # type: Dict[str, Any]
 
     if doc.objtype == 'module':
@@ -198,6 +233,9 @@ def generate_autosummary_content(name: str, obj: Any, parent: Any,
             get_members(obj, {'class'}, imported=imported_members)
         ns['exceptions'], ns['all_exceptions'] = \
             get_members(obj, {'exception'}, imported=imported_members)
+        ispackage = hasattr(obj, '__path__')
+        if ispackage and recursive:
+            ns['modules'], ns['all_modules'] = get_modules(obj)
     elif doc.objtype == 'class':
         ns['members'] = dir(obj)
         ns['inherited_members'] = \
@@ -224,7 +262,7 @@ def generate_autosummary_content(name: str, obj: Any, parent: Any,
     ns['objtype'] = doc.objtype
     ns['underline'] = len(name) * '='
 
-    return template.render(template_name, ns)
+    return template.render(doc.objtype, ns)
 
 
 def generate_autosummary_docs(sources: List[str], output_dir: str = None,
@@ -247,6 +285,14 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None,
     else:
         _warn = logger.warning
 
+    if builder:
+        warnings.warn('builder argument for generate_autosummary_docs() is deprecated.',
+                      RemovedInSphinx50Warning)
+
+    if template_dir:
+        warnings.warn('template_dir argument for generate_autosummary_docs() is deprecated.',
+                      RemovedInSphinx50Warning)
+
     showed_sources = list(sorted(sources))
     if len(showed_sources) > 20:
         showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:]
@@ -259,7 +305,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None,
     if base_path is not None:
         sources = [os.path.join(base_path, filename) for filename in sources]
 
-    template = AutosummaryRenderer(builder, template_dir)
+    template = AutosummaryRenderer(app)
 
     # read
     items = find_autosummary_in_files(sources)
@@ -284,7 +330,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None,
             continue
 
         content = generate_autosummary_content(name, obj, parent, template, entry.template,
-                                               imported_members, app)
+                                               imported_members, app, entry.recursive)
 
         filename = os.path.join(path, name + suffix)
         if os.path.isfile(filename):
@@ -306,8 +352,7 @@ def generate_autosummary_docs(sources: List[str], output_dir: str = None,
     if new_files:
         generate_autosummary_docs(new_files, output_dir=output_dir,
                                   suffix=suffix, warn=warn, info=info,
-                                  base_path=base_path, builder=builder,
+                                  base_path=base_path,
-                                  template_dir=template_dir,
                                   imported_members=imported_members, app=app,
                                   overwrite=overwrite)
 
@@ -369,11 +414,13 @@ def find_autosummary_in_lines(lines: List[str], module: str = None, filename: st
     module_re = re.compile(
         r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$')
     autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?')
+    recursive_arg_re = re.compile(r'^\s+:recursive:\s*$')
     toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$')
     template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$')
 
     documented = []  # type: List[AutosummaryEntry]
 
+    recursive = False
     toctree = None  # type: str
     template = None
     current_module = module
@@ -382,6 +429,11 @@ def find_autosummary_in_lines(lines: List[str], module: str = None, filename: st
 
     for line in lines:
         if in_autosummary:
+            m = recursive_arg_re.match(line)
+            if m:
+                recursive = True
+                continue
+
             m = toctree_arg_re.match(line)
             if m:
                 toctree = m.group(1)
@@ -406,7 +458,7 @@ def find_autosummary_in_lines(lines: List[str], module: str = None, filename: st
                 if current_module and \
                    not name.startswith(current_module + '.'):
                     name = "%s.%s" % (current_module, name)
-                documented.append(AutosummaryEntry(name, toctree, template))
+                documented.append(AutosummaryEntry(name, toctree, template, recursive))
                 continue
 
             if not line.strip() or line.startswith(base_indent + " "):
@@ -418,6 +470,7 @@ def find_autosummary_in_lines(lines: List[str], module: str = None, filename: st
         if m:
             in_autosummary = True
             base_indent = m.group(1)
+            recursive = False
             toctree = None
             template = None
             continue
@@ -483,14 +536,18 @@ The format of the autosummary directive is documented in the
 def main(argv: List[str] = sys.argv[1:]) -> None:
     sphinx.locale.setlocale(locale.LC_ALL, '')
     sphinx.locale.init_console(os.path.join(package_dir, 'locale'), 'sphinx')
+    translator, _ = sphinx.locale.init([], None)
 
-    app = DummyApplication()
+    app = DummyApplication(translator)
     logging.setup(app, sys.stdout, sys.stderr)  # type: ignore
     setup_documenters(app)
     args = get_parser().parse_args(argv)
+
+    if args.templates:
+        app.config.templates_path.append(path.abspath(args.templates))
+
     generate_autosummary_docs(args.source_file, args.output_dir,
                               '.' + args.suffix,
-                              template_dir=args.templates,
                               imported_members=args.imported_members,
                               app=app)
 

sphinx/ext/autosummary/templates/autosummary/module.rst

@@ -34,3 +34,16 @@
    {%- endfor %}
    {% endif %}
    {% endblock %}
+
+{% block modules %}
+{% if modules %}
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :recursive:
+{% for item in modules %}
+   {{ item }}
+{%- endfor %}
+{% endif %}
+{% endblock %}

sphinx/ext/napoleon/iterators.py

@@ -11,7 +11,7 @@
 """
 
 import collections
-from typing import Any, Iterable
+from typing import Any, Iterable, Optional
 
 
 class peek_iter:
@@ -62,7 +62,7 @@ class peek_iter:
     def __next__(self, n: int = None) -> Any:
         return self.next(n)
 
-    def _fillcache(self, n: int) -> None:
+    def _fillcache(self, n: Optional[int]) -> None:
         """Cache `n` items. If `n` is 0 or None, then 1 item is cached."""
         if not n:
             n = 1
@@ -123,7 +123,7 @@ class peek_iter:
             result = [self._cache.popleft() for i in range(n)]
         return result
 
-    def peek(self, n: int = None) -> Any:
+    def peek(self, n: Optional[int] = None) -> Any:
         """Preview the next item or `n` items of the iterator.
 
         The iterator is not advanced when peek is called.
@@ -220,7 +220,7 @@ class modify_iter(peek_iter):
                             'modifier must be callable')
         super().__init__(*args)
 
-    def _fillcache(self, n: int) -> None:
+    def _fillcache(self, n: Optional[int]) -> None:
         """Cache `n` modified items. If `n` is 0 or None, 1 item is cached.
 
         Each item returned by the iterator is passed through the

sphinx/locale/__init__.py

@@ -12,7 +12,7 @@ import gettext
 import locale
 from collections import UserString, defaultdict
 from gettext import NullTranslations
-from typing import Any, Callable, Dict, Iterable, List, Tuple, Union
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 
 class _TranslationProxy(UserString):
@@ -173,7 +173,7 @@ def init_console(locale_dir: str, catalog: str) -> Tuple[NullTranslations, bool]
     """
     try:
         # encoding is ignored
-        language, _ = locale.getlocale(locale.LC_MESSAGES)
+        language, _ = locale.getlocale(locale.LC_MESSAGES)  # type: Tuple[Optional[str], Any]
     except AttributeError:
         # LC_MESSAGES is not always defined. Fallback to the default language
         # in case it is not.

sphinx/pycode/ast.py

@@ -58,7 +58,7 @@ def parse(code: str, mode: str = 'exec') -> "ast.AST":
         return ast.parse(code, mode=mode)
 
 
-def unparse(node: ast.AST) -> str:
+def unparse(node: Optional[ast.AST]) -> Optional[str]:
     """Unparse an AST to string."""
     if node is None:
         return None
@@ -138,7 +138,7 @@ def _unparse_arg(arg: ast.arg, default: Optional[ast.AST]) -> str:
 
 def unparse_arguments(node: ast.arguments) -> str:
     """Unparse an arguments to string."""
-    defaults = list(node.defaults)
+    defaults = list(node.defaults)  # type: List[Optional[ast.AST]]
     positionals = len(node.args)
     posonlyargs = 0
     if hasattr(node, "posonlyargs"):  # for py38+
@@ -147,7 +147,7 @@ def unparse_arguments(node: ast.arguments) -> str:
     for _ in range(len(defaults), positionals):
         defaults.insert(0, None)
 
-    kw_defaults = list(node.kw_defaults)
+    kw_defaults = list(node.kw_defaults)  # type: List[Optional[ast.AST]]
     for _ in range(len(kw_defaults), len(node.kwonlyargs)):
         kw_defaults.insert(0, None)
 

sphinx/themes/basic/genindex.html

@@ -7,6 +7,9 @@
     :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 #}
+{%- extends "layout.html" %}
+{% set title = _('Index') %}
+
 {% macro indexentries(firstname, links) %}
   {%- if links -%}
     <a href="{{ links[0][1] }}">
@@ -26,8 +29,6 @@
   {%- endif %}
 {% endmacro %}
 
-{%- extends "layout.html" %}
-{% set title = _('Index') %}
 {% block body %}
 
 <h1 id="index">{{ _('Index') }}</h1>

sphinx/themes/basic/layout.html

@@ -181,6 +181,7 @@
       {%- endif %}
           <div class="body" role="main">
             {% block body %} {% endblock %}
+            <div class="clearer"></div>
           </div>
       {%- if render_sidebar %}
         </div>
@@ -208,7 +209,7 @@
       {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}
     {%- endif %}
     {%- if show_sphinx %}
-      {% trans sphinx_version=sphinx_version|e %}Created using <a href="http://sphinx-doc.org/">Sphinx</a> {{ sphinx_version }}.{% endtrans %}
+      {% trans sphinx_version=sphinx_version|e %}Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> {{ sphinx_version }}.{% endtrans %}
     {%- endif %}
     </div>
 {%- endblock %}

sphinx/themes/basic/static/basic.css_t

@@ -321,13 +321,14 @@ div.sidebar {
     width: 40%;
     float: right;
     clear: right;
+    overflow-x: auto;
 }
 
 p.sidebar-title {
     font-weight: bold;
 }
 
-div.admonition, div.topic, pre {
+div.admonition, div.topic, pre, div[class|="highlight"] {
     clear: both;
 }
 
@@ -337,6 +338,7 @@ div.topic {
     border: 1px solid #ccc;
     padding: 7px 7px 0 7px;
     margin: 10px 0 10px 0;
+    overflow-x: auto;
 }
 
 p.topic-title {
@@ -351,6 +353,7 @@ div.admonition {
     margin-top: 10px;
     margin-bottom: 10px;
     padding: 7px;
+    overflow-x: auto;
 }
 
 div.admonition dt {

sphinx/util/cfamily.py

@@ -338,10 +338,14 @@ class BaseParser:
         self.pos = self.end
         return rv
 
-    def assert_end(self) -> None:
+    def assert_end(self, *, allowSemicolon: bool = False) -> None:
         self.skip_ws()
-        if not self.eof:
+        if allowSemicolon:
-            self.fail('Expected end of definition.')
+            if not self.eof and self.definition[self.pos:] != ';':
+                self.fail('Expected end of definition or ;.')
+        else:
+            if not self.eof:
+                self.fail('Expected end of definition.')
 
     ################################################################################
 

sphinx/util/images.py

@@ -12,7 +12,7 @@ import base64
 import imghdr
 from collections import OrderedDict
 from os import path
-from typing import IO, NamedTuple, Tuple
+from typing import IO, NamedTuple, Optional, Tuple
 
 import imagesize
 
@@ -36,7 +36,7 @@ DataURI = NamedTuple('DataURI', [('mimetype', str),
                                  ('data', bytes)])
 
 
-def get_image_size(filename: str) -> Tuple[int, int]:
+def get_image_size(filename: str) -> Optional[Tuple[int, int]]:
     try:
         size = imagesize.get(filename)
         if size[0] == -1:
@@ -53,7 +53,7 @@ def get_image_size(filename: str) -> Tuple[int, int]:
         return None
 
 
-def guess_mimetype_for_stream(stream: IO, default: str = None) -> str:
+def guess_mimetype_for_stream(stream: IO, default: Optional[str] = None) -> Optional[str]:
     imgtype = imghdr.what(stream)  # type: ignore
     if imgtype:
         return 'image/' + imgtype
@@ -61,7 +61,7 @@ def guess_mimetype_for_stream(stream: IO, default: str = None) -> str:
         return default
 
 
-def guess_mimetype(filename: str = '', default: str = None) -> str:
+def guess_mimetype(filename: str = '', default: Optional[str] = None) -> Optional[str]:
     _, ext = path.splitext(filename.lower())
     if ext in mime_suffixes:
         return mime_suffixes[ext]
@@ -72,7 +72,7 @@ def guess_mimetype(filename: str = '', default: str = None) -> str:
     return default
 
 
-def get_image_extension(mimetype: str) -> str:
+def get_image_extension(mimetype: str) -> Optional[str]:
     for ext, _mimetype in mime_suffixes.items():
         if mimetype == _mimetype:
             return ext
@@ -80,7 +80,7 @@ def get_image_extension(mimetype: str) -> str:
     return None
 
 
-def parse_data_uri(uri: str) -> DataURI:
+def parse_data_uri(uri: str) -> Optional[DataURI]:
     if not uri.startswith('data:'):
         return None
 
@@ -101,7 +101,7 @@ def parse_data_uri(uri: str) -> DataURI:
     return DataURI(mimetype, charset, image_data)
 
 
-def test_svg(h: bytes, f: IO) -> str:
+def test_svg(h: bytes, f: IO) -> Optional[str]:
     """An additional imghdr library helper; test the header is SVG's or not."""
     try:
         if '<svg' in h.decode().lower():

sphinx/util/inspect.py

@@ -20,7 +20,7 @@ from inspect import (  # NOQA
     Parameter, isclass, ismethod, ismethoddescriptor
 )
 from io import StringIO
-from typing import Any, Callable, Mapping, List, Tuple
+from typing import Any, Callable, Mapping, List, Optional, Tuple
 from typing import cast
 
 from sphinx.deprecation import RemovedInSphinx40Warning, RemovedInSphinx50Warning
@@ -565,7 +565,7 @@ class Signature:
         self.partialmethod_with_noargs = False
 
         try:
-            self.signature = inspect.signature(subject)
+            self.signature = inspect.signature(subject)  # type: Optional[inspect.Signature]
         except IndexError:
             # Until python 3.6.4, cpython has been crashed on inspection for
             # partialmethods not having any arguments.

sphinx/util/osutil.py

@@ -18,7 +18,7 @@ import sys
 import warnings
 from io import StringIO
 from os import path
-from typing import Any, Generator, Iterator, List, Tuple, Type
+from typing import Any, Generator, Iterator, List, Optional, Tuple, Type
 
 from sphinx.deprecation import RemovedInSphinx40Warning
 
@@ -202,7 +202,7 @@ class FileAvoidWrite:
     """
     def __init__(self, path: str) -> None:
         self._path = path
-        self._io = None  # type: StringIO
+        self._io = None  # type: Optional[StringIO]
 
     def write(self, data: str) -> None:
         if not self._io:

sphinx/util/png.py

@@ -10,6 +10,7 @@
 
 import binascii
 import struct
+from typing import Optional
 
 
 LEN_IEND = 12
@@ -20,7 +21,7 @@ DEPTH_CHUNK_START = b'tEXtDepth\x00'
 IEND_CHUNK = b'\x00\x00\x00\x00IEND\xAE\x42\x60\x82'
 
 
-def read_png_depth(filename: str) -> int:
+def read_png_depth(filename: str) -> Optional[int]:
     """Read the special tEXt chunk indicating the depth from a PNG file."""
     with open(filename, 'rb') as f:
         f.seek(- (LEN_IEND + LEN_DEPTH), 2)

sphinx/util/template.py

@@ -10,8 +10,11 @@
 
 import os
 from functools import partial
-from typing import Dict, List, Union
+from os import path
+from typing import Callable, Dict, List, Tuple, Union
 
+from jinja2 import TemplateNotFound
+from jinja2.environment import Environment
 from jinja2.loaders import BaseLoader
 from jinja2.sandbox import SandboxedEnvironment
 
@@ -94,3 +97,36 @@ class ReSTRenderer(SphinxRenderer):
         self.env.filters['e'] = rst.escape
         self.env.filters['escape'] = rst.escape
         self.env.filters['heading'] = rst.heading
+
+
+class SphinxTemplateLoader(BaseLoader):
+    """A loader supporting template inheritance"""
+
+    def __init__(self, confdir: str, templates_paths: List[str],
+                 system_templates_paths: List[str]) -> None:
+        self.loaders = []
+        self.sysloaders = []
+
+        for templates_path in templates_paths:
+            loader = SphinxFileSystemLoader(path.join(confdir, templates_path))
+            self.loaders.append(loader)
+
+        for templates_path in system_templates_paths:
+            loader = SphinxFileSystemLoader(templates_path)
+            self.loaders.append(loader)
+            self.sysloaders.append(loader)
+
+    def get_source(self, environment: Environment, template: str) -> Tuple[str, str, Callable]:
+        if template.startswith('!'):
+            # search a template from ``system_templates_paths``
+            loaders = self.sysloaders
+            template = template[1:]
+        else:
+            loaders = self.loaders
+
+        for loader in loaders:
+            try:
+                return loader.get_source(environment, template)
+            except TemplateNotFound:
+                pass
+        raise TemplateNotFound(template)

tests/roots/test-domain-c/semicolon.rst

@@ -0,0 +1,10 @@
+.. c:member:: int member;
+.. c:var:: int var;
+.. c:function:: void f();
+.. .. c:macro:: NO_SEMICOLON;
+.. c:struct:: Struct;
+.. c:union:: Union;
+.. c:enum:: Enum;
+.. c:enumerator:: Enumerator;
+.. c:type:: Type;
+.. c:type:: int TypeDef;

tests/roots/test-domain-cpp/semicolon.rst

@@ -0,0 +1,14 @@
+.. cpp:class:: Class;
+.. cpp:struct:: Struct;
+.. cpp:union:: Union;
+.. cpp:function:: void f();
+.. cpp:member:: int member;
+.. cpp:var:: int var;
+.. cpp:type:: Type;
+.. cpp:type:: int TypeDef;
+.. cpp:type:: Alias = int;
+.. cpp:concept:: template<typename T> Concept;
+.. cpp:enum:: Enum;
+.. cpp:enum-struct:: EnumStruct;
+.. cpp:enum-class:: EnumClass;
+.. cpp:enumerator:: Enumerator;

tests/roots/test-ext-autodoc/target/private.py

@@ -3,3 +3,9 @@ def private_function(name):
 
     :meta private:
     """
+
+def _public_function(name):
+    """public_function is a docstring().
+
+    :meta public:
+    """

tests/roots/test-ext-autosummary-recursive/conf.py

@@ -0,0 +1,7 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('.'))
+
+extensions = ['sphinx.ext.autosummary']
+autosummary_generate = True

tests/roots/test-ext-autosummary-recursive/index.rst

@@ -0,0 +1,15 @@
+API Reference
+=============
+
+.. rubric:: Packages
+
+.. autosummary::
+   :toctree: generated
+   :recursive:
+
+    package
+
+.. autosummary::
+   :toctree: generated
+
+    package2

tests/roots/test-ext-autosummary-recursive/package/module.py

@@ -0,0 +1,13 @@
+from os import *  # NOQA
+
+
+class Foo:
+    def __init__(self):
+        pass
+
+    def bar(self):
+        pass
+
+    @property
+    def baz(self):
+        pass

tests/roots/test-ext-autosummary-recursive/package/module_importfail.py

@@ -0,0 +1,4 @@
+import sys
+
+# Fail module import in a catastrophic way
+sys.exit(1)

tests/roots/test-ext-autosummary-recursive/package/package/module.py

@@ -0,0 +1,13 @@
+from os import *  # NOQA
+
+
+class Foo:
+    def __init__(self):
+        pass
+
+    def bar(self):
+        pass
+
+    @property
+    def baz(self):
+        pass

tests/roots/test-ext-autosummary-recursive/package2/module.py

@@ -0,0 +1,13 @@
+from os import *  # NOQA
+
+
+class Foo:
+    def __init__(self):
+        pass
+
+    def bar(self):
+        pass
+
+    @property
+    def baz(self):
+        pass

tests/roots/test-ext-autosummary/index.rst

@@ -5,6 +5,7 @@
 
 .. autosummary::
    :toctree: generated
+   :caption: An autosummary
 
    autosummary_dummy_module
    autosummary_dummy_module.Foo

tests/roots/test-latex-theme/theme/custom/theme.conf

@@ -1,4 +1,6 @@
 [theme]
 docclass = book
 wrapperclass = sphinxbook
+papersize = a4paper
+pointsize = 12pt
 toplevel_sectioning = chapter

tests/test_autodoc.py

@@ -1032,14 +1032,12 @@ def test_instance_attributes(app):
         '',
         '   .. py:attribute:: InstAttCls.ia1',
         '      :module: target',
-        '      :value: None',
         '',
         '      Doc comment for instance attribute InstAttCls.ia1',
         '',
         '',
         '   .. py:attribute:: InstAttCls.ia2',
         '      :module: target',
-        '      :value: None',
         '',
         '      Docstring for instance attribute InstAttCls.ia2.',
         ''
@@ -1066,7 +1064,6 @@ def test_instance_attributes(app):
         '',
         '   .. py:attribute:: InstAttCls.ia1',
         '      :module: target',
-        '      :value: None',
         '',
         '      Doc comment for instance attribute InstAttCls.ia1',
         ''
@@ -1485,7 +1482,6 @@ def test_autodoc_typed_instance_variables(app):
         '   .. py:attribute:: Class.attr2',
         '      :module: target.typed_vars',
         '      :type: int',
-        '      :value: None',
         '',
         '',
         '   .. py:attribute:: Class.attr3',
@@ -1497,7 +1493,6 @@ def test_autodoc_typed_instance_variables(app):
         '   .. py:attribute:: Class.attr4',
         '      :module: target.typed_vars',
         '      :type: int',
-        '      :value: None',
         '',
         '      attr4',
         '',
@@ -1505,7 +1500,6 @@ def test_autodoc_typed_instance_variables(app):
         '   .. py:attribute:: Class.attr5',
         '      :module: target.typed_vars',
         '      :type: int',
-        '      :value: None',
         '',
         '      attr5',
         '',
@@ -1513,7 +1507,6 @@ def test_autodoc_typed_instance_variables(app):
         '   .. py:attribute:: Class.attr6',
         '      :module: target.typed_vars',
         '      :type: int',
-        '      :value: None',
         '',
         '      attr6',
         '',
@@ -1529,7 +1522,6 @@ def test_autodoc_typed_instance_variables(app):
         '.. py:data:: attr2',
         '   :module: target.typed_vars',
         '   :type: str',
-        '   :value: None',
         '',
         '   attr2',
         '',

tests/test_build_latex.py

@@ -220,7 +220,29 @@ def test_latex_theme(app, status, warning):
     result = (app.outdir / 'python.tex').read_text(encoding='utf8')
     print(result)
     assert r'\def\sphinxdocclass{book}' in result
-    assert r'\documentclass[letterpaper,10pt,english]{sphinxbook}' in result
+    assert r'\documentclass[a4paper,12pt,english]{sphinxbook}' in result
+
+
+@pytest.mark.sphinx('latex', testroot='latex-theme',
+                    confoverrides={'latex_elements': {'papersize': 'b5paper',
+                                                      'pointsize': '9pt'}})
+def test_latex_theme_papersize(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'python.tex').read_text(encoding='utf8')
+    print(result)
+    assert r'\def\sphinxdocclass{book}' in result
+    assert r'\documentclass[b5paper,9pt,english]{sphinxbook}' in result
+
+
+@pytest.mark.sphinx('latex', testroot='latex-theme',
+                    confoverrides={'latex_theme_options': {'papersize': 'b5paper',
+                                                           'pointsize': '9pt'}})
+def test_latex_theme_options(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'python.tex').read_text(encoding='utf8')
+    print(result)
+    assert r'\def\sphinxdocclass{book}' in result
+    assert r'\documentclass[b5paper,9pt,english]{sphinxbook}' in result
 
 
 @pytest.mark.sphinx('latex', testroot='basic', confoverrides={'language': 'zh'})

tests/test_domain_c.py

@@ -27,10 +27,8 @@ def parse(name, string):
     return ast
 
 
-def check(name, input, idDict, output=None):
+def _check(name, input, idDict, output):
     # first a simple check of the AST
-    if output is None:
-        output = input
     ast = parse(name, input)
     res = str(ast)
     if res != output:
@@ -77,6 +75,16 @@ def check(name, input, idDict, output=None):
         raise DefinitionError("")
 
 
+def check(name, input, idDict, output=None):
+    if output is None:
+        output = input
+    # First, check without semicolon
+    _check(name, input, idDict, output)
+    if name != 'macro':
+        # Second, check with semicolon
+        _check(name, input + ' ;', idDict, output + ';')
+
+
 def test_expressions():
     def exprCheck(expr, output=None):
         class Config:
@@ -469,8 +477,9 @@ def test_build_domain_c(app, status, warning):
     ws = filter_warnings(warning, "index")
     assert len(ws) == 0
 
+
 @pytest.mark.sphinx(testroot='domain-c', confoverrides={'nitpicky': True})
-def test_build_domain_c(app, status, warning):
+def test_build_domain_c_namespace(app, status, warning):
     app.builder.build_all()
     ws = filter_warnings(warning, "namespace")
     assert len(ws) == 0
@@ -478,6 +487,7 @@ def test_build_domain_c(app, status, warning):
     for id_ in ('NS.NSVar', 'NULLVar', 'ZeroVar', 'NS2.NS3.NS2NS3Var', 'PopVar'):
         assert 'id="c.{}"'.format(id_) in t
 
+
 @pytest.mark.sphinx(testroot='domain-c', confoverrides={'nitpicky': True})
 def test_build_domain_c_anon_dup_decl(app, status, warning):
     app.builder.build_all()
@@ -487,6 +497,13 @@ def test_build_domain_c_anon_dup_decl(app, status, warning):
     assert "WARNING: c:identifier reference target not found: @b" in ws[1]
 
 
+@pytest.mark.sphinx(testroot='domain-c', confoverrides={'nitpicky': True})
+def test_build_domain_c_semicolon(app, status, warning):
+    app.builder.build_all()
+    ws = filter_warnings(warning, "semicolon")
+    assert len(ws) == 0
+
+
 def test_cfunction(app):
     text = (".. c:function:: PyObject* "
             "PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)")

tests/test_domain_cpp.py

@@ -32,10 +32,8 @@ def parse(name, string):
     return ast
 
 
-def check(name, input, idDict, output=None):
+def _check(name, input, idDict, output):
     # first a simple check of the AST
-    if output is None:
-        output = input
     ast = parse(name, input)
     res = str(ast)
     if res != output:
@@ -82,6 +80,15 @@ def check(name, input, idDict, output=None):
         raise DefinitionError("")
 
 
+def check(name, input, idDict, output=None):
+    if output is None:
+        output = input
+    # First, check without semicolon
+    _check(name, input, idDict, output)
+    # Second, check with semicolon
+    _check(name, input + ' ;', idDict, output + ';')
+
+
 def test_fundamental_types():
     # see https://en.cppreference.com/w/cpp/language/types
     for t, id_v2 in cppDomain._id_fundamental_v2.items():
@@ -392,7 +399,7 @@ def test_function_definitions():
     x = 'std::vector<std::pair<std::string, int>> &module::test(register int ' \
         'foo, bar, std::string baz = "foobar, blah, bleh") const = 0'
     check('function', x, {1: "module::test__i.bar.ssC",
-          2: "NK6module4testEi3barNSt6stringE"})
+                          2: "NK6module4testEi3barNSt6stringE"})
     check('function', 'void f(std::pair<A, B>)',
           {1: "f__std::pair:A.B:", 2: "1fNSt4pairI1A1BEE"})
     check('function', 'explicit module::myclass::foo::foo()',
@@ -426,6 +433,10 @@ def test_function_definitions():
           {1: "get_valueCE", 2: "9get_valuev"})
     check('function', 'int get_value() const noexcept',
           {1: "get_valueC", 2: "NK9get_valueEv"})
+    check('function', 'int get_value() const noexcept(std::is_nothrow_move_constructible<T>::value)',
+          {1: "get_valueC", 2: "NK9get_valueEv"})
+    check('function', 'int get_value() const noexcept("see below")',
+          {1: "get_valueC", 2: "NK9get_valueEv"})
     check('function', 'int get_value() const noexcept = delete',
           {1: "get_valueC", 2: "NK9get_valueEv"})
     check('function', 'int get_value() volatile const',
@@ -867,7 +878,7 @@ def test_xref_parsing():
 
 
 def filter_warnings(warning, file):
-    lines = warning.getvalue().split("\n");
+    lines = warning.getvalue().split("\n")
     res = [l for l in lines if "domain-cpp" in l and "{}.rst".format(file) in l and
            "WARNING: document isn't included in any toctree" not in l]
     print("Filtered warnings for file '{}':".format(file))
@@ -902,6 +913,13 @@ def test_build_domain_cpp_backslash_ok(app, status, warning):
     assert len(ws) == 0
 
 
+@pytest.mark.sphinx(testroot='domain-cpp', confoverrides={'nitpicky': True})
+def test_build_domain_cpp_semicolon(app, status, warning):
+    app.builder.build_all()
+    ws = filter_warnings(warning, "semicolon")
+    assert len(ws) == 0
+
+
 @pytest.mark.sphinx(testroot='domain-cpp',
                     confoverrides={'nitpicky': True, 'strip_signature_backslash': True})
 def test_build_domain_cpp_backslash_ok(app, status, warning):

tests/test_ext_autodoc_private_members.py

@@ -22,6 +22,14 @@ def test_private_field(app):
         '',
         '.. py:module:: target.private',
         '',
+        '',
+        '.. py:function:: _public_function(name)',
+        '   :module: target.private',
+        '',
+        '   public_function is a docstring().',
+        '',
+        '   :meta public:',
+        '',
     ]
 
 
@@ -36,6 +44,14 @@ def test_private_field_and_private_members(app):
         '.. py:module:: target.private',
         '',
         '',
+        '.. py:function:: _public_function(name)',
+        '   :module: target.private',
+        '',
+        '   public_function is a docstring().',
+        '',
+        '   :meta public:',
+        '',
+        '',
         '.. py:function:: private_function(name)',
         '   :module: target.private',
         '',

tests/test_ext_autosummary.py

@@ -19,9 +19,10 @@ from sphinx import addnodes
 from sphinx.ext.autosummary import (
     autosummary_table, autosummary_toc, mangle_signature, import_by_name, extract_summary
 )
-from sphinx.ext.autosummary.generate import AutosummaryEntry, generate_autosummary_docs
+from sphinx.ext.autosummary.generate import AutosummaryEntry, generate_autosummary_docs, main as autogen_main
 from sphinx.testing.util import assert_node, etree_parse
 from sphinx.util.docutils import new_document
+from sphinx.util.osutil import cd
 
 html_warnfile = StringIO()
 
@@ -197,7 +198,7 @@ def test_autosummary_generate(app, status, warning):
                           nodes.paragraph,
                           addnodes.tabular_col_spec,
                           autosummary_table,
-                          autosummary_toc))
+                          [autosummary_toc, addnodes.toctree]))
     assert_node(doctree[3],
                 [autosummary_table, nodes.table, nodes.tgroup, (nodes.colspec,
                                                                 nodes.colspec,
@@ -205,6 +206,8 @@ def test_autosummary_generate(app, status, warning):
                                                                                nodes.row,
                                                                                nodes.row,
                                                                                nodes.row)])])
+    assert_node(doctree[4][0], addnodes.toctree, caption="An autosummary")
+
     assert doctree[3][0][0][2][0].astext() == 'autosummary_dummy_module\n\n'
     assert doctree[3][0][0][2][1].astext() == 'autosummary_dummy_module.Foo()\n\n'
     assert doctree[3][0][0][2][2].astext() == 'autosummary_dummy_module.bar(x[, y])\n\n'
@@ -259,6 +262,31 @@ def test_autosummary_generate_overwrite2(app_params, make_app):
     assert 'autosummary_dummy_module.rst' not in app._warning.getvalue()
 
 
+@pytest.mark.sphinx('dummy', testroot='ext-autosummary-recursive')
+def test_autosummary_recursive(app, status, warning):
+    app.build()
+
+    # autosummary having :recursive: option
+    assert (app.srcdir / 'generated' / 'package.rst').exists()
+    assert (app.srcdir / 'generated' / 'package.module.rst').exists()
+    assert (app.srcdir / 'generated' / 'package.module_importfail.rst').exists() is False
+    assert (app.srcdir / 'generated' / 'package.package.rst').exists()
+    assert (app.srcdir / 'generated' / 'package.package.module.rst').exists()
+
+    # autosummary not having :recursive: option
+    assert (app.srcdir / 'generated' / 'package2.rst').exists()
+    assert (app.srcdir / 'generated' / 'package2.module.rst').exists() is False
+
+    # Check content of recursively generated stub-files
+    content = (app.srcdir / 'generated' / 'package.rst').read_text()
+    assert 'package.module' in content
+    assert 'package.package' in content
+    assert 'package.module_importfail' in content
+
+    content = (app.srcdir / 'generated' / 'package.package.rst').read_text()
+    assert 'package.package.module' in content
+
+
 @pytest.mark.sphinx('latex', **default_kw)
 def test_autosummary_latex_table_colspec(app, status, warning):
     app.builder.build_all()
@@ -328,7 +356,7 @@ def test_autosummary_imported_members(app, status, warning):
 @pytest.mark.sphinx(testroot='ext-autodoc')
 def test_generate_autosummary_docs_property(app):
     with patch('sphinx.ext.autosummary.generate.find_autosummary_in_files') as mock:
-        mock.return_value = [AutosummaryEntry('target.methods.Base.prop', 'prop', None)]
+        mock.return_value = [AutosummaryEntry('target.methods.Base.prop', 'prop', None, False)]
         generate_autosummary_docs([], output_dir=app.srcdir, builder=app.builder, app=app)
 
     content = (app.srcdir / 'target.methods.Base.prop.rst').read_text()
@@ -361,3 +389,10 @@ def test_empty_autosummary_generate(app, status, warning):
                     confoverrides={'autosummary_generate': ['unknown']})
 def test_invalid_autosummary_generate(app, status, warning):
     assert 'WARNING: autosummary_generate: file not found: unknown.rst' in warning.getvalue()
+
+
+def test_autogen(rootdir, tempdir):
+    with cd(rootdir / 'test-templating'):
+        args = ['-o', tempdir, '-t', '.', 'autosummary_templating.txt']
+        autogen_main(args)
+        assert (tempdir / 'sphinx.application.TemplateBridge.rst').exists()