Merge branch 'master' into feature-autosummary-packages

.travis.yml

@@ -13,13 +13,14 @@ matrix:
   include:
     - python: '3.5'
       env:
-        - TOXENV=du13
+        - TOXENV=du12
     - python: '3.6'
       env:
-        - TOXENV=py36
+        - TOXENV=du13
-        - PYTEST_ADDOPTS="--cov ./ --cov-append --cov-config setup.cfg"
     - python: '3.7'
-      env: TOXENV=py37
+      env:
+        - TOXENV=py37
+        - PYTEST_ADDOPTS="--cov ./ --cov-append --cov-config setup.cfg"
     - python: 'nightly'
       env: TOXENV=py38
     - python: '3.6'

CHANGES

@@ -1,9 +1,51 @@
-Release 2.0.0 (in development)
+Release 2.1.0 (in development)
 ==============================
 
 Dependencies
 ------------
 
+Incompatible changes
+--------------------
+
+Deprecated
+----------
+
+Features added
+--------------
+
+Bugs fixed
+----------
+
+Testing
+--------
+
+Release 2.0.0 beta2 (in development)
+====================================
+
+Dependencies
+------------
+
+Incompatible changes
+--------------------
+
+Deprecated
+----------
+
+Features added
+--------------
+
+Bugs fixed
+----------
+
+Testing
+--------
+
+Release 2.0.0 beta1 (in development)
+====================================
+
+Dependencies
+------------
+
 * LaTeX builder now depends on TeX Live 2015 or above.
 * LaTeX builder (with ``'pdflatex'`` :confval:`latex_engine`) will process
   Unicode Greek letters in text (not in math mark-up) via the text font and
@@ -23,7 +65,9 @@ Dependencies
 
   - sphinxcontrib.applehelp
   - sphinxcontrib.devhelp
+  - sphinxcontrib.htmlhelp
   - sphinxcontrib.jsmath
+  - sphinxcontrib.serializinghtml
   - sphinxcontrib.qthelp
 
 Incompatible changes
@@ -59,12 +103,17 @@ Incompatible changes
   from LaTeX preamble now get overwritten.  Use ``\sphinxtableofcontentshook``
   to insert custom user definitions.  See :ref:`latex-macros`.
 * quickstart: Simplify generated ``conf.py``
+* #4148: quickstart: some questions are removed.  They are still able to specify
+  via command line options
 * websupport: unbundled from sphinx core. Please use sphinxcontrib-websupport
 * C++, the visibility of base classes is now always rendered as present in the
   input. That is, ``private`` is now shown, where it was ellided before.
 * LaTeX: graphics inclusion of oversized images rescales to not exceed
   the text width and height, even if width and/or height option were used.
   (refs: #5956)
+* epub: ``epub_title`` defaults to the :confval:`project` option
+* #4550: All tables and figures without ``align`` option are displayed to center
+* #4587: html: Output HTML5 by default
 
 Deprecated
 ----------
@@ -75,6 +124,10 @@ Deprecated
   ``EpubBuilder.build_container()``, ``EpubBuilder.bulid_content()``,
   ``EpubBuilder.build_toc()`` and ``EpubBuilder.build_epub()``
 * The arguments of ``Epub3Builder.build_navigation_doc()``
+* The config variables
+
+  - :confval:`html_experimental_html5_writer`
+
 * The ``encoding`` argument of ``autodoc.Documenter.get_doc()``,
   ``autodoc.DocstringSignatureMixin.get_doc()``,
   ``autodoc.DocstringSignatureMixin._find_signature()``, and
@@ -108,21 +161,30 @@ Deprecated
 * ``sphinx.io.SphinxFileInput.supported``
 * ``sphinx.io.SphinxRSTFileInput``
 * ``sphinx.registry.SphinxComponentRegistry.add_source_input()``
+* ``sphinx.roles.abbr_role()``
+* ``sphinx.roles.emph_literal_role()``
+* ``sphinx.roles.menusel_role()``
+* ``sphinx.roles.index_role()``
+* ``sphinx.roles.indexmarkup_role()``
 * ``sphinx.testing.util.remove_unicode_literal()``
 * ``sphinx.util.attrdict``
 * ``sphinx.util.force_decode()``
 * ``sphinx.util.get_matching_docs()``
 * ``sphinx.util.inspect.Parameter``
+* ``sphinx.util.jsonimpl``
 * ``sphinx.util.osutil.EEXIST``
 * ``sphinx.util.osutil.EINVAL``
 * ``sphinx.util.osutil.ENOENT``
 * ``sphinx.util.osutil.EPIPE``
 * ``sphinx.util.osutil.walk()``
 * ``sphinx.util.PeekableIterator``
+* ``sphinx.util.pycompat.NoneType``
 * ``sphinx.util.pycompat.TextIOWrapper``
 * ``sphinx.util.pycompat.UnicodeMixin``
 * ``sphinx.util.pycompat.htmlescape``
 * ``sphinx.util.pycompat.indent``
+* ``sphinx.util.pycompat.sys_encoding``
+* ``sphinx.util.pycompat.terminal_safe()``
 * ``sphinx.util.pycompat.u``
 * ``sphinx.writers.latex.ExtBabel``
 * ``sphinx.writers.latex.LaTeXTranslator._make_visit_admonition()``
@@ -154,6 +216,11 @@ Features added
 
 * #4182: autodoc: Support :confval:`suppress_warnings`
 * #5533: autodoc: :confval:`autodoc_default_options` supports ``member-order``
+* #5394: autodoc: Display readable names in type annotations for mocked objects
+* #5459: autodoc: :confval:`autodoc_default_options` accepts ``True`` as a value
+* #1148: autodoc: Add :rst:dir:`autodecorator` directive for decorators
+* #5635: autosummary: Add :confval:`autosummary_mock_imports` to mock external
+  libraries on importing targets
 * #4018: htmlhelp: Add :confval:`htmlhelp_file_suffix` and
   :confval:`htmlhelp_link_suffix`
 * #5559: text: Support complex tables (colspan and rowspan)
@@ -173,10 +240,17 @@ Features added
 * #4611: epub: Show warning for duplicated ToC entries
 * #1851: Allow to omit an argument for :rst:dir:`code-block` directive.  If
   omitted, it follows :rst:dir:`highlight` or :confval:`highlight_language`
+* #4587: html: Add :confval:`html4_writer` to use old HTML4 writer
 * #6016: HTML search: A placeholder for the search summary prevents search
   result links from changing their position when the search terminates.  This
   makes navigating search results easier.
-  
+* #5196: linkcheck also checks remote images exist
+* #5924: githubpages: create CNAME file for custom domains when
+  :confval:`html_baseurl` set
+* #4261: autosectionlabel: restrict the labeled sections by new config value;
+  :confval:`autosectionlabel_maxdepth`
+
+
 Bugs fixed
 ----------
 
@@ -225,6 +299,10 @@ Bugs fixed
 ----------
 
 * LaTeX: Remove extraneous space after author names on PDF title page (refs: #6004)
+* #6026: LaTeX: A cross reference to definition list does not work
+* #6046: LaTeX: ``TypeError`` is raised when invalid latex_elements given
+* #6019: imgconverter: Including multipage PDF fails
+* #6047: autodoc: ``autofunction`` emits a warning for method objects
 
 Testing
 --------

bindep.txt

@@ -0,0 +1,15 @@
+texlive [platform:rpm]
+texlive-fncychap [platform:rpm]
+texlive-titlesec [platform:rpm]
+texlive-tabulary [platform:rpm]
+texlive-framed [platform:rpm]
+texlive-wrapfig [platform:rpm]
+texlive-upquote [platform:rpm]
+texlive-capt-of [platform:rpm]
+texlive-needspace [platform:rpm]
+texlive-polyglossia [platform:rpm]
+texlive-luatex85 [platform:rpm]
+texlive-anyfontsize [platform:rpm]
+texlive-ctablestack [platform:rpm]
+texlive-gnu-freefont [platform:rpm]
+latexmk [platform:rpm]

doc/_themes/sphinx13/static/sphinx13.css

@@ -337,7 +337,7 @@ a tt:hover {
 }
 
 pre {
-    font-family: 'Consolas', 'DejaVu Sans Mono',
+    font-family: 'Consolas', 'Courier New', 'DejaVu Sans Mono',
                  'Bitstream Vera Sans Mono', monospace;
     font-size: 13px;
     letter-spacing: 0.015em;
@@ -388,32 +388,29 @@ div.admonition, div.warning {
     padding: 0;
 }
 
-div.admonition p, div.warning p {
+div.admonition > p, div.warning > p {
     margin: 0.5em 1em 0.5em 1em;
     padding: 0;
 }
 
-div.admonition pre, div.warning pre {
+div.admonition > pre, div.warning > pre {
     margin: 0.4em 1em 0.4em 1em;
 }
 
-div.admonition p.admonition-title,
+div.admonition > p.admonition-title,
-div.warning p.admonition-title {
+div.warning > p.admonition-title {
-    margin-top: 1em;
+    margin-top: 0.5em;
-    padding-top: 0.5em;
     font-weight: bold;
 }
 
 div.warning {
     border: 1px solid #940000;
-/*    background-color: #FFCCCF;*/
 }
 
-div.warning p.admonition-title {
+div.admonition > ul,
-}
+div.admonition > ol,
-
+div.warning > ul,
-div.admonition ul, div.admonition ol,
+div.warning > ol {
-div.warning ul, div.warning ol {
     margin: 0.1em 0.5em 0.5em 3em;
     padding: 0;
 }

doc/extdev/index.rst

@@ -290,11 +290,36 @@ The following is a list of deprecated interfaces.
      - 4.0
      - ``sphinx.builders.epub3.validate_config_values()``
 
+   * - ``sphinx.builders.html.JSONHTMLBuilder``
+     - 2.0
+     - 4.0
+     - ``sphinx.builders.serializinghtml.JSONHTMLBuilder``
+
+   * - ``sphinx.builders.html.PickleHTMLBuilder``
+     - 2.0
+     - 4.0
+     - ``sphinx.builders.serializinghtml.PickleHTMLBuilder``
+
+   * - ``sphinx.builders.html.SerializingHTMLBuilder``
+     - 2.0
+     - 4.0
+     - ``sphinx.builders.serializinghtml.SerializingHTMLBuilder``
+
    * - ``sphinx.builders.html.SingleFileHTMLBuilder``
      - 2.0
      - 4.0
      - ``sphinx.builders.singlehtml.SingleFileHTMLBuilder``
 
+   * - ``sphinx.builders.html.WebHTMLBuilder``
+     - 2.0
+     - 4.0
+     - ``sphinx.builders.serializinghtml.PickleHTMLBuilder``
+
+   * - ``sphinx.builders.htmlhelp``
+     - 2.0
+     - 4.0
+     - ``sphinxcontrib.htmlhelp``
+
    * - ``sphinx.builders.htmlhelp.HTMLHelpBuilder.open_file()``
      - 2.0
      - 4.0
@@ -370,6 +395,31 @@ The following is a list of deprecated interfaces.
      - 4.0
      - ``sphinxcontrib.jsmath``
 
+   * - ``sphinx.roles.abbr_role()``
+     - 2.0
+     - 4.0
+     - ``sphinx.roles.Abbreviation``
+
+   * - ``sphinx.roles.emph_literal_role()``
+     - 2.0
+     - 4.0
+     - ``sphinx.roles.EmphasizedLiteral``
+
+   * - ``sphinx.roles.menusel_role()``
+     - 2.0
+     - 4.0
+     - ``sphinx.roles.GUILabel`` or ``sphinx.roles.MenuSelection``
+
+   * - ``sphinx.roles.index_role()``
+     - 2.0
+     - 4.0
+     - ``sphinx.roles.Index``
+
+   * - ``sphinx.roles.indexmarkup_role()``
+     - 2.0
+     - 4.0
+     - ``sphinx.roles.PEP`` or ``sphinx.roles.RFC``
+
    * - ``sphinx.testing.util.remove_unicode_literal()``
      - 2.0
      - 4.0
@@ -395,6 +445,11 @@ The following is a list of deprecated interfaces.
      - 3.0
      - N/A
 
+   * - ``sphinx.util.jsonimpl``
+     - 2.0
+     - 4.0
+     - ``sphinxcontrib.serializinghtml.jsonimpl``
+
    * - ``sphinx.util.osutil.EEXIST``
      - 2.0
      - 4.0
@@ -420,6 +475,11 @@ The following is a list of deprecated interfaces.
      - 4.0
      - ``os.walk()``
 
+   * - ``sphinx.util.pycompat.NoneType``
+     - 2.0
+     - 4.0
+     - ``sphinx.util.typing.NoneType``
+
    * - ``sphinx.util.pycompat.TextIOWrapper``
      - 2.0
      - 4.0
@@ -440,6 +500,16 @@ The following is a list of deprecated interfaces.
      - 4.0
      - ``textwrap.indent()``
 
+   * - ``sphinx.util.pycompat.sys_encoding``
+     - 2.0
+     - 4.0
+     - ``sys.getdefaultencoding()``
+
+   * - ``sphinx.util.pycompat.terminal_safe()``
+     - 2.0
+     - 4.0
+     - ``sphinx.util.console.terminal_safe()``
+
    * - ``sphinx.util.pycompat.u``
      - 2.0
      - 4.0

doc/extdev/utils.rst

@@ -18,5 +18,11 @@ components (e.g. :class:`.Config`, :class:`.BuildEnvironment` and so on) easily.
 .. autoclass:: sphinx.util.docutils.SphinxDirective
    :members:
 
+.. autoclass:: sphinx.util.docutils.SphinxRole
+   :members:
+
+.. autoclass:: sphinx.util.docutils.ReferenceRole
+   :members:
+
 .. autoclass:: sphinx.transforms.post_transforms.images.ImageConverter
    :members:

doc/usage/builders/index.rst

@@ -28,6 +28,7 @@ The builder's "name" must be given to the **-b** command-line option of
 
    .. autoattribute:: supported_image_types
 
+.. module:: sphinx.builders.dirhtml
 .. class:: DirectoryHTMLBuilder
 
    This is a subclass of the standard HTML builder.  Its output is a directory
@@ -45,6 +46,7 @@ The builder's "name" must be given to the **-b** command-line option of
 
    .. versionadded:: 0.6
 
+.. module:: sphinx.builders.singlehtml
 .. class:: SingleFileHTMLBuilder
 
    This is an HTML builder that combines the whole project in one output file.
@@ -297,7 +299,7 @@ name is ``rinoh``. Refer to the `rinohtype manual`_ for details.
    .. versionadded:: 1.1
 
 
-.. currentmodule:: sphinx.builders.html
+.. currentmodule:: sphinxcontrib.serializinghtml
 .. class:: SerializingHTMLBuilder
 
    This builder uses a module that implements the Python serialization API

doc/usage/configuration.rst

@@ -1331,6 +1331,12 @@ that use Sphinx's HTMLWriter class.
 
    .. versionadded:: 1.6
 
+   .. deprecated:: 2.0
+
+.. confval:: html4_writer
+
+   Output is processed with HTML4 writer.  Default is ``False``.
+
 Options for Single HTML output
 -------------------------------
 
@@ -1561,7 +1567,11 @@ the `Dublin Core metadata <http://dublincore.org/>`_.
 .. confval:: epub_title
 
    The title of the document.  It defaults to the :confval:`html_title` option
-   but can be set independently for epub creation.
+   but can be set independently for epub creation.  It defaults to the
+   :confval:`project` option.
+
+   .. versionchanged:: 2.0
+      It defaults to the ``project`` option.
 
 .. confval:: epub_description
 

doc/usage/extensions/autodoc.rst

@@ -237,6 +237,7 @@ inserting them into the page source under a suitable :rst:dir:`py:module`,
 
 
 .. rst:directive:: autofunction
+                   autodecorator
                    autodata
                    automethod
                    autoattribute
@@ -293,10 +294,11 @@ inserting them into the page source under a suitable :rst:dir:`py:module`,
       docstrings.
    .. versionchanged:: 1.1
       Comment docs are now allowed on the same line after an assignment.
-
    .. versionchanged:: 1.2
       :rst:dir:`autodata` and :rst:dir:`autoattribute` have an ``annotation``
       option.
+   .. versionchanged:: 2.0
+      :rst:dir:`autodecorator` added.
 
    .. note::
 
@@ -376,12 +378,12 @@ There are also new config values that you can set:
            'members': 'var1, var2',
            'member-order': 'bysource',
            'special-members': '__init__',
-           'undoc-members': None,
+           'undoc-members': True,
            'exclude-members': '__weakref__'
        }
 
-   Setting ``None`` is equivalent to giving the option name in the list format
+   Setting ``None`` or ``True`` to the value is equivalent to giving only the
-   (i.e. it means "yes/true/on").
+   option name to the directives.
 
    The supported options are ``'members'``, ``'member-order'``,
    ``'undoc-members'``, ``'private-members'``, ``'special-members'``,
@@ -390,6 +392,9 @@ There are also new config values that you can set:
 
    .. versionadded:: 1.8
 
+   .. versionchanged:: 2.0
+      Accepts ``True`` as a value.
+
 .. confval:: autodoc_docstring_signature
 
    Functions imported from C modules cannot be introspected, and therefore the

doc/usage/extensions/autosectionlabel.rst

@@ -38,3 +38,10 @@ Configuration
    called ``Introduction`` that appears in document ``index.rst``.  Useful for
    avoiding ambiguity when the same section heading appears in different
    documents.
+
+.. confval:: autosectionlabel_maxdepth
+
+   If set, autosectionlabel chooses the sections for labeling by its depth. For
+   example, when set 1 to ``autosectionlabel_maxdepth``, labels are generated
+   only for top level sections, and deeper sections are not labeled.  It
+   defaults to ``None`` (disabled).

doc/usage/extensions/autosummary.rst

@@ -181,6 +181,12 @@ Packages can be explored recursively when generating stub pages.
 
    .. versionadded:: 2.0
 
+.. confval:: autosummary_mock_imports
+
+   This value contains a list of modules to be mocked up.  See
+   :confval:`autodoc_mock_imports` for more details.  It defaults to
+   :confval:`autodoc_mock_imports`.
+
 Customizing templates
 ---------------------
 

doc/usage/extensions/githubpages.rst

@@ -6,5 +6,11 @@
 
 .. versionadded:: 1.4
 
+.. versionchanged:: 2.0
+   Support ``CNAME`` file
+
 This extension creates ``.nojekyll`` file on generated HTML directory to publish
 the document on GitHub Pages.
+
+It also creates a ``CNAME`` file for custom domains when :confval:`html_baseurl`
+set.

setup.py

@@ -18,6 +18,8 @@ install_requires = [
     'sphinxcontrib-applehelp',
     'sphinxcontrib-devhelp',
     'sphinxcontrib-jsmath',
+    'sphinxcontrib-htmlhelp',
+    'sphinxcontrib-serializinghtml',
     'sphinxcontrib-qthelp',
     'Jinja2>=2.3',
     'Pygments>=2.0',

sphinx/__init__.py

@@ -32,8 +32,8 @@ if 'PYTHONWARNINGS' not in os.environ:
 warnings.filterwarnings('ignore', "'U' mode is deprecated",
                         DeprecationWarning, module='docutils.io')
 
-__version__ = '2.0.0+'
+__version__ = '2.1.0+'
-__released__ = '2.0.0'  # used when Sphinx builds its own docs
+__released__ = '2.1.0'  # used when Sphinx builds its own docs
 
 #: Version info for better programmatic use.
 #:
@@ -43,7 +43,7 @@ __released__ = '2.0.0'  # used when Sphinx builds its own docs
 #:
 #: .. versionadded:: 1.2
 #:    Before version 1.2, check the string ``sphinx.__version__``.
-version_info = (2, 0, 0, 'beta', 0)
+version_info = (2, 1, 0, 'beta', 0)
 
 package_dir = path.abspath(path.dirname(__file__))
 

sphinx/application.py

@@ -37,7 +37,6 @@ from sphinx.registry import SphinxComponentRegistry
 from sphinx.util import docutils
 from sphinx.util import import_object, progress_message
 from sphinx.util import logging
-from sphinx.util import pycompat  # noqa: F401
 from sphinx.util.build_phase import BuildPhase
 from sphinx.util.console import bold  # type: ignore
 from sphinx.util.docutils import directive_helper
@@ -68,7 +67,6 @@ builtin_extensions = (
     'sphinx.builders.dummy',
     'sphinx.builders.gettext',
     'sphinx.builders.html',
-    'sphinx.builders.htmlhelp',
     'sphinx.builders.latex',
     'sphinx.builders.linkcheck',
     'sphinx.builders.manpage',
@@ -108,6 +106,8 @@ builtin_extensions = (
     # 1st party extensions
     'sphinxcontrib.applehelp',
     'sphinxcontrib.devhelp',
+    'sphinxcontrib.htmlhelp',
+    'sphinxcontrib.serializinghtml',
     'sphinxcontrib.qthelp',
     # Strictly, alabaster theme is not a builtin extension,
     # but it is loaded automatically to use it as default theme.

sphinx/builders/__init__.py

@@ -19,7 +19,7 @@ from sphinx.environment.adapters.asset import ImageAdapter
 from sphinx.errors import SphinxError
 from sphinx.io import read_doc
 from sphinx.locale import __
-from sphinx.util import i18n, import_object, logging, rst, status_iterator
+from sphinx.util import i18n, import_object, logging, rst, progress_message, status_iterator
 from sphinx.util.build_phase import BuildPhase
 from sphinx.util.console import bold  # type: ignore
 from sphinx.util.docutils import sphinx_domains
@@ -351,16 +351,14 @@ class Builder:
         if updated_docnames:
             # save the environment
             from sphinx.application import ENV_PICKLE_FILENAME
-            logger.info(bold(__('pickling environment... ')), nonl=True)
+            with progress_message(__('pickling environment')):
-            with open(path.join(self.doctreedir, ENV_PICKLE_FILENAME), 'wb') as f:
+                with open(path.join(self.doctreedir, ENV_PICKLE_FILENAME), 'wb') as f:
-                pickle.dump(self.env, f, pickle.HIGHEST_PROTOCOL)
+                    pickle.dump(self.env, f, pickle.HIGHEST_PROTOCOL)
-            logger.info(__('done'))
 
             # global actions
             self.app.phase = BuildPhase.CONSISTENCY_CHECK
-            logger.info(bold(__('checking consistency... ')), nonl=True)
+            with progress_message(__('checking consistency')):
-            self.env.check_consistency()
+                self.env.check_consistency()
-            logger.info(__('done'))
         else:
             if method == 'update' and not docnames:
                 logger.info(bold(__('no targets are out of date.')))
@@ -559,9 +557,8 @@ class Builder:
                     docnames.add(tocdocname)
         docnames.add(self.config.master_doc)
 
-        logger.info(bold(__('preparing documents... ')), nonl=True)
+        with progress_message(__('preparing documents')):
-        self.prepare_writing(docnames)
+            self.prepare_writing(docnames)
-        logger.info(__('done'))
 
         if self.parallel_ok:
             # number of subprocesses is parallel-1 because the main process

sphinx/builders/_epub_base.py

@@ -134,8 +134,6 @@ class EpubBuilder(StandaloneHTMLBuilder):
     html_scaled_image_link = False
     # don't generate search index or include search page
     search = False
-    # use html5 translator by default
-    default_html5_translator = True
 
     coverpage_name = COVERPAGE_NAME
     toctree_template = TOCTREE_TEMPLATE
@@ -655,7 +653,7 @@ class EpubBuilder(StandaloneHTMLBuilder):
         if incr:
             self.playorder += 1
         self.tocid += 1
-        return NavPoint(self.esc('navPoint%d' % self.tocid), self.playorder,
+        return NavPoint('navPoint%d' % self.tocid, self.playorder,
                         node['text'], node['refuri'], [])
 
     def build_navpoints(self, nodes):

sphinx/builders/epub3.py

@@ -269,7 +269,7 @@ def setup(app):
     app.add_config_value('epub_version', 3.0, 'epub')  # experimental
     app.add_config_value('epub_theme', 'epub', 'epub')
     app.add_config_value('epub_theme_options', {}, 'epub')
-    app.add_config_value('epub_title', lambda self: self.html_title, 'epub')
+    app.add_config_value('epub_title', lambda self: self.project, 'epub')
     app.add_config_value('epub_author', lambda self: self.author, 'epub')
     app.add_config_value('epub_language', lambda self: self.language or 'en', 'epub')
     app.add_config_value('epub_publisher', lambda self: self.author, 'epub')

sphinx/builders/html.py

@@ -9,16 +9,13 @@
 """
 
 import html
-import pickle
 import posixpath
 import re
 import sys
-import types
 import warnings
 from hashlib import md5
 from os import path
 
-import docutils
 from docutils import nodes
 from docutils.core import publish_parts
 from docutils.frontend import OptionParser
@@ -26,7 +23,6 @@ from docutils.io import DocTreeInput, StringOutput
 from docutils.utils import relative_path
 
 from sphinx import package_dir, __display_version__
-from sphinx.application import ENV_PICKLE_FILENAME
 from sphinx.builders import Builder
 from sphinx.deprecation import (
     RemovedInSphinx30Warning, RemovedInSphinx40Warning, deprecated_alias
@@ -39,15 +35,14 @@ from sphinx.highlighting import PygmentsBridge
 from sphinx.locale import _, __
 from sphinx.search import js_index
 from sphinx.theming import HTMLThemeFactory
-from sphinx.util import jsonimpl, logging, status_iterator
+from sphinx.util import logging, status_iterator
 from sphinx.util.console import bold  # type: ignore
 from sphinx.util.docutils import is_html5_writer_available, new_document
 from sphinx.util.fileutil import copy_asset
 from sphinx.util.i18n import format_date
 from sphinx.util.inventory import InventoryFile
 from sphinx.util.matching import patmatch, Matcher, DOTFILES
-from sphinx.util.osutil import SEP, os_path, relative_uri, ensuredir, \
+from sphinx.util.osutil import os_path, relative_uri, ensuredir, movefile, copyfile
-    movefile, copyfile
 from sphinx.writers.html import HTMLWriter, HTMLTranslator
 
 if False:
@@ -58,7 +53,7 @@ if False:
     from sphinx.domains import Domain, Index, IndexEntry  # NOQA
     from sphinx.util.tags import Tags  # NOQA
 
-# Experimental HTML5 Writer
+# HTML5 Writer is avialable or not
 if is_html5_writer_available():
     from sphinx.writers.html5 import HTML5Translator
     html5_ready = True
@@ -67,8 +62,6 @@ else:
 
 #: the filename for the inventory of objects
 INVENTORY_FILENAME = 'objects.inv'
-#: the filename for the "last build" file (for serializing builders)
-LAST_BUILD_FILENAME = 'last_build'
 
 logger = logging.getLogger(__name__)
 return_codes_re = re.compile('[\r\n]+')
@@ -116,24 +109,24 @@ class JSContainer(list):
     """The container for JavaScript scripts."""
     def insert(self, index, obj):
         # type: (int, str) -> None
-        warnings.warn('builder.script_files is deprecated. '
+        warnings.warn('To modify script_files in the theme is deprecated. '
-                      'Please use app.add_js_file() instead.',
+                      'Please insert a <script> tag directly in your theme instead.',
-                      RemovedInSphinx30Warning, stacklevel=2)
+                      RemovedInSphinx30Warning, stacklevel=3)
         super().insert(index, obj)
 
     def extend(self, other):  # type: ignore
         # type: (List[str]) -> None
-        warnings.warn('builder.script_files is deprecated. '
+        warnings.warn('To modify script_files in the theme is deprecated. '
-                      'Please use app.add_js_file() instead.',
+                      'Please insert a <script> tag directly in your theme instead.',
-                      RemovedInSphinx30Warning, stacklevel=2)
+                      RemovedInSphinx30Warning, stacklevel=3)
         for item in other:
             self.append(item)
 
     def __iadd__(self, other):  # type: ignore
         # type: (List[str]) -> JSContainer
-        warnings.warn('builder.script_files is deprecated. '
+        warnings.warn('To modify script_files in the theme is deprecated. '
-                      'Please use app.add_js_file() instead.',
+                      'Please insert a <script> tag directly in your theme instead.',
-                      RemovedInSphinx30Warning, stacklevel=2)
+                      RemovedInSphinx30Warning, stacklevel=3)
         for item in other:
             self.append(item)
         return self
@@ -242,8 +235,6 @@ class StandaloneHTMLBuilder(Builder):
     search = True  # for things like HTML help and Apple help: suppress search
     use_index = False
     download_support = True  # enable download role
-    # use html5 translator by default
-    default_html5_translator = False
 
     imgpath = None          # type: str
     domain_indices = []     # type: List[Tuple[str, Type[Index], List[Tuple[str, List[IndexEntry]]], bool]]  # NOQA
@@ -285,11 +276,6 @@ class StandaloneHTMLBuilder(Builder):
 
         self.use_index = self.get_builder_config('use_index', 'html')
 
-        if self.config.html_experimental_html5_writer and not html5_ready:
-            logger.warning(__('html_experimental_html5_writer is set, but current version '
-                              'is old. Docutils\' version should be 0.13 or newer, but %s.'),
-                           docutils.__version__)
-
     def create_build_info(self):
         # type: () -> BuildInfo
         return BuildInfo(self.config, self.tags, ['html'])
@@ -374,14 +360,10 @@ class StandaloneHTMLBuilder(Builder):
     @property
     def default_translator_class(self):  # type: ignore
         # type: () -> Type[nodes.NodeVisitor]
-        use_html5_writer = self.config.html_experimental_html5_writer
+        if not html5_ready or self.config.html4_writer:
-        if use_html5_writer is None:
-            use_html5_writer = self.default_html5_translator
-
-        if use_html5_writer and html5_ready:
-            return HTML5Translator
-        else:
             return HTMLTranslator
+        else:
+            return HTML5Translator
 
     @property
     def math_renderer_name(self):
@@ -562,7 +544,7 @@ class StandaloneHTMLBuilder(Builder):
             'parents': [],
             'logo': logo,
             'favicon': favicon,
-            'html5_doctype': self.config.html_experimental_html5_writer and html5_ready,
+            'html5_doctype': html5_ready and not self.config.html4_writer
         }
         if self.theme:
             self.globalcontext.update(
@@ -1169,140 +1151,6 @@ class StandaloneHTMLBuilder(Builder):
         logger.info(__('done'))
 
 
-class SerializingHTMLBuilder(StandaloneHTMLBuilder):
-    """
-    An abstract builder that serializes the generated HTML.
-    """
-    #: the serializing implementation to use.  Set this to a module that
-    #: implements a `dump`, `load`, `dumps` and `loads` functions
-    #: (pickle, simplejson etc.)
-    implementation = None  # type: Any
-    implementation_dumps_unicode = False
-    #: additional arguments for dump()
-    additional_dump_args = ()  # type: Tuple
-
-    #: the filename for the global context file
-    globalcontext_filename = None  # type: str
-
-    supported_image_types = ['image/svg+xml', 'image/png',
-                             'image/gif', 'image/jpeg']
-
-    def init(self):
-        # type: () -> None
-        self.build_info = BuildInfo(self.config, self.tags)
-        self.imagedir = '_images'
-        self.current_docname = None
-        self.theme = None       # no theme necessary
-        self.templates = None   # no template bridge necessary
-        self.init_templates()
-        self.init_highlighter()
-        self.init_css_files()
-        self.init_js_files()
-        self.use_index = self.get_builder_config('use_index', 'html')
-
-    def get_target_uri(self, docname, typ=None):
-        # type: (str, str) -> str
-        if docname == 'index':
-            return ''
-        if docname.endswith(SEP + 'index'):
-            return docname[:-5]  # up to sep
-        return docname + SEP
-
-    def dump_context(self, context, filename):
-        # type: (Dict, str) -> None
-        if self.implementation_dumps_unicode:
-            with open(filename, 'w', encoding='utf-8') as ft:
-                self.implementation.dump(context, ft, *self.additional_dump_args)
-        else:
-            with open(filename, 'wb') as fb:
-                self.implementation.dump(context, fb, *self.additional_dump_args)
-
-    def handle_page(self, pagename, ctx, templatename='page.html',
-                    outfilename=None, event_arg=None):
-        # type: (str, Dict, str, str, Any) -> None
-        ctx['current_page_name'] = pagename
-        self.add_sidebars(pagename, ctx)
-
-        if not outfilename:
-            outfilename = path.join(self.outdir,
-                                    os_path(pagename) + self.out_suffix)
-
-        # we're not taking the return value here, since no template is
-        # actually rendered
-        self.app.emit('html-page-context', pagename, templatename, ctx, event_arg)
-
-        # make context object serializable
-        for key in list(ctx):
-            if isinstance(ctx[key], types.FunctionType):
-                del ctx[key]
-
-        ensuredir(path.dirname(outfilename))
-        self.dump_context(ctx, outfilename)
-
-        # if there is a source file, copy the source file for the
-        # "show source" link
-        if ctx.get('sourcename'):
-            source_name = path.join(self.outdir, '_sources',
-                                    os_path(ctx['sourcename']))
-            ensuredir(path.dirname(source_name))
-            copyfile(self.env.doc2path(pagename), source_name)
-
-    def handle_finish(self):
-        # type: () -> None
-        # dump the global context
-        outfilename = path.join(self.outdir, self.globalcontext_filename)
-        self.dump_context(self.globalcontext, outfilename)
-
-        # super here to dump the search index
-        super().handle_finish()
-
-        # copy the environment file from the doctree dir to the output dir
-        # as needed by the web app
-        copyfile(path.join(self.doctreedir, ENV_PICKLE_FILENAME),
-                 path.join(self.outdir, ENV_PICKLE_FILENAME))
-
-        # touch 'last build' file, used by the web application to determine
-        # when to reload its environment and clear the cache
-        open(path.join(self.outdir, LAST_BUILD_FILENAME), 'w').close()
-
-
-class PickleHTMLBuilder(SerializingHTMLBuilder):
-    """
-    A Builder that dumps the generated HTML into pickle files.
-    """
-    name = 'pickle'
-    epilog = __('You can now process the pickle files in %(outdir)s.')
-
-    implementation = pickle
-    implementation_dumps_unicode = False
-    additional_dump_args = (pickle.HIGHEST_PROTOCOL,)
-    indexer_format = pickle
-    indexer_dumps_unicode = False
-    out_suffix = '.fpickle'
-    globalcontext_filename = 'globalcontext.pickle'
-    searchindex_filename = 'searchindex.pickle'
-
-
-# compatibility alias
-WebHTMLBuilder = PickleHTMLBuilder
-
-
-class JSONHTMLBuilder(SerializingHTMLBuilder):
-    """
-    A builder that dumps the generated HTML into JSON files.
-    """
-    name = 'json'
-    epilog = __('You can now process the JSON files in %(outdir)s.')
-
-    implementation = jsonimpl
-    implementation_dumps_unicode = True
-    indexer_format = jsonimpl
-    indexer_dumps_unicode = True
-    out_suffix = '.fjson'
-    globalcontext_filename = 'globalcontext.json'
-    searchindex_filename = 'searchindex.json'
-
-
 def convert_html_css_files(app, config):
     # type: (Sphinx, Config) -> None
     """This converts string styled html_css_files to tuple styled one."""
@@ -1386,11 +1234,19 @@ def validate_math_renderer(app):
 # for compatibility
 from sphinx.builders.dirhtml import DirectoryHTMLBuilder  # NOQA
 from sphinx.builders.singlehtml import SingleFileHTMLBuilder  # NOQA
+from sphinxcontrib.serializinghtml import (  # NOQA
+    LAST_BUILD_FILENAME, JSONHTMLBuilder, PickleHTMLBuilder, SerializingHTMLBuilder
+)
 
 deprecated_alias('sphinx.builders.html',
                  {
+                     'LAST_BUILD_FILENAME': LAST_BUILD_FILENAME,
                      'DirectoryHTMLBuilder': DirectoryHTMLBuilder,
+                     'JSONHTMLBuilder': JSONHTMLBuilder,
+                     'PickleHTMLBuilder': PickleHTMLBuilder,
+                     'SerializingHTMLBuilder': SerializingHTMLBuilder,
                      'SingleFileHTMLBuilder': SingleFileHTMLBuilder,
+                     'WebHTMLBuilder': PickleHTMLBuilder,
                  },
                  RemovedInSphinx40Warning)
 
@@ -1399,8 +1255,6 @@ def setup(app):
     # type: (Sphinx) -> Dict[str, Any]
     # builders
     app.add_builder(StandaloneHTMLBuilder)
-    app.add_builder(PickleHTMLBuilder)
-    app.add_builder(JSONHTMLBuilder)
 
     # config values
     app.add_config_value('html_theme', 'alabaster', 'html')
@@ -1440,9 +1294,9 @@ def setup(app):
     app.add_config_value('html_search_options', {}, 'html')
     app.add_config_value('html_search_scorer', '', None)
     app.add_config_value('html_scaled_image_link', True, 'html')
-    app.add_config_value('html_experimental_html5_writer', None, 'html')
     app.add_config_value('html_baseurl', '', 'html')
     app.add_config_value('html_math_renderer', None, 'env')
+    app.add_config_value('html4_writer', False, 'html')
 
     # event handlers
     app.connect('config-inited', convert_html_css_files)

sphinx/builders/htmlhelp.py

@@ -9,364 +9,36 @@
     :license: BSD, see LICENSE for details.
 """
 
-import html
-import os
 import warnings
-from os import path
 
-from docutils import nodes
+from sphinxcontrib.htmlhelp import (
+    chm_locales, chm_htmlescape, HTMLHelpBuilder, default_htmlhelp_basename
+)
+
+from sphinx.deprecation import RemovedInSphinx40Warning, deprecated_alias
 
-from sphinx import addnodes
-from sphinx.builders.html import StandaloneHTMLBuilder
-from sphinx.deprecation import RemovedInSphinx40Warning
-from sphinx.environment.adapters.indexentries import IndexEntries
-from sphinx.locale import __
-from sphinx.util import logging
-from sphinx.util.nodes import NodeMatcher
-from sphinx.util.osutil import make_filename_from_project
 
 if False:
     # For type annotation
-    from typing import Any, Dict, IO, List, Match, Tuple  # NOQA
+    from typing import Any, Dict  # NOQA
     from sphinx.application import Sphinx  # NOQA
-    from sphinx.config import Config  # NOQA
 
 
-logger = logging.getLogger(__name__)
+deprecated_alias('sphinx.builders.devhelp',
-
+                 {
-
+                     'chm_locales': chm_locales,
-# Project file (*.hhp) template.  'outname' is the file basename (like
+                     'chm_htmlescape': chm_htmlescape,
-# the pythlp in pythlp.hhp); 'version' is the doc version number (like
+                     'HTMLHelpBuilder': HTMLHelpBuilder,
-# the 2.2 in Python 2.2).
+                     'default_htmlhelp_basename': default_htmlhelp_basename,
-# The magical numbers in the long line under [WINDOWS] set most of the
+                 },
-# user-visible features (visible buttons, tabs, etc).
+                 RemovedInSphinx40Warning)
-# About 0x10384e:  This defines the buttons in the help viewer.  The
-# following defns are taken from htmlhelp.h.  Not all possibilities
-# actually work, and not all those that work are available from the Help
-# Workshop GUI.  In particular, the Zoom/Font button works and is not
-# available from the GUI.  The ones we're using are marked with 'x':
-#
-#    0x000002   Hide/Show   x
-#    0x000004   Back        x
-#    0x000008   Forward     x
-#    0x000010   Stop
-#    0x000020   Refresh
-#    0x000040   Home        x
-#    0x000080   Forward
-#    0x000100   Back
-#    0x000200   Notes
-#    0x000400   Contents
-#    0x000800   Locate      x
-#    0x001000   Options     x
-#    0x002000   Print       x
-#    0x004000   Index
-#    0x008000   Search
-#    0x010000   History
-#    0x020000   Favorites
-#    0x040000   Jump 1
-#    0x080000   Jump 2
-#    0x100000   Zoom/Font   x
-#    0x200000   TOC Next
-#    0x400000   TOC Prev
-
-project_template = '''\
-[OPTIONS]
-Binary TOC=No
-Binary Index=No
-Compiled file=%(outname)s.chm
-Contents file=%(outname)s.hhc
-Default Window=%(outname)s
-Default topic=%(master_doc)s
-Display compile progress=No
-Full text search stop list file=%(outname)s.stp
-Full-text search=Yes
-Index file=%(outname)s.hhk
-Language=%(lcid)#x
-Title=%(title)s
-
-[WINDOWS]
-%(outname)s="%(title)s","%(outname)s.hhc","%(outname)s.hhk",\
-"%(master_doc)s","%(master_doc)s",,,,,0x63520,220,0x10384e,[0,0,1024,768],,,,,,,0
-
-[FILES]
-'''
-
-contents_header = '''\
-<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
-<HTML>
-<HEAD>
-<meta name="GENERATOR" content="Microsoft&reg; HTML Help Workshop 4.1">
-<!-- Sitemap 1.0 -->
-</HEAD><BODY>
-<OBJECT type="text/site properties">
-        <param name="Window Styles" value="0x801227">
-        <param name="ImageType" value="Folder">
-</OBJECT>
-<UL>
-'''
-
-contents_footer = '''\
-</UL></BODY></HTML>
-'''
-
-object_sitemap = '''\
-<OBJECT type="text/sitemap">
-    <param name="Name" value="%s">
-    <param name="Local" value="%s">
-</OBJECT>
-'''
-
-# List of words the full text search facility shouldn't index.  This
-# becomes file outname.stp.  Note that this list must be pretty small!
-# Different versions of the MS docs claim the file has a maximum size of
-# 256 or 512 bytes (including \r\n at the end of each line).
-# Note that "and", "or", "not" and "near" are operators in the search
-# language, so no point indexing them even if we wanted to.
-stopwords = """
-a  and  are  as  at
-be  but  by
-for
-if  in  into  is  it
-near  no  not
-of  on  or
-such
-that  the  their  then  there  these  they  this  to
-was  will  with
-""".split()
-
-# The following list includes only languages supported by Sphinx. See
-# https://docs.microsoft.com/en-us/previous-versions/windows/embedded/ms930130(v=msdn.10)
-# for more.
-chm_locales = {
-    # lang:   LCID,  encoding
-    'ca':    (0x403, 'cp1252'),
-    'cs':    (0x405, 'cp1250'),
-    'da':    (0x406, 'cp1252'),
-    'de':    (0x407, 'cp1252'),
-    'en':    (0x409, 'cp1252'),
-    'es':    (0x40a, 'cp1252'),
-    'et':    (0x425, 'cp1257'),
-    'fa':    (0x429, 'cp1256'),
-    'fi':    (0x40b, 'cp1252'),
-    'fr':    (0x40c, 'cp1252'),
-    'hr':    (0x41a, 'cp1250'),
-    'hu':    (0x40e, 'cp1250'),
-    'it':    (0x410, 'cp1252'),
-    'ja':    (0x411, 'cp932'),
-    'ko':    (0x412, 'cp949'),
-    'lt':    (0x427, 'cp1257'),
-    'lv':    (0x426, 'cp1257'),
-    'nl':    (0x413, 'cp1252'),
-    'no_NB': (0x414, 'cp1252'),
-    'pl':    (0x415, 'cp1250'),
-    'pt_BR': (0x416, 'cp1252'),
-    'ru':    (0x419, 'cp1251'),
-    'sk':    (0x41b, 'cp1250'),
-    'sl':    (0x424, 'cp1250'),
-    'sv':    (0x41d, 'cp1252'),
-    'tr':    (0x41f, 'cp1254'),
-    'uk_UA': (0x422, 'cp1251'),
-    'zh_CN': (0x804, 'cp936'),
-    'zh_TW': (0x404, 'cp950'),
-}
-
-
-def chm_htmlescape(s, quote=True):
-    # type: (str, bool) -> str
-    """
-    chm_htmlescape() is a wrapper of html.escape().
-    .hhc/.hhk files don't recognize hex escaping, we need convert
-    hex escaping to decimal escaping. for example: ``&#x27;`` -> ``&#39;``
-    html.escape() may generates a hex escaping ``&#x27;`` for single
-    quote ``'``, this wrapper fixes this.
-    """
-    s = html.escape(s, quote)
-    s = s.replace('&#x27;', '&#39;')    # re-escape as decimal
-    return s
-
-
-class HTMLHelpBuilder(StandaloneHTMLBuilder):
-    """
-    Builder that also outputs Windows HTML help project, contents and
-    index files.  Adapted from the original Doc/tools/prechm.py.
-    """
-    name = 'htmlhelp'
-    epilog = __('You can now run HTML Help Workshop with the .htp file in '
-                '%(outdir)s.')
-
-    # don't copy the reST source
-    copysource = False
-    supported_image_types = ['image/png', 'image/gif', 'image/jpeg']
-
-    # don't add links
-    add_permalinks = False
-    # don't add sidebar etc.
-    embedded = True
-
-    # don't generate search index or include search page
-    search = False
-
-    lcid = 0x409
-    encoding = 'cp1252'
-
-    def init(self):
-        # type: () -> None
-        # the output files for HTML help is .html by default
-        self.out_suffix = '.html'
-        self.link_suffix = '.html'
-        super().init()
-        # determine the correct locale setting
-        locale = chm_locales.get(self.config.language)
-        if locale is not None:
-            self.lcid, self.encoding = locale
-
-    def open_file(self, outdir, basename, mode='w'):
-        # type: (str, str, str) -> IO
-        # open a file with the correct encoding for the selected language
-        warnings.warn('HTMLHelpBuilder.open_file() is deprecated.',
-                      RemovedInSphinx40Warning)
-        return open(path.join(outdir, basename), mode, encoding=self.encoding,
-                    errors='xmlcharrefreplace')
-
-    def update_page_context(self, pagename, templatename, ctx, event_arg):
-        # type: (str, str, Dict, str) -> None
-        ctx['encoding'] = self.encoding
-
-    def handle_finish(self):
-        # type: () -> None
-        self.build_hhx(self.outdir, self.config.htmlhelp_basename)
-
-    def write_doc(self, docname, doctree):
-        # type: (str, nodes.document) -> None
-        for node in doctree.traverse(nodes.reference):
-            # add ``target=_blank`` attributes to external links
-            if node.get('internal') is None and 'refuri' in node:
-                node['target'] = '_blank'
-
-        super().write_doc(docname, doctree)
-
-    def build_hhx(self, outdir, outname):
-        # type: (str, str) -> None
-        logger.info(__('dumping stopword list...'))
-        filename = path.join(outdir, outname + '.stp')
-        with open(filename, 'w', encoding=self.encoding, errors='xmlcharrefreplace') as f:
-            for word in sorted(stopwords):
-                print(word, file=f)
-
-        logger.info(__('writing project file...'))
-        filename = path.join(outdir, outname + '.hhp')
-        with open(filename, 'w', encoding=self.encoding, errors='xmlcharrefreplace') as f:
-            f.write(project_template % {
-                'outname': outname,
-                'title': self.config.html_title,
-                'version': self.config.version,
-                'project': self.config.project,
-                'lcid': self.lcid,
-                'master_doc': self.config.master_doc + self.out_suffix
-            })
-            if not outdir.endswith(os.sep):
-                outdir += os.sep
-            olen = len(outdir)
-            for root, dirs, files in os.walk(outdir):
-                dirs.sort()
-                files.sort()
-                staticdir = root.startswith(path.join(outdir, '_static'))
-                for fn in sorted(files):
-                    if (staticdir and not fn.endswith('.js')) or \
-                       fn.endswith('.html'):
-                        print(path.join(root, fn)[olen:].replace(os.sep, '\\'),
-                              file=f)
-
-        logger.info(__('writing TOC file...'))
-        filename = path.join(outdir, outname + '.hhc')
-        with open(filename, 'w', encoding=self.encoding, errors='xmlcharrefreplace') as f:
-            f.write(contents_header)
-            # special books
-            f.write('<LI> ' + object_sitemap % (self.config.html_short_title,
-                                                self.config.master_doc + self.out_suffix))
-            for indexname, indexcls, content, collapse in self.domain_indices:
-                f.write('<LI> ' + object_sitemap % (indexcls.localname,
-                                                    '%s.html' % indexname))
-            # the TOC
-            tocdoc = self.env.get_and_resolve_doctree(
-                self.config.master_doc, self, prune_toctrees=False)
-
-            def write_toc(node, ullevel=0):
-                # type: (nodes.Node, int) -> None
-                if isinstance(node, nodes.list_item):
-                    f.write('<LI> ')
-                    for subnode in node:
-                        write_toc(subnode, ullevel)
-                elif isinstance(node, nodes.reference):
-                    link = node['refuri']
-                    title = chm_htmlescape(node.astext(), True)
-                    f.write(object_sitemap % (title, link))
-                elif isinstance(node, nodes.bullet_list):
-                    if ullevel != 0:
-                        f.write('<UL>\n')
-                    for subnode in node:
-                        write_toc(subnode, ullevel + 1)
-                    if ullevel != 0:
-                        f.write('</UL>\n')
-                elif isinstance(node, addnodes.compact_paragraph):
-                    for subnode in node:
-                        write_toc(subnode, ullevel)
-
-            matcher = NodeMatcher(addnodes.compact_paragraph, toctree=True)
-            for node in tocdoc.traverse(matcher):  # type: addnodes.compact_paragraph
-                write_toc(node)
-            f.write(contents_footer)
-
-        logger.info(__('writing index file...'))
-        index = IndexEntries(self.env).create_index(self)
-        filename = path.join(outdir, outname + '.hhk')
-        with open(filename, 'w', encoding=self.encoding, errors='xmlcharrefreplace') as f:
-            f.write('<UL>\n')
-
-            def write_index(title, refs, subitems):
-                # type: (str, List[Tuple[str, str]], List[Tuple[str, List[Tuple[str, str]]]]) -> None  # NOQA
-                def write_param(name, value):
-                    # type: (str, str) -> None
-                    item = '    <param name="%s" value="%s">\n' % (name, value)
-                    f.write(item)
-                title = chm_htmlescape(title, True)
-                f.write('<LI> <OBJECT type="text/sitemap">\n')
-                write_param('Keyword', title)
-                if len(refs) == 0:
-                    write_param('See Also', title)
-                elif len(refs) == 1:
-                    write_param('Local', refs[0][1])
-                else:
-                    for i, ref in enumerate(refs):
-                        # XXX: better title?
-                        write_param('Name', '[%d] %s' % (i, ref[1]))
-                        write_param('Local', ref[1])
-                f.write('</OBJECT>\n')
-                if subitems:
-                    f.write('<UL> ')
-                    for subitem in subitems:
-                        write_index(subitem[0], subitem[1], [])
-                    f.write('</UL>')
-            for (key, group) in index:
-                for title, (refs, subitems, key_) in group:
-                    write_index(title, refs, subitems)
-            f.write('</UL>\n')
-
-
-def default_htmlhelp_basename(config):
-    # type: (Config) -> str
-    """Better default htmlhelp_basename setting."""
-    return make_filename_from_project(config.project) + 'doc'
 
 
 def setup(app):
     # type: (Sphinx) -> Dict[str, Any]
-    app.setup_extension('sphinx.builders.html')
+    warnings.warn('sphinx.builders.htmlhelp has been moved to sphinxcontrib-htmlhelp.',
-    app.add_builder(HTMLHelpBuilder)
+                  RemovedInSphinx40Warning)
-
+    app.setup_extension('sphinxcontrib.htmlhelp')
-    app.add_config_value('htmlhelp_basename', default_htmlhelp_basename, None)
-    app.add_config_value('htmlhelp_file_suffix', None, 'html', [str])
-    app.add_config_value('htmlhelp_link_suffix', None, 'html', [str])
 
     return {
         'version': 'builtin',

sphinx/builders/latex/__init__.py

@@ -28,7 +28,7 @@ from sphinx.environment.adapters.asset import ImageAdapter
 from sphinx.errors import SphinxError
 from sphinx.locale import _, __
 from sphinx.transforms import SphinxTransformer
-from sphinx.util import texescape, logging, status_iterator
+from sphinx.util import texescape, logging, progress_message, status_iterator
 from sphinx.util.console import bold, darkgreen  # type: ignore
 from sphinx.util.docutils import SphinxFileOutput, new_document
 from sphinx.util.fileutil import copy_asset_file
@@ -250,33 +250,32 @@ class LaTeXBuilder(Builder):
                 toctree_only = entry[5]
             destination = SphinxFileOutput(destination_path=path.join(self.outdir, targetname),
                                            encoding='utf-8', overwrite_if_changed=True)
-            logger.info(__("processing %s..."), targetname, nonl=True)
+            with progress_message(__("processing %s") % targetname):
-            toctrees = self.env.get_doctree(docname).traverse(addnodes.toctree)
+                toctrees = self.env.get_doctree(docname).traverse(addnodes.toctree)
-            if toctrees:
+                if toctrees:
-                if toctrees[0].get('maxdepth') > 0:
+                    if toctrees[0].get('maxdepth') > 0:
-                    tocdepth = toctrees[0].get('maxdepth')
+                        tocdepth = toctrees[0].get('maxdepth')
+                    else:
+                        tocdepth = None
                 else:
                     tocdepth = None
-            else:
+                doctree = self.assemble_doctree(
-                tocdepth = None
+                    docname, toctree_only,
-            doctree = self.assemble_doctree(
+                    appendices=((docclass != 'howto') and self.config.latex_appendices or []))
-                docname, toctree_only,
+                doctree['tocdepth'] = tocdepth
-                appendices=((docclass != 'howto') and self.config.latex_appendices or []))
+                self.apply_transforms(doctree)
-            doctree['tocdepth'] = tocdepth
+                self.post_process_images(doctree)
-            self.apply_transforms(doctree)
+                self.update_doc_context(title, author)
-            self.post_process_images(doctree)
-            self.update_doc_context(title, author)
 
-            logger.info(__("writing... "), nonl=True)
+            with progress_message(__("writing")):
-            docsettings.author = author
+                docsettings.author = author
-            docsettings.title = title
+                docsettings.title = title
-            docsettings.contentsname = self.get_contentsname(docname)
+                docsettings.contentsname = self.get_contentsname(docname)
-            docsettings.docname = docname
+                docsettings.docname = docname
-            docsettings.docclass = docclass
+                docsettings.docclass = docclass
 
-            doctree.settings = docsettings
+                doctree.settings = docsettings
-            docwriter.write(doctree, destination)
+                docwriter.write(doctree, destination)
-            logger.info(__("done"))
 
     def get_contentsname(self, indexfile):
         # type: (str) -> str
@@ -354,8 +353,15 @@ class LaTeXBuilder(Builder):
         # type: () -> None
         self.copy_image_files()
         self.write_message_catalog()
+        self.copy_support_files()
 
-        # copy TeX support files from texinputs
+        if self.config.latex_additional_files:
+            self.copy_latex_additional_files()
+
+    @progress_message(__('copying TeX support files'))
+    def copy_support_files(self):
+        # type: () -> None
+        """copy TeX support files from texinputs."""
         # configure usage of xindy (impacts Makefile and latexmkrc)
         # FIXME: convert this rather to a confval with suitable default
         #        according to language ? but would require extra documentation
@@ -386,21 +392,19 @@ class LaTeXBuilder(Builder):
             copy_asset_file(path.join(staticdirname, 'Makefile_t'),
                             self.outdir, context=context)
 
-        # copy additional files
-        if self.config.latex_additional_files:
-            logger.info(bold(__('copying additional files...')), nonl=True)
-            for filename in self.config.latex_additional_files:
-                logger.info(' ' + filename, nonl=True)
-                copy_asset_file(path.join(self.confdir, filename), self.outdir)
-            logger.info('')
-
         # the logo is handled differently
         if self.config.latex_logo:
             if not path.isfile(path.join(self.confdir, self.config.latex_logo)):
                 raise SphinxError(__('logo file %r does not exist') % self.config.latex_logo)
             else:
                 copy_asset_file(path.join(self.confdir, self.config.latex_logo), self.outdir)
-        logger.info(__('done'))
+
+    @progress_message(__('copying additional files'))
+    def copy_latex_additional_files(self):
+        # type: () -> None
+        for filename in self.config.latex_additional_files:
+            logger.info(' ' + filename, nonl=True)
+            copy_asset_file(path.join(self.confdir, filename), self.outdir)
 
     def copy_image_files(self):
         # type: () -> None
@@ -439,7 +443,7 @@ def validate_config_values(app, config):
     for key in list(config.latex_elements):
         if key not in DEFAULT_SETTINGS:
             msg = __("Unknown configure key: latex_elements[%r]. ignored.")
-            logger.warning(msg % key)
+            logger.warning(msg % (key,))
             config.latex_elements.pop(key)
 
 

sphinx/builders/linkcheck.py

@@ -25,7 +25,7 @@ from sphinx.util import encode_uri, requests, logging
 from sphinx.util.console import (  # type: ignore
     purple, red, darkgreen, darkgray, darkred, turquoise
 )
-from sphinx.util.nodes import traverse_parent
+from sphinx.util.nodes import get_node_line
 from sphinx.util.requests import is_ssl_error
 
 if False:
@@ -271,17 +271,24 @@ class CheckExternalLinksBuilder(Builder):
         # type: (str, nodes.Node) -> None
         logger.info('')
         n = 0
-        for node in doctree.traverse(nodes.reference):
+
-            if 'refuri' not in node:
+        # reference nodes
+        for refnode in doctree.traverse(nodes.reference):
+            if 'refuri' not in refnode:
                 continue
-            uri = node['refuri']
+            uri = refnode['refuri']
-            lineno = None
+            lineno = get_node_line(refnode)
-            for parent in traverse_parent(node):
-                if parent.line:
-                    lineno = parent.line
-                    break
             self.wqueue.put((uri, docname, lineno), False)
             n += 1
+
+        # image nodes
+        for imgnode in doctree.traverse(nodes.image):
+            uri = imgnode['candidates'].get('?')
+            if uri and '://' in uri:
+                lineno = get_node_line(imgnode)
+                self.wqueue.put((uri, docname, lineno), False)
+                n += 1
+
         done = 0
         while done < n:
             self.process_result(self.rqueue.get())

sphinx/builders/manpage.py

@@ -18,7 +18,8 @@ from sphinx.builders import Builder
 from sphinx.environment import NoUri
 from sphinx.locale import __
 from sphinx.util import logging
-from sphinx.util.console import bold, darkgreen  # type: ignore
+from sphinx.util import progress_message
+from sphinx.util.console import darkgreen  # type: ignore
 from sphinx.util.nodes import inline_all_toctrees
 from sphinx.util.osutil import make_filename_from_project
 from sphinx.writers.manpage import ManualPageWriter, ManualPageTranslator
@@ -60,6 +61,7 @@ class ManualPageBuilder(Builder):
             return ''
         raise NoUri
 
+    @progress_message(__('writing'))
     def write(self, *ignored):
         # type: (Any) -> None
         docwriter = ManualPageWriter(self)
@@ -68,8 +70,6 @@ class ManualPageBuilder(Builder):
             components=(docwriter,),
             read_config_files=True).get_default_values()  # type: Any
 
-        logger.info(bold(__('writing... ')), nonl=True)
-
         for info in self.config.man_pages:
             docname, name, description, authors, section = info
             if docname not in self.env.all_docs:
@@ -105,7 +105,6 @@ class ManualPageBuilder(Builder):
                 pendingnode.replace_self(pendingnode.children)
 
             docwriter.write(largetree, destination)
-        logger.info('')
 
     def finish(self):
         # type: () -> None

sphinx/builders/singlehtml.py

@@ -16,7 +16,8 @@ from sphinx.builders.html import StandaloneHTMLBuilder
 from sphinx.environment.adapters.toctree import TocTree
 from sphinx.locale import __
 from sphinx.util import logging
-from sphinx.util.console import bold, darkgreen  # type: ignore
+from sphinx.util import progress_message
+from sphinx.util.console import darkgreen  # type: ignore
 from sphinx.util.nodes import inline_all_toctrees
 
 if False:
@@ -162,24 +163,32 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
         # type: (Any) -> None
         docnames = self.env.all_docs
 
-        logger.info(bold(__('preparing documents... ')), nonl=True)
+        with progress_message(__('preparing documents')):
-        self.prepare_writing(docnames)  # type: ignore
+            self.prepare_writing(docnames)  # type: ignore
-        logger.info(__('done'))
 
-        logger.info(bold(__('assembling single document... ')), nonl=True)
+        with progress_message(__('assembling single document')):
-        doctree = self.assemble_doctree()
+            doctree = self.assemble_doctree()
-        self.env.toc_secnumbers = self.assemble_toc_secnumbers()
+            self.env.toc_secnumbers = self.assemble_toc_secnumbers()
-        self.env.toc_fignumbers = self.assemble_toc_fignumbers()
+            self.env.toc_fignumbers = self.assemble_toc_fignumbers()
-        logger.info('')
+
-        logger.info(bold(__('writing... ')), nonl=True)
+        with progress_message(__('writing')):
-        self.write_doc_serialized(self.config.master_doc, doctree)
+            self.write_doc_serialized(self.config.master_doc, doctree)
-        self.write_doc(self.config.master_doc, doctree)
+            self.write_doc(self.config.master_doc, doctree)
-        logger.info(__('done'))
 
     def finish(self):
+        # type: () -> None
+        self.write_additional_files()
+        self.copy_image_files()
+        self.copy_download_files()
+        self.copy_static_files()
+        self.copy_extra_files()
+        self.write_buildinfo()
+        self.dump_inventory()
+
+    @progress_message(__('writing additional files'))
+    def write_additional_files(self):
         # type: () -> None
         # no indices or search pages are supported
-        logger.info(bold(__('writing additional files...')), nonl=True)
 
         # additional pages from conf.py
         for pagename, template in self.config.html_additional_pages.items():
@@ -191,15 +200,6 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
             fn = path.join(self.outdir, '_static', 'opensearch.xml')
             self.handle_page('opensearch', {}, 'opensearch.xml', outfilename=fn)
 
-        logger.info('')
-
-        self.copy_image_files()
-        self.copy_download_files()
-        self.copy_static_files()
-        self.copy_extra_files()
-        self.write_buildinfo()
-        self.dump_inventory()
-
 
 def setup(app):
     # type: (Sphinx) -> Dict[str, Any]

sphinx/builders/texinfo.py

@@ -22,8 +22,8 @@ from sphinx.environment import NoUri
 from sphinx.environment.adapters.asset import ImageAdapter
 from sphinx.locale import _, __
 from sphinx.util import logging
-from sphinx.util import status_iterator
+from sphinx.util import progress_message, status_iterator
-from sphinx.util.console import bold, darkgreen  # type: ignore
+from sphinx.util.console import darkgreen  # type: ignore
 from sphinx.util.docutils import new_document
 from sphinx.util.fileutil import copy_asset_file
 from sphinx.util.nodes import inline_all_toctrees
@@ -113,28 +113,27 @@ class TexinfoBuilder(Builder):
             destination = FileOutput(
                 destination_path=path.join(self.outdir, targetname),
                 encoding='utf-8')
-            logger.info(__("processing %s..."), targetname, nonl=True)
+            with progress_message(__("processing %s") % targetname):
-            doctree = self.assemble_doctree(
+                appendices = self.config.texinfo_appendices or []
-                docname, toctree_only,
+                doctree = self.assemble_doctree(docname, toctree_only, appendices=appendices)
-                appendices=(self.config.texinfo_appendices or []))
+
-            logger.info(__("writing... "), nonl=True)
+            with progress_message(__("writing")):
-            self.post_process_images(doctree)
+                self.post_process_images(doctree)
-            docwriter = TexinfoWriter(self)
+                docwriter = TexinfoWriter(self)
-            settings = OptionParser(
+                settings = OptionParser(
-                defaults=self.env.settings,
+                    defaults=self.env.settings,
-                components=(docwriter,),
+                    components=(docwriter,),
-                read_config_files=True).get_default_values()  # type: Any
+                    read_config_files=True).get_default_values()  # type: Any
-            settings.author = author
+                settings.author = author
-            settings.title = title
+                settings.title = title
-            settings.texinfo_filename = targetname[:-5] + '.info'
+                settings.texinfo_filename = targetname[:-5] + '.info'
-            settings.texinfo_elements = self.config.texinfo_elements
+                settings.texinfo_elements = self.config.texinfo_elements
-            settings.texinfo_dir_entry = direntry or ''
+                settings.texinfo_dir_entry = direntry or ''
-            settings.texinfo_dir_category = category or ''
+                settings.texinfo_dir_category = category or ''
-            settings.texinfo_dir_description = description or ''
+                settings.texinfo_dir_description = description or ''
-            settings.docname = docname
+                settings.docname = docname
-            doctree.settings = settings
+                doctree.settings = settings
-            docwriter.write(doctree, destination)
+                docwriter.write(doctree, destination)
-            logger.info(__("done"))
 
     def assemble_doctree(self, indexfile, toctree_only, appendices):
         # type: (str, bool, List[str]) -> nodes.document
@@ -182,16 +181,7 @@ class TexinfoBuilder(Builder):
     def finish(self):
         # type: () -> None
         self.copy_image_files()
-
+        self.copy_support_files()
-        logger.info(bold(__('copying Texinfo support files... ')), nonl=True)
-        # copy Makefile
-        fn = path.join(self.outdir, 'Makefile')
-        logger.info(fn, nonl=True)
-        try:
-            copy_asset_file(os.path.join(template_dir, 'Makefile'), fn)
-        except OSError as err:
-            logger.warning(__("error writing file %s: %s"), fn, err)
-        logger.info(__(' done'))
 
     def copy_image_files(self):
         # type: () -> None
@@ -208,6 +198,15 @@ class TexinfoBuilder(Builder):
                     logger.warning(__('cannot copy image file %r: %s'),
                                    path.join(self.srcdir, src), err)
 
+    def copy_support_files(self):
+        # type: () -> None
+        try:
+            with progress_message(__('copying Texinfo support files')):
+                logger.info('Makefile ', nonl=True)
+                copy_asset_file(os.path.join(template_dir, 'Makefile'), self.outdir)
+        except OSError as err:
+            logger.warning(__("error writing file Makefile: %s"), err)
+
 
 def default_texinfo_documents(config):
     # type: (Config) -> List[Tuple[str, str, str, str, str, str, str]]

sphinx/cmd/build.py

@@ -23,9 +23,8 @@ from sphinx.application import Sphinx
 from sphinx.errors import SphinxError
 from sphinx.locale import __
 from sphinx.util import Tee, format_exception_cut_frames, save_traceback
-from sphinx.util.console import red, nocolor, color_terminal  # type: ignore
+from sphinx.util.console import red, nocolor, color_terminal, terminal_safe  # type: ignore
 from sphinx.util.docutils import docutils_namespace, patch_docutils
-from sphinx.util.pycompat import terminal_safe
 
 if False:
     # For type annotation

sphinx/cmd/quickstart.py

@@ -17,7 +17,6 @@ import time
 import warnings
 from collections import OrderedDict
 from os import path
-from urllib.parse import quote
 
 # try to import readline, unix specific enhancement
 try:
@@ -37,11 +36,10 @@ import sphinx.locale
 from sphinx import __display_version__, package_dir
 from sphinx.deprecation import RemovedInSphinx40Warning
 from sphinx.locale import __
-from sphinx.util import texescape
 from sphinx.util.console import (  # type: ignore
     colorize, bold, red, turquoise, nocolor, color_terminal
 )
-from sphinx.util.osutil import ensuredir, make_filename
+from sphinx.util.osutil import ensuredir
 from sphinx.util.template import SphinxRenderer
 
 if False:
@@ -375,25 +373,15 @@ def generate(d, overwrite=True, silent=False, templatedir=None):
     """Generate project based on values in *d*."""
     template = QuickstartRenderer(templatedir=templatedir)
 
-    texescape.init()
-
     if 'mastertoctree' not in d:
         d['mastertoctree'] = ''
     if 'mastertocmaxdepth' not in d:
         d['mastertocmaxdepth'] = 2
 
-    d['PY3'] = True
-    d['project_fn'] = make_filename(d['project'])
-    d['project_url'] = quote(d['project'].encode('idna'))
-    d['project_manpage'] = d['project_fn'].lower()
     d['now'] = time.asctime()
     d['project_underline'] = column_width(d['project']) * '='
     d.setdefault('extensions', [])
     d['copyright'] = time.strftime('%Y') + ', ' + d['author']
-    d['author_texescaped'] = d['author'].translate(texescape.tex_escape_map)
-    d['project_doc'] = d['project'] + ' Documentation'
-    d['project_doc_texescaped'] = (d['project'] + ' Documentation').\
-        translate(texescape.tex_escape_map)
 
     ensuredir(d['path'])
 
@@ -528,7 +516,7 @@ Makefile to be used with sphinx-build.
     group = parser.add_argument_group(__('Structure options'))
     group.add_argument('--sep', action='store_true', default=None,
                        help=__('if specified, separate source and build dirs'))
-    group.add_argument('--dot', metavar='DOT',
+    group.add_argument('--dot', metavar='DOT', default='_',
                        help=__('replacement for dot in _templates etc.'))
 
     group = parser.add_argument_group(__('Project basic options'))
@@ -542,9 +530,9 @@ Makefile to be used with sphinx-build.
                        help=__('release of project'))
     group.add_argument('-l', '--language', metavar='LANGUAGE', dest='language',
                        help=__('document language'))
-    group.add_argument('--suffix', metavar='SUFFIX',
+    group.add_argument('--suffix', metavar='SUFFIX', default='.rst',
                        help=__('source file suffix'))
-    group.add_argument('--master', metavar='MASTER',
+    group.add_argument('--master', metavar='MASTER', default='index',
                        help=__('master document name'))
     group.add_argument('--epub', action='store_true', default=False,
                        help=__('use epub'))
@@ -558,11 +546,11 @@ Makefile to be used with sphinx-build.
                        action='append', help=__('enable arbitrary extensions'))
 
     group = parser.add_argument_group(__('Makefile and Batchfile creation'))
-    group.add_argument('--makefile', action='store_true', dest='makefile', default=None,
+    group.add_argument('--makefile', action='store_true', dest='makefile', default=True,
                        help=__('create makefile'))
     group.add_argument('--no-makefile', action='store_false', dest='makefile',
                        help=__('do not create makefile'))
-    group.add_argument('--batchfile', action='store_true', dest='batchfile', default=None,
+    group.add_argument('--batchfile', action='store_true', dest='batchfile', default=True,
                        help=__('create batchfile'))
     group.add_argument('--no-batchfile', action='store_false',
                        dest='batchfile',
@@ -604,6 +592,13 @@ def main(argv=sys.argv[1:]):
     # delete None or False value
     d = dict((k, v) for k, v in d.items() if v is not None)
 
+    # handle use of CSV-style extension values
+    d.setdefault('extensions', [])
+    for ext in d['extensions'][:]:
+        if ',' in ext:
+            d['extensions'].remove(ext)
+            d['extensions'].extend(ext.split(','))
+
     try:
         if 'quiet' in d:
             if not set(['project', 'author']).issubset(d):
@@ -633,13 +628,6 @@ def main(argv=sys.argv[1:]):
         print('[Interrupted.]')
         return 130  # 128 + SIGINT
 
-    # handle use of CSV-style extension values
-    d.setdefault('extensions', [])
-    for ext in d['extensions'][:]:
-        if ',' in ext:
-            d['extensions'].remove(ext)
-            d['extensions'].extend(ext.split(','))
-
     for variable in d.get('variables', []):
         try:
             name, value = variable.split('=')

sphinx/config.py

@@ -22,7 +22,8 @@ from sphinx.locale import _, __
 from sphinx.util import logging
 from sphinx.util.i18n import format_date
 from sphinx.util.osutil import cd
-from sphinx.util.pycompat import execfile_, NoneType
+from sphinx.util.pycompat import execfile_
+from sphinx.util.typing import NoneType
 
 if False:
     # For type annotation

sphinx/deprecation.py

@@ -48,7 +48,7 @@ class _ModuleWrapper(object):
     def __getattr__(self, name):
         # type: (str) -> Any
         if name in self._objects:
-            warnings.warn("%s.%s is now deprecated. Please refer CHANGES to grasp"
+            warnings.warn("%s.%s is now deprecated. Please refer CHANGES to grasp "
                           "the changes of Sphinx API." % (self._modname, name),
                           self._warning, stacklevel=3)
             return self._objects[name]

sphinx/domains/cpp.py

@@ -7158,7 +7158,7 @@ class CPPDomain(Domain):
         # the non-identifier refs are cross-references, which should be processed:
         # - fix parenthesis due to operator() and add_function_parentheses
         if typ != "identifier":
-            title = contnode.pop(0).astext()  # type: ignore
+            title = contnode.pop(0).astext()
             # If it's operator(), we need to add '()' if explicit function parens
             # are requested. Then the Sphinx machinery will add another pair.
             # Also, if it's an 'any' ref that resolves to a function, we need to add

sphinx/domains/python.py

@@ -151,7 +151,7 @@ class PyXrefMixin:
         delims_re = re.compile(delims)
         sub_targets = re.split(delims, target)
 
-        split_contnode = bool(contnode and contnode.astext() == target)  # type: ignore
+        split_contnode = bool(contnode and contnode.astext() == target)
 
         results = []
         for sub_target in filter(None, sub_targets):

sphinx/ext/autodoc/__init__.py

@@ -72,7 +72,7 @@ INSTANCEATTR = object()
 def members_option(arg):
     # type: (Any) -> Union[object, List[str]]
     """Used to convert the :members: option to auto directives."""
-    if arg is None:
+    if arg is None or arg is True:
         return ALL
     return [x.strip() for x in arg.split(',')]
 
@@ -1002,6 +1002,7 @@ class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # typ
             return None
         try:
             if (not isfunction(self.object) and
+                    not inspect.ismethod(self.object) and
                     not isbuiltin(self.object) and
                     not inspect.isclass(self.object) and
                     hasattr(self.object, '__call__')):
@@ -1032,6 +1033,23 @@ class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # typ
         pass
 
 
+class DecoratorDocumenter(FunctionDocumenter):
+    """
+    Specialized Documenter subclass for decorator functions.
+    """
+    objtype = 'decorator'
+
+    # must be lower than FunctionDocumenter
+    priority = -1
+
+    def format_args(self):
+        args = super().format_args()
+        if ',' in args:
+            return args
+        else:
+            return None
+
+
 class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type: ignore
     """
     Specialized Documenter subclass for classes.
@@ -1461,6 +1479,7 @@ def setup(app):
     app.add_autodocumenter(ExceptionDocumenter)
     app.add_autodocumenter(DataDocumenter)
     app.add_autodocumenter(FunctionDocumenter)
+    app.add_autodocumenter(DecoratorDocumenter)
     app.add_autodocumenter(MethodDocumenter)
     app.add_autodocumenter(AttributeDocumenter)
     app.add_autodocumenter(InstanceAttributeDocumenter)

sphinx/ext/autodoc/importer.py

@@ -32,13 +32,18 @@ logger = logging.getLogger(__name__)
 class _MockObject:
     """Used by autodoc_mock_imports."""
 
+    __display_name__ = '_MockObject'
+
     def __new__(cls, *args, **kwargs):
         # type: (Any, Any) -> Any
-        if len(args) == 3 and isinstance(args[1], tuple) and args[1][-1].__class__ is cls:
+        if len(args) == 3 and isinstance(args[1], tuple):
-            # subclassing MockObject
+            superclass = args[1][-1].__class__
-            return type(args[0], (_MockObject,), args[2], **kwargs)  # type: ignore
+            if superclass is cls:
-        else:
+                # subclassing MockObject
-            return super(_MockObject, cls).__new__(cls)
+                return _make_subclass(args[0], superclass.__display_name__,
+                                      superclass=superclass, attributes=args[2])
+
+        return super(_MockObject, cls).__new__(cls)
 
     def __init__(self, *args, **kwargs):
         # type: (Any, Any) -> None
@@ -62,11 +67,11 @@ class _MockObject:
 
     def __getitem__(self, key):
         # type: (str) -> _MockObject
-        return self
+        return _make_subclass(key, self.__display_name__, self.__class__)()
 
     def __getattr__(self, key):
         # type: (str) -> _MockObject
-        return self
+        return _make_subclass(key, self.__display_name__, self.__class__)()
 
     def __call__(self, *args, **kw):
         # type: (Any, Any) -> Any
@@ -75,6 +80,18 @@ class _MockObject:
             return args[0]
         return self
 
+    def __repr__(self):
+        # type: () -> str
+        return self.__display_name__
+
+
+def _make_subclass(name, module, superclass=_MockObject, attributes=None):
+    # type: (str, str, Any, dict) -> Any
+    attrs = {'__module__': module, '__display_name__': module + '.' + name}
+    attrs.update(attributes or {})
+
+    return type(name, (superclass,), attrs)
+
 
 class _MockModule(ModuleType):
     """Used by autodoc_mock_imports."""
@@ -92,9 +109,11 @@ class _MockModule(ModuleType):
 
     def __getattr__(self, name):
         # type: (str) -> _MockObject
-        o = _MockObject()
+        return _make_subclass(name, self.__name__)()
-        o.__module__ = self.__name__
+
-        return o
+    def __repr__(self):
+        # type: () -> str
+        return self.__name__
 
 
 class _MockImporter(MetaPathFinder):

sphinx/ext/autosectionlabel.py

@@ -30,11 +30,23 @@ if False:
     from sphinx.application import Sphinx  # NOQA
 
 
+def get_node_depth(node):
+    i = 0
+    cur_node = node
+    while cur_node.parent != node.document:
+        cur_node = cur_node.parent
+        i += 1
+    return i
+
+
 def register_sections_as_label(app, document):
     # type: (Sphinx, nodes.Node) -> None
     labels = app.env.domaindata['std']['labels']
     anonlabels = app.env.domaindata['std']['anonlabels']
     for node in document.traverse(nodes.section):
+        if (app.config.autosectionlabel_maxdepth and
+                get_node_depth(node) >= app.config.autosectionlabel_maxdepth):
+            continue
         labelid = node['ids'][0]
         docname = app.env.docname
         title = cast(nodes.title, node[0])
@@ -57,6 +69,7 @@ def register_sections_as_label(app, document):
 def setup(app):
     # type: (Sphinx) -> Dict[str, Any]
     app.add_config_value('autosectionlabel_prefix_document', False, 'env')
+    app.add_config_value('autosectionlabel_maxdepth', None, 'env')
     app.connect('doctree-read', register_sections_as_label)
 
     return {

sphinx/ext/autosummary/__init__.py

@@ -72,7 +72,7 @@ from sphinx.deprecation import RemovedInSphinx40Warning
 from sphinx.environment.adapters.toctree import TocTree
 from sphinx.ext.autodoc import get_documenters
 from sphinx.ext.autodoc.directive import DocumenterBridge, Options
-from sphinx.ext.autodoc.importer import import_module
+from sphinx.ext.autodoc.importer import import_module, mock
 from sphinx.locale import __
 from sphinx.pycode import ModuleAnalyzer, PycodeError
 from sphinx.util import import_object, rst, logging
@@ -286,7 +286,8 @@ class Autosummary(SphinxDirective):
                 display_name = name.split('.')[-1]
 
             try:
-                real_name, obj, parent, modname = import_by_name(name, prefixes=prefixes)
+                with mock(self.config.autosummary_mock_imports):
+                    real_name, obj, parent, modname = import_by_name(name, prefixes=prefixes)
             except ImportError:
                 logger.warning(__('failed to import %s'), name)
                 items.append((name, '', '', name))
@@ -703,11 +704,13 @@ def process_generate_options(app):
         return
 
     recursion_limit = app.config.autosummary_recursion_limit
-    generate_autosummary_docs(genfiles, builder=app.builder,
+  
-                              warn=logger.warning, info=logger.info,
+    with mock(app.config.autosummary_mock_imports):
-                              suffix=suffix, base_path=app.srcdir,
+        generate_autosummary_docs(genfiles, builder=app.builder,
-                              recursion_limit=recursion_limit,
+                                  warn=logger.warning, info=logger.info,
-                              app=app)
+                                  suffix=suffix, base_path=app.srcdir,
+                                  recursion_limit=recursion_limit,
+                                  app=app)
 
 
 def setup(app):
@@ -732,4 +735,6 @@ def setup(app):
     app.connect('builder-inited', process_generate_options)
     app.add_config_value('autosummary_generate', [], True, [bool])
     app.add_config_value('autosummary_recursion_limit', 0, 0)
+    app.add_config_value('autosummary_mock_imports',
+                         lambda config: config.autodoc_mock_imports, 'env')
     return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

sphinx/ext/githubpages.py

@@ -9,6 +9,7 @@
 """
 
 import os
+import urllib
 
 import sphinx
 
@@ -19,14 +20,22 @@ if False:
     from sphinx.environment import BuildEnvironment  # NOQA
 
 
-def create_nojekyll(app, env):
+def create_nojekyll_and_cname(app, env):
     # type: (Sphinx, BuildEnvironment) -> None
     if app.builder.format == 'html':
-        path = os.path.join(app.builder.outdir, '.nojekyll')
+        open(os.path.join(app.builder.outdir, '.nojekyll'), 'wt').close()
-        open(path, 'wt').close()
+
+        html_baseurl = app.config.html_baseurl
+        if html_baseurl:
+            domain = urllib.parse.urlparse(html_baseurl).hostname
+            if domain and not domain.endswith(".github.io"):
+                with open(os.path.join(app.builder.outdir, 'CNAME'), 'wt') as f:
+                    # NOTE: don't write a trailing newline. The `CNAME` file that's
+                    # auto-generated by the Github UI doesn't have one.
+                    f.write(domain)
 
 
 def setup(app):
     # type: (Sphinx) -> Dict[str, Any]
-    app.connect('env-updated', create_nojekyll)
+    app.connect('env-updated', create_nojekyll_and_cname)
     return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

sphinx/ext/imgconverter.py

@@ -54,9 +54,9 @@ class ImagemagickConverter(ImageConverter):
         # type: (str, str) -> bool
         """Converts the image to expected one."""
         try:
-            if _from.lower().endswith('.gif'):
+            # append an index 0 to source filename to pick up the first frame
-                # when target is GIF format, pick the first frame
+            # (or first page) of image (ex. Animation GIF, PDF)
-                _from += '[0]'
+            _from += '[0]'
 
             args = ([self.config.image_converter] +
                     self.config.image_converter_args +

sphinx/ext/imgmath.py

@@ -12,6 +12,7 @@ import posixpath
 import re
 import shutil
 import subprocess
+import sys
 import tempfile
 from hashlib import sha1
 from os import path
@@ -26,7 +27,6 @@ from sphinx.util import logging
 from sphinx.util.math import get_node_equation_number, wrap_displaymath
 from sphinx.util.osutil import ensuredir
 from sphinx.util.png import read_png_depth, write_png_depth
-from sphinx.util.pycompat import sys_encoding
 
 if False:
     # For type annotation
@@ -46,9 +46,9 @@ class MathExtError(SphinxError):
     def __init__(self, msg, stderr=None, stdout=None):
         # type: (str, bytes, bytes) -> None
         if stderr:
-            msg += '\n[stderr]\n' + stderr.decode(sys_encoding, 'replace')
+            msg += '\n[stderr]\n' + stderr.decode(sys.getdefaultencoding(), 'replace')
         if stdout:
-            msg += '\n[stdout]\n' + stdout.decode(sys_encoding, 'replace')
+            msg += '\n[stdout]\n' + stdout.decode(sys.getdefaultencoding(), 'replace')
         super().__init__(msg)
 
 

sphinx/io.py

@@ -21,7 +21,7 @@ from typing import Any, Union  # NOQA
 from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.transforms import (
     ApplySourceWorkaround, ExtraTranslatableNodes, CitationReferences,
-    DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, SortIds,
+    DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, SortIds, FigureAligner,
     AutoNumbering, AutoIndexUpgrader, FilterSystemMessages,
     UnreferencedFootnotesDetector, SphinxSmartQuotes, DoctreeReadEvent, ManpageLink
 )
@@ -96,7 +96,7 @@ class SphinxStandaloneReader(SphinxBaseReader):
     """
     transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, PreserveTranslatableMessages,
                   Locale, CitationReferences, DefaultSubstitutions, MoveModuleTargets,
-                  HandleCodeBlocks, AutoNumbering, AutoIndexUpgrader, SortIds,
+                  HandleCodeBlocks, AutoNumbering, AutoIndexUpgrader, SortIds, FigureAligner,
                   RemoveTranslatableInline, FilterSystemMessages, RefOnlyBulletListTransform,
                   UnreferencedFootnotesDetector, SphinxSmartQuotes, ManpageLink,
                   SphinxDomains, SubstitutionDefinitionsRemover, DoctreeReadEvent,

sphinx/roles.py

@@ -9,13 +9,15 @@
 """
 
 import re
+import warnings
 
 from docutils import nodes, utils
 
 from sphinx import addnodes
-from sphinx.errors import SphinxError
+from sphinx.deprecation import RemovedInSphinx40Warning
 from sphinx.locale import _
 from sphinx.util import ws_re
+from sphinx.util.docutils import ReferenceRole, SphinxRole
 from sphinx.util.nodes import split_explicit_title, process_index_entry, \
     set_role_source_info
 
@@ -44,7 +46,7 @@ generic_docroles = {
 
 # -- generic cross-reference role ----------------------------------------------
 
-class XRefRole:
+class XRefRole(ReferenceRole):
     """
     A generic cross-referencing role.  To create a callable that can be used as
     a role function, create an instance of this class.
@@ -82,8 +84,12 @@ class XRefRole:
         if innernodeclass is not None:
             self.innernodeclass = innernodeclass
 
+        super().__init__()
+
     def _fix_parens(self, env, has_explicit_title, title, target):
         # type: (BuildEnvironment, bool, str, str) -> Tuple[str, str]
+        warnings.warn('XRefRole._fix_parens() is deprecated.',
+                      RemovedInSphinx40Warning, stacklevel=2)
         if not has_explicit_title:
             if title.endswith('()'):
                 # remove parentheses
@@ -96,55 +102,70 @@ class XRefRole:
             target = target[:-2]
         return title, target
 
-    def __call__(self, typ, rawtext, text, lineno, inliner,
+    def update_title_and_target(self, title, target):
-                 options={}, content=[]):
+        # type: (str, str) -> Tuple[str, str]
-        # type: (str, str, str, int, Inliner, Dict, List[str]) -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
+        if not self.has_explicit_title:
-        env = inliner.document.settings.env
+            if title.endswith('()'):
-        if not typ:
+                # remove parentheses
-            typ = env.temp_data.get('default_role')
+                title = title[:-2]
-            if not typ:
+            if self.config.add_function_parentheses:
-                typ = env.config.default_role
+                # add them back to all occurrences if configured
-            if not typ:
+                title += '()'
-                raise SphinxError('cannot determine default role!')
+        # remove parentheses from the target too
+        if target.endswith('()'):
+            target = target[:-2]
+        return title, target
+
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        if ':' not in self.name:
+            self.refdomain, self.reftype = '', self.name
+            self.classes = ['xref', self.reftype]
         else:
-            typ = typ.lower()
+            self.refdomain, self.reftype = self.name.split(':', 1)
-        if ':' not in typ:
+            self.classes = ['xref', self.refdomain, '%s-%s' % (self.refdomain, self.reftype)]
-            domain, role = '', typ
+
-            classes = ['xref', role]
+        if self.text.startswith('!'):
+            # if the first character is a bang, don't cross-reference at all
+            return self.create_non_xref_node()
         else:
-            domain, role = typ.split(':', 1)
+            return self.create_xref_node()
-            classes = ['xref', domain, '%s-%s' % (domain, role)]
+
-        # if the first character is a bang, don't cross-reference at all
+    def create_non_xref_node(self):
-        if text[0:1] == '!':
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
-            text = utils.unescape(text)[1:]
+        text = utils.unescape(self.text[1:])
-            if self.fix_parens:
+        if self.fix_parens:
-                text, tgt = self._fix_parens(env, False, text, "")
+            self.has_explicit_title = False  # treat as implicit
-            innernode = self.innernodeclass(rawtext, text, classes=classes)
+            text, target = self.update_title_and_target(text, "")
-            return self.result_nodes(inliner.document, env, innernode, is_ref=False)
+
-        # split title and target in role content
+        node = self.innernodeclass(self.rawtext, text, classes=self.classes)
-        has_explicit_title, title, target = split_explicit_title(text)
+        return self.result_nodes(self.inliner.document, self.env, node, is_ref=False)
-        title = utils.unescape(title)
+
-        target = utils.unescape(target)
+    def create_xref_node(self):
-        # fix-up title and target
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        target = self.target
+        title = self.title
         if self.lowercase:
             target = target.lower()
         if self.fix_parens:
-            title, target = self._fix_parens(
+            title, target = self.update_title_and_target(title, target)
-                env, has_explicit_title, title, target)
+
         # create the reference node
-        refnode = self.nodeclass(rawtext, reftype=role, refdomain=domain,
+        options = {'refdoc': self.env.docname,
-                                 refexplicit=has_explicit_title)
+                   'refdomain': self.refdomain,
-        # we may need the line number for warnings
+                   'reftype': self.reftype,
-        set_role_source_info(inliner, lineno, refnode)
+                   'refexplicit': self.has_explicit_title,
-        title, target = self.process_link(env, refnode, has_explicit_title, title, target)
+                   'refwarn': self.warn_dangling}
-        # now that the target and title are finally determined, set them
+        refnode = self.nodeclass(self.rawtext, **options)
+        self.set_source_info(refnode)
+
+        # determine the target and title for the class
+        title, target = self.process_link(self.env, refnode, self.has_explicit_title,
+                                          title, target)
         refnode['reftarget'] = target
-        refnode += self.innernodeclass(rawtext, title, classes=classes)
+        refnode += self.innernodeclass(self.rawtext, title, classes=self.classes)
-        # we also need the source document
+
-        refnode['refdoc'] = env.docname
+        return self.result_nodes(self.inliner.document, self.env, refnode, is_ref=True)
-        refnode['refwarn'] = self.warn_dangling
-        # result_nodes allow further modification of return values
-        return self.result_nodes(inliner.document, env, refnode, is_ref=True)
 
     # methods that can be overwritten
 
@@ -179,6 +200,8 @@ class AnyXRefRole(XRefRole):
 def indexmarkup_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     # type: (str, str, str, int, Inliner, Dict, List[str]) -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
     """Role for PEP/RFC references that generate an index entry."""
+    warnings.warn('indexmarkup_role() is deprecated.  Please use PEP or RFC class instead.',
+                  RemovedInSphinx40Warning, stacklevel=2)
     env = inliner.document.settings.env
     if not typ:
         assert env.temp_data['default_role']
@@ -242,11 +265,87 @@ def indexmarkup_role(typ, rawtext, text, lineno, inliner, options={}, content=[]
         raise ValueError('unknown role type: %s' % typ)
 
 
+class PEP(ReferenceRole):
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        target_id = 'index-%s' % self.env.new_serialno('index')
+        entries = [('single', _('Python Enhancement Proposals; PEP %s') % self.target,
+                    target_id, '', None)]
+
+        index = addnodes.index(entries=entries)
+        target = nodes.target('', '', ids=[target_id])
+        self.inliner.document.note_explicit_target(target)
+
+        try:
+            refuri = self.build_uri()
+            reference = nodes.reference('', '', internal=False, refuri=refuri, classes=['pep'])
+            if self.has_explicit_title:
+                reference += nodes.strong(self.title, self.title)
+            else:
+                title = "PEP " + self.title
+                reference += nodes.strong(title, title)
+        except ValueError:
+            msg = self.inliner.reporter.error('invalid PEP number %s' % self.target,
+                                              line=self.lineno)
+            prb = self.inliner.problematic(self.rawtext, self.rawtext, msg)
+            return [prb], [msg]
+
+        return [index, target, reference], []
+
+    def build_uri(self):
+        # type: () -> str
+        base_url = self.inliner.document.settings.pep_base_url
+        ret = self.target.split('#', 1)
+        if len(ret) == 2:
+            return base_url + 'pep-%04d#%s' % (int(ret[0]), ret[1])
+        else:
+            return base_url + 'pep-%04d' % int(ret[0])
+
+
+class RFC(ReferenceRole):
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
+        target_id = 'index-%s' % self.env.new_serialno('index')
+        entries = [('single', 'RFC; RFC %s' % self.target, target_id, '', None)]
+
+        index = addnodes.index(entries=entries)
+        target = nodes.target('', '', ids=[target_id])
+        self.inliner.document.note_explicit_target(target)
+
+        try:
+            refuri = self.build_uri()
+            reference = nodes.reference('', '', internal=False, refuri=refuri, classes=['rfc'])
+            if self.has_explicit_title:
+                reference += nodes.strong(self.title, self.title)
+            else:
+                title = "RFC " + self.title
+                reference += nodes.strong(title, title)
+        except ValueError:
+            msg = self.inliner.reporter.error('invalid RFC number %s' % self.target,
+                                              line=self.lineno)
+            prb = self.inliner.problematic(self.rawtext, self.rawtext, msg)
+            return [prb], [msg]
+
+        return [index, target, reference], []
+
+    def build_uri(self):
+        # type: () -> str
+        base_url = self.inliner.document.settings.rfc_base_url
+        ret = self.target.split('#', 1)
+        if len(ret) == 2:
+            return base_url + self.inliner.rfc_url % int(ret[0]) + '#' + ret[1]
+        else:
+            return base_url + self.inliner.rfc_url % int(ret[0])
+
+
 _amp_re = re.compile(r'(?<!&)&(?![&\s])')
 
 
 def menusel_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     # type: (str, str, str, int, Inliner, Dict, List[str]) -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
+    warnings.warn('menusel_role() is deprecated. '
+                  'Please use MenuSelection or GUILabel class instead.',
+                  RemovedInSphinx40Warning, stacklevel=2)
     env = inliner.document.settings.env
     if not typ:
         assert env.temp_data['default_role']
@@ -279,6 +378,32 @@ def menusel_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     return [node], []
 
 
+class GUILabel(SphinxRole):
+    amp_re = re.compile(r'(?<!&)&(?![&\s])')
+
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        node = nodes.inline(rawtext=self.rawtext, classes=[self.name])
+        spans = self.amp_re.split(self.text)
+        node += nodes.Text(spans.pop(0))
+        for span in spans:
+            span = span.replace('&&', '&')
+
+            letter = nodes.Text(span[0])
+            accelerator = nodes.inline('', '', letter, classes=['accelerator'])
+            node += accelerator
+            node += nodes.Text(span[1:])
+
+        return [node], []
+
+
+class MenuSelection(GUILabel):
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        self.text = self.text.replace('-->', '\N{TRIANGULAR BULLET}')  # type: ignore
+        return super().run()
+
+
 _litvar_re = re.compile('{([^}]+)}')
 parens_re = re.compile(r'(\\*{|\\*})')
 
@@ -286,6 +411,9 @@ parens_re = re.compile(r'(\\*{|\\*})')
 def emph_literal_role(typ, rawtext, text, lineno, inliner,
                       options={}, content=[]):
     # type: (str, str, str, int, Inliner, Dict, List[str]) -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
+    warnings.warn('emph_literal_role() is deprecated. '
+                  'Please use EmphasizedLiteral class instead.',
+                  RemovedInSphinx40Warning, stacklevel=2)
     env = inliner.document.settings.env
     if not typ:
         assert env.temp_data['default_role']
@@ -333,11 +461,65 @@ def emph_literal_role(typ, rawtext, text, lineno, inliner,
     return [retnode], []
 
 
+class EmphasizedLiteral(SphinxRole):
+    parens_re = re.compile(r'(\\\\|\\{|\\}|{|})')
+
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        children = self.parse(self.text)
+        node = nodes.literal(self.rawtext, '', *children,
+                             role=self.name.lower(), classes=[self.name])
+
+        return [node], []
+
+    def parse(self, text):
+        # type: (str) -> List[nodes.Node]
+        result = []  # type: List[nodes.Node]
+
+        stack = ['']
+        for part in self.parens_re.split(text):
+            if part == '\\\\':  # escaped backslash
+                stack[-1] += '\\'
+            elif part == '{':
+                if len(stack) >= 2 and stack[-2] == "{":  # nested
+                    stack[-1] += "{"
+                else:
+                    # start emphasis
+                    stack.append('{')
+                    stack.append('')
+            elif part == '}':
+                if len(stack) == 3 and stack[1] == "{" and len(stack[2]) > 0:
+                    # emphasized word found
+                    if stack[0]:
+                        result.append(nodes.Text(stack[0], stack[0]))
+                    result.append(nodes.emphasis(stack[2], stack[2]))
+                    stack = ['']
+                else:
+                    # emphasized word not found; the rparen is not a special symbol
+                    stack.append('}')
+                    stack = [''.join(stack)]
+            elif part == '\\{':  # escaped left-brace
+                stack[-1] += '{'
+            elif part == '\\}':  # escaped right-brace
+                stack[-1] += '}'
+            else:  # others (containing escaped braces)
+                stack[-1] += part
+
+        if ''.join(stack):
+            # remaining is treated as Text
+            text = ''.join(stack)
+            result.append(nodes.Text(text, text))
+
+        return result
+
+
 _abbr_re = re.compile(r'\((.*)\)$', re.S)
 
 
 def abbr_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     # type: (str, str, str, int, Inliner, Dict, List[str]) -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
+    warnings.warn('abbr_role() is deprecated.  Please use Abbrevation class instead.',
+                  RemovedInSphinx40Warning, stacklevel=2)
     text = utils.unescape(text)
     m = _abbr_re.search(text)
     if m is None:
@@ -349,8 +531,25 @@ def abbr_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     return [nodes.abbreviation(abbr, abbr, **options)], []
 
 
+class Abbreviation(SphinxRole):
+    abbr_re = re.compile(r'\((.*)\)$', re.S)
+
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        matched = self.abbr_re.search(self.text)
+        if matched:
+            text = self.text[:matched.start()].strip()
+            self.options['explanation'] = matched.group(1)
+        else:
+            text = self.text
+
+        return [nodes.abbreviation(self.rawtext, text, **self.options)], []
+
+
 def index_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     # type: (str, str, str, int, Inliner, Dict, List[str]) -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
+    warnings.warn('index_role() is deprecated.  Please use Index class instead.',
+                  RemovedInSphinx40Warning, stacklevel=2)
     # create new reference target
     env = inliner.document.settings.env
     targetid = 'index-%s' % env.new_serialno('index')
@@ -378,20 +577,44 @@ def index_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     return [indexnode, targetnode, textnode], []
 
 
+class Index(ReferenceRole):
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        target_id = 'index-%s' % self.env.new_serialno('index')
+        if self.has_explicit_title:
+            # if an explicit target is given, process it as a full entry
+            title = self.title
+            entries = process_index_entry(self.target, target_id)
+        else:
+            # otherwise we just create a single entry
+            if self.target.startswith('!'):
+                title = self.title[1:]
+                entries = [('single', self.target[1:], target_id, 'main', None)]
+            else:
+                title = self.title
+                entries = [('single', self.target, target_id, '', None)]
+
+        index = addnodes.index(entries=entries)
+        target = nodes.target('', '', ids=[target_id])
+        text = nodes.Text(title, title)
+        self.set_source_info(index)
+        return [index, target, text], []
+
+
 specific_docroles = {
     # links to download references
     'download': XRefRole(nodeclass=addnodes.download_reference),
     # links to anything
     'any': AnyXRefRole(warn_dangling=True),
 
-    'pep': indexmarkup_role,
+    'pep': PEP(),
-    'rfc': indexmarkup_role,
+    'rfc': RFC(),
-    'guilabel': menusel_role,
+    'guilabel': GUILabel(),
-    'menuselection': menusel_role,
+    'menuselection': MenuSelection(),
-    'file': emph_literal_role,
+    'file': EmphasizedLiteral(),
-    'samp': emph_literal_role,
+    'samp': EmphasizedLiteral(),
-    'abbr': abbr_role,
+    'abbr': Abbreviation(),
-    'index': index_role,
+    'index': Index(),
 }  # type: Dict[str, RoleFunction]
 
 

sphinx/search/__init__.py

@@ -18,8 +18,8 @@ from docutils import nodes
 from sphinx import addnodes
 from sphinx import package_dir
 from sphinx.deprecation import RemovedInSphinx40Warning
-from sphinx.util import jsdump, rpartition
 from sphinx.search.jssplitter import splitter_code
+from sphinx.util import jsdump, rpartition
 
 if False:
     # For type annotation

sphinx/templates/htmlhelp/project.hhc

@@ -0,0 +1,31 @@
+{%- macro sitemap(name, docname) -%}
+<OBJECT type="text/sitemap">
+  <PARAM name="Name" value="{{ name|e }}" />
+  <PARAM name="Local" value="{{ docname|e }}{{ suffix }}" />
+</OBJECT>
+{%- endmacro -%}
+
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+<HTML>
+  <HEAD>
+    <META name="GENERATOR" content="Microsoft&reg; HTML Help Workshop 4.1" />
+    <!-- Sitemap 1.0 -->
+  </HEAD>
+  <BODY>
+    <OBJECT type="text/site properties">
+      <PARAM name="Window Styles" value="0x801227" />
+      <PARAM name="ImageType" value="Folder" />
+    </OBJECT>
+    <UL>
+      <LI>
+        {{ sitemap(short_title, master_doc)|indent(8) }}
+      </LI>
+      {%- for indexname, indexcls, content, collapse in domain_indices %}
+      <LI>
+        {{ sitemap(indexcls.localname, indexname)|indent(8) }}
+      </LI>
+      {%- endfor %}
+      {{ body|indent(6) }}
+    </UL>
+  </BODY>
+</HTML>

sphinx/templates/htmlhelp/project.hhp

@@ -0,0 +1,21 @@
+[OPTIONS]
+Binary TOC=No
+Binary Index=No
+Compiled file={{ outname }}.chm
+Contents file={{ outname }}.hhc
+Default Window={{ outname }}
+Default topic={{ master_doc }}
+Display compile progress=No
+Full text search stop list file={{ outname }}.stp
+Full-text search=Yes
+Index file={{ outname }}.hhk
+Language={{ "%#x"|format(lcid) }}
+Title={{ title }}
+
+[WINDOWS]
+{{ outname }}="{{ title }}","{{ outname }}.hhc","{{ outname }}.hhk","{{ master_doc }}","{{ master_doc }}",,,,,0x63520,220,0x10384e,[0,0,1024,768],,,,,,,0
+
+[FILES]
+{%- for filename in files %}
+{{ filename }}
+{%- endfor %}

sphinx/templates/htmlhelp/project.stp

@@ -0,0 +1,33 @@
+a
+and
+are
+as
+at
+be
+but
+by
+for
+if
+in
+into
+is
+it
+near
+no
+not
+of
+on
+or
+such
+that
+the
+their
+then
+there
+these
+they
+this
+to
+was
+will
+with

sphinx/templates/quickstart/conf.py_t

@@ -30,10 +30,16 @@ project = {{ project | repr }}
 copyright = {{ copyright | repr }}
 author = {{ author | repr }}
 
+{%- if version %}
+
 # The short X.Y version
 version = {{ version | repr }}
+{%- endif %}
+{%- if release %}
+
 # The full version, including alpha/beta/rc tags
 release = {{ release | repr }}
+{%- endif %}
 
 
 # -- General configuration ---------------------------------------------------
@@ -50,17 +56,20 @@ extensions = [
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['{{ dot }}templates']
 
+{% if suffix != '.rst' -%}
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
 source_suffix = {{ suffix | repr }}
 
-{% if master_doc != 'index' -%}
+{% endif -%}
+{% if master != 'index' -%}
 # The master toctree document.
 master_doc = {{ master | repr }}
 
 {% endif -%}
+{% if language -%}
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
 #
@@ -68,6 +77,7 @@ master_doc = {{ master | repr }}
 # Usually you set "language" from the command line for these cases.
 language = {{ language | repr }}
 
+{% endif -%}
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
@@ -81,44 +91,10 @@ exclude_patterns = [{{ exclude_patterns }}]
 #
 html_theme = 'alabaster'
 
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['{{ dot }}static']
-
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# The default sidebars (for documents that don't match any pattern) are
-# defined by theme itself.  Builtin themes are using these templates by
-# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
-# 'searchbox.html']``.
-#
-# html_sidebars = {}
-
-
-# -- Options for Epub output -------------------------------------------------
-
-# Bibliographic Dublin Core info.
-epub_title = project
-
-# The unique identifier of the text. This can be a ISBN number
-# or the project homepage.
-#
-# epub_identifier = ''
-
-# A unique identification for the text.
-#
-# epub_uid = ''
-
-# A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
 {%- if extensions %}
 
 

sphinx/themes/basic/static/basic.css_t

@@ -231,6 +231,16 @@ a.headerlink {
     visibility: hidden;
 }
 
+a.brackets:before,
+span.brackets > a:before{
+    content: "[";
+}
+
+a.brackets:after,
+span.brackets > a:after {
+    content: "]";
+}
+
 h1:hover > a.headerlink,
 h2:hover > a.headerlink,
 h3:hover > a.headerlink,
@@ -391,6 +401,14 @@ table.citation td {
     border-bottom: none;
 }
 
+td > p:first-child {
+    margin-top: 0px;
+}
+
+td > p:only-child {
+    margin-bottom: 0px;
+}
+
 /* -- figures --------------------------------------------------------------- */
 
 div.figure {
@@ -460,11 +478,57 @@ ol.upperroman {
     list-style: upper-roman;
 }
 
+li > p:first-child {
+    margin-top: 0px;
+}
+
+li > p:only-child {
+    margin-bottom: 0px;
+}
+
+dl.footnote > dt,
+dl.citation > dt {
+    float: left;
+}
+
+dl.footnote > dd,
+dl.citation > dd {
+    margin-bottom: 0em;
+}
+
+dl.footnote > dd:after,
+dl.citation > dd:after {
+    content: "";
+    clear: both;
+}
+
+dl.field-list {
+    display: flex;
+    flex-wrap: wrap;
+}
+
+dl.field-list > dt {
+    flex-basis: 20%;
+    font-weight: bold;
+    word-break: break-word;
+}
+
+dl.field-list > dt:after {
+    content: ":";
+}
+
+dl.field-list > dd {
+    flex-basis: 70%;
+    padding-left: 1em;
+    margin-left: 0em;
+    margin-bottom: 0em;
+}
+
 dl {
     margin-bottom: 15px;
 }
 
-dd p {
+dd > p:first-child {
     margin-top: 0px;
 }
 

sphinx/transforms/__init__.py

@@ -23,7 +23,7 @@ from sphinx.locale import _, __
 from sphinx.util import logging
 from sphinx.util.docutils import new_document
 from sphinx.util.i18n import format_date
-from sphinx.util.nodes import apply_source_workaround, is_smartquotable
+from sphinx.util.nodes import NodeMatcher, apply_source_workaround, is_smartquotable
 
 if False:
     # For type annotation
@@ -309,6 +309,19 @@ class UnreferencedFootnotesDetector(SphinxTransform):
                                location=node)
 
 
+class FigureAligner(SphinxTransform):
+    """
+    Align figures to center by default.
+    """
+    default_priority = 700
+
+    def apply(self, **kwargs):
+        # type: (Any) -> None
+        matcher = NodeMatcher(nodes.table, nodes.figure)
+        for node in self.document.traverse(matcher):  # type: nodes.Element
+            node.setdefault('align', 'center')
+
+
 class FilterSystemMessages(SphinxTransform):
     """Filter system messages from a doctree."""
     default_priority = 999

sphinx/util/console.py

@@ -27,6 +27,12 @@ _ansi_re = re.compile('\x1b\\[(\\d\\d;){0,2}\\d\\dm')
 codes = {}  # type: Dict[str, str]
 
 
+def terminal_safe(s):
+    # type: (str) -> str
+    """safely encode a string for printing to the terminal."""
+    return s.encode('ascii', 'backslashreplace').decode('ascii')
+
+
 def get_terminal_width():
     # type: () -> int
     """Borrowed from the py lib."""

sphinx/util/docutils.py

@@ -23,10 +23,10 @@ from docutils import nodes
 from docutils.io import FileOutput
 from docutils.parsers.rst import Directive, directives, roles, convert_directive_function
 from docutils.statemachine import StateMachine
-from docutils.utils import Reporter
+from docutils.utils import Reporter, unescape
 
 from sphinx.deprecation import RemovedInSphinx30Warning
-from sphinx.errors import ExtensionError
+from sphinx.errors import ExtensionError, SphinxError
 from sphinx.locale import __
 from sphinx.util import logging
 
@@ -36,7 +36,8 @@ report_re = re.compile('^(.+?:(?:\\d+)?): \\((DEBUG|INFO|WARNING|ERROR|SEVERE)/(
 if False:
     # For type annotation
     from types import ModuleType  # NOQA
-    from typing import Any, Callable, Generator, List, Set, Tuple, Type  # NOQA
+    from typing import Any, Callable, Dict, Generator, List, Set, Tuple, Type  # NOQA
+    from docutils.parsers.rst.states import Inliner  # NOQA
     from docutils.statemachine import State, StringList  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.config import Config  # NOQA
@@ -383,6 +384,99 @@ class SphinxDirective(Directive):
         return self.env.config
 
 
+class SphinxRole:
+    """A base class for Sphinx roles.
+
+    This class provides helper methods for Sphinx roles.
+
+    .. note:: The subclasses of this class might not work with docutils.
+              This class is strongly coupled with Sphinx.
+    """
+    name = None     #: The role name actually used in the document.
+    rawtext = None  #: A string containing the entire interpreted text input.
+    text = None     #: The interpreted text content.
+    lineno = None   #: The line number where the interpreted text begins.
+    inliner = None  #: The ``docutils.parsers.rst.states.Inliner`` object.
+    options = None  #: A dictionary of directive options for customization
+                    #: (from the "role" directive).
+    content = None  #: A list of strings, the directive content for customization
+                    #: (from the "role" directive).
+
+    def __call__(self, name, rawtext, text, lineno, inliner, options={}, content=[]):
+        # type: (str, str, str, int, Inliner, Dict, List[str]) -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
+        self.rawtext = rawtext
+        self.text = unescape(text)
+        self.lineno = lineno
+        self.inliner = inliner
+        self.options = options
+        self.content = content
+
+        # guess role type
+        if name:
+            self.name = name.lower()
+        else:
+            self.name = self.env.temp_data.get('default_role')
+            if not self.name:
+                self.name = self.env.config.default_role
+            if not self.name:
+                raise SphinxError('cannot determine default role!')
+
+        return self.run()
+
+    def run(self):
+        # type: () -> Tuple[List[nodes.Node], List[nodes.system_message]]
+        raise NotImplementedError
+
+    @property
+    def env(self):
+        # type: () -> BuildEnvironment
+        """Reference to the :class:`.BuildEnvironment` object."""
+        return self.inliner.document.settings.env
+
+    @property
+    def config(self):
+        # type: () -> Config
+        """Reference to the :class:`.Config` object."""
+        return self.env.config
+
+    def set_source_info(self, node, lineno=None):
+        # type: (nodes.Node, int) -> None
+        if lineno is None:
+            lineno = self.lineno
+
+        source_info = self.inliner.reporter.get_source_and_line(lineno)  # type: ignore
+        node.source, node.line = source_info
+
+
+class ReferenceRole(SphinxRole):
+    """A base class for reference roles.
+
+    The reference roles can accpet ``link title <target>`` style as a text for
+    the role.  The parsed result; link title and target will be stored to
+    ``self.title`` and ``self.target``.
+    """
+    has_explicit_title = None   #: A boolean indicates the role has explicit title or not.
+    title = None                #: The link title for the interpreted text.
+    target = None               #: The link target for the interpreted text.
+
+    # \x00 means the "<" was backslash-escaped
+    explicit_title_re = re.compile(r'^(.+?)\s*(?<!\x00)<(.*?)>$', re.DOTALL)
+
+    def __call__(self, name, rawtext, text, lineno, inliner, options={}, content=[]):
+        # type: (str, str, str, int, Inliner, Dict, List[str]) -> Tuple[List[nodes.Node], List[nodes.system_message]]  # NOQA
+        matched = self.explicit_title_re.match(text)
+        if matched:
+            self.has_explicit_title = True
+            self.title = unescape(matched.group(1))
+            self.target = unescape(matched.group(2))
+        else:
+            self.has_explicit_title = False
+            self.title = unescape(text)
+            self.target = unescape(text)
+
+        return super().__call__(name, rawtext, text, lineno, inliner, options, content)
+
+
 class SphinxTranslator(nodes.NodeVisitor):
     """A base class for Sphinx translators.
 

sphinx/util/inspect.py

@@ -20,7 +20,7 @@ from io import StringIO
 
 from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.util import logging
-from sphinx.util.pycompat import NoneType
+from sphinx.util.typing import NoneType
 
 if False:
     # For type annotation

sphinx/util/jsonimpl.py

@@ -9,13 +9,20 @@
 """
 
 import json
+import warnings
 from collections import UserString
 
+from sphinx.deprecation import RemovedInSphinx40Warning
+
 if False:
     # For type annotation
     from typing import Any, IO  # NOQA
 
 
+warnings.warn('sphinx.util.jsonimpl is deprecated',
+              RemovedInSphinx40Warning, stacklevel=2)
+
+
 class SphinxJSONEncoder(json.JSONEncoder):
     """JSONEncoder subclass that forces translation proxies."""
     def default(self, obj):

sphinx/util/nodes.py

@@ -285,6 +285,14 @@ def find_source_node(node):
     return None
 
 
+def get_node_line(node):
+    # type: (nodes.Element) -> int
+    for pnode in traverse_parent(node):
+        if pnode.line:
+            return pnode.line
+    return None
+
+
 def traverse_parent(node, cls=None):
     # type: (nodes.Element, Any) -> Iterable[nodes.Element]
     while node:
@@ -293,6 +301,15 @@ def traverse_parent(node, cls=None):
         node = node.parent
 
 
+def get_prev_node(node):
+    # type: (nodes.Node) -> nodes.Node
+    pos = node.parent.index(node)
+    if pos > 0:
+        return node.parent[pos - 1]
+    else:
+        return None
+
+
 def traverse_translatable_index(doctree):
     # type: (nodes.Element) -> Iterable[Tuple[nodes.Element, List[str]]]
     """Traverse translatable index node from a document tree."""

sphinx/util/pycompat.py

@@ -17,6 +17,8 @@ import warnings
 from sphinx.deprecation import RemovedInSphinx40Warning, deprecated_alias
 from sphinx.locale import __
 from sphinx.util import logging
+from sphinx.util.console import terminal_safe
+from sphinx.util.typing import NoneType
 
 if False:
     # For type annotation
@@ -26,22 +28,9 @@ if False:
 logger = logging.getLogger(__name__)
 
 
-NoneType = type(None)
-
 # ------------------------------------------------------------------------------
 # Python 2/3 compatibility
 
-# sys_encoding: some kind of default system encoding; should be used with
-# a lenient error handler
-sys_encoding = sys.getdefaultencoding()
-
-
-# terminal_safe(): safely encode a string for printing to the terminal
-def terminal_safe(s):
-    # type: (str) -> str
-    return s.encode('ascii', 'backslashreplace').decode('ascii')
-
-
 # convert_with_2to3():
 # support for running 2to3 over config files
 def convert_with_2to3(filepath):
@@ -99,9 +88,12 @@ def execfile_(filepath, _globals, open=open):
 
 deprecated_alias('sphinx.util.pycompat',
                  {
+                     'NoneType': NoneType,  # type: ignore
                      'TextIOWrapper': io.TextIOWrapper,
                      'htmlescape': html.escape,
                      'indent': textwrap.indent,
+                     'terminal_safe': terminal_safe,
+                     'sys_encoding': sys.getdefaultencoding(),
                      'u': '',
                  },
                  RemovedInSphinx40Warning)

sphinx/util/typing.py

@@ -20,6 +20,9 @@ DirectiveOption = Callable[[str], Any]
 # Text like nodes which are initialized with text and rawsource
 TextlikeNode = Union[nodes.Text, nodes.TextElement]
 
+# type of None
+NoneType = type(None)
+
 # common role functions
 RoleFunction = Callable[[str, str, str, int, Inliner, Dict, List[str]],
                         Tuple[List[nodes.Node], List[nodes.system_message]]]

sphinx/writers/latex.py

@@ -30,7 +30,7 @@ from sphinx.errors import SphinxError
 from sphinx.locale import admonitionlabels, _, __
 from sphinx.util import split_into, logging
 from sphinx.util.docutils import SphinxTranslator
-from sphinx.util.nodes import clean_astext
+from sphinx.util.nodes import clean_astext, get_prev_node
 from sphinx.util.template import LaTeXRenderer
 from sphinx.util.texescape import tex_escape_map, tex_replace_map
 
@@ -1752,9 +1752,15 @@ class LaTeXTranslator(SphinxTranslator):
         elif domain.get_enumerable_node_type(next_node) and domain.get_numfig_title(next_node):
             return
 
-        if 'refuri' in node or 'refid' in node or 'refname' in node:
+        if 'refuri' in node:
-            # skip indirect targets (external hyperlink and internal links)
             return
+        if node.get('refid'):
+            prev_node = get_prev_node(node)
+            if isinstance(prev_node, nodes.reference) and node['refid'] == prev_node['refid']:
+                # a target for a hyperlink reference having alias
+                pass
+            else:
+                add_target(node['refid'])
         for id in node['ids']:
             add_target(id)
 

sphinx/writers/texinfo.py

@@ -247,7 +247,7 @@ class TexinfoTranslator(SphinxTranslator):
         title = self.settings.title  # type: str
         if not title:
             title_node = self.document.next_node(nodes.title)
-            title = (title and title_node.astext()) or '<untitled>'
+            title = (title_node and title_node.astext()) or '<untitled>'
         elements['title'] = self.escape_id(title) or '<untitled>'
         # filename
         if not elements['filename']:

tests/roots/test-build-htmlhelp/conf.py

@@ -1 +0,0 @@
-project = 'test'

tests/roots/test-build-htmlhelp/index.rst

@@ -1,19 +0,0 @@
-Index markup
-------------
-
-.. index::
-   single: entry
-   pair: entry; pair
-   double: entry; double
-   triple: index; entry; triple
-   keyword: with
-   see: from; to
-   seealso: fromalso; toalso
-
-.. index::
-   !Main, !Other
-   !single: entry; pair
-
-.. index:: triple-quoted string, Unicode Consortium, raw string
-   single: """; string literal
-   single: '''; string literal

tests/roots/test-build-htmlhelp/make.bat

@@ -1,64 +0,0 @@
-@echo off
-setlocal
-
-pushd %~dp0
-
-set this=%~n0
-
-if not defined PYTHON set PYTHON=py
-
-if not defined SPHINXBUILD (
-    %PYTHON% -c "import sphinx" > nul 2> nul
-    if errorlevel 1 (
-        echo Installing sphinx with %PYTHON%
-        %PYTHON% -m pip install sphinx
-        if errorlevel 1 exit /B
-    )
-    set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())"
-)
-
-rem Search for HHC in likely places
-set HTMLHELP=
-where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch
-where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
-if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
-if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
-if not exist "%HTMLHELP%" (
-    echo.
-    echo.The HTML Help Workshop was not found.  Set the HTMLHELP variable
-    echo.to the path to hhc.exe or download and install it from
-    echo.http://msdn.microsoft.com/en-us/library/ms669985
-    exit /B 1
-)
-echo hhc.exe path: %HTMLHELP%
-
-if "%BUILDDIR%" EQU "" set BUILDDIR=build
-
-%SPHINXBUILD% >nul 2> nul
-if errorlevel 9009 (
-    echo.
-    echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-    echo.installed, then set the SPHINXBUILD environment variable to point
-    echo.to the full path of the 'sphinx-build' executable. Alternatively you
-    echo.may add the Sphinx directory to PATH.
-    popd
-    exit /B 1
-)
-
-set SPHINXOPTS=-D html_theme_options.body_max_width=none %SPHINXOPTS%
-
-cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% -bhtmlhelp -dbuild\doctrees . "%BUILDDIR%\htmlhelp"
-
-"%HTMLHELP%" "%BUILDDIR%\htmlhelp\test.hhp"
-rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2
-if not errorlevel 2 cmd /C exit /b 0
-
-echo.
-if errorlevel 1 (
-    echo.Build failed (exit code %ERRORLEVEL%^), check for error messages
-    echo.above.  Any output will be found in %BUILDDIR%\%1
-) else (
-    echo.Build succeeded. All output should be in %BUILDDIR%\%1
-)
-
-popd

tests/roots/test-docutilsconf/conf.py

@@ -1,4 +0,0 @@
-project = 'Sphinx docutils conf <Tests>'
-source_suffix = '.txt'
-keep_warnings = True
-exclude_patterns = ['_build']

tests/roots/test-docutilsconf/index.rst

@@ -0,0 +1,6 @@
+test-docutilsconf
+==================
+
+Sphinx [1]_
+
+.. [1] Python Documentation Generator

tests/roots/test-docutilsconf/index.txt

@@ -1,15 +0,0 @@
-docutils conf
-=============
-
-field-name-limit
-----------------
-
-:short: desc
-:long long long long: long title
-
-option-limit
-------------
-
---short                short desc
---long-long-long-long  long desc
-

tests/roots/test-domain-cpp/conf.py

@@ -1,2 +1 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']

tests/roots/test-domain-js/conf.py

@@ -1,2 +1 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']

tests/roots/test-domain-py/conf.py

@@ -1,2 +1 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']

tests/roots/test-ext-autodoc/target/callable.py

@@ -4,5 +4,10 @@ class Callable():
     def __call__(self, arg1, arg2, **kwargs):
         pass
 
+    def method(self, arg1, arg2):
+        """docstring of Callable.method()."""
+        pass
+
 
 function = Callable()
+method = function.method

tests/roots/test-ext-autodoc/target/decorator.py

@@ -0,0 +1,16 @@
+def deco1(func):
+    """docstring for deco1"""
+    def wrapper():
+        return func()
+
+    return wrapper
+
+
+def deco2(condition, message):
+    """docstring for deco2"""
+    def decorator(func):
+        def wrapper():
+            return func()
+
+        return wrapper
+    return decorator

tests/roots/test-ext-autodoc/target/need_mocks.py

@@ -15,6 +15,11 @@ def decoratedFunction():
     return None
 
 
+def func(arg: missing_module.Class):
+    """a function takes mocked object as an argument"""
+    pass
+
+
 class TestAutodoc(object):
     """TestAutodoc docstring."""
     @missing_name

tests/roots/test-ext-autosectionlabel-prefix-document/index.rst

@@ -15,6 +15,12 @@ For Windows users
 For UNIX users
 --------------
 
+Linux
+^^^^^
+
+FreeBSD
+^^^^^^^
+
 This one's got an apostrophe
 ----------------------------
 
@@ -26,4 +32,6 @@ References
 * :ref:`index:Installation`
 * :ref:`index:For Windows users`
 * :ref:`index:For UNIX users`
+* :ref:`index:Linux`
+* :ref:`index:FreeBSD`
 * :ref:`index:This one's got an apostrophe`

tests/roots/test-ext-autosectionlabel/index.rst

@@ -15,15 +15,23 @@ For Windows users
 For UNIX users
 --------------
 
+Linux
+^^^^^
+
+FreeBSD
+^^^^^^^
+
 This one's got an apostrophe
 ----------------------------
 
-
 References
 ==========
 
+* :ref:`test-ext-autosectionlabel`
 * :ref:`Introduce of Sphinx`
 * :ref:`Installation`
 * :ref:`For Windows users`
 * :ref:`For UNIX users`
+* :ref:`Linux`
+* :ref:`FreeBSD`
 * :ref:`This one's got an apostrophe`

tests/roots/test-ext-autosummary-mock_imports/conf.py

@@ -0,0 +1,7 @@
+import os
+import sys
+sys.path.insert(0, os.path.abspath('.'))
+
+extensions = ['sphinx.ext.autosummary']
+autosummary_generate = True
+autosummary_mock_imports = ['unknown']

tests/roots/test-ext-autosummary-mock_imports/foo.py

@@ -0,0 +1,6 @@
+import unknown
+
+
+class Foo(unknown.Class):
+    """Foo class"""
+    pass

tests/roots/test-ext-autosummary-mock_imports/index.rst

@@ -0,0 +1,7 @@
+test-ext-autosummary-mock_imports
+=================================
+
+.. autosummary::
+   :toctree: generated
+
+   foo

tests/roots/test-footnotes/conf.py

@@ -1,2 +1 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']

tests/roots/test-html_entity/index.rst

@@ -7,7 +7,7 @@ Empty cell
 ----------
 
 .. list-table::
-   :header-rows: 1
+
    - * un
      *
      * trois

tests/roots/test-latex-table/expects/longtable.tex

@@ -1,6 +1,6 @@
 \label{\detokenize{longtable:longtable}}
 
-\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|l|l|}
+\begin{savenotes}\sphinxatlongtablestart\begin{longtable}[c]{|l|l|}
 \hline
 \sphinxstyletheadfamily 
 header1

tests/roots/test-latex-table/expects/longtable_having_caption.tex

@@ -1,6 +1,6 @@
 \label{\detokenize{longtable:longtable-having-caption}}
 
-\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|l|l|}
+\begin{savenotes}\sphinxatlongtablestart\begin{longtable}[c]{|l|l|}
 \sphinxthelongtablecaptionisattop
 \caption{caption for longtable\strut}\label{\detokenize{longtable:id1}}\\*[\sphinxlongtablecapskipadjust]
 \hline

tests/roots/test-latex-table/expects/longtable_having_problematic_cell.tex

@@ -1,6 +1,6 @@
 \label{\detokenize{longtable:longtable-having-problematic-cell}}
 
-\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|*{2}{\X{1}{2}|}}
+\begin{savenotes}\sphinxatlongtablestart\begin{longtable}[c]{|*{2}{\X{1}{2}|}}
 \hline
 \sphinxstyletheadfamily 
 header1

tests/roots/test-latex-table/expects/longtable_having_stub_columns_and_problematic_cell.tex

@@ -1,6 +1,6 @@
 \label{\detokenize{longtable:longtable-having-both-stub-columns-and-problematic-cell}}
 
-\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|*{3}{\X{1}{3}|}}
+\begin{savenotes}\sphinxatlongtablestart\begin{longtable}[c]{|*{3}{\X{1}{3}|}}
 \hline
 \sphinxstyletheadfamily 
 header1

tests/roots/test-latex-table/expects/longtable_having_verbatim.tex

@@ -1,6 +1,6 @@
 \label{\detokenize{longtable:longtable-having-verbatim}}
 
-\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|*{2}{\X{1}{2}|}}
+\begin{savenotes}\sphinxatlongtablestart\begin{longtable}[c]{|*{2}{\X{1}{2}|}}
 \hline
 \sphinxstyletheadfamily 
 header1

tests/roots/test-latex-table/expects/longtable_having_widths.tex

@@ -1,6 +1,6 @@
 \label{\detokenize{longtable:longtable-having-widths-option}}
 
-\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{30}{100}|\X{70}{100}|}
+\begin{savenotes}\sphinxatlongtablestart\begin{longtable}[c]{|\X{30}{100}|\X{70}{100}|}
 \hline\noalign{\phantomsection\label{\detokenize{longtable:namedlongtable}}\label{\detokenize{longtable:mylongtable}}}%
 \sphinxstyletheadfamily 
 header1

tests/roots/test-latex-table/expects/longtable_having_widths_and_problematic_cell.tex

@@ -1,6 +1,6 @@
 \label{\detokenize{longtable:longtable-having-both-widths-and-problematic-cell}}
 
-\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|\X{30}{100}|\X{70}{100}|}
+\begin{savenotes}\sphinxatlongtablestart\begin{longtable}[c]{|\X{30}{100}|\X{70}{100}|}
 \hline
 \sphinxstyletheadfamily 
 header1

tests/roots/test-latex-table/expects/longtable_with_tabularcolumn.tex

@@ -1,6 +1,6 @@
 \label{\detokenize{longtable:longtable-with-tabularcolumn}}
 
-\begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|c|c|}
+\begin{savenotes}\sphinxatlongtablestart\begin{longtable}[c]{|c|c|}
 \hline
 \sphinxstyletheadfamily 
 header1

tests/roots/test-linkcheck/links.txt

@@ -11,3 +11,6 @@ Some additional anchors to exercise ignore code
 * `Example Bar invalid <http://example.com/#top>`_
 * `Example anchor invalid <http://www.sphinx-doc.org/en/1.7/intro.html#does-not-exist>`_
 * `Complete nonsense <https://localhost:7777/doesnotexist>`_
+
+.. image:: http://example.com/image.png
+.. figure:: http://example.com/image2.png

tests/roots/test-manpage_url/conf.py

@@ -1,2 +1 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']

tests/roots/test-maxlistdepth/conf.py

@@ -1,4 +1,3 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']
 
 latex_elements = {

tests/roots/test-numfig/conf.py

@@ -1,2 +1 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']

tests/roots/test-root/autodoc.txt

@@ -3,8 +3,6 @@ Autodoc tests
 
 Just testing a few autodoc possibilities...
 
-.. automodule:: util
-
 .. automodule:: autodoc_target
    :members:
 
@@ -28,12 +26,6 @@ Just testing a few autodoc possibilities...
    :members:
 
 
-.. automodule:: autodoc_fodder
-   :noindex:
-
-   .. autoclass:: MarkupError
-
-
 .. currentmodule:: autodoc_target
 
 .. autoclass:: InstAttCls

tests/roots/test-root/autodoc_fodder.py

@@ -1,7 +0,0 @@
-
-class MarkupError(object):
-    """
-    .. note:: This is a docstring with a
-    small markup error which should have
-    correct location information.
-    """

tests/roots/test-root/images.txt

@@ -4,9 +4,6 @@ Sphinx image handling
 .. first, a simple test with direct filename
 .. image:: img.png
 
-.. a non-existing image with direct filename
-.. image:: foo.png
-
 .. an image with path name (relative to this directory!)
 .. image:: subdir/img.png
    :height: 100

tests/roots/test-root/includes.txt

@@ -3,7 +3,6 @@ Testing downloadable files
 
 Download :download:`img.png` here.
 Download :download:`this <subdir/img.png>` there.
-Don't download :download:`this <nonexisting.png>`.
 
 Test file and literal inclusion
 ===============================

tests/roots/test-root/subdir/include.inc

@@ -2,4 +2,4 @@
 
 .. Paths in included files are relative to the file that
    includes them
-.. image:: ../root/img.png
+.. image:: subdir/img.png

tests/roots/test-toctree-empty/conf.py

@@ -1,3 +1,2 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']
 templates_path = ['_templates']

tests/roots/test-toctree-glob/conf.py

@@ -1,2 +1 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']

tests/roots/test-toctree-maxdepth/conf.py

@@ -1,2 +1 @@
-html_theme = 'classic'
 exclude_patterns = ['_build']

tests/test_api_translator.py

@@ -12,6 +12,8 @@ import sys
 
 import pytest
 
+from sphinx.util import docutils
+
 
 @pytest.fixture(scope='module', autouse=True)
 def setup_module(rootdir):
@@ -26,7 +28,10 @@ def test_html_translator(app, status, warning):
     # no set_translator()
     translator_class = app.builder.get_translator_class()
     assert translator_class
-    assert translator_class.__name__ == 'HTMLTranslator'
+    if docutils.__version_info__ < (0, 13):
+        assert translator_class.__name__ == 'HTMLTranslator'
+    else:
+        assert translator_class.__name__ == 'HTML5Translator'
 
 
 @pytest.mark.sphinx('html', testroot='api-set-translator')

tests/test_autodoc.py

@@ -602,6 +602,29 @@ def test_generate():
                            'Class.meth', more_content=add_content)
 
 
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autodoc_decorator(app):
+    actual = do_autodoc(app, 'decorator', 'target.decorator.deco1')
+    assert list(actual) == [
+        '',
+        '.. py:decorator:: deco1',
+        '   :module: target.decorator',
+        '',
+        '   docstring for deco1',
+        '   '
+    ]
+
+    actual = do_autodoc(app, 'decorator', 'target.decorator.deco2')
+    assert list(actual) == [
+        '',
+        '.. py:decorator:: deco2(condition, message)',
+        '   :module: target.decorator',
+        '',
+        '   docstring for deco2',
+        '   '
+    ]
+
+
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
 def test_autodoc_exception(app):
     actual = do_autodoc(app, 'exception', 'target.CustomEx')
@@ -1342,10 +1365,23 @@ def test_autofunction_for_callable(app):
     ]
 
 
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autofunction_for_method(app):
+    actual = do_autodoc(app, 'function', 'target.callable.method')
+    assert list(actual) == [
+        '',
+        '.. py:function:: method(arg1, arg2)',
+        '   :module: target.callable',
+        '',
+        '   docstring of Callable.method().',
+        '   '
+    ]
+
+
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
 def test_mocked_module_imports(app, warning):
     # no autodoc_mock_imports
-    options = {"members": 'TestAutodoc,decoratedFunction'}
+    options = {"members": 'TestAutodoc,decoratedFunction,func'}
     actual = do_autodoc(app, 'module', 'target.need_mocks', options)
     assert list(actual) == []
     assert "autodoc: failed to import module 'need_mocks'" in warning.getvalue()
@@ -1382,6 +1418,12 @@ def test_mocked_module_imports(app, warning):
         '   :module: target.need_mocks',
         '',
         '   decoratedFunction docstring',
+        '   ',
+        '',
+        '.. py:function:: func(arg: missing_module.Class)',
+        '   :module: target.need_mocks',
+        '',
+        '   a function takes mocked object as an argument',
         '   '
     ]
     assert warning.getvalue() == ''
@@ -1511,6 +1553,12 @@ def test_autodoc_default_options(app):
     assert '   .. py:attribute:: EnumCls.val1' in actual
     assert '   .. py:attribute:: EnumCls.val4' not in actual
 
+    # with :members: = True
+    app.config.autodoc_default_options = {'members': True}
+    actual = do_autodoc(app, 'class', 'target.enum.EnumCls')
+    assert '   .. py:attribute:: EnumCls.val1' in actual
+    assert '   .. py:attribute:: EnumCls.val4' not in actual
+
     # with :members: and :undoc-members:
     app.config.autodoc_default_options = {
         'members': None,

tests/test_build.py

@@ -8,7 +8,6 @@
     :license: BSD, see LICENSE for details.
 """
 
-import pickle
 import sys
 from textwrap import dedent
 
@@ -58,11 +57,7 @@ def nonascii_srcdir(request, rootdir, sphinx_test_tempdir):
 #       (html, epub, latex, texinfo and manpage)
 @pytest.mark.parametrize(
     "buildername",
-    [
+    ['dirhtml', 'singlehtml', 'text', 'changes', 'xml', 'pseudoxml', 'linkcheck'],
-        # note: no 'html' - if it's ok with dirhtml it's ok with html
-        'dirhtml', 'singlehtml', 'pickle', 'json', 'text', 'htmlhelp',
-        'changes', 'xml', 'pseudoxml', 'linkcheck',
-    ],
 )
 @mock.patch('sphinx.builders.linkcheck.requests.head',
             side_effect=request_session_head)
@@ -110,7 +105,7 @@ def test_image_glob(app, status, warning):
     app.builder.build_all()
 
     # index.rst
-    doctree = pickle.loads((app.doctreedir / 'index.doctree').bytes())
+    doctree = app.env.get_doctree('index')
 
     assert isinstance(doctree[0][1], nodes.image)
     assert doctree[0][1]['candidates'] == {'*': 'rimg.png'}
@@ -135,7 +130,7 @@ def test_image_glob(app, status, warning):
     assert doctree[0][4][0]['uri'] == 'img.*'
 
     # subdir/index.rst
-    doctree = pickle.loads((app.doctreedir / 'subdir/index.doctree').bytes())
+    doctree = app.env.get_doctree('subdir/index')
 
     assert isinstance(doctree[0][1], nodes.image)
     sub = path('subdir')

tests/test_build_epub.py

@@ -15,6 +15,8 @@ from xml.etree import ElementTree
 
 import pytest
 
+from sphinx.util import docutils
+
 
 # check given command is runnable
 def runnable(command):
@@ -70,7 +72,7 @@ def test_build_epub(app):
 
     # toc.ncx
     toc = EPUBElementTree.fromstring((app.outdir / 'toc.ncx').text())
-    assert toc.find("./ncx:docTitle/ncx:text").text == 'Python  documentation'
+    assert toc.find("./ncx:docTitle/ncx:text").text == 'Python'
 
     # toc.ncx / head
     meta = list(toc.find("./ncx:head"))
@@ -94,7 +96,7 @@ def test_build_epub(app):
     # content.opf / metadata
     metadata = opf.find("./idpf:metadata")
     assert metadata.find("./dc:language").text == 'en'
-    assert metadata.find("./dc:title").text == 'Python  documentation'
+    assert metadata.find("./dc:title").text == 'Python'
     assert metadata.find("./dc:description").text == 'unknown'
     assert metadata.find("./dc:creator").text == 'unknown'
     assert metadata.find("./dc:contributor").text == 'unknown'
@@ -174,7 +176,7 @@ def test_nested_toc(app):
 
     # toc.ncx
     toc = EPUBElementTree.fromstring((app.outdir / 'toc.ncx').bytes())
-    assert toc.find("./ncx:docTitle/ncx:text").text == 'Python  documentation'
+    assert toc.find("./ncx:docTitle/ncx:text").text == 'Python'
 
     # toc.ncx / navPoint
     def navinfo(elem):
@@ -229,8 +231,7 @@ def test_escaped_toc(app):
 
     # toc.ncx
     toc = EPUBElementTree.fromstring((app.outdir / 'toc.ncx').bytes())
-    assert toc.find("./ncx:docTitle/ncx:text").text == ('need <b>"escaped"</b> '
+    assert toc.find("./ncx:docTitle/ncx:text").text == 'need <b>"escaped"</b> project'
-                                                        'project  documentation')
 
     # toc.ncx / navPoint
     def navinfo(elem):
@@ -351,6 +352,8 @@ def test_epub_css_files(app):
             'href="https://example.com/custom.css" />' not in content)
 
 
+@pytest.mark.skipif(docutils.__version_info__ < (0, 13),
+                    reason='docutils-0.13 or above is required')
 @pytest.mark.sphinx('epub', testroot='roles-download')
 def test_html_download_role(app, status, warning):
     app.build()