Merge branch '5.x'

# Conflicts:
#	CHANGES
#	doc/conf.py
#	sphinx/__init__.py
#	sphinx/builders/html/__init__.py
#	sphinx/domains/python.py
#	tests/test_build_html.py

CHANGES

@@ -21,16 +21,12 @@ Bugs fixed
 Testing
 --------
 
-Release 5.2.0 (in development)
+Release 5.3.0 (in development)
 ==============================
 
 Dependencies
 ------------
 
-* #10356: Sphinx now uses declarative metadata with ``pyproject.toml`` to
-  create packages, using PyPA's ``build`` project as a build backend. Patch by
-  Adam Turner.
-
 Incompatible changes
 --------------------
 
@@ -40,6 +36,47 @@ Deprecated
 Features added
 --------------
 
+Bugs fixed
+----------
+
+Testing
+--------
+
+Release 5.2.1 (released Sep 24, 2022)
+=====================================
+
+Bugs fixed
+----------
+
+* #10861: Always normalise the ``pycon3`` lexer to ``pycon``.
+* Fix using ``sphinx.ext.autosummary`` with modules containing titles in the
+  module-level docstring.
+
+Release 5.2.0.post0 (released Sep 24, 2022)
+===========================================
+
+* Recreated source tarballs for Debian maintainers.
+
+Release 5.2.0 (released Sep 24, 2022)
+=====================================
+
+Dependencies
+------------
+
+* #10356: Sphinx now uses declarative metadata with ``pyproject.toml`` to
+  create packages, using PyPA's ``flit`` project as a build backend. Patch by
+  Adam Turner.
+
+Deprecated
+----------
+
+* #10843: Support for HTML 4 output. Patch by Adam Turner.
+
+Features added
+--------------
+
+* #10738: napoleon: Add support for docstring types using 'of', like
+  ``type of type``. Example: ``tuple of int``.
 * #10286: C++, support requires clauses not just between the template
   parameter lists and the declaration.
 * #10755: linkcheck: Check the source URL of raw directives that use the ``url``
@@ -53,44 +90,18 @@ Features added
 * #6692: HTML Search: Include explicit :rst:dir:`index` directive index entries
   in the search index and search results. Patch by Adam Turner
 * #10816: imgmath: Allow embedding images in HTML as base64
-
+* #10854: HTML Search: Use browser localstorage for highlight control, stop
-Bugs fixed
+  storing highlight parameters in URL query strings. Patch by Adam Turner.
-----------
-
-* #10257: C++, ensure consistent non-specialization template argument
-  representation.
-* #10729: C++, fix parsing of certain non-type template parameter packs.
-* #10715: Revert #10520: "Fix" use of sidebar classes in ``agogo.css_t``
-
-Testing
---------
-
-Release 5.1.2 (in development)
-==============================
-
-Dependencies
-------------
-
-Incompatible changes
---------------------
-
-Deprecated
-----------
-
-Features added
---------------
-
-* #10738: napoleon: Add support for docstring types using 'of', like
-  ``type of type``. Example: ``tuple of int``.
 
 Bugs fixed
 ----------
 
 * #10723: LaTeX: 5.1.0 has made the 'sphinxsetup' ``verbatimwithframe=false``
   become without effect.
-
+* #10257: C++, ensure consistent non-specialization template argument
-Testing
+  representation.
---------
+* #10729: C++, fix parsing of certain non-type template parameter packs.
+* #10715: Revert #10520: "Fix" use of sidebar classes in ``agogo.css_t``
 
 Release 5.1.1 (released Jul 26, 2022)
 =====================================

doc/changes.rst

@@ -11,5 +11,12 @@ Changelog
 .. raw:: latex
 
    \addtocontents{toc}{\protect\setcounter{tocdepth}{1}}%
+   \makeatletter
+   \addtocontents{toc}%
+   {\def\string\l@section{\string\@dottedtocline{1}{1.5em}{3.3em}}}
+   \addtocontents{toc}%
+   {\def\string\l@subsection{\string\@dottedtocline{2}{4.8em}{4em}}}
+   \makeatother
+
 
 .. include:: ../CHANGES

doc/conf.py

@@ -48,6 +48,7 @@ epub_post_files = [('usage/installation.xhtml', 'Installing Sphinx'),
                    ('develop.xhtml', 'Sphinx development')]
 epub_exclude_files = ['_static/opensearch.xml', '_static/doctools.js',
                       '_static/searchtools.js',
+                      '_static/sphinx_highlight.js',
                       '_static/basic.css',
                       '_static/language_data.js',
                       'search.html', '_static/websupport.js']

doc/extdev/deprecated.rst

@@ -22,6 +22,11 @@ The following is a list of deprecated interfaces.
      - (will be) Removed
      - Alternatives
 
+   * - HTML 4 support
+     - 5.2
+     - 7.0
+     - N/A
+
    * - ``sphinx.util.path_stabilize``
      - 5.1
      - 7.0

doc/latex.rst

@@ -1300,6 +1300,12 @@ Macros
      ``\sphinxtableofcontentshook``.  This macro is also executed by the
      ``'howto'`` docclass, but defaults to empty with it.
 
+  .. hint::
+
+     If adding to preamble the loading of ``tocloft`` package, also add to
+     preamble ``\renewcommand\sphinxtableofcontentshook{}`` else it will reset
+     ``\l@section`` and ``\l@subsection`` cancelling ``tocloft`` customization.
+
 - ``\sphinxmaketitle``: Used as the default setting of the ``'maketitle'``
   :confval:`latex_elements` key.
   Defined in the class files :file:`sphinxmanual.cls` and

karma.conf.js

@@ -15,8 +15,10 @@ module.exports = function(config) {
 
     // list of files / patterns to load in the browser
     files: [
+      'tests/js/documentation_options.js',
       'sphinx/themes/basic/static/doctools.js',
       'sphinx/themes/basic/static/searchtools.js',
+      'sphinx/themes/basic/static/sphinx_highlight.js',
       'tests/js/*.js'
     ],
 

pyproject.toml

@@ -150,6 +150,7 @@ show_error_context = true
 strict_optional = true
 warn_redundant_casts = true
 warn_unused_ignores = true
+disallow_any_generics = true
 
 [[tool.mypy.overrides]]
 module = [
@@ -181,6 +182,30 @@ module = [
 ]
 strict_optional = false
 
+[[tool.mypy.overrides]]
+module = [
+    "sphinx.application",
+    "sphinx.builders.*",
+    "sphinx.cmd.*",
+    "sphinx.config",
+    "sphinx.deprecation",
+    "sphinx.domains.*",
+    "sphinx.environment.*",
+    "sphinx.events",
+    "sphinx.ext.*",
+    "sphinx.highlighting",
+    "sphinx.jinja2glue",
+    "sphinx.locale",
+    "sphinx.pycode.*",
+    "sphinx.registry",
+    "sphinx.roles",
+    "sphinx.search.*",
+    "sphinx.testing.*",
+    "sphinx.util.*",
+    "sphinx.writers.*",
+]
+disallow_any_generics = false
+
 [tool.pytest.ini_options]
 filterwarnings = [
     "all",

sphinx/addnodes.py

@@ -1,4 +1,4 @@
-"""Additional docutils nodes."""
+"""Document tree nodes that Sphinx defines on top of those in Docutils."""
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence
 
@@ -119,7 +119,7 @@ class toctree(nodes.General, nodes.Element, translatable):
 #############################################################
 
 class _desc_classes_injector(nodes.Element, not_smartquotable):
-    """Helper base class for injecting a fixes list of classes.
+    """Helper base class for injecting a fixed list of classes.
 
     Use as the first base class.
     """
@@ -390,7 +390,7 @@ class index(nodes.Invisible, nodes.Inline, nodes.TextElement):
 
 
 class centered(nodes.Part, nodes.TextElement):
-    """Deprecated."""
+    """This node is deprecated."""
 
 
 class acks(nodes.Element):
@@ -455,13 +455,18 @@ class pending_xref(nodes.Inline, nodes.Element):
 
 
 class pending_xref_condition(nodes.Inline, nodes.TextElement):
-    """Node for cross-references that are used to choose appropriate
+    """Node representing a potential way to create a cross-reference and the
-    content of the reference by conditions on the resolving phase.
+    condition in which this way should be used.
 
-    When the :py:class:`pending_xref` node contains one or more
+    This node is only allowed to be placed under a :py:class:`pending_xref`
-    **pending_xref_condition** nodes, the cross-reference resolver
+    node.  A **pending_xref** node must contain either no **pending_xref_condition**
-    should choose the content of the reference using defined conditions
+    nodes or it must only contains **pending_xref_condition** nodes.
-    in ``condition`` attribute of each pending_xref_condition nodes::
+
+    The cross-reference resolver will replace a :py:class:`pending_xref` which
+    contains **pending_xref_condition** nodes by the content of exactly one of
+    those **pending_xref_condition** nodes' content. It uses the **condition**
+    attribute to decide which **pending_xref_condition** node's content to
+    use. For example, let us consider how the cross-reference resolver acts on::
 
         <pending_xref refdomain="py" reftarget="io.StringIO ...>
             <pending_xref_condition condition="resolved">
@@ -471,32 +476,26 @@ class pending_xref_condition(nodes.Inline, nodes.TextElement):
                 <literal>
                     io.StringIO
 
-    After the processing of cross-reference resolver, one of the content node
+    If the cross-reference resolver successfully resolves the cross-reference,
-    under pending_xref_condition node is chosen by its condition and to be
+    then it rewrites the **pending_xref** as::
-    removed all of pending_xref_condition nodes::
 
-        # When resolved the cross-reference successfully
         <reference>
             <literal>
                 StringIO
 
-        # When resolution is failed
+    Otherwise, if the cross-reference resolution failed, it rewrites the
+    **pending_xref** as::
+
         <reference>
             <literal>
                 io.StringIO
 
-    .. note:: This node is only allowed to be placed under pending_xref node.
+    The **pending_xref_condition** node should have **condition** attribute.
-              It is not allows to place it under other nodes.  In addition,
-              pending_xref node must contain only pending_xref_condition
-              nodes if it contains one or more pending_xref_condition nodes.
-
-    The pending_xref_condition node should have **condition** attribute.
     Domains can be store their individual conditions into the attribute to
     filter contents on resolving phase.  As a reserved condition name,
     ``condition="*"`` is used for the fallback of resolution failure.
     Additionally, as a recommended condition name, ``condition="resolved"``
-    is used for the representation of resolstion success in the intersphinx
+    represents a resolution success in the intersphinx module.
-    module.
 
     .. versionadded:: 4.0
     """

sphinx/application.py

@@ -636,8 +636,9 @@ class Sphinx:
 
         :param name: The name of the directive
         :param cls: A directive class
-        :param override: If true, install the directive forcedly even if another directive
+        :param override: If false, do not install it if another directive
                          is already installed as the same name
+                         If true, unconditionally install the directive.
 
         For example, a custom directive named ``my-directive`` would be added
         like this:
@@ -684,8 +685,9 @@ class Sphinx:
 
         :param name: The name of role
         :param role: A role function
-        :param override: If true, install the role forcedly even if another role is already
+        :param override: If false, do not install it if another role
-                         installed as the same name
+                         is already installed as the same name
+                         If true, unconditionally install the role.
 
         For more details about role functions, see `the Docutils docs
         <https://docutils.sourceforge.io/docs/howto/rst-roles.html>`__ .
@@ -705,8 +707,9 @@ class Sphinx:
         Register a Docutils role that does nothing but wrap its contents in the
         node given by *nodeclass*.
 
-        If *override* is True, the given *nodeclass* is forcedly installed even if
+        :param override: If false, do not install it if another role
-        a role named as *name* is already installed.
+                         is already installed as the same name
+                         If true, unconditionally install the role.
 
         .. versionadded:: 0.6
         .. versionchanged:: 1.8
@@ -725,8 +728,9 @@ class Sphinx:
         """Register a domain.
 
         :param domain: A domain class
-        :param override: If true, install the domain forcedly even if another domain
+        :param override: If false, do not install it if another domain
                          is already installed as the same name
+                         If true, unconditionally install the domain.
 
         .. versionadded:: 1.0
         .. versionchanged:: 1.8
@@ -744,8 +748,9 @@ class Sphinx:
         :param domain: The name of target domain
         :param name: A name of directive
         :param cls: A directive class
-        :param override: If true, install the directive forcedly even if another directive
+        :param override: If false, do not install it if another directive
                          is already installed as the same name
+                         If true, unconditionally install the directive.
 
         .. versionadded:: 1.0
         .. versionchanged:: 1.8
@@ -763,8 +768,9 @@ class Sphinx:
         :param domain: The name of the target domain
         :param name: The name of the role
         :param role: The role function
-        :param override: If true, install the role forcedly even if another role is already
+        :param override: If false, do not install it if another role
-                         installed as the same name
+                         is already installed as the same name
+                         If true, unconditionally install the role.
 
         .. versionadded:: 1.0
         .. versionchanged:: 1.8
@@ -780,8 +786,9 @@ class Sphinx:
 
         :param domain: The name of the target domain
         :param index: The index class
-        :param override: If true, install the index forcedly even if another index is
+        :param override: If false, do not install it if another index
-                         already installed as the same name
+                         is already installed as the same name
+                         If true, unconditionally install the index.
 
         .. versionadded:: 1.0
         .. versionchanged:: 1.8
@@ -886,8 +893,10 @@ class Sphinx:
         (Of course, the element following the ``topic`` directive needn't be a
         section.)
 
-        If *override* is True, the given crossref_type is forcedly installed even if
+
-        a crossref_type having the same name is already installed.
+        :param override: If false, do not install it if another cross-reference type
+                         is already installed as the same name
+                         If true, unconditionally install the cross-reference type.
 
         .. versionchanged:: 1.8
            Add *override* keyword.
@@ -946,20 +955,22 @@ class Sphinx:
                     loading_method: Optional[str] = None, **kwargs: Any) -> None:
         """Register a JavaScript file to include in the HTML output.
 
-        :param filename: The filename of the JavaScript file.  It must be relative to the HTML
+        :param filename: The name of a JavaScript file that the default HTML
-                         static path, a full URI with scheme, or ``None`` value.  The ``None``
+                         template will include. It must be relative to the HTML
-                         value is used to create inline ``<script>`` tag.  See the description
+                         static path, or a full URI with scheme, or ``None`` .
-                         of *kwargs* below.
+                         The ``None`` value is used to create an inline
-        :param priority: The priority to determine the order of ``<script>`` tag for
+                         ``<script>`` tag.  See the description of *kwargs*
-                         JavaScript files.  See list of "prority range for JavaScript
+                         below.
-                         files" below.  If the priority of the JavaScript files it the same
+        :param priority: Files are included in ascending order of priority. If
-                         as others, the JavaScript files will be loaded in order of
+                         multiple JavaScript files have the same priority,
-                         registration.
+                         those files will be included in order of registration.
-        :param loading_method: The loading method of the JavaScript file.  ``'async'`` or
+                         See list of "prority range for JavaScript files" below.
-                               ``'defer'`` is allowed.
+        :param loading_method: The loading method for the JavaScript file.
-        :param kwargs: Extra keyword arguments are included as attributes of the ``<script>``
+                               Either ``'async'`` or ``'defer'`` are allowed.
-                       tag.  A special keyword argument ``body`` is given, its value will be
+        :param kwargs: Extra keyword arguments are included as attributes of the
-                       added between the ``<script>`` tag.
+                       ``<script>`` tag.  If the special keyword argument
+                       ``body`` is given, its value will be added as the content
+                       of the  ``<script>`` tag.
 
         Example::
 
@@ -1012,14 +1023,15 @@ class Sphinx:
     def add_css_file(self, filename: str, priority: int = 500, **kwargs: Any) -> None:
         """Register a stylesheet to include in the HTML output.
 
-        :param filename: The filename of the CSS file.  It must be relative to the HTML
+        :param filename: The name of a CSS file that the default HTML
+                         template will include. It must be relative to the HTML
                          static path, or a full URI with scheme.
-        :param priority: The priority to determine the order of ``<link>`` tag for the
+        :param priority: Files are included in ascending order of priority. If
-                         CSS files.  See list of "prority range for CSS files" below.
+                         multiple CSS files have the same priority,
-                         If the priority of the CSS files it the same as others, the
+                         those files will be included in order of registration.
-                         CSS files will be loaded in order of registration.
+                         See list of "prority range for CSS files" below.
-        :param kwargs: Extra keyword arguments are included as attributes of the ``<link>``
+        :param kwargs: Extra keyword arguments are included as attributes of the
-                       tag.
+                       ``<link>`` tag.
 
         Example::
 
@@ -1167,8 +1179,9 @@ class Sphinx:
         Same as :confval:`source_suffix`.  The users can override this
         using the config setting.
 
-        If *override* is True, the given *suffix* is forcedly installed even if
+        :param override: If false, do not install it the same suffix
-        the same suffix is already installed.
+                         is already installed.
+                         If true, unconditionally install the suffix.
 
         .. versionadded:: 1.8
         """
@@ -1177,8 +1190,9 @@ class Sphinx:
     def add_source_parser(self, parser: Type[Parser], override: bool = False) -> None:
         """Register a parser class.
 
-        If *override* is True, the given *parser* is forcedly installed even if
+        :param override: If false, do not install it if another parser
-        a parser for the same suffix is already installed.
+                         is already installed for the same suffix.
+                         If true, unconditionally install the parser.
 
         .. versionadded:: 1.4
         .. versionchanged:: 1.8

sphinx/builders/__init__.py

@@ -10,6 +10,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Sequence
 
 from docutils import nodes
 from docutils.nodes import Node
+from docutils.utils import DependencyList
 
 from sphinx.config import Config
 from sphinx.deprecation import RemovedInSphinx70Warning
@@ -490,6 +491,9 @@ class Builder:
         filename = self.env.doc2path(docname)
         filetype = get_filetype(self.app.config.source_suffix, filename)
         publisher = self.app.registry.get_publisher(self.app, filetype)
+        # record_dependencies is mutable even though it is in settings,
+        # explicitly re-initialise for each document
+        publisher.settings.record_dependencies = DependencyList()
         with sphinx_domains(self.env), rst.default_role(docname, self.config.default_role):
             # set up error_handler for the target document
             codecs.register_error('sphinx', UnicodeDecodeErrorHandler(docname))  # type: ignore

sphinx/builders/html/__init__.py

@@ -349,6 +349,7 @@ class StandaloneHTMLBuilder(Builder):
         self.add_js_file('documentation_options.js', id="documentation_options",
                          data_url_root='', priority=200)
         self.add_js_file('doctools.js', priority=200)
+        self.add_js_file('sphinx_highlight.js', priority=200)
 
         for filename, attrs in self.app.registry.js_files:
             self.add_js_file(filename, **attrs)
@@ -369,7 +370,7 @@ class StandaloneHTMLBuilder(Builder):
     @property
     def default_translator_class(self) -> Type[nodes.NodeVisitor]:  # type: ignore
         if self.config.html4_writer:
-            return HTMLTranslator
+            return HTMLTranslator  # RemovedInSphinx70Warning
         else:
             return HTML5Translator
 
@@ -1303,6 +1304,15 @@ def validate_html_favicon(app: Sphinx, config: Config) -> None:
         config.html_favicon = None  # type: ignore
 
 
+def deprecate_html_4(_app: Sphinx, config: Config) -> None:
+    """Warn on HTML 4."""
+    # RemovedInSphinx70Warning
+    if config.html4_writer:
+        logger.warning(_('Support for emitting HTML 4 output is deprecated and '
+                         'will be removed in Sphinx 7. ("html4_writer=True '
+                         'detected in configuration options)'))
+
+
 # for compatibility
 import sphinxcontrib.serializinghtml  # NOQA
 
@@ -1375,6 +1385,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.connect('config-inited', validate_html_static_path, priority=800)
     app.connect('config-inited', validate_html_logo, priority=800)
     app.connect('config-inited', validate_html_favicon, priority=800)
+    app.connect('config-inited', deprecate_html_4, priority=800)
     app.connect('builder-inited', validate_math_renderer)
     app.connect('html-page-context', setup_css_tag_helper)
     app.connect('html-page-context', setup_js_tag_helper)

sphinx/builders/latex/transforms.py

@@ -30,7 +30,7 @@ class FootnoteDocnameUpdater(SphinxTransform):
 
 
 class SubstitutionDefinitionsRemover(SphinxPostTransform):
-    """Remove ``substitution_definition node from doctrees."""
+    """Remove ``substitution_definition`` nodes from doctrees."""
 
     # should be invoked after Substitutions process
     default_priority = Substitutions.default_priority + 1

sphinx/domains/std.py

@@ -493,12 +493,12 @@ class ProductionList(SphinxDirective):
         lines = nl_escape_re.sub('', self.arguments[0]).split('\n')
 
         productionGroup = ""
-        i = 0
+        first_rule_seen = False
         for rule in lines:
-            if i == 0 and ':' not in rule:
+            if not first_rule_seen and ':' not in rule:
                 productionGroup = rule.strip()
                 continue
-            i += 1
+            first_rule_seen = True
             try:
                 name, tokens = rule.split(':', 1)
             except ValueError:

sphinx/ext/autodoc/__init__.py

@@ -939,6 +939,7 @@ class ModuleDocumenter(Documenter):
     objtype = 'module'
     content_indent = ''
     titles_allowed = True
+    _extra_indent = '   '
 
     option_spec: OptionSpec = {
         'members': members_option, 'undoc-members': bool_option,
@@ -958,7 +959,7 @@ class ModuleDocumenter(Documenter):
 
     def add_content(self, more_content: Optional[StringList]) -> None:
         old_indent = self.indent
-        self.indent += '   '
+        self.indent += self._extra_indent
         super().add_content(None)
         self.indent = old_indent
         if more_content:

sphinx/ext/autosummary/__init__.py

@@ -362,6 +362,9 @@ class Autosummary(SphinxDirective):
 
             # -- Grab the summary
 
+            # bodge for ModuleDocumenter
+            documenter._extra_indent = ''  # type: ignore[attr-defined]
+
             documenter.add_content(None)
             summary = extract_summary(self.bridge.result.data[:], self.state.document)
 

sphinx/ext/doctest.py

@@ -94,15 +94,9 @@ class TestDirective(SphinxDirective):
             # only save if it differs from code
             node['test'] = test
         if self.name == 'doctest':
-            if self.config.highlight_language in ('py', 'python'):
+            node['language'] = 'pycon'
-                node['language'] = 'pycon'
-            else:
-                node['language'] = 'pycon3'  # default
         elif self.name == 'testcode':
-            if self.config.highlight_language in ('py', 'python'):
+            node['language'] = 'python'
-                node['language'] = 'python'
-            else:
-                node['language'] = 'python3'  # default
         elif self.name == 'testoutput':
             # don't try to highlight output
             node['language'] = 'none'

sphinx/ext/viewcode.py

@@ -244,7 +244,7 @@ def collect_pages(app: Sphinx) -> Generator[Tuple[str, Dict[str, Any], str], Non
         # construct a page name for the highlighted source
         pagename = posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/'))
         # highlight the source using the builder's highlighter
-        if env.config.highlight_language in ('python3', 'default', 'none'):
+        if env.config.highlight_language in {'default', 'none'}:
             lexer = env.config.highlight_language
         else:
             lexer = 'python'

sphinx/extension.py

@@ -34,7 +34,14 @@ class Extension:
 
 
 def verify_needs_extensions(app: "Sphinx", config: Config) -> None:
-    """Verify the required Sphinx extensions are loaded."""
+    """Check that extensions mentioned in :confval:`needs_extensions` satisfy the version
+    requirement, and warn if an extension is not loaded.
+
+    Warns if an extension in :confval:`needs_extension` is not loaded.
+
+    :raises VersionRequirementError: if the version of an extension in
+    :confval:`needs_extension` is unknown or older than the required version.
+    """
     if config.needs_extensions is None:
         return
 

sphinx/highlighting.py

@@ -120,6 +120,8 @@ class PygmentsBridge:
                 lang = 'pycon'
             else:
                 lang = 'python'
+        if lang == 'pycon3':
+            lang = 'pycon'
 
         if lang in lexers:
             # just return custom lexers here (without installing raiseonerror filter)

sphinx/search/__init__.py

@@ -290,9 +290,11 @@ class IndexBuilder:
         self._titles = dict(zip(index2fn, frozen['titles']))
         self._all_titles = {}
 
+        for docname in self._titles.keys():
+            self._all_titles[docname] = []
         for title, doc_tuples in frozen['alltitles'].items():
             for doc, titleid in doc_tuples:
-                self._all_titles.setdefault(index2fn[doc], []).append((title, titleid))
+                self._all_titles[index2fn[doc]].append((title, titleid))
 
         def load_terms(mapping: Dict[str, Any]) -> Dict[str, Set[str]]:
             rv = {}
@@ -380,12 +382,12 @@ class IndexBuilder:
         alltitles: Dict[str, List[Tuple[int, str]]] = {}
         for docname, titlelist in self._all_titles.items():
             for title, titleid in titlelist:
-                alltitles.setdefault(title, []).append((fn2index[docname],  titleid))
+                alltitles.setdefault(title, []).append((fn2index[docname], titleid))
 
         index_entries: Dict[str, List[Tuple[int, str]]] = {}
         for docname, entries in self._index_entries.items():
             for entry, entry_id, main_entry in entries:
-                index_entries.setdefault(entry.lower(), []).append((fn2index[docname],  entry_id))
+                index_entries.setdefault(entry.lower(), []).append((fn2index[docname], entry_id))
 
         return dict(docnames=docnames, filenames=filenames, titles=titles, terms=terms,
                     objects=objects, objtypes=objtypes, objnames=objnames,

sphinx/themes/basic/static/doctools.js

@@ -10,6 +10,13 @@
  */
 "use strict";
 
+const BLACKLISTED_KEY_CONTROL_ELEMENTS = new Set([
+  "TEXTAREA",
+  "INPUT",
+  "SELECT",
+  "BUTTON",
+]);
+
 const _ready = (callback) => {
   if (document.readyState !== "loading") {
     callback();
@@ -18,73 +25,11 @@ const _ready = (callback) => {
   }
 };
 
-/**
- * highlight a given string on a node by wrapping it in
- * span elements with the given class name.
- */
-const _highlight = (node, addItems, text, className) => {
-  if (node.nodeType === Node.TEXT_NODE) {
-    const val = node.nodeValue;
-    const parent = node.parentNode;
-    const pos = val.toLowerCase().indexOf(text);
-    if (
-      pos >= 0 &&
-      !parent.classList.contains(className) &&
-      !parent.classList.contains("nohighlight")
-    ) {
-      let span;
-
-      const closestNode = parent.closest("body, svg, foreignObject");
-      const isInSVG = closestNode && closestNode.matches("svg");
-      if (isInSVG) {
-        span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
-      } else {
-        span = document.createElement("span");
-        span.classList.add(className);
-      }
-
-      span.appendChild(document.createTextNode(val.substr(pos, text.length)));
-      parent.insertBefore(
-        span,
-        parent.insertBefore(
-          document.createTextNode(val.substr(pos + text.length)),
-          node.nextSibling
-        )
-      );
-      node.nodeValue = val.substr(0, pos);
-
-      if (isInSVG) {
-        const rect = document.createElementNS(
-          "http://www.w3.org/2000/svg",
-          "rect"
-        );
-        const bbox = parent.getBBox();
-        rect.x.baseVal.value = bbox.x;
-        rect.y.baseVal.value = bbox.y;
-        rect.width.baseVal.value = bbox.width;
-        rect.height.baseVal.value = bbox.height;
-        rect.setAttribute("class", className);
-        addItems.push({ parent: parent, target: rect });
-      }
-    }
-  } else if (node.matches && !node.matches("button, select, textarea")) {
-    node.childNodes.forEach((el) => _highlight(el, addItems, text, className));
-  }
-};
-const _highlightText = (thisNode, text, className) => {
-  let addItems = [];
-  _highlight(thisNode, addItems, text, className);
-  addItems.forEach((obj) =>
-    obj.parent.insertAdjacentElement("beforebegin", obj.target)
-  );
-};
-
 /**
  * Small JavaScript module for the documentation.
  */
 const Documentation = {
   init: () => {
-    Documentation.highlightSearchWords();
     Documentation.initDomainIndexTable();
     Documentation.initOnKeyListeners();
   },
@@ -126,51 +71,6 @@ const Documentation = {
     Documentation.LOCALE = catalog.locale;
   },
 
-  /**
-   * highlight the search words provided in the url in the text
-   */
-  highlightSearchWords: () => {
-    const highlight =
-      new URLSearchParams(window.location.search).get("highlight") || "";
-    const terms = highlight.toLowerCase().split(/\s+/).filter(x => x);
-    if (terms.length === 0) return; // nothing to do
-
-    // There should never be more than one element matching "div.body"
-    const divBody = document.querySelectorAll("div.body");
-    const body = divBody.length ? divBody[0] : document.querySelector("body");
-    window.setTimeout(() => {
-      terms.forEach((term) => _highlightText(body, term, "highlighted"));
-    }, 10);
-
-    const searchBox = document.getElementById("searchbox");
-    if (searchBox === null) return;
-    searchBox.appendChild(
-      document
-        .createRange()
-        .createContextualFragment(
-          '<p class="highlight-link">' +
-            '<a href="javascript:Documentation.hideSearchWords()">' +
-            Documentation.gettext("Hide Search Matches") +
-            "</a></p>"
-        )
-    );
-  },
-
-  /**
-   * helper function to hide the search marks again
-   */
-  hideSearchWords: () => {
-    document
-      .querySelectorAll("#searchbox .highlight-link")
-      .forEach((el) => el.remove());
-    document
-      .querySelectorAll("span.highlighted")
-      .forEach((el) => el.classList.remove("highlighted"));
-    const url = new URL(window.location);
-    url.searchParams.delete("highlight");
-    window.history.replaceState({}, "", url);
-  },
-
   /**
    * helper function to focus on search bar
    */
@@ -210,15 +110,11 @@ const Documentation = {
     )
       return;
 
-    const blacklistedElements = new Set([
-      "TEXTAREA",
-      "INPUT",
-      "SELECT",
-      "BUTTON",
-    ]);
     document.addEventListener("keydown", (event) => {
-      if (blacklistedElements.has(document.activeElement.tagName)) return; // bail for input elements
+      // bail for input elements
-      if (event.altKey || event.ctrlKey || event.metaKey) return; // bail with special keys
+      if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return;
+      // bail with special keys
+      if (event.altKey || event.ctrlKey || event.metaKey) return;
 
       if (!event.shiftKey) {
         switch (event.key) {
@@ -240,10 +136,6 @@ const Documentation = {
               event.preventDefault();
             }
             break;
-          case "Escape":
-            if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) break;
-            Documentation.hideSearchWords();
-            event.preventDefault();
         }
       }
 

sphinx/themes/basic/static/searchtools.js

@@ -57,7 +57,7 @@ const _removeChildren = (element) => {
 const _escapeRegExp = (string) =>
   string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string
 
-const _displayItem = (item, highlightTerms, searchTerms) => {
+const _displayItem = (item, searchTerms) => {
   const docBuilder = DOCUMENTATION_OPTIONS.BUILDER;
   const docUrlRoot = DOCUMENTATION_OPTIONS.URL_ROOT;
   const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX;
@@ -82,10 +82,8 @@ const _displayItem = (item, highlightTerms, searchTerms) => {
     requestUrl = docUrlRoot + docName + docFileSuffix;
     linkUrl = docName + docLinkSuffix;
   }
-  const params = new URLSearchParams();
-  params.set("highlight", [...highlightTerms].join(" "));
   let linkEl = listItem.appendChild(document.createElement("a"));
-  linkEl.href = linkUrl + "?" + params.toString() + anchor;
+  linkEl.href = linkUrl + anchor;
   linkEl.dataset.score = score;
   linkEl.innerHTML = title;
   if (descr)
@@ -97,7 +95,7 @@ const _displayItem = (item, highlightTerms, searchTerms) => {
       .then((data) => {
         if (data)
           listItem.appendChild(
-            Search.makeSearchSummary(data, searchTerms, highlightTerms)
+            Search.makeSearchSummary(data, searchTerms)
           );
       });
   Search.output.appendChild(listItem);
@@ -117,15 +115,14 @@ const _finishSearch = (resultCount) => {
 const _displayNextItem = (
   results,
   resultCount,
-  highlightTerms,
   searchTerms
 ) => {
   // results left, load the summary and display it
   // this is intended to be dynamic (don't sub resultsCount)
   if (results.length) {
-    _displayItem(results.pop(), highlightTerms, searchTerms);
+    _displayItem(results.pop(), searchTerms);
     setTimeout(
-      () => _displayNextItem(results, resultCount, highlightTerms, searchTerms),
+      () => _displayNextItem(results, resultCount, searchTerms),
       5
     );
   }
@@ -271,6 +268,10 @@ const Search = {
       }
     });
 
+    if (SPHINX_HIGHLIGHT_ENABLED) {  // set in sphinx_highlight.js
+      localStorage.setItem("sphinx_highlight_terms", [...highlightTerms].join(" "))
+    }
+
     // console.debug("SEARCH: searching for:");
     // console.info("required: ", [...searchTerms]);
     // console.info("excluded: ", [...excludedTerms]);
@@ -286,7 +287,7 @@ const Search = {
           let score = Math.round(100 * queryLower.length / title.length)
           results.push([
             docNames[file],
-            `${titles[file]} > ${title}`,
+            titles[file] !== title ? `${titles[file]} > ${title}` : title,
             id !== null ? "#" + id : "",
             null,
             score,
@@ -359,7 +360,7 @@ const Search = {
     // console.info("search results:", Search.lastresults);
 
     // print the results
-    _displayNextItem(results, results.length, highlightTerms, searchTerms);
+    _displayNextItem(results, results.length, searchTerms);
   },
 
   /**
@@ -538,11 +539,9 @@ const Search = {
   /**
    * helper function to return a node containing the
    * search summary for a given text. keywords is a list
-   * of stemmed words, highlightWords is the list of normal, unstemmed
+   * of stemmed words.
-   * words. the first one is used to find the occurrence, the
-   * latter for highlighting it.
    */
-  makeSearchSummary: (htmlText, keywords, highlightWords) => {
+  makeSearchSummary: (htmlText, keywords) => {
     const text = Search.htmlToText(htmlText);
     if (text === "") return null;
 
@@ -560,10 +559,6 @@ const Search = {
     summary.classList.add("context");
     summary.textContent = top + text.substr(startWithContext, 240).trim() + tail;
 
-    highlightWords.forEach((highlightWord) =>
-      _highlightText(summary, highlightWord, "highlighted")
-    );
-
     return summary;
   },
 };

sphinx/themes/basic/static/sphinx_highlight.js

@@ -0,0 +1,144 @@
+/* Highlighting utilities for Sphinx HTML documentation. */
+"use strict";
+
+const SPHINX_HIGHLIGHT_ENABLED = true
+
+/**
+ * highlight a given string on a node by wrapping it in
+ * span elements with the given class name.
+ */
+const _highlight = (node, addItems, text, className) => {
+  if (node.nodeType === Node.TEXT_NODE) {
+    const val = node.nodeValue;
+    const parent = node.parentNode;
+    const pos = val.toLowerCase().indexOf(text);
+    if (
+      pos >= 0 &&
+      !parent.classList.contains(className) &&
+      !parent.classList.contains("nohighlight")
+    ) {
+      let span;
+
+      const closestNode = parent.closest("body, svg, foreignObject");
+      const isInSVG = closestNode && closestNode.matches("svg");
+      if (isInSVG) {
+        span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
+      } else {
+        span = document.createElement("span");
+        span.classList.add(className);
+      }
+
+      span.appendChild(document.createTextNode(val.substr(pos, text.length)));
+      parent.insertBefore(
+        span,
+        parent.insertBefore(
+          document.createTextNode(val.substr(pos + text.length)),
+          node.nextSibling
+        )
+      );
+      node.nodeValue = val.substr(0, pos);
+
+      if (isInSVG) {
+        const rect = document.createElementNS(
+          "http://www.w3.org/2000/svg",
+          "rect"
+        );
+        const bbox = parent.getBBox();
+        rect.x.baseVal.value = bbox.x;
+        rect.y.baseVal.value = bbox.y;
+        rect.width.baseVal.value = bbox.width;
+        rect.height.baseVal.value = bbox.height;
+        rect.setAttribute("class", className);
+        addItems.push({ parent: parent, target: rect });
+      }
+    }
+  } else if (node.matches && !node.matches("button, select, textarea")) {
+    node.childNodes.forEach((el) => _highlight(el, addItems, text, className));
+  }
+};
+const _highlightText = (thisNode, text, className) => {
+  let addItems = [];
+  _highlight(thisNode, addItems, text, className);
+  addItems.forEach((obj) =>
+    obj.parent.insertAdjacentElement("beforebegin", obj.target)
+  );
+};
+
+/**
+ * Small JavaScript module for the documentation.
+ */
+const SphinxHighlight = {
+
+  /**
+   * highlight the search words provided in localstorage in the text
+   */
+  highlightSearchWords: () => {
+    if (!SPHINX_HIGHLIGHT_ENABLED) return;  // bail if no highlight
+
+    // get and clear terms from localstorage
+    const url = new URL(window.location);
+    const highlight =
+        localStorage.getItem("sphinx_highlight_terms")
+        || url.searchParams.get("highlight")
+        || "";
+    localStorage.removeItem("sphinx_highlight_terms")
+    url.searchParams.delete("highlight");
+    window.history.replaceState({}, "", url);
+
+    // get individual terms from highlight string
+    const terms = highlight.toLowerCase().split(/\s+/).filter(x => x);
+    if (terms.length === 0) return; // nothing to do
+
+    // There should never be more than one element matching "div.body"
+    const divBody = document.querySelectorAll("div.body");
+    const body = divBody.length ? divBody[0] : document.querySelector("body");
+    window.setTimeout(() => {
+      terms.forEach((term) => _highlightText(body, term, "highlighted"));
+    }, 10);
+
+    const searchBox = document.getElementById("searchbox");
+    if (searchBox === null) return;
+    searchBox.appendChild(
+      document
+        .createRange()
+        .createContextualFragment(
+          '<p class="highlight-link">' +
+            '<a href="javascript:SphinxHighlight.hideSearchWords()">' +
+            _("Hide Search Matches") +
+            "</a></p>"
+        )
+    );
+  },
+
+  /**
+   * helper function to hide the search marks again
+   */
+  hideSearchWords: () => {
+    document
+      .querySelectorAll("#searchbox .highlight-link")
+      .forEach((el) => el.remove());
+    document
+      .querySelectorAll("span.highlighted")
+      .forEach((el) => el.classList.remove("highlighted"));
+    localStorage.removeItem("sphinx_highlight_terms")
+  },
+
+  initEscapeListener: () => {
+    // only install a listener if it is really needed
+    if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return;
+
+    document.addEventListener("keydown", (event) => {
+      // bail for input elements
+      if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return;
+      // bail with special keys
+      if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return;
+      if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) {
+        SphinxHighlight.hideSearchWords();
+        event.preventDefault();
+      }
+    });
+  },
+};
+
+_ready(SphinxHighlight.highlightSearchWords);
+_ready(SphinxHighlight.initEscapeListener);

sphinx/transforms/post_transforms/code.py

@@ -109,9 +109,9 @@ class TrimDoctestFlagsTransform(SphinxTransform):
             return False  # skip parsed-literal node
 
         language = node.get('language')
-        if language in ('pycon', 'pycon3'):
+        if language in {'pycon', 'pycon3'}:
             return True
-        elif language in ('py', 'py3', 'python', 'python3', 'default'):
+        elif language in {'py', 'python', 'py3', 'python3', 'default'}:
             return node.rawsource.startswith('>>>')
         elif language == 'guess':
             try:

sphinx/writers/html.py

@@ -65,6 +65,7 @@ class HTMLWriter(Writer):
         self.clean_meta = ''.join(self.visitor.meta[2:])
 
 
+# RemovedInSphinx70Warning
 class HTMLTranslator(SphinxTranslator, BaseTranslator):
     """
     Our custom HTML translator.

tests/js/documentation_options.js

@@ -0,0 +1 @@
+const DOCUMENTATION_OPTIONS = {};

tests/js/sphinx_highlight.js

@@ -1,5 +1,3 @@
-const DOCUMENTATION_OPTIONS = {};
-
 describe('highlightText', function() {
 
   const cyrillicTerm = 'шеллы';

tests/roots/test-environment-record-dependencies/api.rst

@@ -0,0 +1,4 @@
+API
+===
+
+.. automodule:: example_module

tests/roots/test-environment-record-dependencies/conf.py

@@ -0,0 +1,5 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('.'))
+extensions = ['sphinx.ext.autodoc']

tests/roots/test-environment-record-dependencies/example_module.py

@@ -0,0 +1,2 @@
+def example_function():
+    return 42

tests/roots/test-environment-record-dependencies/index.rst

@@ -0,0 +1,3 @@
+.. toctree::
+
+    api

tests/test_build_html.py

@@ -131,6 +131,16 @@ def test_html4_output(app, status, warning):
     app.build()
 
 
+def test_html4_deprecation(make_app, tempdir):
+    (tempdir / 'conf.py').write_text('', encoding='utf-8')
+    app = make_app(
+        buildername='html',
+        srcdir=tempdir,
+        confoverrides={'html4_writer': True},
+    )
+    assert 'HTML 4 output is deprecated and will be removed' in app._warning.getvalue()
+
+
 @pytest.mark.parametrize("fname,expect", flat_dict({
     'images.html': [
         (".//img[@src='_images/img.png']", ''),
@@ -1222,7 +1232,8 @@ def test_assets_order(app):
 
     # js_files
     expected = ['_static/early.js',
-                '_static/doctools.js', 'https://example.com/script.js', '_static/normal.js',
+                '_static/doctools.js', '_static/sphinx_highlight.js',
+                'https://example.com/script.js', '_static/normal.js',
                 '_static/late.js', '_static/js/custom.js', '_static/lazy.js']
     pattern = '.*'.join('src="%s"' % f for f in expected)
     assert re.search(pattern, content, re.S)

tests/test_environment_record_dependencies.py

@@ -0,0 +1,10 @@
+"""Tests for ``record_dependencies``."""
+
+import pytest
+
+
+@pytest.mark.sphinx('html', testroot='environment-record-dependencies')
+def test_record_dependencies_cleared(app):
+    app.builder.read()
+    assert app.env.dependencies['index'] == set()
+    assert app.env.dependencies['api'] == {'example_module.py'}

tests/test_ext_doctest.py

@@ -29,16 +29,16 @@ def test_highlight_language_default(app, status, warning):
     app.build()
     doctree = app.env.get_doctree('doctest')
     for node in doctree.findall(nodes.literal_block):
-        assert node['language'] in ('python3', 'pycon3', 'none')
+        assert node['language'] in {'python', 'pycon', 'none'}
 
 
 @pytest.mark.sphinx('dummy', testroot='ext-doctest',
                     confoverrides={'highlight_language': 'python'})
-def test_highlight_language_python2(app, status, warning):
+def test_highlight_language_python3(app, status, warning):
     app.build()
     doctree = app.env.get_doctree('doctest')
     for node in doctree.findall(nodes.literal_block):
-        assert node['language'] in ('python', 'pycon', 'none')
+        assert node['language'] in {'python', 'pycon', 'none'}
 
 
 def test_is_allowed_version():

tests/test_highlighting.py

@@ -81,14 +81,23 @@ def test_default_highlight(logger):
     ret = bridge.highlight_block('reST ``like`` text', 'default')
     assert ret == '<div class="highlight"><pre><span></span>reST ``like`` text\n</pre></div>\n'
 
-    # python3: highlights as python3
+    # python: highlights as python3
-    ret = bridge.highlight_block('print "Hello sphinx world"', 'python3')
+    ret = bridge.highlight_block('print("Hello sphinx world")', 'python')
-    assert ret == ('<div class="highlight"><pre><span></span><span class="nb">print</span> '
+    assert ret == ('<div class="highlight"><pre><span></span><span class="nb">print</span>'
-                   '<span class="s2">&quot;Hello sphinx world&quot;</span>\n</pre></div>\n')
+                   '<span class="p">(</span>'
+                   '<span class="s2">&quot;Hello sphinx world&quot;</span>'
+                   '<span class="p">)</span>\n</pre></div>\n')
 
-    # python3: raises error if highlighting failed
+    # python3: highlights as python3
-    ret = bridge.highlight_block('reST ``like`` text', 'python3')
+    ret = bridge.highlight_block('print("Hello sphinx world")', 'python3')
+    assert ret == ('<div class="highlight"><pre><span></span><span class="nb">print</span>'
+                   '<span class="p">(</span>'
+                   '<span class="s2">&quot;Hello sphinx world&quot;</span>'
+                   '<span class="p">)</span>\n</pre></div>\n')
+
+    # python: raises error if highlighting failed
+    ret = bridge.highlight_block('reST ``like`` text', 'python')
     logger.warning.assert_called_with('Could not lex literal_block as "%s". '
-                                      'Highlighting skipped.', 'python3',
+                                      'Highlighting skipped.', 'python',
                                       type='misc', subtype='highlighting_failure',
                                       location=None)

utils/release-checklist

@@ -14,7 +14,7 @@ for stable releases
 * ``twine upload dist/Sphinx-* --sign --identity [your GPG key]``
 * open https://pypi.org/project/Sphinx/ and check there are no obvious errors
 * ``sh utils/bump_docker.sh X.Y.Z``
-* ``git tag vX.Y.Z``
+* ``git tag vX.Y.Z -m "Sphinx X.Y.Z"``
 * ``python utils/bump_version.py --in-develop X.Y.Zb0`` (ex. 1.5.3b0)
 * Check diff by ``git diff``
 * ``git commit -am 'Bump version'``
@@ -37,7 +37,7 @@ for first beta releases
 * ``python -m build .``
 * ``twine upload dist/Sphinx-* --sign --identity [your GPG key]``
 * open https://pypi.org/project/Sphinx/ and check there are no obvious errors
-* ``git tag vX.Y.0b1``
+* ``git tag vX.Y.0b1 -m "Sphinx X.Y.0b1"``
 * ``python utils/bump_version.py --in-develop X.Y.0b2`` (ex. 1.6.0b2)
 * Check diff by ``git diff``
 * ``git commit -am 'Bump version'``
@@ -65,7 +65,7 @@ for other beta releases
 * ``python -m build .``
 * ``twine upload dist/Sphinx-* --sign --identity [your GPG key]``
 * open https://pypi.org/project/Sphinx/ and check there are no obvious errors
-* ``git tag vX.Y.0bN``
+* ``git tag vX.Y.0bN -m "Sphinx X.Y.0bN"``
 * ``python utils/bump_version.py --in-develop X.Y.0bM`` (ex. 1.6.0b3)
 * Check diff by `git diff``
 * ``git commit -am 'Bump version'``
@@ -91,7 +91,7 @@ for major releases
 * ``twine upload dist/Sphinx-* --sign --identity [your GPG key]``
 * open https://pypi.org/project/Sphinx/ and check there are no obvious errors
 * ``sh utils/bump_docker.sh X.Y.Z``
-* ``git tag vX.Y.0``
+* ``git tag vX.Y.0 -m "Sphinx X.Y.0"``
 * ``python utils/bump_version.py --in-develop X.Y.1b0`` (ex. 1.6.1b0)
 * Check diff by ``git diff``
 * ``git commit -am 'Bump version'``
@@ -101,7 +101,7 @@ for major releases
 * ``git push origin master``
 * open https://github.com/sphinx-doc/sphinx/settings/branches and make ``A.B`` branch *not* protected
 * ``git checkout A.B`` (checkout old stable)
-* Run ``git tag A.B`` to paste a tag instead branch
+* Run ``git tag A.B -m "Sphinx A.B"`` to paste a tag instead branch
 * Run ``git push origin :A.B --tags`` to remove old stable branch
 * open https://readthedocs.org/dashboard/sphinx/versions/ and enable the released version
 * Add new version/milestone to tracker categories