Merge branch '3.x' into latex_hyperlinked_caption_footnotes_on_3.x

CHANGES

@@ -18,9 +18,12 @@ Deprecated
 ----------
 
 * pending_xref node for viewcode extension
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
 * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
 * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good``
 * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected``
+* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
 * ``sphinx.builders.linkcheck.node_line_or_0()``
 * ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()``
 * ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter``
@@ -50,6 +53,7 @@ Features added
 * #6550: html: Allow to use HTML permalink texts via
   :confval:`html_permalinks_icon`
 * #1638: html: Add permalink icons to glossary terms
+* #8852: i18n: Allow to translate heading syntax in MyST-Parser
 * #8649: imgconverter: Skip availability check if builder supports the image
   type
 * #8573: napoleon: Allow to change the style of custom sections using
@@ -119,6 +123,8 @@ Bugs fixed
   an hyperlink
 * #7576: LaTeX with French babel and memoir crash: "Illegal parameter number
   in definition of ``\FNH@prefntext``"
+* #8055: LaTeX (docs): A potential display bug with the LaTeX generation step
+  in Sphinx (how to generate one-column index)
 * #8072: LaTeX: Directive :rst:dir:`hlist` not implemented in LaTeX
 * #8214: LaTeX: The :rst:role:`index` role and the glossary generate duplicate
   entries in the LaTeX index (if both used for same term)
@@ -131,6 +137,9 @@ Bugs fixed
 * #8780: LaTeX: long words in narrow columns may not be hyphenated
 * #8788: LaTeX: ``\titleformat`` last argument in sphinx.sty should be
   bracketed, not braced (and is anyhow not needed) 
+* #8849: LaTex: code-block printed out of margin (see the opt-in LaTeX syntax
+  boolean :ref:`verbatimforcewraps <latexsphinxsetupforcewraps>` for use via
+  the :ref:`'sphinxsetup' <latexsphinxsetup>` key of ``latex_elements``)
 
 Testing
 --------

doc/conf.py

@@ -24,6 +24,7 @@ html_theme_path = ['_themes']
 modindex_common_prefix = ['sphinx.']
 html_static_path = ['_static']
 html_sidebars = {'index': ['indexsidebar.html', 'searchbox.html']}
+html_title = 'Sphinx documentation'
 html_additional_pages = {'index': 'index.html'}
 html_use_opensearch = 'https://www.sphinx-doc.org/en/master'
 html_baseurl = 'https://www.sphinx-doc.org/en/master/'

doc/extdev/deprecated.rst

@@ -27,6 +27,16 @@ The following is a list of deprecated interfaces.
      - 5.0
      - ``sphinx.ext.viewcode.viewcode_anchor``
 
+   * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore``
+     - 3.5
+     - 5.0
+     - N/A
+
+   * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth``
+     - 3.5
+     - 5.0
+     - N/A
+
    * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken``
      - 3.5
      - 5.0
@@ -42,6 +52,11 @@ The following is a list of deprecated interfaces.
      - 5.0
      - N/A
 
+   * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore``
+     - 3.5
+     - 5.0
+     - N/A
+
    * - ``sphinx.builders.linkcheck.node_line_or_0()``
      - 3.5
      - 5.0

doc/latex.rst

@@ -497,11 +497,22 @@ Keys that don't need to be overridden unless in special cases are:
    .. versionchanged:: 1.6
       Remove unneeded ``{}`` after ``\\hrule``.
 
+``'makeindex'``
+   "makeindex" call, the last thing before ``\begin{document}``. With
+   ``'\\usepackage[columns=1]{idxlayout}\\makeindex'`` the index will use
+   only one column. You may have to install ``idxlayout`` LaTeX package.
+
+   Default: ``'\\makeindex'``
+
 ``'printindex'``
    "printindex" call, the last thing in the file. Override if you want to
-   generate the index differently or append some content after the index. For
-   example ``'\\footnotesize\\raggedright\\printindex'`` is advisable when the
-   index is full of long entries.
+   generate the index differently, append some content after the index, or
+   change the font. As LaTeX uses two-column mode for the index it is
+   often advisable to set this key to
+   ``'\\footnotesize\\raggedright\\printindex'``. Or, to obtain a one-column
+   index, use ``'\\def\\twocolumn[#1]{#1}\\printindex'`` (this trick may fail
+   if using a custom document class; then try the ``idxlayout`` approach
+   described in the documentation of the ``'makeindex'`` key).
 
    Default: ``'\\printindex'``
 
@@ -527,7 +538,6 @@ Keys that are set by other options and therefore should not be overridden are:
 ``'title'``
 ``'release'``
 ``'author'``
-``'makeindex'``
 
 
 .. _latexsphinxsetup:
@@ -585,7 +595,9 @@ The below is included at the end of the chapter::
 
      \endgroup
 
-LaTeX boolean keys require *lowercase* ``true`` or ``false`` values.
+LaTeX syntax for boolean keys requires *lowercase* ``true`` or ``false``
+e.g ``'sphinxsetup': "verbatimwrapslines=false"``.  If setting the
+boolean key to ``true``, ``=true`` is optional.
 Spaces around the commas and equal signs are ignored, spaces inside LaTeX
 macros may be significant.
 
@@ -636,14 +648,68 @@ macros may be significant.
     Boolean to specify if long lines in :rst:dir:`code-block`\ 's contents are
     wrapped.
 
+    If ``true``, line breaks may happen at spaces (the last space before the
+    line break will be rendered using a special symbol), and at ascii
+    punctuation characters (i.e. not at letters or digits). Whenever a long
+    string has no break points, it is moved to next line. If its length is
+    longer than the line width it will overflow.
+
     Default: ``true``
 
-``literalblockcappos``
-    Decides the caption position: either ``b`` ("bottom") or ``t`` ("top").
+.. _latexsphinxsetupforcewraps:
 
-    Default: ``t``
+``verbatimforcewraps``
+    Boolean to specify if long lines in :rst:dir:`code-block`\ 's contents
+    should be forcefully wrapped to never overflow due to long strings.
 
-    .. versionadded:: 1.7
+    .. note::
+
+       It is assumed that the Pygments_ LaTeXFormatter has not been used with
+       its ``texcomments`` or similar options which allow additional
+       (arbitrary) LaTeX mark-up.
+
+       Also, in case of :confval:`latex_engine` set to ``'pdflatex'``, only
+       the default LaTeX handling of Unicode code points, i.e. ``utf8`` not
+       ``utf8x`` is allowed.
+
+    .. _Pygments: https://pygments.org/
+
+    Default: ``false``
+
+    .. versionadded:: 3.5.0
+
+``verbatimmaxoverfull``
+    A number. If an unbreakable long string has length larger than the total
+    linewidth plus this number of characters, and if ``verbatimforcewraps``
+    mode is on, the input line will be reset using the forceful algorithm
+    which applies breakpoints at each character.
+
+    Default: ``3``
+
+    .. versionadded:: 3.5.0
+
+``verbatimmaxunderfull``
+    A number. If ``verbatimforcewraps`` mode applies, and if after applying
+    the line wrapping at spaces and punctuation, the first part of the split
+    line is lacking at least that number of characters to fill the available
+    width, then the input line will be reset using the forceful algorithm.
+
+    As the default is set to a high value, the forceful algorithm is triggered
+    only in overfull case, i.e. in presence of a string longer than full
+    linewidth. Set this to ``0`` to force all input lines to be hard wrapped
+    at the current avaiable linewidth::
+
+      latex_elements = {
+          'sphinxsetup': "verbatimforcewraps, verbatimmaxunderfull=0",
+      }
+
+    This can be done locally for a given code-block via the use of raw latex
+    directives to insert suitable ``\sphinxsetup`` (before and after) into the
+    latex file.
+
+    Default: ``100``
+
+    .. versionadded:: 3.5.0
 
 ``verbatimhintsturnover``
     Boolean to specify if code-blocks display "continued on next page" and

sphinx/builders/linkcheck.py

@@ -12,14 +12,14 @@ import json
 import queue
 import re
 import socket
-import threading
 import time
 import warnings
 from datetime import datetime, timezone
 from email.utils import parsedate_to_datetime
 from html.parser import HTMLParser
 from os import path
-from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast
+from threading import Thread
+from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple, cast
 from urllib.parse import unquote, urlparse
 
 from docutils import nodes
@@ -114,11 +114,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
 
     def init(self) -> None:
         self.hyperlinks = {}    # type: Dict[str, Hyperlink]
-        self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
-        self.anchors_ignore = [re.compile(x)
-                               for x in self.config.linkcheck_anchors_ignore]
-        self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
-                     in self.config.linkcheck_auth]
         self._good = set()       # type: Set[str]
         self._broken = {}        # type: Dict[str, str]
         self._redirected = {}    # type: Dict[str, Tuple[str, int]]
@@ -129,15 +124,43 @@ class CheckExternalLinksBuilder(DummyBuilder):
         self.rate_limits = {}  # type: Dict[str, RateLimit]
         self.wqueue = queue.PriorityQueue()  # type: queue.PriorityQueue
         self.rqueue = queue.Queue()  # type: queue.Queue
-        self.workers = []  # type: List[threading.Thread]
+        self.workers = []  # type: List[Thread]
         for i in range(self.config.linkcheck_workers):
-            thread = threading.Thread(target=self.check_thread, daemon=True)
+            thread = HyperlinkAvailabilityCheckWorker(self)
             thread.start()
             self.workers.append(thread)
 
     def is_ignored_uri(self, uri: str) -> bool:
         return any(pat.match(uri) for pat in self.to_ignore)
 
+    @property
+    def anchors_ignore(self) -> List[Pattern]:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "anchors_ignore"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        return [re.compile(x) for x in self.config.linkcheck_anchors_ignore]
+
+    @property
+    def auth(self) -> List[Tuple[Pattern, Any]]:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "auth"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        return [(re.compile(pattern), auth_info) for pattern, auth_info
+                in self.config.linkcheck_auth]
+
+    @property
+    def to_ignore(self) -> List[Pattern]:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "to_ignore"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        return [re.compile(x) for x in self.config.linkcheck_ignore]
+
     @property
     def good(self) -> Set[str]:
         warnings.warn(
@@ -166,6 +189,136 @@ class CheckExternalLinksBuilder(DummyBuilder):
         return self._redirected
 
     def check_thread(self) -> None:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "check_thread"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        # do nothing.
+
+    def limit_rate(self, response: Response) -> Optional[float]:
+        warnings.warn(
+            "%s.%s is deprecated." % (self.__class__.__name__, "limit_rate"),
+            RemovedInSphinx50Warning,
+            stacklevel=2,
+        )
+        return HyperlinkAvailabilityCheckWorker(self).limit_rate(response)
+
+    def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None:
+        uri, docname, lineno, status, info, code = result
+
+        filename = self.env.doc2path(docname, None)
+        linkstat = dict(filename=filename, lineno=lineno,
+                        status=status, code=code, uri=uri,
+                        info=info)
+        if status == 'unchecked':
+            self.write_linkstat(linkstat)
+            return
+        if status == 'working' and info == 'old':
+            self.write_linkstat(linkstat)
+            return
+        if lineno:
+            logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
+        if status == 'ignored':
+            if info:
+                logger.info(darkgray('-ignored- ') + uri + ': ' + info)
+            else:
+                logger.info(darkgray('-ignored- ') + uri)
+            self.write_linkstat(linkstat)
+        elif status == 'local':
+            logger.info(darkgray('-local-   ') + uri)
+            self.write_entry('local', docname, filename, lineno, uri)
+            self.write_linkstat(linkstat)
+        elif status == 'working':
+            logger.info(darkgreen('ok        ') + uri + info)
+            self.write_linkstat(linkstat)
+        elif status == 'broken':
+            if self.app.quiet or self.app.warningiserror:
+                logger.warning(__('broken link: %s (%s)'), uri, info,
+                               location=(filename, lineno))
+            else:
+                logger.info(red('broken    ') + uri + red(' - ' + info))
+            self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
+            self.write_linkstat(linkstat)
+        elif status == 'redirected':
+            try:
+                text, color = {
+                    301: ('permanently', purple),
+                    302: ('with Found', purple),
+                    303: ('with See Other', purple),
+                    307: ('temporarily', turquoise),
+                    308: ('permanently', purple),
+                }[code]
+            except KeyError:
+                text, color = ('with unknown code', purple)
+            linkstat['text'] = text
+            logger.info(color('redirect  ') + uri + color(' - ' + text + ' to ' + info))
+            self.write_entry('redirected ' + text, docname, filename,
+                             lineno, uri + ' to ' + info)
+            self.write_linkstat(linkstat)
+        else:
+            raise ValueError("Unknown status %s." % status)
+
+    def write_entry(self, what: str, docname: str, filename: str, line: int,
+                    uri: str) -> None:
+        self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
+
+    def write_linkstat(self, data: dict) -> None:
+        self.json_outfile.write(json.dumps(data))
+        self.json_outfile.write('\n')
+
+    def finish(self) -> None:
+        logger.info('')
+
+        with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
+             open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
+            total_links = 0
+            for hyperlink in self.hyperlinks.values():
+                if self.is_ignored_uri(hyperlink.uri):
+                    self.process_result(
+                        CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno,
+                                    'ignored', '', 0))
+                else:
+                    self.wqueue.put(hyperlink, False)
+                    total_links += 1
+
+            done = 0
+            while done < total_links:
+                self.process_result(self.rqueue.get())
+                done += 1
+
+        if self._broken:
+            self.app.statuscode = 1
+
+        self.wqueue.join()
+        # Shutdown threads.
+        for worker in self.workers:
+            self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
+
+
+class HyperlinkAvailabilityCheckWorker(Thread):
+    """A worker class for checking the availability of hyperlinks."""
+
+    def __init__(self, builder: CheckExternalLinksBuilder) -> None:
+        self.config = builder.config
+        self.env = builder.env
+        self.rate_limits = builder.rate_limits
+        self.rqueue = builder.rqueue
+        self.wqueue = builder.wqueue
+
+        self.anchors_ignore = [re.compile(x)
+                               for x in self.config.linkcheck_anchors_ignore]
+        self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
+                     in self.config.linkcheck_auth]
+        self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore]
+
+        self._good = builder._good
+        self._broken = builder._broken
+        self._redirected = builder._redirected
+
+        super().__init__(daemon=True)
+
+    def run(self) -> None:
         kwargs = {}
         if self.config.linkcheck_timeout:
             kwargs['timeout'] = self.config.linkcheck_timeout
@@ -378,97 +531,6 @@ class CheckExternalLinksBuilder(DummyBuilder):
         self.rate_limits[netloc] = RateLimit(delay, next_check)
         return next_check
 
-    def process_result(self, result: CheckResult) -> None:
-        uri, docname, lineno, status, info, code = result
-
-        filename = self.env.doc2path(docname, None)
-        linkstat = dict(filename=filename, lineno=lineno,
-                        status=status, code=code, uri=uri,
-                        info=info)
-        if status == 'unchecked':
-            self.write_linkstat(linkstat)
-            return
-        if status == 'working' and info == 'old':
-            self.write_linkstat(linkstat)
-            return
-        if lineno:
-            logger.info('(%16s: line %4d) ', docname, lineno, nonl=True)
-        if status == 'ignored':
-            if info:
-                logger.info(darkgray('-ignored- ') + uri + ': ' + info)
-            else:
-                logger.info(darkgray('-ignored- ') + uri)
-            self.write_linkstat(linkstat)
-        elif status == 'local':
-            logger.info(darkgray('-local-   ') + uri)
-            self.write_entry('local', docname, filename, lineno, uri)
-            self.write_linkstat(linkstat)
-        elif status == 'working':
-            logger.info(darkgreen('ok        ') + uri + info)
-            self.write_linkstat(linkstat)
-        elif status == 'broken':
-            if self.app.quiet or self.app.warningiserror:
-                logger.warning(__('broken link: %s (%s)'), uri, info,
-                               location=(filename, lineno))
-            else:
-                logger.info(red('broken    ') + uri + red(' - ' + info))
-            self.write_entry('broken', docname, filename, lineno, uri + ': ' + info)
-            self.write_linkstat(linkstat)
-        elif status == 'redirected':
-            try:
-                text, color = {
-                    301: ('permanently', purple),
-                    302: ('with Found', purple),
-                    303: ('with See Other', purple),
-                    307: ('temporarily', turquoise),
-                    308: ('permanently', purple),
-                }[code]
-            except KeyError:
-                text, color = ('with unknown code', purple)
-            linkstat['text'] = text
-            logger.info(color('redirect  ') + uri + color(' - ' + text + ' to ' + info))
-            self.write_entry('redirected ' + text, docname, filename,
-                             lineno, uri + ' to ' + info)
-            self.write_linkstat(linkstat)
-        else:
-            raise ValueError("Unknown status %s." % status)
-
-    def write_entry(self, what: str, docname: str, filename: str, line: int,
-                    uri: str) -> None:
-        self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri))
-
-    def write_linkstat(self, data: dict) -> None:
-        self.json_outfile.write(json.dumps(data))
-        self.json_outfile.write('\n')
-
-    def finish(self) -> None:
-        logger.info('')
-
-        with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\
-             open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile:
-            total_links = 0
-            for hyperlink in self.hyperlinks.values():
-                if self.is_ignored_uri(hyperlink.uri):
-                    self.process_result(
-                        CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno,
-                                    'ignored', '', 0))
-                else:
-                    self.wqueue.put(hyperlink, False)
-                    total_links += 1
-
-            done = 0
-            while done < total_links:
-                self.process_result(self.rqueue.get())
-                done += 1
-
-        if self._broken:
-            self.app.statuscode = 1
-
-        self.wqueue.join()
-        # Shutdown threads.
-        for worker in self.workers:
-            self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False)
-
 
 class HyperlinkCollector(SphinxPostTransform):
     builders = ('linkcheck',)

sphinx/texinputs/sphinx.sty

@@ -334,6 +334,9 @@
 % verbatim
 \DeclareBoolOption[true]{verbatimwithframe}
 \DeclareBoolOption[true]{verbatimwrapslines}
+\DeclareBoolOption[false]{verbatimforcewraps}
+\DeclareStringOption[3]{verbatimmaxoverfull}
+\DeclareStringOption[100]{verbatimmaxunderfull}
 \DeclareBoolOption[true]{verbatimhintsturnover}
 \DeclareBoolOption[true]{inlineliteralwraps}
 \DeclareStringOption[t]{literalblockcappos}
@@ -414,7 +417,7 @@
 \DisableKeyvalOption{sphinx}{mathnumfig}
 % To allow hyphenation of first word in narrow contexts; no option,
 % customization to be done via 'preamble' key
-\newcommand*\sphinxAtStartPar{\nobreak\hskip\z@skip}
+\newcommand*\sphinxAtStartPar{\hskip\z@skip}
 % No need for the \hspace{0pt} trick (\hskip\z@skip) with luatex
 \ifdefined\directlua\let\sphinxAtStartPar\@empty\fi
 % user interface: options can be changed midway in a document!
@@ -1193,13 +1196,188 @@
   % no need to restore \fboxsep here, as this ends up in a \hbox from fancyvrb
 }%
 % \sphinxVerbatimFormatLine will be set locally to one of those two:
-\newcommand\sphinxVerbatimFormatLineWrap[1]{%
-    \hsize\linewidth
+\newcommand\sphinxVerbatimFormatLineWrap{%
+  \hsize\linewidth
+  \ifspx@opt@verbatimforcewraps
+       \expandafter\spx@verb@FormatLineForceWrap
+  \else\expandafter\spx@verb@FormatLineWrap
+  \fi
+}%
+\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}%
+\long\def\spx@verb@FormatLineWrap#1{%
     \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@
           \doublehyphendemerits\z@\finalhyphendemerits\z@
           \strut #1\strut}%
 }%
-\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}%
+%
+% The normal line wrapping allows breaks at spaces and ascii non
+% letters, non digits. The \raggedright above means there will be
+% an overfilled line only if some non-breakable "word" was
+% encountered, which is longer than a line (it is moved always to
+% be on its own on a new line).
+% 
+% The "forced" line wrapping will parse the tokens to add potential
+% breakpoints at each character. As some strings are highlighted,
+% we have to apply the highlighting character per character, which
+% requires to manipulate the output of the Pygments LaTeXFormatter.
+%
+% Doing this at latex level is complicated. The contents should
+% be as expected: i.e. some active characters from
+% \sphinxbreaksviaactive, some Pygments character escapes such as
+% \PYGZdl{}, and the highlighting \PYG macro with always 2
+% arguments. No other macros should be there, except perhaps
+% zero-parameter macros. In particular:
+% - the texcomments Pygments option must be set to False
+%
+% With pdflatex, Unicode input gives multi-bytes characters
+% where the first byte is active. We support the "utf8" macros
+% only. "utf8x" is not supported.
+%
+% The highlighting macro \PYG will be applied character per
+% character. Highlighting via a colored background gives thus a
+% chain of small colored boxes which may cause some artefact in
+% some pdf viewers. Can't do anything here if we do want the line
+% break to be possible.
+%
+% First a measurement step is done of what would the standard line
+% wrapping give (i.e line breaks only at spaces and non-letter,
+% non-digit ascii characters), cf TeX by Topic for the basic
+% dissecting technique: TeX unfortunately when building a vertical
+% box does not store in an accessible way what was the maximal
+% line-width during paragraph building.
+%
+% If the max width exceeds the linewidth by more than verbatimmaxoverfull
+% character widths, or if the min width plus verbatimmaxunderfull character
+% widths is inferior to linewidth, then we apply the "force wrapping" with
+% potential line break at each character, else we don't.
+\long\def\spx@verb@FormatLineForceWrap#1{%
+    % \spx@image@box is a scratch box register that we can use here
+    \global\let\spx@verb@maxwidth\z@
+    \global\let\spx@verb@minwidth\linewidth
+    \setbox\spx@image@box
+    \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@
+          \doublehyphendemerits\z@\finalhyphendemerits\z@
+          \strut #1\strut\@@par
+          \spx@verb@getwidths}%
+    \ifdim\spx@verb@maxwidth>
+          \dimexpr\linewidth+\spx@opt@verbatimmaxoverfull\fontcharwd\font`X \relax
+      \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}%
+    \else
+      \ifdim\spx@verb@minwidth<
+            \dimexpr\linewidth-\spx@opt@verbatimmaxunderfull\fontcharwd\font`X \relax
+        \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}%
+      \else
+        \spx@verb@FormatLineWrap{#1}%
+    \fi\fi
+}%
+% auxiliary paragraph dissector to get max and min widths
+\newbox\spx@scratchbox
+\def\spx@verb@getwidths {%
+    \unskip\unpenalty
+    \setbox\spx@scratchbox\lastbox
+    \ifvoid\spx@scratchbox
+    \else
+       \setbox\spx@scratchbox\hbox{\unhbox\spx@scratchbox}%
+       \ifdim\spx@verb@maxwidth<\wd\spx@scratchbox
+          \xdef\spx@verb@maxwidth{\number\wd\spx@scratchbox sp}%
+       \fi
+       \ifdim\spx@verb@minwidth>\wd\spx@scratchbox
+          \xdef\spx@verb@minwidth{\number\wd\spx@scratchbox sp}%
+       \fi
+       \expandafter\spx@verb@getwidths
+    \fi
+}%
+% auxiliary macros to implement "cut long line even in middle of word"
+\catcode`Z=3 % safe delimiter
+\def\spx@verb@wrapPYG{%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@i
+}%
+\def\spx@verb@wrapPYG@i{%
+    \ifx\spx@nexttoken\spx@verb@wrapPYG\let\next=\@gobble\else
+    \ifx\spx@nexttoken\PYG\let\next=\spx@verb@wrapPYG@PYG@onebyone\else
+      \discretionary{}{\sphinxafterbreak}{}%
+      \let\next\spx@verb@wrapPYG@ii
+    \fi\fi
+    \next
+}%
+% Let's recognize active characters. We don't support utf8x only utf8.
+% And here #1 should not have picked up (non empty) braced contents
+\long\def\spx@verb@wrapPYG@ii#1{%
+    \ifcat\noexpand~\noexpand#1\relax% active character
+      \expandafter\spx@verb@wrapPYG@active
+    \else % non-active character, control sequence such as \PYGZdl, or empty
+      \expandafter\spx@verb@wrapPYG@one
+    \fi {#1}%
+}%
+\long\def\spx@verb@wrapPYG@active#1{%
+% Let's hope expansion of active character does not really require arguments,
+% as we certainly don't want to go into expanding upfront token stream anyway.
+    \expandafter\spx@verb@wrapPYG@iii#1{}{}{}{}{}{}{}{}{}Z#1%
+}%
+\long\def\spx@verb@wrapPYG@iii#1#2Z{%
+    \ifx\UTFviii@four@octets#1\let\next=\spx@verb@wrapPYG@four\else
+    \ifx\UTFviii@three@octets#1\let\next=\spx@verb@wrapPYG@three\else
+    \ifx\UTFviii@two@octets#1\let\next=\spx@verb@wrapPYG@two\else
+    \let\next=\spx@verb@wrapPYG@one
+    \fi\fi\fi
+    \next
+}%
+\long\def\spx@verb@wrapPYG@one   #1{#1\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@two   #1#2{#1#2\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@three #1#2#3{#1#2#3\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@four  #1#2#3#4{#1#2#3#4\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+% Replace \PYG by itself applied one character at a time! This way breakpoints
+% can be inserted.
+\def\spx@verb@wrapPYG@PYG@onebyone#1#2#3{% #1 = \PYG, #2 = highlight spec, #3 = tokens
+    \def\spx@verb@wrapPYG@PYG@spec{{#2}}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i#3Z%
+}%
+\def\spx@verb@wrapPYG@PYG@i{%
+    \ifx\spx@nexttokenZ\let\next=\spx@verb@wrapPYG@PYG@done\else
+      \discretionary{}{\sphinxafterbreak}{}%
+      \let\next\spx@verb@wrapPYG@PYG@ii
+    \fi
+    \next
+}%
+\def\spx@verb@wrapPYG@PYG@doneZ{\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@PYG@ii#1{%
+    \ifcat\noexpand~\noexpand#1\relax% active character
+      \expandafter\spx@verb@wrapPYG@PYG@active
+    \else % non-active character, control sequence such as \PYGZdl, or empty
+      \expandafter\spx@verb@wrapPYG@PYG@one
+    \fi {#1}%
+}%
+\long\def\spx@verb@wrapPYG@PYG@active#1{%
+% Let's hope expansion of active character does not really require arguments,
+% as we certainly don't want to go into expanding upfront token stream anyway.
+    \expandafter\spx@verb@wrapPYG@PYG@iii#1{}{}{}{}{}{}{}{}{}Z#1%
+}%
+\long\def\spx@verb@wrapPYG@PYG@iii#1#2Z{%
+    \ifx\UTFviii@four@octets#1\let\next=\spx@verb@wrapPYG@PYG@four\else
+    \ifx\UTFviii@three@octets#1\let\next=\spx@verb@wrapPYG@PYG@three\else
+    \ifx\UTFviii@two@octets#1\let\next=\spx@verb@wrapPYG@PYG@two\else
+    \let\next=\spx@verb@wrapPYG@PYG@one
+    \fi\fi\fi
+    \next
+}%
+\long\def\spx@verb@wrapPYG@PYG@one#1{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@two#1#2{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@three#1#2#3{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2#3}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@four#1#2#3#4{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2#3#4}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\catcode`Z 11%
+%
 \g@addto@macro\FV@SetupFont{%
     \sbox\sphinxcontinuationbox {\spx@opt@verbatimcontinued}%
     \sbox\sphinxvisiblespacebox {\spx@opt@verbatimvisiblespace}%

sphinx/transforms/i18n.py

@@ -263,7 +263,7 @@ class Locale(SphinxTransform):
             # see: http://docutils.sourceforge.net/docs/ref/doctree.html#structural-subelements
             if isinstance(node, nodes.title):
                 # This generates: <section ...><title>msgstr</title></section>
-                msgstr = msgstr + '\n' + '-' * len(msgstr) * 2
+                msgstr = msgstr + '\n' + '=' * len(msgstr) * 2
 
             patch = publish_msgstr(self.app, msgstr, source,
                                    node.line, self.config, settings)

sphinx/writers/latex.py

@@ -374,9 +374,8 @@ class LaTeXTranslator(SphinxTranslator):
                 'fncychap' not in self.config.latex_elements):
             # use Sonny style if any language specified (except English)
             self.elements['fncychap'] = ('\\usepackage[Sonny]{fncychap}\n'
-                                         '\\ChNameVar{\\Large\\normalfont'
-                                         '\\sffamily}\n\\ChTitleVar{\\Large'
-                                         '\\normalfont\\sffamily}')
+                                         '\\ChNameVar{\\Large\\normalfont\\sffamily}\n'
+                                         '\\ChTitleVar{\\Large\\normalfont\\sffamily}')
 
         self.babel = self.builder.babel
         if self.config.language and not self.babel.is_supported_language():
@@ -614,7 +613,8 @@ class LaTeXTranslator(SphinxTranslator):
         pass
 
     def visit_productionlist(self, node: Element) -> None:
-        self.body.append('\n\n\\begin{productionlist}\n')
+        self.body.append('\n\n')
+        self.body.append('\\begin{productionlist}\n')
         self.in_production_list = 1
 
     def depart_productionlist(self, node: Element) -> None:
@@ -705,9 +705,11 @@ class LaTeXTranslator(SphinxTranslator):
 
     def visit_desc(self, node: Element) -> None:
         if self.config.latex_show_urls == 'footnote':
-            self.body.append('\n\n\\begin{savenotes}\\begin{fulllineitems}\n')
+            self.body.append('\n\n')
+            self.body.append('\\begin{savenotes}\\begin{fulllineitems}\n')
         else:
-            self.body.append('\n\n\\begin{fulllineitems}\n')
+            self.body.append('\n\n')
+            self.body.append('\\begin{fulllineitems}\n')
         if self.table:
             self.table.has_problematic = True
 
@@ -737,13 +739,15 @@ class LaTeXTranslator(SphinxTranslator):
         if not node.get('is_multiline'):
             self._visit_signature_line(node)
         else:
-            self.body.append('%\n\\pysigstartmultiline\n')
+            self.body.append('%\n')
+            self.body.append('\\pysigstartmultiline\n')
 
     def depart_desc_signature(self, node: Element) -> None:
         if not node.get('is_multiline'):
             self._depart_signature_line(node)
         else:
-            self.body.append('%\n\\pysigstopmultiline')
+            self.body.append('%\n')
+            self.body.append('\\pysigstopmultiline')
 
     def visit_desc_signature_line(self, node: Element) -> None:
         self._visit_signature_line(node)
@@ -821,8 +825,9 @@ class LaTeXTranslator(SphinxTranslator):
         pass
 
     def visit_seealso(self, node: Element) -> None:
-        self.body.append('\n\n\\sphinxstrong{%s:}\n\\nopagebreak\n\n'
-                         % admonitionlabels['seealso'])
+        self.body.append('\n\n')
+        self.body.append('\\sphinxstrong{%s:}\n' % admonitionlabels['seealso'])
+        self.body.append('\\nopagebreak\n\n')
 
     def depart_seealso(self, node: Element) -> None:
         self.body.append("\n\n")
@@ -846,7 +851,8 @@ class LaTeXTranslator(SphinxTranslator):
         if self.in_parsed_literal:
             self.body.append('\\begin{footnote}[%s]' % label.astext())
         else:
-            self.body.append('%%\n\\begin{footnote}[%s]' % label.astext())
+            self.body.append('%\n')
+            self.body.append('\\begin{footnote}[%s]' % label.astext())
         if 'auto' not in node.attributes:
             self.body.append('\\phantomsection'
                              '\\label{\\thesphinxscope.%s}%%\n' % label.astext())
@@ -856,7 +862,8 @@ class LaTeXTranslator(SphinxTranslator):
         if self.in_parsed_literal:
             self.body.append('\\end{footnote}')
         else:
-            self.body.append('%\n\\end{footnote}')
+            self.body.append('%\n')
+            self.body.append('\\end{footnote}')
         self.in_footnote -= 1
 
     def visit_label(self, node: Element) -> None:
@@ -1318,10 +1325,11 @@ class LaTeXTranslator(SphinxTranslator):
             if 'width' in node:
                 length = self.latex_image_length(node['width'])
                 if length:
-                    self.body.append('\\begin{sphinxfigure-in-table}[%s]\n'
-                                     '\\centering\n' % length)
+                    self.body.append('\\begin{sphinxfigure-in-table}[%s]\n' % length)
+                    self.body.append('\\centering\n')
             else:
-                self.body.append('\\begin{sphinxfigure-in-table}\n\\centering\n')
+                self.body.append('\\begin{sphinxfigure-in-table}\n')
+                self.body.append('\\centering\n')
             if any(isinstance(child, nodes.caption) for child in node):
                 self.body.append('\\capstart')
             self.context.append('\\end{sphinxfigure-in-table}\\relax\n')
@@ -1333,14 +1341,16 @@ class LaTeXTranslator(SphinxTranslator):
                 length = self.latex_image_length(node[0]['width'])
             self.body.append('\n\n')    # Insert a blank line to prevent infinite loop
                                         # https://github.com/sphinx-doc/sphinx/issues/7059
-            self.body.append('\\begin{wrapfigure}{%s}{%s}\n\\centering' %
+            self.body.append('\\begin{wrapfigure}{%s}{%s}\n' %
                              ('r' if node['align'] == 'right' else 'l', length or '0pt'))
+            self.body.append('\\centering')
             self.context.append('\\end{wrapfigure}\n')
         elif self.in_minipage:
             self.body.append('\n\\begin{center}')
             self.context.append('\\end{center}\n')
         else:
-            self.body.append('\n\\begin{figure}[%s]\n\\centering\n' % align)
+            self.body.append('\n\\begin{figure}[%s]\n' % align)
+            self.body.append('\\centering\n')
             if any(isinstance(child, nodes.caption) for child in node):
                 self.body.append('\\capstart\n')
             self.context.append('\\end{figure}\n')
@@ -1742,13 +1752,16 @@ class LaTeXTranslator(SphinxTranslator):
 
     def visit_footnotetext(self, node: Element) -> None:
         label = cast(nodes.label, node[0])
-        self.body.append('%%\n\\begin{footnotetext}[%s]'
+        self.body.append('%\n')
+        self.body.append('\\begin{footnotetext}[%s]'
                          '\\phantomsection\\label{\\thesphinxscope.%s}%%\n'
-                         '\\sphinxAtStartFootnote\n' % (label.astext(), label.astext()))
+                         % (label.astext(), label.astext()))
+        self.body.append('\\sphinxAtStartFootnote\n')
 
     def depart_footnotetext(self, node: Element) -> None:
         # the \ignorespaces in particular for after table header use
-        self.body.append('%\n\\end{footnotetext}\\ignorespaces ')
+        self.body.append('%\n')
+        self.body.append('\\end{footnotetext}\\ignorespaces ')
 
     def visit_captioned_literal_block(self, node: Element) -> None:
         pass
@@ -1821,8 +1834,8 @@ class LaTeXTranslator(SphinxTranslator):
 
     def visit_line_block(self, node: Element) -> None:
         if isinstance(node.parent, nodes.line_block):
-            self.body.append('\\item[]\n'
-                             '\\begin{DUlineblock}{\\DUlineblockindent}\n')
+            self.body.append('\\item[]\n')
+            self.body.append('\\begin{DUlineblock}{\\DUlineblockindent}\n')
         else:
             self.body.append('\n\\begin{DUlineblock}{0em}\n')
         if self.table:

tests/test_build_linkcheck.py

@@ -21,7 +21,8 @@ from unittest import mock
 import pytest
 import requests
 
-from sphinx.builders.linkcheck import CheckExternalLinksBuilder, RateLimit
+from sphinx.builders.linkcheck import (CheckExternalLinksBuilder,
+                                       HyperlinkAvailabilityCheckWorker, RateLimit)
 from sphinx.util.console import strip_colors
 
 from .utils import CERT_FILE, http_server, https_server
@@ -536,40 +537,50 @@ class FakeResponse:
 
 def test_limit_rate_default_sleep(app):
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     checker.rate_limits = {}
+    worker = HyperlinkAvailabilityCheckWorker(checker)
     with mock.patch('time.time', return_value=0.0):
-        next_check = checker.limit_rate(FakeResponse())
+        next_check = worker.limit_rate(FakeResponse())
     assert next_check == 60.0
 
 
 def test_limit_rate_user_max_delay(app):
     app.config.linkcheck_rate_limit_timeout = 0.0
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     checker.rate_limits = {}
-    next_check = checker.limit_rate(FakeResponse())
+    worker = HyperlinkAvailabilityCheckWorker(checker)
+    next_check = worker.limit_rate(FakeResponse())
     assert next_check is None
 
 
 def test_limit_rate_doubles_previous_wait_time(app):
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
+    worker = HyperlinkAvailabilityCheckWorker(checker)
     with mock.patch('time.time', return_value=0.0):
-        next_check = checker.limit_rate(FakeResponse())
+        next_check = worker.limit_rate(FakeResponse())
     assert next_check == 120.0
 
 
 def test_limit_rate_clips_wait_time_to_max_time(app):
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     app.config.linkcheck_rate_limit_timeout = 90.0
     checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)}
+    worker = HyperlinkAvailabilityCheckWorker(checker)
     with mock.patch('time.time', return_value=0.0):
-        next_check = checker.limit_rate(FakeResponse())
+        next_check = worker.limit_rate(FakeResponse())
     assert next_check == 90.0
 
 
 def test_limit_rate_bails_out_after_waiting_max_time(app):
     checker = CheckExternalLinksBuilder(app)
+    checker.init()
     app.config.linkcheck_rate_limit_timeout = 90.0
     checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)}
-    next_check = checker.limit_rate(FakeResponse())
+    worker = HyperlinkAvailabilityCheckWorker(checker)
+    next_check = worker.limit_rate(FakeResponse())
     assert next_check is None