From f02fb7a8cc23a44a392597109d9e6276b5100583 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 31 Jan 2021 17:45:38 +0900 Subject: [PATCH 01/19] refactor: linkcheck: Separate worker feature from builder class To reduce the complexity of the linkcheck builder, this separates the worker feature from the builder class. --- sphinx/builders/linkcheck.py | 225 ++++++++++++++++++++-------------- tests/test_build_linkcheck.py | 23 +++- 2 files changed, 148 insertions(+), 100 deletions(-) diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index 8877e2ed5..0c452ae4a 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -12,13 +12,13 @@ import json import queue import re import socket -import threading import time import warnings from datetime import datetime, timezone from email.utils import parsedate_to_datetime from html.parser import HTMLParser from os import path +from threading import Thread from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast from urllib.parse import unquote, urlparse @@ -129,9 +129,9 @@ class CheckExternalLinksBuilder(DummyBuilder): self.rate_limits = {} # type: Dict[str, RateLimit] self.wqueue = queue.PriorityQueue() # type: queue.PriorityQueue self.rqueue = queue.Queue() # type: queue.Queue - self.workers = [] # type: List[threading.Thread] + self.workers = [] # type: List[Thread] for i in range(self.config.linkcheck_workers): - thread = threading.Thread(target=self.check_thread, daemon=True) + thread = HyperlinkAvailabilityCheckWorker(self) thread.start() self.workers.append(thread) @@ -166,6 +166,134 @@ class CheckExternalLinksBuilder(DummyBuilder): return self._redirected def check_thread(self) -> None: + warnings.warn( + "%s.%s is deprecated." % (self.__class__.__name__, "check_thread"), + RemovedInSphinx50Warning, + stacklevel=2, + ) + # do nothing. + + def limit_rate(self, response: Response) -> Optional[float]: + warnings.warn( + "%s.%s is deprecated." % (self.__class__.__name__, "limit_rate"), + RemovedInSphinx50Warning, + stacklevel=2, + ) + return HyperlinkAvailabilityCheckWorker(self).limit_rate(response) + + def process_result(self, result: Tuple[str, str, int, str, str, int]) -> None: + uri, docname, lineno, status, info, code = result + + filename = self.env.doc2path(docname, None) + linkstat = dict(filename=filename, lineno=lineno, + status=status, code=code, uri=uri, + info=info) + if status == 'unchecked': + self.write_linkstat(linkstat) + return + if status == 'working' and info == 'old': + self.write_linkstat(linkstat) + return + if lineno: + logger.info('(%16s: line %4d) ', docname, lineno, nonl=True) + if status == 'ignored': + if info: + logger.info(darkgray('-ignored- ') + uri + ': ' + info) + else: + logger.info(darkgray('-ignored- ') + uri) + self.write_linkstat(linkstat) + elif status == 'local': + logger.info(darkgray('-local- ') + uri) + self.write_entry('local', docname, filename, lineno, uri) + self.write_linkstat(linkstat) + elif status == 'working': + logger.info(darkgreen('ok ') + uri + info) + self.write_linkstat(linkstat) + elif status == 'broken': + if self.app.quiet or self.app.warningiserror: + logger.warning(__('broken link: %s (%s)'), uri, info, + location=(filename, lineno)) + else: + logger.info(red('broken ') + uri + red(' - ' + info)) + self.write_entry('broken', docname, filename, lineno, uri + ': ' + info) + self.write_linkstat(linkstat) + elif status == 'redirected': + try: + text, color = { + 301: ('permanently', purple), + 302: ('with Found', purple), + 303: ('with See Other', purple), + 307: ('temporarily', turquoise), + 308: ('permanently', purple), + }[code] + except KeyError: + text, color = ('with unknown code', purple) + linkstat['text'] = text + logger.info(color('redirect ') + uri + color(' - ' + text + ' to ' + info)) + self.write_entry('redirected ' + text, docname, filename, + lineno, uri + ' to ' + info) + self.write_linkstat(linkstat) + else: + raise ValueError("Unknown status %s." % status) + + def write_entry(self, what: str, docname: str, filename: str, line: int, + uri: str) -> None: + self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri)) + + def write_linkstat(self, data: dict) -> None: + self.json_outfile.write(json.dumps(data)) + self.json_outfile.write('\n') + + def finish(self) -> None: + logger.info('') + + with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\ + open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile: + total_links = 0 + for hyperlink in self.hyperlinks.values(): + if self.is_ignored_uri(hyperlink.uri): + self.process_result( + CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno, + 'ignored', '', 0)) + else: + self.wqueue.put(hyperlink, False) + total_links += 1 + + done = 0 + while done < total_links: + self.process_result(self.rqueue.get()) + done += 1 + + if self._broken: + self.app.statuscode = 1 + + self.wqueue.join() + # Shutdown threads. + for worker in self.workers: + self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False) + + +class HyperlinkAvailabilityCheckWorker(Thread): + """A worker class for checing the availability of hyperlinks.""" + + def __init__(self, builder: CheckExternalLinksBuilder) -> None: + self.app = builder.app + self.anchors_ignore = builder.anchors_ignore + self.auth = builder.auth + self.config = builder.config + self.env = builder.env + self.rate_limits = builder.rate_limits + self.rqueue = builder.rqueue + self.to_ignore = builder.to_ignore + self.wqueue = builder.wqueue + + self._good = builder._good + self._broken = builder._broken + self._redirected = builder._redirected + + super().__init__(daemon=True) + + def run(self) -> None: kwargs = {} if self.config.linkcheck_timeout: kwargs['timeout'] = self.config.linkcheck_timeout @@ -378,97 +506,6 @@ class CheckExternalLinksBuilder(DummyBuilder): self.rate_limits[netloc] = RateLimit(delay, next_check) return next_check - def process_result(self, result: CheckResult) -> None: - uri, docname, lineno, status, info, code = result - - filename = self.env.doc2path(docname, None) - linkstat = dict(filename=filename, lineno=lineno, - status=status, code=code, uri=uri, - info=info) - if status == 'unchecked': - self.write_linkstat(linkstat) - return - if status == 'working' and info == 'old': - self.write_linkstat(linkstat) - return - if lineno: - logger.info('(%16s: line %4d) ', docname, lineno, nonl=True) - if status == 'ignored': - if info: - logger.info(darkgray('-ignored- ') + uri + ': ' + info) - else: - logger.info(darkgray('-ignored- ') + uri) - self.write_linkstat(linkstat) - elif status == 'local': - logger.info(darkgray('-local- ') + uri) - self.write_entry('local', docname, filename, lineno, uri) - self.write_linkstat(linkstat) - elif status == 'working': - logger.info(darkgreen('ok ') + uri + info) - self.write_linkstat(linkstat) - elif status == 'broken': - if self.app.quiet or self.app.warningiserror: - logger.warning(__('broken link: %s (%s)'), uri, info, - location=(filename, lineno)) - else: - logger.info(red('broken ') + uri + red(' - ' + info)) - self.write_entry('broken', docname, filename, lineno, uri + ': ' + info) - self.write_linkstat(linkstat) - elif status == 'redirected': - try: - text, color = { - 301: ('permanently', purple), - 302: ('with Found', purple), - 303: ('with See Other', purple), - 307: ('temporarily', turquoise), - 308: ('permanently', purple), - }[code] - except KeyError: - text, color = ('with unknown code', purple) - linkstat['text'] = text - logger.info(color('redirect ') + uri + color(' - ' + text + ' to ' + info)) - self.write_entry('redirected ' + text, docname, filename, - lineno, uri + ' to ' + info) - self.write_linkstat(linkstat) - else: - raise ValueError("Unknown status %s." % status) - - def write_entry(self, what: str, docname: str, filename: str, line: int, - uri: str) -> None: - self.txt_outfile.write("%s:%s: [%s] %s\n" % (filename, line, what, uri)) - - def write_linkstat(self, data: dict) -> None: - self.json_outfile.write(json.dumps(data)) - self.json_outfile.write('\n') - - def finish(self) -> None: - logger.info('') - - with open(path.join(self.outdir, 'output.txt'), 'w') as self.txt_outfile,\ - open(path.join(self.outdir, 'output.json'), 'w') as self.json_outfile: - total_links = 0 - for hyperlink in self.hyperlinks.values(): - if self.is_ignored_uri(hyperlink.uri): - self.process_result( - CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno, - 'ignored', '', 0)) - else: - self.wqueue.put(hyperlink, False) - total_links += 1 - - done = 0 - while done < total_links: - self.process_result(self.rqueue.get()) - done += 1 - - if self._broken: - self.app.statuscode = 1 - - self.wqueue.join() - # Shutdown threads. - for worker in self.workers: - self.wqueue.put((CHECK_IMMEDIATELY, None, None, None), False) - class HyperlinkCollector(SphinxPostTransform): builders = ('linkcheck',) diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index 60b62435c..e297d42c4 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -21,7 +21,8 @@ from unittest import mock import pytest import requests -from sphinx.builders.linkcheck import CheckExternalLinksBuilder, RateLimit +from sphinx.builders.linkcheck import (CheckExternalLinksBuilder, + HyperlinkAvailabilityCheckWorker, RateLimit) from sphinx.util.console import strip_colors from .utils import CERT_FILE, http_server, https_server @@ -536,40 +537,50 @@ class FakeResponse: def test_limit_rate_default_sleep(app): checker = CheckExternalLinksBuilder(app) + checker.init() checker.rate_limits = {} + worker = HyperlinkAvailabilityCheckWorker(checker) with mock.patch('time.time', return_value=0.0): - next_check = checker.limit_rate(FakeResponse()) + next_check = worker.limit_rate(FakeResponse()) assert next_check == 60.0 def test_limit_rate_user_max_delay(app): app.config.linkcheck_rate_limit_timeout = 0.0 checker = CheckExternalLinksBuilder(app) + checker.init() checker.rate_limits = {} - next_check = checker.limit_rate(FakeResponse()) + worker = HyperlinkAvailabilityCheckWorker(checker) + next_check = worker.limit_rate(FakeResponse()) assert next_check is None def test_limit_rate_doubles_previous_wait_time(app): checker = CheckExternalLinksBuilder(app) + checker.init() checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)} + worker = HyperlinkAvailabilityCheckWorker(checker) with mock.patch('time.time', return_value=0.0): - next_check = checker.limit_rate(FakeResponse()) + next_check = worker.limit_rate(FakeResponse()) assert next_check == 120.0 def test_limit_rate_clips_wait_time_to_max_time(app): checker = CheckExternalLinksBuilder(app) + checker.init() app.config.linkcheck_rate_limit_timeout = 90.0 checker.rate_limits = {"localhost": RateLimit(60.0, 0.0)} + worker = HyperlinkAvailabilityCheckWorker(checker) with mock.patch('time.time', return_value=0.0): - next_check = checker.limit_rate(FakeResponse()) + next_check = worker.limit_rate(FakeResponse()) assert next_check == 90.0 def test_limit_rate_bails_out_after_waiting_max_time(app): checker = CheckExternalLinksBuilder(app) + checker.init() app.config.linkcheck_rate_limit_timeout = 90.0 checker.rate_limits = {"localhost": RateLimit(90.0, 0.0)} - next_check = checker.limit_rate(FakeResponse()) + worker = HyperlinkAvailabilityCheckWorker(checker) + next_check = worker.limit_rate(FakeResponse()) assert next_check is None From 84130fff40102c29e4969444ae7b536c6ce4d7a3 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 6 Feb 2021 01:24:16 +0900 Subject: [PATCH 02/19] Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: François Freitag --- sphinx/builders/linkcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index 0c452ae4a..590eec201 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -274,7 +274,7 @@ class CheckExternalLinksBuilder(DummyBuilder): class HyperlinkAvailabilityCheckWorker(Thread): - """A worker class for checing the availability of hyperlinks.""" + """A worker class for checking the availability of hyperlinks.""" def __init__(self, builder: CheckExternalLinksBuilder) -> None: self.app = builder.app From ad5b0babd7405f8740be9e1cb65a11feaf60915e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 6 Feb 2021 01:31:22 +0900 Subject: [PATCH 03/19] refactor: linkcheck: Remove unused attribute HyperlinkAvailabilityCheckWorker.app --- sphinx/builders/linkcheck.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index 590eec201..659305d32 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -277,7 +277,6 @@ class HyperlinkAvailabilityCheckWorker(Thread): """A worker class for checking the availability of hyperlinks.""" def __init__(self, builder: CheckExternalLinksBuilder) -> None: - self.app = builder.app self.anchors_ignore = builder.anchors_ignore self.auth = builder.auth self.config = builder.config From f0763355989891a79796f9ecc548add515293a48 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 6 Feb 2021 20:59:04 +0900 Subject: [PATCH 04/19] refactor: LaTeX: separate self.body.append() call to single line output --- sphinx/writers/latex.py | 54 +++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index b3ec5de56..8319cbbfe 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -374,9 +374,8 @@ class LaTeXTranslator(SphinxTranslator): 'fncychap' not in self.config.latex_elements): # use Sonny style if any language specified (except English) self.elements['fncychap'] = ('\\usepackage[Sonny]{fncychap}\n' - '\\ChNameVar{\\Large\\normalfont' - '\\sffamily}\n\\ChTitleVar{\\Large' - '\\normalfont\\sffamily}') + '\\ChNameVar{\\Large\\normalfont\\sffamily}\n' + '\\ChTitleVar{\\Large\\normalfont\\sffamily}') self.babel = self.builder.babel if self.config.language and not self.babel.is_supported_language(): @@ -614,7 +613,8 @@ class LaTeXTranslator(SphinxTranslator): pass def visit_productionlist(self, node: Element) -> None: - self.body.append('\n\n\\begin{productionlist}\n') + self.body.append('\n\n') + self.body.append('\\begin{productionlist}\n') self.in_production_list = 1 def depart_productionlist(self, node: Element) -> None: @@ -705,9 +705,11 @@ class LaTeXTranslator(SphinxTranslator): def visit_desc(self, node: Element) -> None: if self.config.latex_show_urls == 'footnote': - self.body.append('\n\n\\begin{savenotes}\\begin{fulllineitems}\n') + self.body.append('\n\n') + self.body.append('\\begin{savenotes}\\begin{fulllineitems}\n') else: - self.body.append('\n\n\\begin{fulllineitems}\n') + self.body.append('\n\n') + self.body.append('\\begin{fulllineitems}\n') if self.table: self.table.has_problematic = True @@ -737,13 +739,15 @@ class LaTeXTranslator(SphinxTranslator): if not node.get('is_multiline'): self._visit_signature_line(node) else: - self.body.append('%\n\\pysigstartmultiline\n') + self.body.append('%\n') + self.body.append('\\pysigstartmultiline\n') def depart_desc_signature(self, node: Element) -> None: if not node.get('is_multiline'): self._depart_signature_line(node) else: - self.body.append('%\n\\pysigstopmultiline') + self.body.append('%\n') + self.body.append('\\pysigstopmultiline') def visit_desc_signature_line(self, node: Element) -> None: self._visit_signature_line(node) @@ -821,8 +825,9 @@ class LaTeXTranslator(SphinxTranslator): pass def visit_seealso(self, node: Element) -> None: - self.body.append('\n\n\\sphinxstrong{%s:}\n\\nopagebreak\n\n' - % admonitionlabels['seealso']) + self.body.append('\n\n') + self.body.append('\\sphinxstrong{%s:}\n' % admonitionlabels['seealso']) + self.body.append('\\nopagebreak\n\n') def depart_seealso(self, node: Element) -> None: self.body.append("\n\n") @@ -844,14 +849,16 @@ class LaTeXTranslator(SphinxTranslator): if self.in_parsed_literal: self.body.append('\\begin{footnote}[%s]' % label.astext()) else: - self.body.append('%%\n\\begin{footnote}[%s]' % label.astext()) + self.body.append('%\n') + self.body.append('\\begin{footnote}[%s]' % label.astext()) self.body.append('\\sphinxAtStartFootnote\n') def depart_footnote(self, node: Element) -> None: if self.in_parsed_literal: self.body.append('\\end{footnote}') else: - self.body.append('%\n\\end{footnote}') + self.body.append('%\n') + self.body.append('\\end{footnote}') self.in_footnote -= 1 def visit_label(self, node: Element) -> None: @@ -1313,10 +1320,11 @@ class LaTeXTranslator(SphinxTranslator): if 'width' in node: length = self.latex_image_length(node['width']) if length: - self.body.append('\\begin{sphinxfigure-in-table}[%s]\n' - '\\centering\n' % length) + self.body.append('\\begin{sphinxfigure-in-table}[%s]\n' % length) + self.body.append('\\centering\n') else: - self.body.append('\\begin{sphinxfigure-in-table}\n\\centering\n') + self.body.append('\\begin{sphinxfigure-in-table}\n') + self.body.append('\\centering\n') if any(isinstance(child, nodes.caption) for child in node): self.body.append('\\capstart') self.context.append('\\end{sphinxfigure-in-table}\\relax\n') @@ -1328,14 +1336,16 @@ class LaTeXTranslator(SphinxTranslator): length = self.latex_image_length(node[0]['width']) self.body.append('\n\n') # Insert a blank line to prevent infinite loop # https://github.com/sphinx-doc/sphinx/issues/7059 - self.body.append('\\begin{wrapfigure}{%s}{%s}\n\\centering' % + self.body.append('\\begin{wrapfigure}{%s}{%s}\n' % ('r' if node['align'] == 'right' else 'l', length or '0pt')) + self.body.append('\\centering') self.context.append('\\end{wrapfigure}\n') elif self.in_minipage: self.body.append('\n\\begin{center}') self.context.append('\\end{center}\n') else: - self.body.append('\n\\begin{figure}[%s]\n\\centering\n' % align) + self.body.append('\n\\begin{figure}[%s]\n' % align) + self.body.append('\\centering\n') if any(isinstance(child, nodes.caption) for child in node): self.body.append('\\capstart\n') self.context.append('\\end{figure}\n') @@ -1737,12 +1747,14 @@ class LaTeXTranslator(SphinxTranslator): def visit_footnotetext(self, node: Element) -> None: label = cast(nodes.label, node[0]) - self.body.append('%%\n\\begin{footnotetext}[%s]' + self.body.append('%\n') + self.body.append('\\begin{footnotetext}[%s]' '\\sphinxAtStartFootnote\n' % label.astext()) def depart_footnotetext(self, node: Element) -> None: # the \ignorespaces in particular for after table header use - self.body.append('%\n\\end{footnotetext}\\ignorespaces ') + self.body.append('%\n') + self.body.append('\\end{footnotetext}\\ignorespaces ') def visit_captioned_literal_block(self, node: Element) -> None: pass @@ -1815,8 +1827,8 @@ class LaTeXTranslator(SphinxTranslator): def visit_line_block(self, node: Element) -> None: if isinstance(node.parent, nodes.line_block): - self.body.append('\\item[]\n' - '\\begin{DUlineblock}{\\DUlineblockindent}\n') + self.body.append('\\item[]\n') + self.body.append('\\begin{DUlineblock}{\\DUlineblockindent}\n') else: self.body.append('\n\\begin{DUlineblock}{0em}\n') if self.table: From 38b14bbfef5d5d498cdcd2a9a069934992a06007 Mon Sep 17 00:00:00 2001 From: jfbu Date: Sat, 6 Feb 2021 16:35:16 +0100 Subject: [PATCH 05/19] Revert "Add \nobreak inside \sphinxAtStartPar" This reverts commit 17642a5e6bbcccc5616da381c10a068c9e09b6d7. Fixes #8838. For some reason the \nobreak causes breakage in table vertical spacing for merge cells in tabular and longtable, and regarding tabulary for all cells... Reverting this will cause as described in the reverted commit message some much less annoying problem in certain circumstances when a long word has no hyphenation point. --- sphinx/texinputs/sphinx.sty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index eb5617dda..da03ff989 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -414,7 +414,7 @@ \DisableKeyvalOption{sphinx}{mathnumfig} % To allow hyphenation of first word in narrow contexts; no option, % customization to be done via 'preamble' key -\newcommand*\sphinxAtStartPar{\nobreak\hskip\z@skip} +\newcommand*\sphinxAtStartPar{\hskip\z@skip} % No need for the \hspace{0pt} trick (\hskip\z@skip) with luatex \ifdefined\directlua\let\sphinxAtStartPar\@empty\fi % user interface: options can be changed midway in a document! From 899ccfd40ead50d35f6c7244ea7dad367ab71073 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 4 Feb 2021 01:56:16 +0900 Subject: [PATCH 06/19] refactor: linkcheck: Deprecate attributes of linkcheck builders Move anchors_ignore, auth and to_ignore to HyperlinkAvailabilityCheckWorker and become deprecated. --- CHANGES | 3 +++ doc/extdev/deprecated.rst | 15 ++++++++++++ sphinx/builders/linkcheck.py | 44 ++++++++++++++++++++++++++++-------- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/CHANGES b/CHANGES index 9e3e29fe0..b394c54e3 100644 --- a/CHANGES +++ b/CHANGES @@ -18,9 +18,12 @@ Deprecated ---------- * pending_xref node for viewcode extension +* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore`` +* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth`` * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken`` * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.good`` * ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.redirected`` +* ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore`` * ``sphinx.builders.linkcheck.node_line_or_0()`` * ``sphinx.ext.autodoc.AttributeDocumenter.isinstanceattribute()`` * ``sphinx.ext.autodoc.directive.DocumenterBridge.reporter`` diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index 1350085ef..dbebb840c 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -27,6 +27,16 @@ The following is a list of deprecated interfaces. - 5.0 - ``sphinx.ext.viewcode.viewcode_anchor`` + * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.anchors_ignore`` + - 3.5 + - 5.0 + - N/A + + * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.auth`` + - 3.5 + - 5.0 + - N/A + * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.broken`` - 3.5 - 5.0 @@ -42,6 +52,11 @@ The following is a list of deprecated interfaces. - 5.0 - N/A + * - ``sphinx.builders.linkcheck.CheckExternalLinksBuilder.to_ignore`` + - 3.5 + - 5.0 + - N/A + * - ``sphinx.builders.linkcheck.node_line_or_0()`` - 3.5 - 5.0 diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index 659305d32..900346def 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -19,7 +19,7 @@ from email.utils import parsedate_to_datetime from html.parser import HTMLParser from os import path from threading import Thread -from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, cast +from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Set, Tuple, cast from urllib.parse import unquote, urlparse from docutils import nodes @@ -114,11 +114,6 @@ class CheckExternalLinksBuilder(DummyBuilder): def init(self) -> None: self.hyperlinks = {} # type: Dict[str, Hyperlink] - self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore] - self.anchors_ignore = [re.compile(x) - for x in self.config.linkcheck_anchors_ignore] - self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info - in self.config.linkcheck_auth] self._good = set() # type: Set[str] self._broken = {} # type: Dict[str, str] self._redirected = {} # type: Dict[str, Tuple[str, int]] @@ -138,6 +133,34 @@ class CheckExternalLinksBuilder(DummyBuilder): def is_ignored_uri(self, uri: str) -> bool: return any(pat.match(uri) for pat in self.to_ignore) + @property + def anchors_ignore(self) -> List[Pattern]: + warnings.warn( + "%s.%s is deprecated." % (self.__class__.__name__, "anchors_ignore"), + RemovedInSphinx50Warning, + stacklevel=2, + ) + return [re.compile(x) for x in self.config.linkcheck_anchors_ignore] + + @property + def auth(self) -> List[Tuple[Pattern, Any]]: + warnings.warn( + "%s.%s is deprecated." % (self.__class__.__name__, "auth"), + RemovedInSphinx50Warning, + stacklevel=2, + ) + return [(re.compile(pattern), auth_info) for pattern, auth_info + in self.config.linkcheck_auth] + + @property + def to_ignore(self) -> List[Pattern]: + warnings.warn( + "%s.%s is deprecated." % (self.__class__.__name__, "to_ignore"), + RemovedInSphinx50Warning, + stacklevel=2, + ) + return [re.compile(x) for x in self.config.linkcheck_ignore] + @property def good(self) -> Set[str]: warnings.warn( @@ -277,15 +300,18 @@ class HyperlinkAvailabilityCheckWorker(Thread): """A worker class for checking the availability of hyperlinks.""" def __init__(self, builder: CheckExternalLinksBuilder) -> None: - self.anchors_ignore = builder.anchors_ignore - self.auth = builder.auth self.config = builder.config self.env = builder.env self.rate_limits = builder.rate_limits self.rqueue = builder.rqueue - self.to_ignore = builder.to_ignore self.wqueue = builder.wqueue + self.anchors_ignore = [re.compile(x) + for x in self.config.linkcheck_anchors_ignore] + self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info + in self.config.linkcheck_auth] + self.to_ignore = [re.compile(x) for x in self.config.linkcheck_ignore] + self._good = builder._good self._broken = builder._broken self._redirected = builder._redirected From 4cbb2925fdeac66f836776d28c2f51cc2173c8bf Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 7 Feb 2021 21:23:41 +0900 Subject: [PATCH 07/19] Fix #8837: doc: Remove version info from html_title --- doc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/conf.py b/doc/conf.py index 53f036d3e..a3d2f5f12 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -24,6 +24,7 @@ html_theme_path = ['_themes'] modindex_common_prefix = ['sphinx.'] html_static_path = ['_static'] html_sidebars = {'index': ['indexsidebar.html', 'searchbox.html']} +html_title = 'Sphinx documentation' html_additional_pages = {'index': 'index.html'} html_use_opensearch = 'https://www.sphinx-doc.org/en/master' html_baseurl = 'https://www.sphinx-doc.org/en/master/' From a984f5f7cc429785882bf8bcf663b0ee2f4dc58d Mon Sep 17 00:00:00 2001 From: jfbu Date: Mon, 8 Feb 2021 14:55:30 +0100 Subject: [PATCH 08/19] LaTeX: improve docs of 'printindex' and 'makeindex' Closes #8055 --- CHANGES | 2 ++ doc/latex.rst | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index b394c54e3..55f1b8876 100644 --- a/CHANGES +++ b/CHANGES @@ -119,6 +119,8 @@ Bugs fixed specified * #7576: LaTeX with French babel and memoir crash: "Illegal parameter number in definition of ``\FNH@prefntext``" +* #8055: LaTeX (docs): A potential display bug with the LaTeX generation step + in Sphinx (how to generate one-column index) * #8072: LaTeX: Directive :rst:dir:`hlist` not implemented in LaTeX * #8214: LaTeX: The :rst:role:`index` role and the glossary generate duplicate entries in the LaTeX index (if both used for same term) diff --git a/doc/latex.rst b/doc/latex.rst index 39d4b8147..82016ed67 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -497,11 +497,22 @@ Keys that don't need to be overridden unless in special cases are: .. versionchanged:: 1.6 Remove unneeded ``{}`` after ``\\hrule``. +``'makeindex'`` + "makeindex" call, the last thing before ``\begin{document}``. With + ``'\\usepackage[columns=1]{idxlayout}\\makeindex'`` the index will use + only one column. You may have to install ``idxlayout`` LaTeX package. + + Default: ``'\\makeindex'`` + ``'printindex'`` "printindex" call, the last thing in the file. Override if you want to - generate the index differently or append some content after the index. For - example ``'\\footnotesize\\raggedright\\printindex'`` is advisable when the - index is full of long entries. + generate the index differently, append some content after the index, or + change the font. As LaTeX uses two-column mode for the index it is + often advisable to set this key to + ``'\\footnotesize\\raggedright\\printindex'``. Or, to obtain a one-column + index, use ``'\\def\\twocolumn[#1]{#1}\\printindex'`` (this trick may fail + if using a custom document class; then try the ``idxlayout`` approach + described in the documentation of the ``'makeindex'`` key). Default: ``'\\printindex'`` @@ -527,7 +538,6 @@ Keys that are set by other options and therefore should not be overridden are: ``'title'`` ``'release'`` ``'author'`` -``'makeindex'`` .. _latexsphinxsetup: From 4917dd5f84eca86f8964e55431babb6065bbdc14 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 8 Feb 2021 13:21:28 -0500 Subject: [PATCH 09/19] i18n: Locale transform: Change heading syntax to work for both RST and Markdown --- sphinx/transforms/i18n.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py index d588f0411..6bea6c6ee 100644 --- a/sphinx/transforms/i18n.py +++ b/sphinx/transforms/i18n.py @@ -263,7 +263,7 @@ class Locale(SphinxTransform): # see: http://docutils.sourceforge.net/docs/ref/doctree.html#structural-subelements if isinstance(node, nodes.title): # This generates:
msgstr
- msgstr = msgstr + '\n' + '-' * len(msgstr) * 2 + msgstr = msgstr + '\n' + '=' * len(msgstr) * 2 patch = publish_msgstr(self.app, msgstr, source, node.line, self.config, settings) From 702545da1c075d3b19e6a5f1749e9812bceb7e49 Mon Sep 17 00:00:00 2001 From: jfbu Date: Mon, 8 Feb 2021 23:13:46 +0100 Subject: [PATCH 10/19] LaTeX: optionally apply a second forceful wrapping of long code lines Closes #8849 --- CHANGES | 1 + doc/latex.rst | 12 +++++ sphinx/texinputs/sphinx.sty | 96 +++++++++++++++++++++++++++++++++++-- 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index b394c54e3..e39180cd3 100644 --- a/CHANGES +++ b/CHANGES @@ -131,6 +131,7 @@ Bugs fixed * #8780: LaTeX: long words in narrow columns may not be hyphenated * #8788: LaTeX: ``\titleformat`` last argument in sphinx.sty should be bracketed, not braced (and is anyhow not needed) +* #8849: LaTex: code-block printed out of margin Testing -------- diff --git a/doc/latex.rst b/doc/latex.rst index 39d4b8147..636781753 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -638,6 +638,18 @@ macros may be significant. Default: ``true`` +``verbatimforcewraps`` + Boolean to specify if long lines in :rst:dir:`code-block`\ 's contents + which the wrapping algorithm could not reduce to at most an excess of 3 + characters on a line will be cut forcefully to achieve this maximal excess + of 3 characters on each line. (*this is possibly fragile, so by default is + not done; please try it out and report issues to the maintainers to help + improve and decide whether to make this default*) + + Default: ``false`` + + .. versionadded:: 3.5.0 + ``literalblockcappos`` Decides the caption position: either ``b`` ("bottom") or ``t`` ("top"). diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index da03ff989..85f67270d 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -334,6 +334,7 @@ % verbatim \DeclareBoolOption[true]{verbatimwithframe} \DeclareBoolOption[true]{verbatimwrapslines} +\DeclareBoolOption[false]{verbatimforcewraps} \DeclareBoolOption[true]{verbatimhintsturnover} \DeclareBoolOption[true]{inlineliteralwraps} \DeclareStringOption[t]{literalblockcappos} @@ -1171,13 +1172,102 @@ % no need to restore \fboxsep here, as this ends up in a \hbox from fancyvrb }% % \sphinxVerbatimFormatLine will be set locally to one of those two: -\newcommand\sphinxVerbatimFormatLineWrap[1]{% - \hsize\linewidth +\newcommand\sphinxVerbatimFormatLineWrap{% + \hsize\linewidth + \ifspx@opt@verbatimforcewraps + \expandafter\spx@verb@FormatLineForceWrap + \else\expandafter\spx@verb@FormatLineWrap + \fi +}% +\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}% +\long\def\spx@verb@FormatLineWrap#1{% \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@ \doublehyphendemerits\z@\finalhyphendemerits\z@ \strut #1\strut}% }% -\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}% +% We implement an alternative to wrapping long code lines. This +% alternative works only if the contents are in the expected +% Pygments mark-up: i.e. some character escapes such as \PYGZdl{} +% and the highlighting \PYG macro with always 2 arguments, and no +% other macros. This means: +% - the command prefix *must* be PYG +% - the texcomments Pygments option *must* be set to False (it could +% work by luck if True) +% For non-highlighted tokens a break point is installed at each of them. +% For highlighted tokens (i.e. in 2nd argument of \PYG) every four such +% characters. \PYGZdl{} etc will count for 2, although corresponding to +% only one. The result is that no line should be more than 3 characters +% overfull. +% First a measurement step is done of what would our standard wrapping +% approach give. This is a bit tricky, cf TeX by Topic for the basic +% dissecting technique, because TeX unfortunately when building a +% vertical box does not store in an accessible way what was the maximal +% line-width: the width of the box will be the set \hsize. +% Anyway, if the max width exceed the linewidth by at least 4 character +% widths, then we apply the "force wrapping" of previous paragraph, +% else we apply our "standard wrapping". +\long\def\spx@verb@FormatLineForceWrap#1{% + % \spx@image@box is a scratch box register that we can use here + \global\let\spx@verb@maxwidth\z@ + \setbox\spx@image@box + \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@ + \doublehyphendemerits\z@\finalhyphendemerits\z@ + \strut #1\strut\@@par + \spx@verb@getmaxwidth}% + \ifdim\spx@verb@maxwidth>\dimexpr\linewidth+3\fontcharwd\font`X\relax + \spx@verb@FormatLineWrap{\spx@forcewrapPYG #1\spx@forcewrapPYG}% + \else + \spx@verb@FormatLineWrap{#1}% + \fi +}% +% auxiliary paragraph dissector to get max width +\newbox\spx@line@box +\def\spx@verb@getmaxwidth {% + \unskip\unpenalty + \setbox\spx@line@box\lastbox + \ifvoid\spx@line@box + \else + \setbox\spx@line@box\hbox{\unhbox\spx@line@box}% + \ifdim\spx@verb@maxwidth<\wd\spx@line@box + \xdef\spx@verb@maxwidth{\number\wd\spx@line@box sp}% + \fi + \expandafter\spx@verb@getmaxwidth + \fi +}% +% auxiliary macros to implement "cut long line even in middle of word" +\def\spx@forcewrapPYG{% + \futurelet\spx@nexttoken\spx@forcewrapPYG@i +}% +\def\spx@forcewrapPYG@i{% + \ifx\spx@nexttoken\spx@forcewrapPYG\let\next=\@gobble\else + \ifx\spx@nexttoken\PYG\let\next=\spx@forcewrapPYG@PYG\else + \discretionary{}{\sphinxafterbreak}{}% + \let\next=\spx@forcewrapPYG@ii + \fi\fi + \next +}% +\def\spx@forcewrapPYG@ii#1{#1\futurelet\spx@nexttoken\spx@forcewrapPYG@i}% +% Replace \PYG by itself applied to short strings of 4 characters at a time +% and insert breakpoints in-between +\def\spx@forcewrapPYG@PYG\PYG#1#2{% + \def\spx@PYGspec{{#1}}% + \spx@PYG#2\@empty\@empty\@empty\@empty\relax +}% +\def\spx@PYG#1#2#3#4{% + \discretionary{}{\sphinxafterbreak}{}% + \expandafter\PYG\spx@PYGspec{#1#2#3#4}% +% I assume here contents never contain \@empty. If #4={} originally then +% it is empty here and the \ifx will compare \@empty to \relax and choose +% the else branch, i.e. to continue applying \PYG repeatedly + \ifx#4\@empty\relax + \expandafter\spx@PYG@done + \else + \expandafter\spx@PYG + \fi +}% +% Once \PYG is handled we get back to our forward scan token by token +\def\spx@PYG@done#1\relax{\futurelet\spx@nexttoken\spx@forcewrapPYG@i}% +% \g@addto@macro\FV@SetupFont{% \sbox\sphinxcontinuationbox {\spx@opt@verbatimcontinued}% \sbox\sphinxvisiblespacebox {\spx@opt@verbatimvisiblespace}% From 0076ad3fd3163a93cd8e300894eebe561d02a0a9 Mon Sep 17 00:00:00 2001 From: jfbu Date: Mon, 8 Feb 2021 23:57:10 +0100 Subject: [PATCH 11/19] Indicated force wrapping code lines is fragile with Unicode input --- doc/latex.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/latex.rst b/doc/latex.rst index 636781753..8b31f9206 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -642,9 +642,13 @@ macros may be significant. Boolean to specify if long lines in :rst:dir:`code-block`\ 's contents which the wrapping algorithm could not reduce to at most an excess of 3 characters on a line will be cut forcefully to achieve this maximal excess - of 3 characters on each line. (*this is possibly fragile, so by default is - not done; please try it out and report issues to the maintainers to help - improve and decide whether to make this default*) + of 3 characters on each line. (*this is possibly fragile, and in + particular will break with* ``'pdflatex'``, *if the code-block contains + Unicode input, so by default is not activated; however it will not have + this problem with Unicode engines. Please try it out and report further + issues to the maintainers; injecting* ``\sphinxsetup{verbatimforcewraps}`` + *and* ``\sphinxsetup{verbatimforcewraps=false}`` *via* ``.. raw:: latex`` + *directives will localize usage to only certain code-blocks*) Default: ``false`` From dffb819ec49f6daf386a46477ccfc9eb6f035eee Mon Sep 17 00:00:00 2001 From: jfbu Date: Tue, 9 Feb 2021 11:24:24 +0100 Subject: [PATCH 12/19] Support Unicode input when forcefully wrapping code lines This needs special coding only for pdflatex, Unicode TeX engines already handle Unicode characters as one token. The utf8x LaTeX input encoding is not supported, only utf8. --- sphinx/texinputs/sphinx.sty | 180 +++++++++++++++++++++++++----------- 1 file changed, 128 insertions(+), 52 deletions(-) diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index 85f67270d..953dd0298 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -1185,27 +1185,46 @@ \doublehyphendemerits\z@\finalhyphendemerits\z@ \strut #1\strut}% }% -% We implement an alternative to wrapping long code lines. This -% alternative works only if the contents are in the expected -% Pygments mark-up: i.e. some character escapes such as \PYGZdl{} -% and the highlighting \PYG macro with always 2 arguments, and no -% other macros. This means: -% - the command prefix *must* be PYG -% - the texcomments Pygments option *must* be set to False (it could -% work by luck if True) -% For non-highlighted tokens a break point is installed at each of them. -% For highlighted tokens (i.e. in 2nd argument of \PYG) every four such -% characters. \PYGZdl{} etc will count for 2, although corresponding to -% only one. The result is that no line should be more than 3 characters -% overfull. -% First a measurement step is done of what would our standard wrapping -% approach give. This is a bit tricky, cf TeX by Topic for the basic -% dissecting technique, because TeX unfortunately when building a -% vertical box does not store in an accessible way what was the maximal -% line-width: the width of the box will be the set \hsize. -% Anyway, if the max width exceed the linewidth by at least 4 character -% widths, then we apply the "force wrapping" of previous paragraph, -% else we apply our "standard wrapping". +% +% The normal line wrapping allows breaks at spaces and ascii non +% letters, non digits. The \raggedright above means there will be +% an overfilled line only if some non-breakable "word" was +% encountered, which is longer than a line (it is moved always to +% be on its own on a new line). +% +% The "forced" line wrapping will parse the tokens to add potential +% breakpoints at each character. As some strings are highlighted, +% we have to apply the highlighting character per character, which +% requires to manipulate the output of the Pygments LaTeXFormatter. +% +% Doing this at latex level is complicated. The contents should +% be as expected: i.e. some active characters from +% \sphinxbreaksviaactive, some Pygments character escapes such as +% \PYGZdl{}, and the highlighting \PYG macro with always 2 +% arguments. No other macros should be there, except perhaps +% zero-parameter macros. In particular: +% - the texcomments Pygments option must be set to False +% +% With pdflatex, Unicode input gives multi-bytes characters +% where the first byte is active. We support the "utf8" macros +% only. "utf8x" is not supported. +% +% The highlighting macro \PYG will be applied character per +% character. Highlighting via a colored background gives thus a +% chain of small colored boxes which may cause some artefact in +% some pdf viewers. Can't do anything here if we do want the line +% break to be possible. +% +% First a measurement step is done of what would the standard line +% wrapping give (i.e line breaks only at spaces and non-letter, +% non-digit ascii characters), cf TeX by Topic for the basic +% dissecting technique: TeX unfortunately when building a vertical +% box does not store in an accessible way what was the maximal +% line-width during paragraph building. +% +% If the max width exceed the linewidth by at least 4 character +% widths, then we apply the "force wrapping" with potential line +% break at each character, else we don't. \long\def\spx@verb@FormatLineForceWrap#1{% % \spx@image@box is a scratch box register that we can use here \global\let\spx@verb@maxwidth\z@ @@ -1215,58 +1234,115 @@ \strut #1\strut\@@par \spx@verb@getmaxwidth}% \ifdim\spx@verb@maxwidth>\dimexpr\linewidth+3\fontcharwd\font`X\relax - \spx@verb@FormatLineWrap{\spx@forcewrapPYG #1\spx@forcewrapPYG}% + \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}% \else \spx@verb@FormatLineWrap{#1}% \fi }% % auxiliary paragraph dissector to get max width -\newbox\spx@line@box +\newbox\spx@scratchbox \def\spx@verb@getmaxwidth {% \unskip\unpenalty - \setbox\spx@line@box\lastbox - \ifvoid\spx@line@box + \setbox\spx@scratchbox\lastbox + \ifvoid\spx@scratchbox \else - \setbox\spx@line@box\hbox{\unhbox\spx@line@box}% - \ifdim\spx@verb@maxwidth<\wd\spx@line@box - \xdef\spx@verb@maxwidth{\number\wd\spx@line@box sp}% + \setbox\spx@scratchbox\hbox{\unhbox\spx@scratchbox}% + \ifdim\spx@verb@maxwidth<\wd\spx@scratchbox + \xdef\spx@verb@maxwidth{\number\wd\spx@scratchbox sp}% \fi \expandafter\spx@verb@getmaxwidth \fi }% % auxiliary macros to implement "cut long line even in middle of word" -\def\spx@forcewrapPYG{% - \futurelet\spx@nexttoken\spx@forcewrapPYG@i +\catcode`Z=3 % safe delimiter +\def\spx@verb@wrapPYG{% + \futurelet\spx@nexttoken\spx@verb@wrapPYG@i }% -\def\spx@forcewrapPYG@i{% - \ifx\spx@nexttoken\spx@forcewrapPYG\let\next=\@gobble\else - \ifx\spx@nexttoken\PYG\let\next=\spx@forcewrapPYG@PYG\else +\def\spx@verb@wrapPYG@i{% + \ifx\spx@nexttoken\spx@verb@wrapPYG\let\next=\@gobble\else + \ifx\spx@nexttoken\PYG\let\next=\spx@verb@wrapPYG@PYG@onebyone\else \discretionary{}{\sphinxafterbreak}{}% - \let\next=\spx@forcewrapPYG@ii + \let\next\spx@verb@wrapPYG@ii \fi\fi \next }% -\def\spx@forcewrapPYG@ii#1{#1\futurelet\spx@nexttoken\spx@forcewrapPYG@i}% -% Replace \PYG by itself applied to short strings of 4 characters at a time -% and insert breakpoints in-between -\def\spx@forcewrapPYG@PYG\PYG#1#2{% - \def\spx@PYGspec{{#1}}% - \spx@PYG#2\@empty\@empty\@empty\@empty\relax +% Let's recognize active characters. We don't support utf8x only utf8. +% And here #1 should not have picked up (non empty) braced contents +\long\def\spx@verb@wrapPYG@ii#1{% + \ifcat\noexpand~\noexpand#1\relax% active character + \expandafter\spx@verb@wrapPYG@active + \else % non-active character, control sequence such as \PYGZdl, or empty + \expandafter\spx@verb@wrapPYG@one + \fi {#1}% }% -\def\spx@PYG#1#2#3#4{% - \discretionary{}{\sphinxafterbreak}{}% - \expandafter\PYG\spx@PYGspec{#1#2#3#4}% -% I assume here contents never contain \@empty. If #4={} originally then -% it is empty here and the \ifx will compare \@empty to \relax and choose -% the else branch, i.e. to continue applying \PYG repeatedly - \ifx#4\@empty\relax - \expandafter\spx@PYG@done - \else - \expandafter\spx@PYG +\long\def\spx@verb@wrapPYG@active#1{% +% Let's hope expansion of active character does not really require arguments, +% as we certainly don't want to go into expanding upfront token stream anyway. + \expandafter\spx@verb@wrapPYG@iii#1{}{}{}{}{}{}{}{}{}Z#1% +}% +\long\def\spx@verb@wrapPYG@iii#1#2Z{% + \ifx\UTFviii@four@octets#1\let\next=\spx@verb@wrapPYG@four\else + \ifx\UTFviii@three@octets#1\let\next=\spx@verb@wrapPYG@three\else + \ifx\UTFviii@two@octets#1\let\next=\spx@verb@wrapPYG@two\else + \let\next=\spx@verb@wrapPYG@one + \fi\fi\fi + \next +}% +\long\def\spx@verb@wrapPYG@one #1{#1\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}% +\long\def\spx@verb@wrapPYG@two #1#2{#1#2\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}% +\long\def\spx@verb@wrapPYG@three #1#2#3{#1#2#3\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}% +\long\def\spx@verb@wrapPYG@four #1#2#3#4{#1#2#3#4\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}% +% Replace \PYG by itself applied one character at a time! This way breakpoints +% can be inserted. +\def\spx@verb@wrapPYG@PYG@onebyone#1#2#3{% #1 = \PYG, #2 = highlight spec, #3 = tokens + \def\spx@verb@wrapPYG@PYG@spec{{#2}}% + \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i#3Z% +}% +\def\spx@verb@wrapPYG@PYG@i{% + \ifx\spx@nexttokenZ\let\next=\spx@verb@wrapPYG@PYG@done\else + \discretionary{}{\sphinxafterbreak}{}% + \let\next\spx@verb@wrapPYG@PYG@ii \fi + \next }% -% Once \PYG is handled we get back to our forward scan token by token -\def\spx@PYG@done#1\relax{\futurelet\spx@nexttoken\spx@forcewrapPYG@i}% +\def\spx@verb@wrapPYG@PYG@doneZ{\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}% +\long\def\spx@verb@wrapPYG@PYG@ii#1{% + \ifcat\noexpand~\noexpand#1\relax% active character + \expandafter\spx@verb@wrapPYG@PYG@active + \else % non-active character, control sequence such as \PYGZdl, or empty + \expandafter\spx@verb@wrapPYG@PYG@one + \fi {#1}% +}% +\long\def\spx@verb@wrapPYG@PYG@active#1{% +% Let's hope expansion of active character does not really require arguments, +% as we certainly don't want to go into expanding upfront token stream anyway. + \expandafter\spx@verb@wrapPYG@PYG@iii#1{}{}{}{}{}{}{}{}{}Z#1% +}% +\long\def\spx@verb@wrapPYG@PYG@iii#1#2Z{% + \ifx\UTFviii@four@octets#1\let\next=\spx@verb@wrapPYG@PYG@four\else + \ifx\UTFviii@three@octets#1\let\next=\spx@verb@wrapPYG@PYG@three\else + \ifx\UTFviii@two@octets#1\let\next=\spx@verb@wrapPYG@PYG@two\else + \let\next=\spx@verb@wrapPYG@PYG@one + \fi\fi\fi + \next +}% +\long\def\spx@verb@wrapPYG@PYG@one#1{% + \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1}% + \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i +}% +\long\def\spx@verb@wrapPYG@PYG@two#1#2{% + \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2}% + \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i +}% +\long\def\spx@verb@wrapPYG@PYG@three#1#2#3{% + \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2#3}% + \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i +}% +\long\def\spx@verb@wrapPYG@PYG@four#1#2#3#4{% + \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2#3#4}% + \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i +}% +\catcode`Z 11% % \g@addto@macro\FV@SetupFont{% \sbox\sphinxcontinuationbox {\spx@opt@verbatimcontinued}% From c6a8fb1625a16c44edebd0aa706792510c64239d Mon Sep 17 00:00:00 2001 From: jfbu Date: Tue, 9 Feb 2021 12:03:55 +0100 Subject: [PATCH 13/19] Add verbatimmaxoverfull and verbatimmaxunderfull --- doc/latex.rst | 63 ++++++++++++++++++++++++++++--------- sphinx/texinputs/sphinx.sty | 13 ++++++-- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/doc/latex.rst b/doc/latex.rst index 8b31f9206..a95e130f4 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -585,7 +585,9 @@ The below is included at the end of the chapter:: \endgroup -LaTeX boolean keys require *lowercase* ``true`` or ``false`` values. +LaTeX syntax for boolean keys require *lowercase* ``true`` or ``false`` +e.g ``'sphinxsetup': "verbatimwrapslines=false"``. If setting the +boolean key to ``true``, ``=true`` is optional. Spaces around the commas and equal signs are ignored, spaces inside LaTeX macros may be significant. @@ -636,30 +638,63 @@ macros may be significant. Boolean to specify if long lines in :rst:dir:`code-block`\ 's contents are wrapped. + If ``true``, line breaks may happen at spaces (the last space before the + line break will be rendered using a special symbol), and at ascii + punctuation characters (i.e. not at letters or digits). Whenever a long + string has no break points, it is moved to next line. If its length is + longer than the line width it will overflow. + Default: ``true`` ``verbatimforcewraps`` Boolean to specify if long lines in :rst:dir:`code-block`\ 's contents - which the wrapping algorithm could not reduce to at most an excess of 3 - characters on a line will be cut forcefully to achieve this maximal excess - of 3 characters on each line. (*this is possibly fragile, and in - particular will break with* ``'pdflatex'``, *if the code-block contains - Unicode input, so by default is not activated; however it will not have - this problem with Unicode engines. Please try it out and report further - issues to the maintainers; injecting* ``\sphinxsetup{verbatimforcewraps}`` - *and* ``\sphinxsetup{verbatimforcewraps=false}`` *via* ``.. raw:: latex`` - *directives will localize usage to only certain code-blocks*) + should be forcefully wrapped to never overflow due to long strings. + + .. notice:: + + It is assumed that the Pygments_ LaTeXFormatter has not been used with + its ``texcomments`` or similar options which allow additional + (arbitrary) LaTeX mark-up. + + Also, in case of :confval:`latex_engine` set to ``'pdflatex'``, only + the default LaTeX handling of Unicode code points, i.e. ``utf8`` not + ``utf8x`` is allowed. Default: ``false`` .. versionadded:: 3.5.0 -``literalblockcappos`` - Decides the caption position: either ``b`` ("bottom") or ``t`` ("top"). +``verbatimmaxoverfull`` + A number. If an unbreakable long string has length larger than the total + linewidth plus this number of characters, and if ``verbatimforcewraps`` + mode is on, the input line will be reset using the forceful algorithm + which applies breakpoints at each character. - Default: ``t`` + Default: ``3`` - .. versionadded:: 1.7 + .. versionadded:: 3.5.0 + +``verbatimmaxunderfull`` + A number. If ``verbatimforcewraps`` mode applies, and if after applying + the line wrapping at spaces and punctuation, the first part of the split + line is lacking at least that number of characters to fill the available + width, then the input line will be reset using the forceful algorithm. + + As the default is set to a high value, the forceful algorithm is triggered + only in overfull case, i.e. in presence of a string longer than full + linewidth. Set this to ``0`` to force all input lines to be hard wrapped + at the current avaiable linewidth:: + + latex_elements = { + 'sphinxsetup': "verbatimforcewraps, verbatimmaxunderfull=0", + } + + This can be done locally for a given code-block via the use of raw latex + directives to insert suitable ``\sphinxsetup`` into the latex file. + + Default: ``100`` + + .. versionadded:: 3.5.0 ``verbatimhintsturnover`` Boolean to specify if code-blocks display "continued on next page" and diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index 953dd0298..b2c855acb 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -335,6 +335,8 @@ \DeclareBoolOption[true]{verbatimwithframe} \DeclareBoolOption[true]{verbatimwrapslines} \DeclareBoolOption[false]{verbatimforcewraps} +\DeclareStringOption[3]{verbatimmaxoverfull} +\DeclareStringOption[100]{verbatimmaxunderfull} \DeclareBoolOption[true]{verbatimhintsturnover} \DeclareBoolOption[true]{inlineliteralwraps} \DeclareStringOption[t]{literalblockcappos} @@ -1233,11 +1235,16 @@ \doublehyphendemerits\z@\finalhyphendemerits\z@ \strut #1\strut\@@par \spx@verb@getmaxwidth}% - \ifdim\spx@verb@maxwidth>\dimexpr\linewidth+3\fontcharwd\font`X\relax + \ifdim\spx@verb@maxwidth> + \dimexpr\linewidth+\spx@opt@verbatimmaxoverfull\fontcharwd\font`X \relax \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}% \else - \spx@verb@FormatLineWrap{#1}% - \fi + \ifdim\spx@verb@maxwidth< + \dimexpr\linewidth-\spx@opt@verbatimmaxunderfull\fontcharwd\font`X \relax + \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}% + \else + \spx@verb@FormatLineWrap{#1}% + \fi\fi }% % auxiliary paragraph dissector to get max width \newbox\spx@scratchbox From abddafa98a1221a669c35a3955a8e3d3c6aa2f66 Mon Sep 17 00:00:00 2001 From: jfbu Date: Tue, 9 Feb 2021 12:28:03 +0100 Subject: [PATCH 14/19] Fix inexistent directive "notice", it is "note"... --- doc/latex.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/latex.rst b/doc/latex.rst index a95e130f4..59fcc3850 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -650,7 +650,7 @@ macros may be significant. Boolean to specify if long lines in :rst:dir:`code-block`\ 's contents should be forcefully wrapped to never overflow due to long strings. - .. notice:: + .. note:: It is assumed that the Pygments_ LaTeXFormatter has not been used with its ``texcomments`` or similar options which allow additional From eee17ff61a79994886caa4943cecd4c86ee9f46e Mon Sep 17 00:00:00 2001 From: jfbu Date: Tue, 9 Feb 2021 12:33:16 +0100 Subject: [PATCH 15/19] Fix forgotten link to Pygments home site, mmmmpf --- doc/latex.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/latex.rst b/doc/latex.rst index 59fcc3850..69a0a470d 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -660,6 +660,8 @@ macros may be significant. the default LaTeX handling of Unicode code points, i.e. ``utf8`` not ``utf8x`` is allowed. + .. _Pygments: https://pygments.org/ + Default: ``false`` .. versionadded:: 3.5.0 From 7175d11f05a25fc6c7be0cf7e2037510e10f3b93 Mon Sep 17 00:00:00 2001 From: jfbu Date: Tue, 9 Feb 2021 13:16:12 +0100 Subject: [PATCH 16/19] Add forgotten minwidth computation --- sphinx/texinputs/sphinx.sty | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index b2c855acb..99f7664bd 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -1230,25 +1230,26 @@ \long\def\spx@verb@FormatLineForceWrap#1{% % \spx@image@box is a scratch box register that we can use here \global\let\spx@verb@maxwidth\z@ + \global\let\spx@verb@minwidth\linewidth \setbox\spx@image@box \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@ \doublehyphendemerits\z@\finalhyphendemerits\z@ \strut #1\strut\@@par - \spx@verb@getmaxwidth}% + \spx@verb@getwidths}% \ifdim\spx@verb@maxwidth> \dimexpr\linewidth+\spx@opt@verbatimmaxoverfull\fontcharwd\font`X \relax \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}% \else - \ifdim\spx@verb@maxwidth< + \ifdim\spx@verb@minwidth< \dimexpr\linewidth-\spx@opt@verbatimmaxunderfull\fontcharwd\font`X \relax \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}% \else \spx@verb@FormatLineWrap{#1}% \fi\fi }% -% auxiliary paragraph dissector to get max width +% auxiliary paragraph dissector to get max and min widths \newbox\spx@scratchbox -\def\spx@verb@getmaxwidth {% +\def\spx@verb@getwidths {% \unskip\unpenalty \setbox\spx@scratchbox\lastbox \ifvoid\spx@scratchbox @@ -1257,7 +1258,10 @@ \ifdim\spx@verb@maxwidth<\wd\spx@scratchbox \xdef\spx@verb@maxwidth{\number\wd\spx@scratchbox sp}% \fi - \expandafter\spx@verb@getmaxwidth + \ifdim\spx@verb@minwidth>\wd\spx@scratchbox + \xdef\spx@verb@minwidth{\number\wd\spx@scratchbox sp}% + \fi + \expandafter\spx@verb@getwidths \fi }% % auxiliary macros to implement "cut long line even in middle of word" From 71df21dd4f2365f64a057c4495718946ede003f5 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 9 Feb 2021 22:19:05 +0900 Subject: [PATCH 17/19] Update CHANGES for PR #8853 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index b394c54e3..7c13946ef 100644 --- a/CHANGES +++ b/CHANGES @@ -53,6 +53,7 @@ Features added * #6550: html: Allow to use HTML permalink texts via :confval:`html_permalinks_icon` * #1638: html: Add permalink icons to glossary terms +* #8852: i18n: Allow to translate heading syntax in MyST-Parser * #8649: imgconverter: Skip availability check if builder supports the image type * #8573: napoleon: Allow to change the style of custom sections using From 035d13fd248f386073390578672802fe91e8395a Mon Sep 17 00:00:00 2001 From: jfbu Date: Tue, 9 Feb 2021 15:47:15 +0100 Subject: [PATCH 18/19] Update CHANGES and docs relative to new verbatimforcewraps --- CHANGES | 4 +++- doc/latex.rst | 5 ++++- sphinx/texinputs/sphinx.sty | 7 ++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index e39180cd3..fb8013888 100644 --- a/CHANGES +++ b/CHANGES @@ -131,7 +131,9 @@ Bugs fixed * #8780: LaTeX: long words in narrow columns may not be hyphenated * #8788: LaTeX: ``\titleformat`` last argument in sphinx.sty should be bracketed, not braced (and is anyhow not needed) -* #8849: LaTex: code-block printed out of margin +* #8849: LaTex: code-block printed out of margin (see the opt-in LaTeX syntax + boolean :ref:`verbatimforcewraps ` for use via + the :ref:`'sphinxsetup' ` key of ``latex_elements``) Testing -------- diff --git a/doc/latex.rst b/doc/latex.rst index 69a0a470d..ed99c8994 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -646,6 +646,8 @@ macros may be significant. Default: ``true`` +.. _latexsphinxsetupforcewraps: + ``verbatimforcewraps`` Boolean to specify if long lines in :rst:dir:`code-block`\ 's contents should be forcefully wrapped to never overflow due to long strings. @@ -692,7 +694,8 @@ macros may be significant. } This can be done locally for a given code-block via the use of raw latex - directives to insert suitable ``\sphinxsetup`` into the latex file. + directives to insert suitable ``\sphinxsetup`` (before and after) into the + latex file. Default: ``100`` diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index 99f7664bd..adc9a23a7 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -1224,9 +1224,10 @@ % box does not store in an accessible way what was the maximal % line-width during paragraph building. % -% If the max width exceed the linewidth by at least 4 character -% widths, then we apply the "force wrapping" with potential line -% break at each character, else we don't. +% If the max width exceeds the linewidth by more than verbatimmaxoverfull +% character widths, or if the min width plus verbatimmaxunderfull character +% widths is inferior to linewidth, then we apply the "force wrapping" with +% potential line break at each character, else we don't. \long\def\spx@verb@FormatLineForceWrap#1{% % \spx@image@box is a scratch box register that we can use here \global\let\spx@verb@maxwidth\z@ From aaac7211049d79ef0a71a4cf516eedd63cf0a72b Mon Sep 17 00:00:00 2001 From: jfbu Date: Tue, 9 Feb 2021 16:04:01 +0100 Subject: [PATCH 19/19] Typo --- doc/latex.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/latex.rst b/doc/latex.rst index ed99c8994..ffd5c274e 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -585,7 +585,7 @@ The below is included at the end of the chapter:: \endgroup -LaTeX syntax for boolean keys require *lowercase* ``true`` or ``false`` +LaTeX syntax for boolean keys requires *lowercase* ``true`` or ``false`` e.g ``'sphinxsetup': "verbatimwrapslines=false"``. If setting the boolean key to ``true``, ``=true`` is optional. Spaces around the commas and equal signs are ignored, spaces inside LaTeX