diff --git a/AUTHORS b/AUTHORS index 1cc77b640..8505b644d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -36,7 +36,7 @@ Other contributors, listed alphabetically, are: * Hernan Grecco -- search improvements * Horst Gutmann -- internationalization support * Martin Hans -- autodoc improvements -* Zac Hatfield-Dodds -- doctest reporting improvements +* Zac Hatfield-Dodds -- doctest reporting improvements, intersphinx performance * Doug Hellmann -- graphviz improvements * Tim Hoffmann -- theme improvements * Antti Kaihola -- doctest extension (skipif option) diff --git a/CHANGES b/CHANGES index dfd58c717..cc48cb401 100644 --- a/CHANGES +++ b/CHANGES @@ -9,10 +9,13 @@ Incompatible changes * #6742: ``end-before`` option of :rst:dir:`literalinclude` directive does not match the first line of the code block. +* #1331: Change default User-Agent header to ``"Sphinx/X.Y.Z requests/X.Y.Z + python/X.Y.Z"``. It can be changed via :confval:`user_agent`. Deprecated ---------- +* ``sphinx.builders.gettext.POHEADER`` * ``sphinx.io.SphinxStandaloneReader.app`` * ``sphinx.io.SphinxStandaloneReader.env`` @@ -23,6 +26,13 @@ Features added * #267: html: Eliminate prompt characters of doctest block from copyable text * #6729: html theme: agogo theme now supports ``rightsidebar`` option * #6780: Add PEP-561 Support +* #6762: latex: Allow to load additonal LaTeX packages via ``extrapackages`` key + of :confval:`latex_elements` +* #1331: Add new config variable: :confval:`user_agent` +* #6000: LaTeX: have backslash also be an inline literal word wrap break + character +* #6812: Improve a warning message when extensions are not parallel safe +* #6818: Improve Intersphinx performance for multiple remote inventories. Bugs fixed ---------- @@ -32,6 +42,9 @@ Bugs fixed .. _latex3/latex2e#173: https://github.com/latex3/latex2e/issues/173 * #6618: LaTeX: Avoid section names at the end of a page +* #6738: LaTeX: Do not replace unicode characters by LaTeX macros on unicode + supported LaTeX engines: ¶, §, €, ∞, ±, →, ‣, –, superscript and subscript + digits go through "as is" (as default OpenType font supports them) * #6704: linkcheck: Be defensive and handle newly defined HTTP error code * #6655: image URLs containing ``data:`` causes gettext builder crashed * #6584: i18n: Error when compiling message catalogs on Hindi @@ -42,6 +55,11 @@ Bugs fixed * #5070: epub: Wrong internal href fragment links * #6712: Allow not to install sphinx.testing as runtime (mainly for ALT Linux) * #6741: html: search result was broken with empty :confval:`html_file_suffix` +* #6001: LaTeX does not wrap long code lines at backslash character +* #6804: LaTeX: PDF build breaks if admonition of danger type contains + code-block long enough not to fit on one page +* #6809: LaTeX: code-block in a danger type admonition can easily spill over + bottom of page * #6793: texinfo: Code examples broken following "sidebar" Testing @@ -65,6 +83,8 @@ Features added Bugs fixed ---------- +* #6776: LaTeX: 2019-10-01 LaTeX release breaks :file:`sphinxcyrillic.sty` + Testing -------- diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index edf98cfed..8c201b525 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -26,6 +26,11 @@ The following is a list of deprecated interfaces. - (will be) Removed - Alternatives + * - ``sphinx.builders.gettext.POHEADER`` + - 2.3 + - 4.0 + - ``sphinx/templates/gettext/message.pot_t`` (template file) + * - ``sphinx.io.SphinxStandaloneReader.app`` - 2.3 - 4.0 diff --git a/doc/latex.rst b/doc/latex.rst index 2501d1594..01c0e0cb8 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -226,6 +226,25 @@ into the generated ``.tex`` files. Its ``'sphinxsetup'`` key is described .. versionadded:: 1.5 + ``'extrapackages'`` + Additional LaTeX packages. For example: + + .. code-block:: python + + latex_elements = { + 'packages': r'\usepackage{isodate}' + } + + It defaults to empty. + + The specified LaTeX packages will be loaded before + hyperref package and packages loaded from Sphinx extensions. + + .. hint:: If you'd like to load additional LaTeX packages after hyperref, use + ``'preamble'`` key instead. + + .. versionadded:: 2.3 + ``'footer'`` Additional footer content (before the indices), default empty. @@ -600,12 +619,15 @@ macros may be significant. default ``true``. Allows linebreaks inside inline literals: but extra potential break-points (additionally to those allowed by LaTeX at spaces or for hyphenation) are currently inserted only after the characters - ``. , ; ? ! /``. Due to TeX internals, white space in the line will be - stretched (or shrunk) in order to accomodate the linebreak. + ``. , ; ? ! /`` and ``\``. Due to TeX internals, white space in the line + will be stretched (or shrunk) in order to accomodate the linebreak. .. versionadded:: 1.5 set this option value to ``false`` to recover former behaviour. + .. versionchanged:: 2.3.0 + added potential breakpoint at ``\`` characters. + ``verbatimvisiblespace`` default ``\textcolor{red}{\textvisiblespace}``. When a long code line is split, the last space character from the source code line right before the diff --git a/doc/usage/configuration.rst b/doc/usage/configuration.rst index 02b40256d..e61c09cb2 100644 --- a/doc/usage/configuration.rst +++ b/doc/usage/configuration.rst @@ -510,6 +510,14 @@ General configuration .. versionadded:: 1.6.6 +.. confval:: user_agent + + A User-Agent of Sphinx. It is used for a header on HTTP access (ex. + linkcheck, intersphinx and so on). Default is ``"Sphinx/X.Y.Z + requests/X.Y.Z python/X.Y.Z"``. + + .. versionadded:: 2.3 + .. confval:: tls_verify If true, Sphinx verifies server certifications. Default is ``True``. diff --git a/setup.py b/setup.py index 0a75278b4..202523fe0 100644 --- a/setup.py +++ b/setup.py @@ -176,6 +176,10 @@ setup( description='Python documentation generator', long_description=long_desc, long_description_content_type='text/x-rst', + project_urls={ + "Code": "https://github.com/sphinx-doc/sphinx", + "Issue tracker": "https://github.com/sphinx-doc/sphinx/issues", + }, zip_safe=False, classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/sphinx/application.py b/sphinx/application.py index 1c7a05357..641ce9893 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -1192,26 +1192,30 @@ class Sphinx: """ if typ == 'read': attrname = 'parallel_read_safe' - message = __("the %s extension does not declare if it is safe " - "for parallel reading, assuming it isn't - please " - "ask the extension author to check and make it " - "explicit") + message_not_declared = __("the %s extension does not declare if it " + "is safe for parallel reading, assuming " + "it isn't - please ask the extension author " + "to check and make it explicit") + message_not_safe = __("the %s extension is not safe for parallel reading") elif typ == 'write': attrname = 'parallel_write_safe' - message = __("the %s extension does not declare if it is safe " - "for parallel writing, assuming it isn't - please " - "ask the extension author to check and make it " - "explicit") + message_not_declared = __("the %s extension does not declare if it " + "is safe for parallel writing, assuming " + "it isn't - please ask the extension author " + "to check and make it explicit") + message_not_safe = __("the %s extension is not safe for parallel writing") else: raise ValueError('parallel type %s is not supported' % typ) for ext in self.extensions.values(): allowed = getattr(ext, attrname, None) if allowed is None: - logger.warning(message, ext.name) + logger.warning(message_not_declared, ext.name) logger.warning(__('doing serial %s'), typ) return False elif not allowed: + logger.warning(message_not_safe, ext.name) + logger.warning(__('doing serial %s'), typ) return False return True diff --git a/sphinx/builders/gettext.py b/sphinx/builders/gettext.py index 763399413..ea6fa61a9 100644 --- a/sphinx/builders/gettext.py +++ b/sphinx/builders/gettext.py @@ -11,16 +11,16 @@ from codecs import open from collections import defaultdict, OrderedDict from datetime import datetime, tzinfo, timedelta -from io import StringIO from os import path, walk, getenv from time import time -from typing import Any, Dict, Iterable, List, Set, Tuple, Union +from typing import Any, Dict, Iterable, Generator, List, Set, Tuple, Union from uuid import uuid4 from docutils import nodes from docutils.nodes import Element from sphinx import addnodes +from sphinx import package_dir from sphinx.application import Sphinx from sphinx.builders import Builder from sphinx.domains.python import pairindextypes @@ -30,8 +30,9 @@ from sphinx.util import split_index_msg, logging, status_iterator from sphinx.util.console import bold # type: ignore from sphinx.util.i18n import CatalogInfo, docname_to_domain from sphinx.util.nodes import extract_messages, traverse_translatable_index -from sphinx.util.osutil import relpath, ensuredir, canon_path +from sphinx.util.osutil import ensuredir, canon_path from sphinx.util.tags import Tags +from sphinx.util.template import SphinxRenderer if False: # For type annotation @@ -58,7 +59,15 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"""[1:] +"""[1:] # RemovedInSphinx40Warning + + +class Message: + """An entry of translatable message.""" + def __init__(self, text: str, locations: List[Tuple[str, int]], uuids: List[str]): + self.text = text + self.locations = locations + self.uuids = uuids class Catalog: @@ -80,6 +89,12 @@ class Catalog: self.metadata[msg] = [] self.metadata[msg].append((origin.source, origin.line, origin.uid)) # type: ignore + def __iter__(self) -> Generator[Message, None, None]: + for message in self.messages: + positions = [(source, line) for source, line, uuid in self.metadata[message]] + uuids = [uuid for source, line, uuid in self.metadata[message]] + yield Message(message, positions, uuids) + class MsgOrigin: """ @@ -92,6 +107,22 @@ class MsgOrigin: self.uid = uuid4().hex +class GettextRenderer(SphinxRenderer): + def __init__(self, template_path: str = None) -> None: + if template_path is None: + template_path = path.join(package_dir, 'templates', 'gettext') + super().__init__(template_path) + + def escape(s: str) -> str: + s = s.replace('\\', r'\\') + s = s.replace('"', r'\"') + return s.replace('\n', '\\n"\n"') + + # use texescape as escape filter + self.env.filters['e'] = escape + self.env.filters['escape'] = escape + + class I18nTags(Tags): """Dummy tags module for I18nBuilder. @@ -247,12 +278,13 @@ class MessageCatalogBuilder(I18nBuilder): def finish(self) -> None: super().finish() - data = { + context = { 'version': self.config.version, 'copyright': self.config.copyright, 'project': self.config.project, - 'ctime': datetime.fromtimestamp( - timestamp, ltz).strftime('%Y-%m-%d %H:%M%z'), + 'ctime': datetime.fromtimestamp(timestamp, ltz).strftime('%Y-%m-%d %H:%M%z'), + 'display_location': self.config.gettext_location, + 'display_uuid': self.config.gettext_uuid, } for textdomain, catalog in status_iterator(self.catalogs.items(), __("writing message catalogs... "), @@ -262,30 +294,10 @@ class MessageCatalogBuilder(I18nBuilder): # noop if config.gettext_compact is set ensuredir(path.join(self.outdir, path.dirname(textdomain))) + context['messages'] = list(catalog) + content = GettextRenderer().render('message.pot_t', context) + pofn = path.join(self.outdir, textdomain + '.pot') - output = StringIO() - output.write(POHEADER % data) - - for message in catalog.messages: - positions = catalog.metadata[message] - - if self.config.gettext_location: - # generate "#: file1:line1\n#: file2:line2 ..." - output.write("#: %s\n" % "\n#: ".join( - "%s:%s" % (canon_path(relpath(source, self.outdir)), line) - for source, line, _ in positions)) - if self.config.gettext_uuid: - # generate "# uuid1\n# uuid2\n ..." - output.write("# %s\n" % "\n# ".join(uid for _, _, uid in positions)) - - # message contains *one* line of text ready for translation - message = message.replace('\\', r'\\'). \ - replace('"', r'\"'). \ - replace('\n', '\\n"\n"') - output.write('msgid "%s"\nmsgstr ""\n\n' % message) - - content = output.getvalue() - if should_write(pofn, content): with open(pofn, 'w', encoding='utf-8') as pofile: pofile.write(content) diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index 1be2041bd..635d9df98 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -101,7 +101,6 @@ class CheckExternalLinksBuilder(Builder): 'allow_redirects': True, 'headers': { 'Accept': 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8', - 'User-Agent': requests.useragent_header[0][1], }, } if self.app.config.linkcheck_timeout: diff --git a/sphinx/config.py b/sphinx/config.py index 5ba2c2a3d..d8cce1b3d 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -148,6 +148,7 @@ class Config: 'math_numfig': (True, 'env', []), 'tls_verify': (True, 'env', []), 'tls_cacerts': (None, 'env', []), + 'user_agent': (None, 'env', [str]), 'smartquotes': (True, 'env', []), 'smartquotes_action': ('qDe', 'env', []), 'smartquotes_excludes': ({'languages': ['ja'], diff --git a/sphinx/ext/intersphinx.py b/sphinx/ext/intersphinx.py index cb74ef6f1..7745d43d4 100644 --- a/sphinx/ext/intersphinx.py +++ b/sphinx/ext/intersphinx.py @@ -23,6 +23,7 @@ :license: BSD, see LICENSE for details. """ +import concurrent.futures import functools import posixpath import sys @@ -187,21 +188,18 @@ def fetch_inventory(app: Sphinx, uri: str, inv: Any) -> Any: return invdata -def load_mappings(app: Sphinx) -> None: - """Load all intersphinx mappings into the environment.""" - now = int(time.time()) +def fetch_inventory_group( + name: str, uri: str, invs: Any, cache: Any, app: Any, now: float +) -> bool: cache_time = now - app.config.intersphinx_cache_limit * 86400 - inventories = InventoryAdapter(app.builder.env) - update = False - for key, (name, (uri, invs)) in app.config.intersphinx_mapping.items(): - failures = [] + failures = [] + try: for inv in invs: if not inv: inv = posixpath.join(uri, INVENTORY_FILENAME) # decide whether the inventory must be read: always read local # files; remote ones only if the cache time is expired - if '://' not in inv or uri not in inventories.cache \ - or inventories.cache[uri][1] < cache_time: + if '://' not in inv or uri not in cache or cache[uri][1] < cache_time: safe_inv_url = _get_safe_url(inv) logger.info(__('loading intersphinx inventory from %s...'), safe_inv_url) try: @@ -209,12 +207,11 @@ def load_mappings(app: Sphinx) -> None: except Exception as err: failures.append(err.args) continue - if invdata: - inventories.cache[uri] = (name, now, invdata) - update = True - break - + cache[uri] = (name, now, invdata) + return True + return False + finally: if failures == []: pass elif len(failures) < len(invs): @@ -227,7 +224,21 @@ def load_mappings(app: Sphinx) -> None: logger.warning(__("failed to reach any of the inventories " "with the following issues:") + "\n" + issues) - if update: + +def load_mappings(app: Sphinx) -> None: + """Load all intersphinx mappings into the environment.""" + now = int(time.time()) + inventories = InventoryAdapter(app.builder.env) + + with concurrent.futures.ThreadPoolExecutor() as pool: + futures = [] + for name, (uri, invs) in app.config.intersphinx_mapping.values(): + futures.append(pool.submit( + fetch_inventory_group, name, uri, invs, inventories.cache, app, now + )) + updated = [f.result() for f in concurrent.futures.as_completed(futures)] + + if any(updated): inventories.clear() # Duplicate values in different inventories will shadow each @@ -374,6 +385,7 @@ def inspect_main(argv: List[str]) -> None: class MockConfig: intersphinx_timeout = None # type: int tls_verify = False + user_agent = None class MockApp: srcdir = '' diff --git a/sphinx/ext/todo.py b/sphinx/ext/todo.py index 5da0342f8..d2a8a666d 100644 --- a/sphinx/ext/todo.py +++ b/sphinx/ext/todo.py @@ -30,7 +30,7 @@ from sphinx.locale import _, __ from sphinx.util import logging from sphinx.util.docutils import SphinxDirective from sphinx.util.nodes import make_refnode -from sphinx.util.texescape import tex_escape_map +from sphinx.util.texescape import get_escape_func from sphinx.writers.html import HTMLTranslator from sphinx.writers.latex import LaTeXTranslator @@ -299,10 +299,11 @@ def depart_todo_node(self: HTMLTranslator, node: todo_node) -> None: def latex_visit_todo_node(self: LaTeXTranslator, node: todo_node) -> None: if self.config.todo_include_todos: + escape = get_escape_func(self.config.latex_engine) self.body.append('\n\\begin{sphinxadmonition}{note}{') self.body.append(self.hypertarget_to(node)) title_node = cast(nodes.title, node[0]) - self.body.append('%s:}' % title_node.astext().translate(tex_escape_map)) + self.body.append('%s:}' % escape(title_node.astext())) node.pop(0) else: raise nodes.SkipNode diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py index 977b71f28..0a6cf8d6e 100644 --- a/sphinx/highlighting.py +++ b/sphinx/highlighting.py @@ -28,7 +28,7 @@ from sphinx.ext import doctest from sphinx.locale import __ from sphinx.pygments_styles import SphinxStyle, NoneStyle from sphinx.util import logging -from sphinx.util.texescape import tex_hl_escape_map_new +from sphinx.util.texescape import get_hlescape_func, tex_hl_escape_map_new if False: # For type annotation @@ -68,9 +68,11 @@ class PygmentsBridge: html_formatter = HtmlFormatter latex_formatter = LatexFormatter - def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=None): - # type: (str, str, bool) -> None + def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=None, + latex_engine=None): + # type: (str, str, bool, str) -> None self.dest = dest + self.latex_engine = latex_engine style = self.get_style(stylename) self.formatter_args = {'style': style} # type: Dict[str, Any] @@ -192,7 +194,8 @@ class PygmentsBridge: if self.dest == 'html': return hlsource else: - return hlsource.translate(tex_hl_escape_map_new) + escape = get_hlescape_func(self.latex_engine) + return escape(hlsource) def get_stylesheet(self): # type: () -> str diff --git a/sphinx/templates/gettext/message.pot_t b/sphinx/templates/gettext/message.pot_t new file mode 100644 index 000000000..90df27175 --- /dev/null +++ b/sphinx/templates/gettext/message.pot_t @@ -0,0 +1,33 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) {{ copyright }} +# This file is distributed under the same license as the {{ project }} package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: {{ project|e }} {{ version|e }}\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: {{ ctime|e }}\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +{% for message in messages %} +{% if display_location -%} +{% for source, line in message.locations -%} +#: {{ source }}:{{ line }} +{% endfor -%} +{% endif -%} + +{% if display_uuid -%} +{% for uuid in message.uuids -%} +#: {{ uuid }} +{% endfor -%} +{% endif -%} + +msgid "{{ message.text|e }}" +msgstr "" +{% endfor -%} diff --git a/sphinx/templates/latex/latex.tex_t b/sphinx/templates/latex/latex.tex_t index 2beac82c9..a0a5a26b1 100644 --- a/sphinx/templates/latex/latex.tex_t +++ b/sphinx/templates/latex/latex.tex_t @@ -35,6 +35,7 @@ <%= sphinxsetup %> <%= fvset %> <%= geometry %> +<%= extrapackages %> <%- for name, option in packages %> <%- if option %> diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index 6065be81b..5d9adb5bb 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -1054,7 +1054,7 @@ % Take advantage of the already applied Pygments mark-up to insert % potential linebreaks for TeX processing. % {, <, #, %, $, ' and ": go to next line. -% _, }, ^, &, >, - and ~: stay at end of broken line. +% _, }, ^, &, >, -, ~, and \: stay at end of broken line. % Use of \textquotesingle for straight quote. % FIXME: convert this to package options ? \newcommand*\sphinxbreaksbeforelist {% @@ -1066,6 +1066,7 @@ \newcommand*\sphinxbreaksafterlist {% \do\PYGZus\_\do\PYGZcb\}\do\PYGZca\^\do\PYGZam\&% _, }, ^, &, \do\PYGZgt\>\do\PYGZhy\-\do\PYGZti\~% >, -, ~ + \do\PYGZbs\\% \ } \newcommand*\sphinxbreaksatspecials {% \def\do##1##2% @@ -1110,6 +1111,9 @@ \newcommand*\sphinxVerbatimTitle {} % This box to typeset the caption before framed.sty multiple passes for framing. \newbox\sphinxVerbatim@TitleBox +% This box to measure contents if nested as inner \MakeFramed requires then +% minipage encapsulation but too long contents then break outer \MakeFramed +\newbox\sphinxVerbatim@ContentsBox % This is a workaround to a "feature" of French lists, when literal block % follows immediately; usable generally (does only \par then), a priori... \newcommand*\sphinxvspacefixafterfrenchlists{% @@ -1256,17 +1260,23 @@ \itemsep \z@skip \topsep \z@skip \partopsep \z@skip - % trivlist will set \parsep to \parskip = zero + % trivlist will set \parsep to \parskip (which itself is set to zero above) % \leftmargin will be set to zero by trivlist \rightmargin\z@ \parindent \z@% becomes \itemindent. Default zero, but perhaps overwritten. \trivlist\item\relax - \ifsphinxverbatimwithminipage\spx@inframedtrue\fi - % use a minipage if we are already inside a framed environment - \ifspx@inframed\noindent\begin{minipage}{\linewidth}\fi - \MakeFramed {% adapted over from framed.sty's snugshade environment + \ifspx@inframed\setbox\sphinxVerbatim@ContentsBox\vbox\bgroup + \@setminipage\hsize\linewidth + % use bulk of minipage paragraph shape restores (this is needed + % in indented contexts, at least for some) + \textwidth\hsize \columnwidth\hsize \@totalleftmargin\z@ + \leftskip\z@skip \rightskip\z@skip \@rightskip\z@skip + \else + \ifsphinxverbatimwithminipage\noindent\begin{minipage}{\linewidth}\fi + \MakeFramed {% adapted over from framed.sty's snugshade environment \advance\hsize-\width\@totalleftmargin\z@\linewidth\hsize\@setminipage }% + \fi % For grid placement from \strut's in \FancyVerbFormatLine \lineskip\z@skip % active comma should not be overwritten by \@noligs @@ -1278,8 +1288,49 @@ } {% \endOriginalVerbatim - \par\unskip\@minipagefalse\endMakeFramed % from framed.sty snugshade - \ifspx@inframed\end{minipage}\fi + \ifspx@inframed + \egroup % finish \sphinxVerbatim@ContentsBox vbox + \nobreak % update page totals + \ifdim\dimexpr\ht\sphinxVerbatim@ContentsBox+ + \dp\sphinxVerbatim@ContentsBox+ + \ht\sphinxVerbatim@TitleBox+ + \dp\sphinxVerbatim@TitleBox+ + 2\fboxsep+2\fboxrule+ + % try to account for external frame parameters + \FrameSep+\FrameRule+ + % Usage here of 2 baseline distances is empirical. + % In border case where code-block fits barely in remaining space, + % it gets framed and looks good but the outer frame may continue + % on top of next page and give (if no contents after code-block) + % an empty framed line, as testing showed. + 2\baselineskip+ + % now add all to accumulated page totals and compare to \pagegoal + \pagetotal+\pagedepth>\pagegoal + % long contents: do not \MakeFramed. Do make a caption (either before or + % after) if title exists. Continuation hints across pagebreaks dropped. + % FIXME? a bottom caption may end up isolated at top of next page + % (no problem with a top caption, which is default) + \spx@opt@verbatimwithframefalse + \def\sphinxVerbatim@Title{\noindent\box\sphinxVerbatim@TitleBox\par}% + \sphinxVerbatim@Before + \noindent\unvbox\sphinxVerbatim@ContentsBox\par + \sphinxVerbatim@After + \else + % short enough contents: use \MakeFramed. As it is nested, this requires + % minipage encapsulation. + \noindent\begin{minipage}{\linewidth}% + \MakeFramed {% Use it now with the fetched contents + \advance\hsize-\width\@totalleftmargin\z@\linewidth\hsize\@setminipage + }% + \unvbox\sphinxVerbatim@ContentsBox + % some of this may be superfluous: + \par\unskip\@minipagefalse\endMakeFramed + \end{minipage}% + \fi + \else % non-nested \MakeFramed + \par\unskip\@minipagefalse\endMakeFramed % from framed.sty snugshade + \ifsphinxverbatimwithminipage\end{minipage}\fi + \fi \endtrivlist } \newenvironment {sphinxVerbatimNoFrame} @@ -1314,6 +1365,7 @@ {\def##1{\discretionary{\char`##2}{\sphinxafterbreak}{\char`##2}}}% \do\_\_\do\}\}\do\textasciicircum\^\do\&\&% _, }, ^, &, \do\textgreater\>\do\textasciitilde\~% >, ~ + \do\textbackslash\\% \ } \newcommand*\sphinxbreaksviaactiveinparsedliteral{% \sphinxbreaksviaactive % by default handles . , ; ? ! / @@ -1736,13 +1788,25 @@ % to obtain straight quotes we execute \@noligs as patched by upquote, and % \scantokens is needed in cases where it would be too late for the macro to % first set catcodes and then fetch its argument. We also make the contents -% breakable at non-escaped . , ; ? ! / using \sphinxbreaksviaactive. +% breakable at non-escaped . , ; ? ! / using \sphinxbreaksviaactive, +% and also at \ character (which is escaped to \textbackslash{}). +\protected\def\sphinxtextbackslashbreakbefore + {\discretionary{}{\sphinxafterbreak\sphinx@textbackslash}{\sphinx@textbackslash}} +\protected\def\sphinxtextbackslashbreakafter + {\discretionary{\sphinx@textbackslash}{\sphinxafterbreak}{\sphinx@textbackslash}} +\let\sphinxtextbackslash\sphinxtextbackslashbreakafter % the macro must be protected if it ends up used in moving arguments, % in 'alltt' \@noligs is done already, and the \scantokens must be avoided. \protected\def\sphinxupquote#1{{\def\@tempa{alltt}% \ifx\@tempa\@currenvir\else \ifspx@opt@inlineliteralwraps - \sphinxbreaksviaactive\let\sphinxafterbreak\empty + % break at . , ; ? ! / + \sphinxbreaksviaactive + % break also at \ + \let\sphinx@textbackslash\textbackslash + \let\textbackslash\sphinxtextbackslash + % do not typeset a continuation symbol on next line + \let\sphinxafterbreak\sphinxafterbreakofinlineliteral % do not overwrite the comma set-up \let\verbatim@nolig@list\sphinx@literal@nolig@list \fi @@ -1754,6 +1818,7 @@ \def\sphinx@do@noligs #1{\catcode`#1\active\begingroup\lccode`\~`#1\relax \lowercase{\endgroup\def~{\leavevmode\kern\z@\char`#1 }}} \def\sphinx@literal@nolig@list {\do\`\do\<\do\>\do\'\do\-}% +\let\sphinxafterbreakofinlineliteral\empty % Some custom font markup commands. \protected\def\sphinxstrong#1{\textbf{#1}} diff --git a/sphinx/texinputs/sphinxcyrillic.sty b/sphinx/texinputs/sphinxcyrillic.sty index 1a14c7b24..482b4e3f7 100644 --- a/sphinx/texinputs/sphinxcyrillic.sty +++ b/sphinx/texinputs/sphinxcyrillic.sty @@ -11,7 +11,7 @@ \ProcessLocalKeyvalOptions* % ignore class options \ifspx@cyropt@Xtwo -% original code by tex.sx user egreg: +% original code by tex.sx user egreg (updated 2019/10/28): % https://tex.stackexchange.com/a/460325/ % 159 Cyrillic glyphs as available in X2 TeX 8bit font encoding % This assumes inputenc loaded with utf8 option, or LaTeX release @@ -27,7 +27,9 @@ {Ӎ}{ӎ}{Ӕ}{ӕ}{Ә}{ә}{Ӡ}{ӡ}{Ө}{ө}\do {% \begingroup\def\IeC{\protect\DeclareTextSymbolDefault}% - \protected@edef\@temp{\endgroup\next{X2}}\@temp + \protected@edef\@temp{\endgroup + \@ifl@t@r{\fmtversion}{2019/10/01}{\csname u8:\next\endcsname}{\next}}% + \@temp{X2}% }% \else \ifspx@cyropt@TtwoA diff --git a/sphinx/util/docutils.py b/sphinx/util/docutils.py index e455d0aad..233c1a2aa 100644 --- a/sphinx/util/docutils.py +++ b/sphinx/util/docutils.py @@ -187,9 +187,8 @@ class sphinx_domains: def __enter__(self) -> None: self.enable() - def __exit__(self, exc_type: "Type[Exception]", exc_value: Exception, traceback: Any) -> bool: # type: ignore # NOQA + def __exit__(self, exc_type: "Type[Exception]", exc_value: Exception, traceback: Any) -> None: # NOQA self.disable() - return False def enable(self) -> None: self.directive_func = directives.directive diff --git a/sphinx/util/requests.py b/sphinx/util/requests.py index a279b4eb4..4cc73a85f 100644 --- a/sphinx/util/requests.py +++ b/sphinx/util/requests.py @@ -8,6 +8,7 @@ :license: BSD, see LICENSE for details. """ +import sys import warnings from contextlib import contextmanager from typing import Generator, Union @@ -16,6 +17,7 @@ from urllib.parse import urlsplit import pkg_resources import requests +import sphinx from sphinx.config import Config try: @@ -105,14 +107,28 @@ def _get_tls_cacert(url: str, config: Config) -> Union[str, bool]: return certs.get(hostname, True) +def _get_user_agent(config: Config) -> str: + if config.user_agent: + return config.user_agent + else: + return ' '.join([ + 'Sphinx/%s' % sphinx.__version__, + 'requests/%s' % requests.__version__, + 'python/%s' % '.'.join(map(str, sys.version_info[:3])), + ]) + + def get(url: str, **kwargs) -> requests.Response: """Sends a GET request like requests.get(). This sets up User-Agent header and TLS verification automatically.""" - kwargs.setdefault('headers', dict(useragent_header)) + headers = kwargs.setdefault('headers', {}) config = kwargs.pop('config', None) if config: kwargs.setdefault('verify', _get_tls_cacert(url, config)) + headers.setdefault('User-Agent', _get_user_agent(config)) + else: + headers.setdefault('User-Agent', useragent_header[0][1]) with ignore_insecure_warning(**kwargs): return requests.get(url, **kwargs) @@ -122,10 +138,13 @@ def head(url: str, **kwargs) -> requests.Response: """Sends a HEAD request like requests.head(). This sets up User-Agent header and TLS verification automatically.""" - kwargs.setdefault('headers', dict(useragent_header)) + headers = kwargs.setdefault('headers', {}) config = kwargs.pop('config', None) if config: kwargs.setdefault('verify', _get_tls_cacert(url, config)) + headers.setdefault('User-Agent', _get_user_agent(config)) + else: + headers.setdefault('User-Agent', useragent_header[0][1]) with ignore_insecure_warning(**kwargs): return requests.get(url, **kwargs) diff --git a/sphinx/util/template.py b/sphinx/util/template.py index fd8886944..3a43db9a5 100644 --- a/sphinx/util/template.py +++ b/sphinx/util/template.py @@ -63,14 +63,14 @@ class SphinxRenderer(FileRenderer): class LaTeXRenderer(SphinxRenderer): - def __init__(self, template_path: str = None) -> None: + def __init__(self, template_path: str = None, latex_engine: str = None) -> None: if template_path is None: template_path = os.path.join(package_dir, 'templates', 'latex') super().__init__(template_path) # use texescape as escape filter - self.env.filters['e'] = texescape.escape - self.env.filters['escape'] = texescape.escape + self.env.filters['e'] = texescape.get_escape_func(latex_engine) + self.env.filters['escape'] = texescape.get_escape_func(latex_engine) self.env.filters['eabbr'] = texescape.escape_abbr # use JSP/eRuby like tagging instead because curly bracket; the default diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py index 408ec1253..e3c45adb4 100644 --- a/sphinx/util/texescape.py +++ b/sphinx/util/texescape.py @@ -9,7 +9,7 @@ """ import re -from typing import Dict +from typing import Callable, Dict tex_replacements = [ # map TeX special chars @@ -20,15 +20,38 @@ tex_replacements = [ ('_', r'\_'), ('{', r'\{'), ('}', r'\}'), - ('[', r'{[}'), - (']', r'{]}'), - ('`', r'{}`'), ('\\', r'\textbackslash{}'), ('~', r'\textasciitilde{}'), + ('^', r'\textasciicircum{}'), + # map chars to avoid mis-interpretation in LaTeX + ('[', r'{[}'), + (']', r'{]}'), + # map chars to avoid TeX ligatures + # 1. ' - and , not here for some legacy reason + # 2. no effect with lualatex (done otherwise: #5790) + ('`', r'{}`'), ('<', r'\textless{}'), ('>', r'\textgreater{}'), - ('^', r'\textasciicircum{}'), + # map char for some unknown reason. TODO: remove this? + ('|', r'\textbar{}'), # map special Unicode characters to TeX commands + ('✓', r'\(\checkmark\)'), + ('✔', r'\(\pmb{\checkmark}\)'), + # used to separate -- in options + ('', r'{}'), + # map some special Unicode characters to similar ASCII ones + # (even for Unicode LaTeX as may not be supported by OpenType font) + ('⎽', r'\_'), + ('ℯ', r'e'), + ('ⅈ', r'i'), + # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc + # OHM SIGN U+2126 is handled by LaTeX textcomp package +] + +# A map Unicode characters to LaTeX representation +# (for LaTeX engines which don't support unicode) +unicode_tex_replacements = [ + # map some more common Unicode characters to TeX commands ('¶', r'\P{}'), ('§', r'\S{}'), ('€', r'\texteuro{}'), @@ -36,16 +59,8 @@ tex_replacements = [ ('±', r'\(\pm\)'), ('→', r'\(\rightarrow\)'), ('‣', r'\(\rightarrow\)'), - ('✓', r'\(\checkmark\)'), - ('✔', r'\(\pmb{\checkmark}\)'), - # used to separate -- in options - ('', r'{}'), - # map some special Unicode characters to similar ASCII ones - ('⎽', r'\_'), ('–', r'\textendash{}'), - ('|', r'\textbar{}'), - ('ℯ', r'e'), - ('ⅈ', r'i'), + # superscript ('⁰', r'\(\sp{\text{0}}\)'), ('¹', r'\(\sp{\text{1}}\)'), ('²', r'\(\sp{\text{2}}\)'), @@ -56,6 +71,7 @@ tex_replacements = [ ('⁷', r'\(\sp{\text{7}}\)'), ('⁸', r'\(\sp{\text{8}}\)'), ('⁹', r'\(\sp{\text{9}}\)'), + # subscript ('₀', r'\(\sb{\text{0}}\)'), ('₁', r'\(\sb{\text{1}}\)'), ('₂', r'\(\sb{\text{2}}\)'), @@ -66,13 +82,21 @@ tex_replacements = [ ('₇', r'\(\sb{\text{7}}\)'), ('₈', r'\(\sb{\text{8}}\)'), ('₉', r'\(\sb{\text{9}}\)'), - # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc - # OHM SIGN U+2126 is handled by LaTeX textcomp package ] tex_escape_map = {} # type: Dict[int, str] +tex_escape_map_without_unicode = {} # type: Dict[int, str] tex_replace_map = {} -tex_hl_escape_map_new = {} +tex_hl_escape_map_new = {} # type: Dict[int, str] +tex_hl_escape_map_new_without_unicode = {} # type: Dict[int, str] + + +def get_escape_func(latex_engine: str) -> Callable[[str], str]: + """Get escape() function for given latex_engine.""" + if latex_engine in ('lualatex', 'xelatex'): + return escape_for_unicode_latex_engine + else: + return escape def escape(s: str) -> str: @@ -80,6 +104,29 @@ def escape(s: str) -> str: return s.translate(tex_escape_map) +def escape_for_unicode_latex_engine(s: str) -> str: + """Escape text for unicode supporting LaTeX engine.""" + return s.translate(tex_escape_map_without_unicode) + + +def get_hlescape_func(latex_engine: str) -> Callable[[str], str]: + """Get hlescape() function for given latex_engine.""" + if latex_engine in ('lualatex', 'xelatex'): + return hlescape_for_unicode_latex_engine + else: + return hlescape + + +def hlescape(s: str) -> str: + """Escape text for LaTeX highlighter.""" + return s.translate(tex_hl_escape_map_new) + + +def hlescape_for_unicode_latex_engine(s: str) -> str: + """Escape text for unicode supporting LaTeX engine.""" + return s.translate(tex_hl_escape_map_new_without_unicode) + + def escape_abbr(text: str) -> str: """Adjust spacing after abbreviations. Works with @ letter or other.""" return re.sub(r'\.(?=\s|$)', r'.\@{}', text) @@ -87,6 +134,11 @@ def escape_abbr(text: str) -> str: def init() -> None: for a, b in tex_replacements: + tex_escape_map[ord(a)] = b + tex_escape_map_without_unicode[ord(a)] = b + tex_replace_map[ord(a)] = '_' + + for a, b in unicode_tex_replacements: tex_escape_map[ord(a)] = b tex_replace_map[ord(a)] = '_' @@ -94,3 +146,7 @@ def init() -> None: if a in '[]{}\\': continue tex_hl_escape_map_new[ord(a)] = b + tex_hl_escape_map_new_without_unicode[ord(a)] = b + + for a, b in unicode_tex_replacements: + tex_hl_escape_map_new[ord(a)] = b diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 832d34dfa..5facdc40c 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -32,7 +32,7 @@ from sphinx.util import split_into, logging from sphinx.util.docutils import SphinxTranslator from sphinx.util.nodes import clean_astext, get_prev_node from sphinx.util.template import LaTeXRenderer -from sphinx.util.texescape import tex_escape_map, tex_replace_map +from sphinx.util.texescape import get_escape_func, tex_replace_map try: from docutils.utils.roman import toRoman @@ -155,6 +155,7 @@ DEFAULT_SETTINGS = { '% Set up styles of URL: it should be placed after hyperref.\n' '\\urlstyle{same}'), 'contentsname': '', + 'extrapackages': '', 'preamble': '', 'title': '', 'release': '', @@ -499,6 +500,9 @@ class LaTeXTranslator(SphinxTranslator): self.compact_list = 0 self.first_param = 0 + # escape helper + self.escape = get_escape_func(self.config.latex_engine) + # sort out some elements self.elements = self.builder.context.copy() @@ -649,7 +653,8 @@ class LaTeXTranslator(SphinxTranslator): self.elements['classoptions'] += ',' + \ self.elements['extraclassoptions'] - self.highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style) + self.highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style, + latex_engine=self.config.latex_engine) self.context = [] # type: List[Any] self.descstack = [] # type: List[str] self.table = None # type: Table @@ -757,8 +762,7 @@ class LaTeXTranslator(SphinxTranslator): for i, (letter, entries) in enumerate(content): if i > 0: ret.append('\\indexspace\n') - ret.append('\\bigletter{%s}\n' % - str(letter).translate(tex_escape_map)) + ret.append('\\bigletter{%s}\n' % self.escape(letter)) for entry in entries: if not entry[3]: continue @@ -793,13 +797,14 @@ class LaTeXTranslator(SphinxTranslator): def render(self, template_name, variables): # type: (str, Dict) -> str + renderer = LaTeXRenderer(latex_engine=self.config.latex_engine) for template_dir in self.builder.config.templates_path: template = path.join(self.builder.confdir, template_dir, template_name) if path.exists(template): - return LaTeXRenderer().render(template, variables) + return renderer.render(template, variables) - return LaTeXRenderer().render(template_name, variables) + return renderer.render(template_name, variables) def visit_document(self, node): # type: (nodes.Element) -> None @@ -913,14 +918,13 @@ class LaTeXTranslator(SphinxTranslator): if not self.elements['title']: # text needs to be escaped since it is inserted into # the output literally - self.elements['title'] = node.astext().translate(tex_escape_map) + self.elements['title'] = self.escape(node.astext()) self.this_is_the_title = 0 raise nodes.SkipNode else: short = '' if node.traverse(nodes.image): - short = ('[%s]' % - ' '.join(clean_astext(node).split()).translate(tex_escape_map)) + short = ('[%s]' % self.escape(' '.join(clean_astext(node).split()))) try: self.body.append(r'\%s%s{' % (self.sectionnames[self.sectionlevel], short)) @@ -1954,8 +1958,7 @@ class LaTeXTranslator(SphinxTranslator): else: id = node.get('refuri', '')[1:].replace('#', ':') - title = node.get('title', '%s') - title = str(title).translate(tex_escape_map).replace('\\%s', '%s') + title = self.escape(node.get('title', '%s')).replace('\\%s', '%s') if '\\{name\\}' in title or '\\{number\\}' in title: # new style format (cf. "Fig.%{number}") title = title.replace('\\{name\\}', '{name}').replace('\\{number\\}', '{number}') @@ -2403,7 +2406,7 @@ class LaTeXTranslator(SphinxTranslator): def encode(self, text): # type: (str) -> str - text = str(text).translate(tex_escape_map) + text = self.escape(text) if self.literal_whitespace: # Insert a blank before the newline, to avoid # ! LaTeX Error: There's no line here to end. @@ -2614,33 +2617,31 @@ class LaTeXTranslator(SphinxTranslator): ret = [] # type: List[str] figure = self.builder.config.numfig_format['figure'].split('%s', 1) if len(figure) == 1: - ret.append('\\def\\fnum@figure{%s}\n' % - str(figure[0]).strip().translate(tex_escape_map)) + ret.append('\\def\\fnum@figure{%s}\n' % self.escape(figure[0]).strip()) else: - definition = escape_abbr(str(figure[0]).translate(tex_escape_map)) + definition = escape_abbr(self.escape(figure[0])) ret.append(self.babel_renewcommand('\\figurename', definition)) ret.append('\\makeatletter\n') ret.append('\\def\\fnum@figure{\\figurename\\thefigure{}%s}\n' % - str(figure[1]).translate(tex_escape_map)) + self.escape(figure[1])) ret.append('\\makeatother\n') table = self.builder.config.numfig_format['table'].split('%s', 1) if len(table) == 1: - ret.append('\\def\\fnum@table{%s}\n' % - str(table[0]).strip().translate(tex_escape_map)) + ret.append('\\def\\fnum@table{%s}\n' % self.escape(table[0]).strip()) else: - definition = escape_abbr(str(table[0]).translate(tex_escape_map)) + definition = escape_abbr(self.escape(table[0])) ret.append(self.babel_renewcommand('\\tablename', definition)) ret.append('\\makeatletter\n') ret.append('\\def\\fnum@table{\\tablename\\thetable{}%s}\n' % - str(table[1]).translate(tex_escape_map)) + self.escape(table[1])) ret.append('\\makeatother\n') codeblock = self.builder.config.numfig_format['code-block'].split('%s', 1) if len(codeblock) == 1: pass # FIXME else: - definition = str(codeblock[0]).strip().translate(tex_escape_map) + definition = self.escape(codeblock[0]).strip() ret.append(self.babel_renewcommand('\\literalblockname', definition)) if codeblock[1]: pass # FIXME diff --git a/tests/roots/test-latex-unicode/conf.py b/tests/roots/test-latex-unicode/conf.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-latex-unicode/index.rst b/tests/roots/test-latex-unicode/index.rst new file mode 100644 index 000000000..2abeca98f --- /dev/null +++ b/tests/roots/test-latex-unicode/index.rst @@ -0,0 +1,7 @@ +test-latex-unicode +================== + +* script small e: ℯ +* double struck italic small i: ⅈ +* superscript: ⁰, ¹ +* subscript: ₀, ₁ diff --git a/tests/test_application.py b/tests/test_application.py index f10592b51..a268f492f 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -105,6 +105,8 @@ def test_add_is_parallel_allowed(app, status, warning): app.setup_extension('read_serial') assert app.is_parallel_allowed('read') is False + assert "the read_serial extension is not safe for parallel reading" in warning.getvalue() + warning.truncate(0) # reset warnings assert app.is_parallel_allowed('write') is True assert warning.getvalue() == '' app.extensions.pop('read_serial') diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py index 8410bbd03..0ebcd0b62 100644 --- a/tests/test_build_latex.py +++ b/tests/test_build_latex.py @@ -1437,3 +1437,35 @@ def test_index_on_title(app, status, warning): '\\label{\\detokenize{contents:test-for-index-in-top-level-title}}' '\\index{index@\\spxentry{index}}\n' in result) + + +@pytest.mark.sphinx('latex', testroot='latex-unicode', + confoverrides={'latex_engine': 'pdflatex'}) +def test_texescape_for_non_unicode_supported_engine(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'python.tex').text() + print(result) + assert 'script small e: e' in result + assert 'double struck italic small i: i' in result + assert r'superscript: \(\sp{\text{0}}\), \(\sp{\text{1}}\)' in result + assert r'subscript: \(\sb{\text{0}}\), \(\sb{\text{1}}\)' in result + + +@pytest.mark.sphinx('latex', testroot='latex-unicode', + confoverrides={'latex_engine': 'xelatex'}) +def test_texescape_for_unicode_supported_engine(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'python.tex').text() + print(result) + assert 'script small e: e' in result + assert 'double struck italic small i: i' in result + assert 'superscript: ⁰, ¹' in result + assert 'subscript: ₀, ₁' in result + + +@pytest.mark.sphinx('latex', testroot='basic', + confoverrides={'latex_elements': {'extrapackages': r'\usepackage{foo}'}}) +def test_latex_elements_extrapackages(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'test.tex').text() + assert r'\usepackage{foo}' in result diff --git a/tests/test_markup.py b/tests/test_markup.py index b8c9b66d9..94d1af951 100644 --- a/tests/test_markup.py +++ b/tests/test_markup.py @@ -312,6 +312,24 @@ def test_inline(get_verifier, type, rst, html_expected, latex_expected): verifier(rst, html_expected, latex_expected) +@pytest.mark.sphinx(confoverrides={'latex_engine': 'xelatex'}) +@pytest.mark.parametrize('type,rst,html_expected,latex_expected', [ + ( + # in verbatim code fragments + 'verify', + '::\n\n @Γ\\∞${}', + None, + ('\\begin{sphinxVerbatim}[commandchars=\\\\\\{\\}]\n' + '@Γ\\PYGZbs{}∞\\PYGZdl{}\\PYGZob{}\\PYGZcb{}\n' + '\\end{sphinxVerbatim}'), + ), +]) +def test_inline_for_unicode_latex_engine(get_verifier, type, rst, + html_expected, latex_expected): + verifier = get_verifier(type) + verifier(rst, html_expected, latex_expected) + + def test_samp_role(parse): # no braces text = ':samp:`a{b}c`'