From ee45bb310bd76b5565af7092cfda279f92e19e9a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 20 Oct 2019 18:54:41 +0900 Subject: [PATCH 01/23] Don't return False always on __exit__() According to the python/mypy#7214, mypy-0.740 prefers a return value of __exit__() method which does not swallow exceptions is None instead of bool. mypy#7214: https://github.com/python/mypy/issues/7214 --- sphinx/util/docutils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sphinx/util/docutils.py b/sphinx/util/docutils.py index e455d0aad..233c1a2aa 100644 --- a/sphinx/util/docutils.py +++ b/sphinx/util/docutils.py @@ -187,9 +187,8 @@ class sphinx_domains: def __enter__(self) -> None: self.enable() - def __exit__(self, exc_type: "Type[Exception]", exc_value: Exception, traceback: Any) -> bool: # type: ignore # NOQA + def __exit__(self, exc_type: "Type[Exception]", exc_value: Exception, traceback: Any) -> None: # NOQA self.disable() - return False def enable(self) -> None: self.directive_func = directives.directive From a499b7d83ed90ee5802dc5e39aceb56021d5addc Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 20 Oct 2019 21:06:17 +0900 Subject: [PATCH 02/23] Add LaTeXTranslator.escape(); TeX escape helper method --- sphinx/writers/latex.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 832d34dfa..0d2b2bc97 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -32,7 +32,7 @@ from sphinx.util import split_into, logging from sphinx.util.docutils import SphinxTranslator from sphinx.util.nodes import clean_astext, get_prev_node from sphinx.util.template import LaTeXRenderer -from sphinx.util.texescape import tex_escape_map, tex_replace_map +from sphinx.util.texescape import escape, tex_replace_map try: from docutils.utils.roman import toRoman @@ -499,6 +499,9 @@ class LaTeXTranslator(SphinxTranslator): self.compact_list = 0 self.first_param = 0 + # escape helper + self.escape = escape + # sort out some elements self.elements = self.builder.context.copy() @@ -757,8 +760,7 @@ class LaTeXTranslator(SphinxTranslator): for i, (letter, entries) in enumerate(content): if i > 0: ret.append('\\indexspace\n') - ret.append('\\bigletter{%s}\n' % - str(letter).translate(tex_escape_map)) + ret.append('\\bigletter{%s}\n' % self.escape(letter)) for entry in entries: if not entry[3]: continue @@ -913,14 +915,13 @@ class LaTeXTranslator(SphinxTranslator): if not self.elements['title']: # text needs to be escaped since it is inserted into # the output literally - self.elements['title'] = node.astext().translate(tex_escape_map) + self.elements['title'] = self.escape(node.astext()) self.this_is_the_title = 0 raise nodes.SkipNode else: short = '' if node.traverse(nodes.image): - short = ('[%s]' % - ' '.join(clean_astext(node).split()).translate(tex_escape_map)) + short = ('[%s]' % self.escape(' '.join(clean_astext(node).split()))) try: self.body.append(r'\%s%s{' % (self.sectionnames[self.sectionlevel], short)) @@ -1954,8 +1955,7 @@ class LaTeXTranslator(SphinxTranslator): else: id = node.get('refuri', '')[1:].replace('#', ':') - title = node.get('title', '%s') - title = str(title).translate(tex_escape_map).replace('\\%s', '%s') + title = self.escape(node.get('title', '%s')).replace('\\%s', '%s') if '\\{name\\}' in title or '\\{number\\}' in title: # new style format (cf. "Fig.%{number}") title = title.replace('\\{name\\}', '{name}').replace('\\{number\\}', '{number}') @@ -2403,7 +2403,7 @@ class LaTeXTranslator(SphinxTranslator): def encode(self, text): # type: (str) -> str - text = str(text).translate(tex_escape_map) + text = self.escape(text) if self.literal_whitespace: # Insert a blank before the newline, to avoid # ! LaTeX Error: There's no line here to end. @@ -2614,33 +2614,31 @@ class LaTeXTranslator(SphinxTranslator): ret = [] # type: List[str] figure = self.builder.config.numfig_format['figure'].split('%s', 1) if len(figure) == 1: - ret.append('\\def\\fnum@figure{%s}\n' % - str(figure[0]).strip().translate(tex_escape_map)) + ret.append('\\def\\fnum@figure{%s}\n' % self.escape(figure[0]).strip()) else: - definition = escape_abbr(str(figure[0]).translate(tex_escape_map)) + definition = escape_abbr(self.escape(figure[0])) ret.append(self.babel_renewcommand('\\figurename', definition)) ret.append('\\makeatletter\n') ret.append('\\def\\fnum@figure{\\figurename\\thefigure{}%s}\n' % - str(figure[1]).translate(tex_escape_map)) + self.escape(figure[1])) ret.append('\\makeatother\n') table = self.builder.config.numfig_format['table'].split('%s', 1) if len(table) == 1: - ret.append('\\def\\fnum@table{%s}\n' % - str(table[0]).strip().translate(tex_escape_map)) + ret.append('\\def\\fnum@table{%s}\n' % self.escape(table[0]).strip()) else: - definition = escape_abbr(str(table[0]).translate(tex_escape_map)) + definition = escape_abbr(self.escape(table[0])) ret.append(self.babel_renewcommand('\\tablename', definition)) ret.append('\\makeatletter\n') ret.append('\\def\\fnum@table{\\tablename\\thetable{}%s}\n' % - str(table[1]).translate(tex_escape_map)) + self.escape(table[1])) ret.append('\\makeatother\n') codeblock = self.builder.config.numfig_format['code-block'].split('%s', 1) if len(codeblock) == 1: pass # FIXME else: - definition = str(codeblock[0]).strip().translate(tex_escape_map) + definition = self.escape(codeblock[0]).strip() ret.append(self.babel_renewcommand('\\literalblockname', definition)) if codeblock[1]: pass # FIXME From 56bbb08b2cc4925e4796752ac798fe0001267a49 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 20 Oct 2019 22:29:46 +0900 Subject: [PATCH 03/23] Do not replace unicode characters by LaTeX macros on unicode supported LaTeX engines --- CHANGES | 2 ++ sphinx/util/template.py | 6 ++--- sphinx/util/texescape.py | 32 +++++++++++++++++++++--- sphinx/writers/latex.py | 9 ++++--- tests/roots/test-latex-unicode/conf.py | 0 tests/roots/test-latex-unicode/index.rst | 7 ++++++ tests/test_build_latex.py | 24 ++++++++++++++++++ 7 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 tests/roots/test-latex-unicode/conf.py create mode 100644 tests/roots/test-latex-unicode/index.rst diff --git a/CHANGES b/CHANGES index 26ea1afb6..15b8368ac 100644 --- a/CHANGES +++ b/CHANGES @@ -28,6 +28,8 @@ Bugs fixed .. _latex3/latex2e#173: https://github.com/latex3/latex2e/issues/173 * #6618: LaTeX: Avoid section names at the end of a page +* #6738: LaTeX: Do not replace unicode characters by LaTeX macros on unicode + supported LaTeX engines * #6704: linkcheck: Be defensive and handle newly defined HTTP error code * #6655: image URLs containing ``data:`` causes gettext builder crashed * #6584: i18n: Error when compiling message catalogs on Hindi diff --git a/sphinx/util/template.py b/sphinx/util/template.py index fd8886944..3a43db9a5 100644 --- a/sphinx/util/template.py +++ b/sphinx/util/template.py @@ -63,14 +63,14 @@ class SphinxRenderer(FileRenderer): class LaTeXRenderer(SphinxRenderer): - def __init__(self, template_path: str = None) -> None: + def __init__(self, template_path: str = None, latex_engine: str = None) -> None: if template_path is None: template_path = os.path.join(package_dir, 'templates', 'latex') super().__init__(template_path) # use texescape as escape filter - self.env.filters['e'] = texescape.escape - self.env.filters['escape'] = texescape.escape + self.env.filters['e'] = texescape.get_escape_func(latex_engine) + self.env.filters['escape'] = texescape.get_escape_func(latex_engine) self.env.filters['eabbr'] = texescape.escape_abbr # use JSP/eRuby like tagging instead because curly bracket; the default diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py index 408ec1253..4e7055119 100644 --- a/sphinx/util/texescape.py +++ b/sphinx/util/texescape.py @@ -9,7 +9,7 @@ """ import re -from typing import Dict +from typing import Callable, Dict tex_replacements = [ # map TeX special chars @@ -46,6 +46,14 @@ tex_replacements = [ ('|', r'\textbar{}'), ('ℯ', r'e'), ('ⅈ', r'i'), + # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc + # OHM SIGN U+2126 is handled by LaTeX textcomp package +] + +# A map Unicode characters to LaTeX representation +# (for LaTeX engines which don't support unicode) +unicode_tex_replacements = [ + # superscript ('⁰', r'\(\sp{\text{0}}\)'), ('¹', r'\(\sp{\text{1}}\)'), ('²', r'\(\sp{\text{2}}\)'), @@ -56,6 +64,7 @@ tex_replacements = [ ('⁷', r'\(\sp{\text{7}}\)'), ('⁸', r'\(\sp{\text{8}}\)'), ('⁹', r'\(\sp{\text{9}}\)'), + # subscript ('₀', r'\(\sb{\text{0}}\)'), ('₁', r'\(\sb{\text{1}}\)'), ('₂', r'\(\sb{\text{2}}\)'), @@ -66,20 +75,32 @@ tex_replacements = [ ('₇', r'\(\sb{\text{7}}\)'), ('₈', r'\(\sb{\text{8}}\)'), ('₉', r'\(\sb{\text{9}}\)'), - # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc - # OHM SIGN U+2126 is handled by LaTeX textcomp package ] tex_escape_map = {} # type: Dict[int, str] +tex_escape_map_without_unicode = {} # type: Dict[int, str] tex_replace_map = {} tex_hl_escape_map_new = {} +def get_escape_func(latex_engine: str) -> Callable[[str], str]: + """Get escape() function for given latex_engine.""" + if latex_engine in ('lualatex', 'xelatex'): + return escape_for_unicode_latex_engine + else: + return escape + + def escape(s: str) -> str: """Escape text for LaTeX output.""" return s.translate(tex_escape_map) +def escape_for_unicode_latex_engine(s: str) -> str: + """Escape text for unicode supporting LaTeX engine.""" + return s.translate(tex_escape_map_without_unicode) + + def escape_abbr(text: str) -> str: """Adjust spacing after abbreviations. Works with @ letter or other.""" return re.sub(r'\.(?=\s|$)', r'.\@{}', text) @@ -87,6 +108,11 @@ def escape_abbr(text: str) -> str: def init() -> None: for a, b in tex_replacements: + tex_escape_map[ord(a)] = b + tex_escape_map_without_unicode[ord(a)] = b + tex_replace_map[ord(a)] = '_' + + for a, b in unicode_tex_replacements: tex_escape_map[ord(a)] = b tex_replace_map[ord(a)] = '_' diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 0d2b2bc97..bb84bba7a 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -32,7 +32,7 @@ from sphinx.util import split_into, logging from sphinx.util.docutils import SphinxTranslator from sphinx.util.nodes import clean_astext, get_prev_node from sphinx.util.template import LaTeXRenderer -from sphinx.util.texescape import escape, tex_replace_map +from sphinx.util.texescape import get_escape_func, tex_replace_map try: from docutils.utils.roman import toRoman @@ -500,7 +500,7 @@ class LaTeXTranslator(SphinxTranslator): self.first_param = 0 # escape helper - self.escape = escape + self.escape = get_escape_func(self.config.latex_engine) # sort out some elements self.elements = self.builder.context.copy() @@ -795,13 +795,14 @@ class LaTeXTranslator(SphinxTranslator): def render(self, template_name, variables): # type: (str, Dict) -> str + renderer = LaTeXRenderer(latex_engine=self.config.latex_engine) for template_dir in self.builder.config.templates_path: template = path.join(self.builder.confdir, template_dir, template_name) if path.exists(template): - return LaTeXRenderer().render(template, variables) + return renderer.render(template, variables) - return LaTeXRenderer().render(template_name, variables) + return renderer.render(template_name, variables) def visit_document(self, node): # type: (nodes.Element) -> None diff --git a/tests/roots/test-latex-unicode/conf.py b/tests/roots/test-latex-unicode/conf.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-latex-unicode/index.rst b/tests/roots/test-latex-unicode/index.rst new file mode 100644 index 000000000..2abeca98f --- /dev/null +++ b/tests/roots/test-latex-unicode/index.rst @@ -0,0 +1,7 @@ +test-latex-unicode +================== + +* script small e: ℯ +* double struck italic small i: ⅈ +* superscript: ⁰, ¹ +* subscript: ₀, ₁ diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py index 8410bbd03..80869dea3 100644 --- a/tests/test_build_latex.py +++ b/tests/test_build_latex.py @@ -1437,3 +1437,27 @@ def test_index_on_title(app, status, warning): '\\label{\\detokenize{contents:test-for-index-in-top-level-title}}' '\\index{index@\\spxentry{index}}\n' in result) + + +@pytest.mark.sphinx('latex', testroot='latex-unicode', + confoverrides={'latex_engine': 'pdflatex'}) +def test_texescape_for_non_unicode_supported_engine(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'python.tex').text() + print(result) + assert 'script small e: e' in result + assert 'double struck italic small i: i' in result + assert r'superscript: \(\sp{\text{0}}\), \(\sp{\text{1}}\)' in result + assert r'subscript: \(\sb{\text{0}}\), \(\sb{\text{1}}\)' in result + + +@pytest.mark.sphinx('latex', testroot='latex-unicode', + confoverrides={'latex_engine': 'xelatex'}) +def test_texescape_for_unicode_supported_engine(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'python.tex').text() + print(result) + assert 'script small e: e' in result + assert 'double struck italic small i: i' in result + assert 'superscript: ⁰, ¹' in result + assert 'subscript: ₀, ₁' in result From 32763520a369b2c137be5e9b7440b42c28e65fef Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 23 Oct 2019 00:11:44 +0900 Subject: [PATCH 04/23] Close #1331: Change default User-Agent header --- CHANGES | 4 ++++ doc/usage/configuration.rst | 8 ++++++++ sphinx/builders/linkcheck.py | 1 - sphinx/config.py | 1 + sphinx/ext/intersphinx.py | 1 + sphinx/util/requests.py | 23 +++++++++++++++++++++-- 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 33d5ff2b5..da542ef7a 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,9 @@ Dependencies Incompatible changes -------------------- +* #1331: Change default User-Agent header to ``"Sphinx/X.Y.Z requests/X.Y.Z + python/X.Y.Z"``. It can be changed via :confval:`user_agent`. + Deprecated ---------- @@ -19,6 +22,7 @@ Features added * #6707: C++, support bit-fields. * #267: html: Eliminate prompt characters of doctest block from copyable text * #6729: html theme: agogo theme now supports ``rightsidebar`` option +* #1331: Add new config variable: :confval:`user_agent` Bugs fixed ---------- diff --git a/doc/usage/configuration.rst b/doc/usage/configuration.rst index 02b40256d..e61c09cb2 100644 --- a/doc/usage/configuration.rst +++ b/doc/usage/configuration.rst @@ -510,6 +510,14 @@ General configuration .. versionadded:: 1.6.6 +.. confval:: user_agent + + A User-Agent of Sphinx. It is used for a header on HTTP access (ex. + linkcheck, intersphinx and so on). Default is ``"Sphinx/X.Y.Z + requests/X.Y.Z python/X.Y.Z"``. + + .. versionadded:: 2.3 + .. confval:: tls_verify If true, Sphinx verifies server certifications. Default is ``True``. diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index 1be2041bd..635d9df98 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -101,7 +101,6 @@ class CheckExternalLinksBuilder(Builder): 'allow_redirects': True, 'headers': { 'Accept': 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8', - 'User-Agent': requests.useragent_header[0][1], }, } if self.app.config.linkcheck_timeout: diff --git a/sphinx/config.py b/sphinx/config.py index 5ba2c2a3d..d8cce1b3d 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -148,6 +148,7 @@ class Config: 'math_numfig': (True, 'env', []), 'tls_verify': (True, 'env', []), 'tls_cacerts': (None, 'env', []), + 'user_agent': (None, 'env', [str]), 'smartquotes': (True, 'env', []), 'smartquotes_action': ('qDe', 'env', []), 'smartquotes_excludes': ({'languages': ['ja'], diff --git a/sphinx/ext/intersphinx.py b/sphinx/ext/intersphinx.py index cb74ef6f1..4389c5a44 100644 --- a/sphinx/ext/intersphinx.py +++ b/sphinx/ext/intersphinx.py @@ -374,6 +374,7 @@ def inspect_main(argv: List[str]) -> None: class MockConfig: intersphinx_timeout = None # type: int tls_verify = False + user_agent = None class MockApp: srcdir = '' diff --git a/sphinx/util/requests.py b/sphinx/util/requests.py index a279b4eb4..4cc73a85f 100644 --- a/sphinx/util/requests.py +++ b/sphinx/util/requests.py @@ -8,6 +8,7 @@ :license: BSD, see LICENSE for details. """ +import sys import warnings from contextlib import contextmanager from typing import Generator, Union @@ -16,6 +17,7 @@ from urllib.parse import urlsplit import pkg_resources import requests +import sphinx from sphinx.config import Config try: @@ -105,14 +107,28 @@ def _get_tls_cacert(url: str, config: Config) -> Union[str, bool]: return certs.get(hostname, True) +def _get_user_agent(config: Config) -> str: + if config.user_agent: + return config.user_agent + else: + return ' '.join([ + 'Sphinx/%s' % sphinx.__version__, + 'requests/%s' % requests.__version__, + 'python/%s' % '.'.join(map(str, sys.version_info[:3])), + ]) + + def get(url: str, **kwargs) -> requests.Response: """Sends a GET request like requests.get(). This sets up User-Agent header and TLS verification automatically.""" - kwargs.setdefault('headers', dict(useragent_header)) + headers = kwargs.setdefault('headers', {}) config = kwargs.pop('config', None) if config: kwargs.setdefault('verify', _get_tls_cacert(url, config)) + headers.setdefault('User-Agent', _get_user_agent(config)) + else: + headers.setdefault('User-Agent', useragent_header[0][1]) with ignore_insecure_warning(**kwargs): return requests.get(url, **kwargs) @@ -122,10 +138,13 @@ def head(url: str, **kwargs) -> requests.Response: """Sends a HEAD request like requests.head(). This sets up User-Agent header and TLS verification automatically.""" - kwargs.setdefault('headers', dict(useragent_header)) + headers = kwargs.setdefault('headers', {}) config = kwargs.pop('config', None) if config: kwargs.setdefault('verify', _get_tls_cacert(url, config)) + headers.setdefault('User-Agent', _get_user_agent(config)) + else: + headers.setdefault('User-Agent', useragent_header[0][1]) with ignore_insecure_warning(**kwargs): return requests.get(url, **kwargs) From 0c6ffa42a504bc3b4bd104360513e634464224bc Mon Sep 17 00:00:00 2001 From: jfbu Date: Mon, 28 Oct 2019 17:10:50 +0100 Subject: [PATCH 05/23] Fix #6776: 2019-10-01 LaTeX release breaks sphinxcyrillic.sty --- CHANGES | 2 ++ sphinx/texinputs/sphinxcyrillic.sty | 6 ++++-- tests/roots/test-root/conf.py | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index f49b2fe83..bef374204 100644 --- a/CHANGES +++ b/CHANGES @@ -16,6 +16,8 @@ Features added Bugs fixed ---------- +* #6776: LaTeX: 2019-10-01 LaTeX release breaks :file:`sphinxcyrillic.sty` + Testing -------- diff --git a/sphinx/texinputs/sphinxcyrillic.sty b/sphinx/texinputs/sphinxcyrillic.sty index 1a14c7b24..482b4e3f7 100644 --- a/sphinx/texinputs/sphinxcyrillic.sty +++ b/sphinx/texinputs/sphinxcyrillic.sty @@ -11,7 +11,7 @@ \ProcessLocalKeyvalOptions* % ignore class options \ifspx@cyropt@Xtwo -% original code by tex.sx user egreg: +% original code by tex.sx user egreg (updated 2019/10/28): % https://tex.stackexchange.com/a/460325/ % 159 Cyrillic glyphs as available in X2 TeX 8bit font encoding % This assumes inputenc loaded with utf8 option, or LaTeX release @@ -27,7 +27,9 @@ {Ӎ}{ӎ}{Ӕ}{ӕ}{Ә}{ә}{Ӡ}{ӡ}{Ө}{ө}\do {% \begingroup\def\IeC{\protect\DeclareTextSymbolDefault}% - \protected@edef\@temp{\endgroup\next{X2}}\@temp + \protected@edef\@temp{\endgroup + \@ifl@t@r{\fmtversion}{2019/10/01}{\csname u8:\next\endcsname}{\next}}% + \@temp{X2}% }% \else \ifspx@cyropt@TtwoA diff --git a/tests/roots/test-root/conf.py b/tests/roots/test-root/conf.py index b3cc12ae0..5e6e6cc63 100644 --- a/tests/roots/test-root/conf.py +++ b/tests/roots/test-root/conf.py @@ -41,6 +41,7 @@ html_last_updated_fmt = '%b %d, %Y' html_context = {'hckey': 'hcval', 'hckey_co': 'wrong_hcval_co'} latex_additional_files = ['svgimg.svg'] +latex_elements = {'fontenc':'\\usepackage[X2,LGR,T1]{fontenc}'} coverage_c_path = ['special/*.h'] coverage_c_regexes = {'function': r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'} From ca06005cafa95ea1a3cfdecbe8b3f2c5fac052d4 Mon Sep 17 00:00:00 2001 From: jfbu Date: Mon, 28 Oct 2019 18:52:07 +0100 Subject: [PATCH 06/23] Drop addition of X2 to LaTeX test to avoid extra dependencies for CI This can be reverted if adding the extra dependencies is deemed worth it, but it means having to look at Debian packaging and currently means adding texlive-lang-cyrillic as requirement --- tests/roots/test-root/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/roots/test-root/conf.py b/tests/roots/test-root/conf.py index 5e6e6cc63..b3cc12ae0 100644 --- a/tests/roots/test-root/conf.py +++ b/tests/roots/test-root/conf.py @@ -41,7 +41,6 @@ html_last_updated_fmt = '%b %d, %Y' html_context = {'hckey': 'hcval', 'hckey_co': 'wrong_hcval_co'} latex_additional_files = ['svgimg.svg'] -latex_elements = {'fontenc':'\\usepackage[X2,LGR,T1]{fontenc}'} coverage_c_path = ['special/*.h'] coverage_c_regexes = {'function': r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'} From 4ec563fcff7ea97a4daf63a78051cefbfe450d83 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 26 Oct 2019 18:33:59 +0900 Subject: [PATCH 07/23] Close #6762: latex: Allow to load additonal LaTeX packages --- CHANGES | 2 ++ doc/latex.rst | 19 +++++++++++++++++++ sphinx/templates/latex/latex.tex_t | 1 + sphinx/writers/latex.py | 1 + tests/test_build_latex.py | 8 ++++++++ 5 files changed, 31 insertions(+) diff --git a/CHANGES b/CHANGES index 5497d94a2..6473ed44d 100644 --- a/CHANGES +++ b/CHANGES @@ -23,6 +23,8 @@ Features added * #267: html: Eliminate prompt characters of doctest block from copyable text * #6729: html theme: agogo theme now supports ``rightsidebar`` option * #6780: Add PEP-561 Support +* #6762: latex: Allow to load additonal LaTeX packages via ``extrapackages`` key + of :confval:`latex_elements` Bugs fixed ---------- diff --git a/doc/latex.rst b/doc/latex.rst index 2501d1594..7321f964b 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -226,6 +226,25 @@ into the generated ``.tex`` files. Its ``'sphinxsetup'`` key is described .. versionadded:: 1.5 + ``'extrapackages'`` + Additional LaTeX packages. For example: + + .. code-block:: python + + latex_elements = { + 'packages': r'\usepackage{isodate}' + } + + It defaults to empty. + + The specified LaTeX packages will be loaded before + hyperref package and packages loaded from Sphinx extensions. + + .. hint:: If you'd like to load additional LaTeX packages after hyperref, use + ``'preamble'`` key instead. + + .. versionadded:: 2.3 + ``'footer'`` Additional footer content (before the indices), default empty. diff --git a/sphinx/templates/latex/latex.tex_t b/sphinx/templates/latex/latex.tex_t index 2beac82c9..a0a5a26b1 100644 --- a/sphinx/templates/latex/latex.tex_t +++ b/sphinx/templates/latex/latex.tex_t @@ -35,6 +35,7 @@ <%= sphinxsetup %> <%= fvset %> <%= geometry %> +<%= extrapackages %> <%- for name, option in packages %> <%- if option %> diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 832d34dfa..364067497 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -155,6 +155,7 @@ DEFAULT_SETTINGS = { '% Set up styles of URL: it should be placed after hyperref.\n' '\\urlstyle{same}'), 'contentsname': '', + 'extrapackages': '', 'preamble': '', 'title': '', 'release': '', diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py index 8410bbd03..c48399f1b 100644 --- a/tests/test_build_latex.py +++ b/tests/test_build_latex.py @@ -1437,3 +1437,11 @@ def test_index_on_title(app, status, warning): '\\label{\\detokenize{contents:test-for-index-in-top-level-title}}' '\\index{index@\\spxentry{index}}\n' in result) + + +@pytest.mark.sphinx('latex', testroot='basic', + confoverrides={'latex_elements': {'extrapackages': r'\usepackage{foo}'}}) +def test_latex_elements_extrapackages(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'test.tex').text() + assert r'\usepackage{foo}' in result From 27f0828420348cfc60921b4f6e04fdf174ab7a0c Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 6 Nov 2019 00:56:08 +0900 Subject: [PATCH 08/23] Fix #6793: texinfo: Code examples broken following "sidebar" --- CHANGES | 1 + sphinx/writers/texinfo.py | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index 5497d94a2..dfd58c717 100644 --- a/CHANGES +++ b/CHANGES @@ -42,6 +42,7 @@ Bugs fixed * #5070: epub: Wrong internal href fragment links * #6712: Allow not to install sphinx.testing as runtime (mainly for ALT Linux) * #6741: html: search result was broken with empty :confval:`html_file_suffix` +* #6793: texinfo: Code examples broken following "sidebar" Testing -------- diff --git a/sphinx/writers/texinfo.py b/sphinx/writers/texinfo.py index 0164465f7..b7a590cad 100644 --- a/sphinx/writers/texinfo.py +++ b/sphinx/writers/texinfo.py @@ -1284,6 +1284,7 @@ class TexinfoTranslator(SphinxTranslator): title = cast(nodes.title, node[0]) self.visit_rubric(title) self.body.append('%s\n' % self.escape(title.astext())) + self.depart_rubric(title) def depart_topic(self, node): # type: (nodes.Element) -> None From a550c7075fcbeb9bcb37ec5546a42f811bb2ee1d Mon Sep 17 00:00:00 2001 From: jfbu Date: Wed, 6 Nov 2019 18:42:52 +0100 Subject: [PATCH 09/23] LaTeX: Allow linebreaks at \ (in literals, code-blocks, parsed-literal) Default configuration is to allow the linebreak after the backslash. It is possible at user side via 'preamble' key and some LaTeX code to allow the linebreak before the backslash, but each of inline-literal, code-block, and parsed-literal has its more or less cumbersome way for that. Closes: #6000, #6001 This commit handles only the \ character, which had been left aside by accident so far. Inline-literals could break at more characters but this is not yet done in this commit. In this context, in future it could be useful to modify sphinx.util.texescape to use mark-up such as \sphinxtextbackslash rather than naked \textbackslash and similarly for some other characters. This could then be used to allow linebreaks even in regular text paragraphs. --- CHANGES | 3 +++ doc/latex.rst | 7 +++++-- sphinx/texinputs/sphinx.sty | 21 ++++++++++++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 313ef49eb..d6f0caefb 100644 --- a/CHANGES +++ b/CHANGES @@ -28,6 +28,8 @@ Features added * #6762: latex: Allow to load additonal LaTeX packages via ``extrapackages`` key of :confval:`latex_elements` * #1331: Add new config variable: :confval:`user_agent` +* #6000: LaTeX: have backslash also be an inline literal word wrap break + character Bugs fixed ---------- @@ -47,6 +49,7 @@ Bugs fixed * #5070: epub: Wrong internal href fragment links * #6712: Allow not to install sphinx.testing as runtime (mainly for ALT Linux) * #6741: html: search result was broken with empty :confval:`html_file_suffix` +* #6001: LaTeX does not wrap long code lines at backslash character Testing -------- diff --git a/doc/latex.rst b/doc/latex.rst index 7321f964b..01c0e0cb8 100644 --- a/doc/latex.rst +++ b/doc/latex.rst @@ -619,12 +619,15 @@ macros may be significant. default ``true``. Allows linebreaks inside inline literals: but extra potential break-points (additionally to those allowed by LaTeX at spaces or for hyphenation) are currently inserted only after the characters - ``. , ; ? ! /``. Due to TeX internals, white space in the line will be - stretched (or shrunk) in order to accomodate the linebreak. + ``. , ; ? ! /`` and ``\``. Due to TeX internals, white space in the line + will be stretched (or shrunk) in order to accomodate the linebreak. .. versionadded:: 1.5 set this option value to ``false`` to recover former behaviour. + .. versionchanged:: 2.3.0 + added potential breakpoint at ``\`` characters. + ``verbatimvisiblespace`` default ``\textcolor{red}{\textvisiblespace}``. When a long code line is split, the last space character from the source code line right before the diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index 6065be81b..6023e6d22 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -1054,7 +1054,7 @@ % Take advantage of the already applied Pygments mark-up to insert % potential linebreaks for TeX processing. % {, <, #, %, $, ' and ": go to next line. -% _, }, ^, &, >, - and ~: stay at end of broken line. +% _, }, ^, &, >, -, ~, and \: stay at end of broken line. % Use of \textquotesingle for straight quote. % FIXME: convert this to package options ? \newcommand*\sphinxbreaksbeforelist {% @@ -1066,6 +1066,7 @@ \newcommand*\sphinxbreaksafterlist {% \do\PYGZus\_\do\PYGZcb\}\do\PYGZca\^\do\PYGZam\&% _, }, ^, &, \do\PYGZgt\>\do\PYGZhy\-\do\PYGZti\~% >, -, ~ + \do\PYGZbs\\% \ } \newcommand*\sphinxbreaksatspecials {% \def\do##1##2% @@ -1314,6 +1315,7 @@ {\def##1{\discretionary{\char`##2}{\sphinxafterbreak}{\char`##2}}}% \do\_\_\do\}\}\do\textasciicircum\^\do\&\&% _, }, ^, &, \do\textgreater\>\do\textasciitilde\~% >, ~ + \do\textbackslash\\% \ } \newcommand*\sphinxbreaksviaactiveinparsedliteral{% \sphinxbreaksviaactive % by default handles . , ; ? ! / @@ -1736,13 +1738,25 @@ % to obtain straight quotes we execute \@noligs as patched by upquote, and % \scantokens is needed in cases where it would be too late for the macro to % first set catcodes and then fetch its argument. We also make the contents -% breakable at non-escaped . , ; ? ! / using \sphinxbreaksviaactive. +% breakable at non-escaped . , ; ? ! / using \sphinxbreaksviaactive, +% and also at \ character (which is escaped to \textbackslash{}). +\protected\def\sphinxtextbackslashbreakbefore + {\discretionary{}{\sphinxafterbreak\sphinx@textbackslash}{\sphinx@textbackslash}} +\protected\def\sphinxtextbackslashbreakafter + {\discretionary{\sphinx@textbackslash}{\sphinxafterbreak}{\sphinx@textbackslash}} +\let\sphinxtextbackslash\sphinxtextbackslashbreakafter % the macro must be protected if it ends up used in moving arguments, % in 'alltt' \@noligs is done already, and the \scantokens must be avoided. \protected\def\sphinxupquote#1{{\def\@tempa{alltt}% \ifx\@tempa\@currenvir\else \ifspx@opt@inlineliteralwraps - \sphinxbreaksviaactive\let\sphinxafterbreak\empty + % break at . , ; ? ! / + \sphinxbreaksviaactive + % break also at \ + \let\sphinx@textbackslash\textbackslash + \let\textbackslash\sphinxtextbackslash + % do not typeset a continuation symbol on next line + \let\sphinxafterbreak\sphinxafterbreakofinlineliteral % do not overwrite the comma set-up \let\verbatim@nolig@list\sphinx@literal@nolig@list \fi @@ -1754,6 +1768,7 @@ \def\sphinx@do@noligs #1{\catcode`#1\active\begingroup\lccode`\~`#1\relax \lowercase{\endgroup\def~{\leavevmode\kern\z@\char`#1 }}} \def\sphinx@literal@nolig@list {\do\`\do\<\do\>\do\'\do\-}% +\let\sphinxafterbreakofinlineliteral\empty % Some custom font markup commands. \protected\def\sphinxstrong#1{\textbf{#1}} From 399f773d9dcd14d1ba27a74448ba9da5da7c7389 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 8 Nov 2019 00:36:57 +0900 Subject: [PATCH 10/23] gettext: Use template file to generate message catalog --- CHANGES | 1 + doc/extdev/deprecated.rst | 5 ++ sphinx/builders/gettext.py | 72 +++++++++++++++----------- sphinx/templates/gettext/message.pot_t | 33 ++++++++++++ 4 files changed, 81 insertions(+), 30 deletions(-) create mode 100644 sphinx/templates/gettext/message.pot_t diff --git a/CHANGES b/CHANGES index 313ef49eb..126413b14 100644 --- a/CHANGES +++ b/CHANGES @@ -15,6 +15,7 @@ Incompatible changes Deprecated ---------- +* ``sphinx.builders.gettext.POHEADER`` * ``sphinx.io.SphinxStandaloneReader.app`` * ``sphinx.io.SphinxStandaloneReader.env`` diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst index edf98cfed..8c201b525 100644 --- a/doc/extdev/deprecated.rst +++ b/doc/extdev/deprecated.rst @@ -26,6 +26,11 @@ The following is a list of deprecated interfaces. - (will be) Removed - Alternatives + * - ``sphinx.builders.gettext.POHEADER`` + - 2.3 + - 4.0 + - ``sphinx/templates/gettext/message.pot_t`` (template file) + * - ``sphinx.io.SphinxStandaloneReader.app`` - 2.3 - 4.0 diff --git a/sphinx/builders/gettext.py b/sphinx/builders/gettext.py index 763399413..ea6fa61a9 100644 --- a/sphinx/builders/gettext.py +++ b/sphinx/builders/gettext.py @@ -11,16 +11,16 @@ from codecs import open from collections import defaultdict, OrderedDict from datetime import datetime, tzinfo, timedelta -from io import StringIO from os import path, walk, getenv from time import time -from typing import Any, Dict, Iterable, List, Set, Tuple, Union +from typing import Any, Dict, Iterable, Generator, List, Set, Tuple, Union from uuid import uuid4 from docutils import nodes from docutils.nodes import Element from sphinx import addnodes +from sphinx import package_dir from sphinx.application import Sphinx from sphinx.builders import Builder from sphinx.domains.python import pairindextypes @@ -30,8 +30,9 @@ from sphinx.util import split_index_msg, logging, status_iterator from sphinx.util.console import bold # type: ignore from sphinx.util.i18n import CatalogInfo, docname_to_domain from sphinx.util.nodes import extract_messages, traverse_translatable_index -from sphinx.util.osutil import relpath, ensuredir, canon_path +from sphinx.util.osutil import ensuredir, canon_path from sphinx.util.tags import Tags +from sphinx.util.template import SphinxRenderer if False: # For type annotation @@ -58,7 +59,15 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"""[1:] +"""[1:] # RemovedInSphinx40Warning + + +class Message: + """An entry of translatable message.""" + def __init__(self, text: str, locations: List[Tuple[str, int]], uuids: List[str]): + self.text = text + self.locations = locations + self.uuids = uuids class Catalog: @@ -80,6 +89,12 @@ class Catalog: self.metadata[msg] = [] self.metadata[msg].append((origin.source, origin.line, origin.uid)) # type: ignore + def __iter__(self) -> Generator[Message, None, None]: + for message in self.messages: + positions = [(source, line) for source, line, uuid in self.metadata[message]] + uuids = [uuid for source, line, uuid in self.metadata[message]] + yield Message(message, positions, uuids) + class MsgOrigin: """ @@ -92,6 +107,22 @@ class MsgOrigin: self.uid = uuid4().hex +class GettextRenderer(SphinxRenderer): + def __init__(self, template_path: str = None) -> None: + if template_path is None: + template_path = path.join(package_dir, 'templates', 'gettext') + super().__init__(template_path) + + def escape(s: str) -> str: + s = s.replace('\\', r'\\') + s = s.replace('"', r'\"') + return s.replace('\n', '\\n"\n"') + + # use texescape as escape filter + self.env.filters['e'] = escape + self.env.filters['escape'] = escape + + class I18nTags(Tags): """Dummy tags module for I18nBuilder. @@ -247,12 +278,13 @@ class MessageCatalogBuilder(I18nBuilder): def finish(self) -> None: super().finish() - data = { + context = { 'version': self.config.version, 'copyright': self.config.copyright, 'project': self.config.project, - 'ctime': datetime.fromtimestamp( - timestamp, ltz).strftime('%Y-%m-%d %H:%M%z'), + 'ctime': datetime.fromtimestamp(timestamp, ltz).strftime('%Y-%m-%d %H:%M%z'), + 'display_location': self.config.gettext_location, + 'display_uuid': self.config.gettext_uuid, } for textdomain, catalog in status_iterator(self.catalogs.items(), __("writing message catalogs... "), @@ -262,30 +294,10 @@ class MessageCatalogBuilder(I18nBuilder): # noop if config.gettext_compact is set ensuredir(path.join(self.outdir, path.dirname(textdomain))) + context['messages'] = list(catalog) + content = GettextRenderer().render('message.pot_t', context) + pofn = path.join(self.outdir, textdomain + '.pot') - output = StringIO() - output.write(POHEADER % data) - - for message in catalog.messages: - positions = catalog.metadata[message] - - if self.config.gettext_location: - # generate "#: file1:line1\n#: file2:line2 ..." - output.write("#: %s\n" % "\n#: ".join( - "%s:%s" % (canon_path(relpath(source, self.outdir)), line) - for source, line, _ in positions)) - if self.config.gettext_uuid: - # generate "# uuid1\n# uuid2\n ..." - output.write("# %s\n" % "\n# ".join(uid for _, _, uid in positions)) - - # message contains *one* line of text ready for translation - message = message.replace('\\', r'\\'). \ - replace('"', r'\"'). \ - replace('\n', '\\n"\n"') - output.write('msgid "%s"\nmsgstr ""\n\n' % message) - - content = output.getvalue() - if should_write(pofn, content): with open(pofn, 'w', encoding='utf-8') as pofile: pofile.write(content) diff --git a/sphinx/templates/gettext/message.pot_t b/sphinx/templates/gettext/message.pot_t new file mode 100644 index 000000000..90df27175 --- /dev/null +++ b/sphinx/templates/gettext/message.pot_t @@ -0,0 +1,33 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) {{ copyright }} +# This file is distributed under the same license as the {{ project }} package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: {{ project|e }} {{ version|e }}\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: {{ ctime|e }}\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +{% for message in messages %} +{% if display_location -%} +{% for source, line in message.locations -%} +#: {{ source }}:{{ line }} +{% endfor -%} +{% endif -%} + +{% if display_uuid -%} +{% for uuid in message.uuids -%} +#: {{ uuid }} +{% endfor -%} +{% endif -%} + +msgid "{{ message.text|e }}" +msgstr "" +{% endfor -%} From d247299a884b3d3826a9941d1b63cc611b12fcd7 Mon Sep 17 00:00:00 2001 From: jfbu Date: Sat, 9 Nov 2019 18:52:21 +0100 Subject: [PATCH 11/23] LaTeX: fix #6804 In case a code-block is in an environment already using the "framed" package, measure the vertical size needed for the code lines and use a nested "framed" only if short enough (i.e. at most 90% of text area on a page). Long nested contents will drop the inner frame. They lose the background color as well as the continuation hints across pagebreaks. But the caption if present is typeset normally. And highlighted lines and line numbers are rendered with no alteration. Short nested contents (should) render exactly as before this commit: they use "framed" but this must be encapsulated in an unbreakable "minipage". All decoration including caption are obeyed but of course not the continuation hints are these never apply. --- sphinx/texinputs/sphinx.sty | 50 +++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index 6023e6d22..2ae4f5e1c 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -1111,6 +1111,9 @@ \newcommand*\sphinxVerbatimTitle {} % This box to typeset the caption before framed.sty multiple passes for framing. \newbox\sphinxVerbatim@TitleBox +% This box to measure contents if nested as inner \MakeFramed requires then +% minipage encapsulation but too long contents then break outer \MakeFramed +\newbox\sphinxVerbatim@ContentsBox % This is a workaround to a "feature" of French lists, when literal block % follows immediately; usable generally (does only \par then), a priori... \newcommand*\sphinxvspacefixafterfrenchlists{% @@ -1257,17 +1260,23 @@ \itemsep \z@skip \topsep \z@skip \partopsep \z@skip - % trivlist will set \parsep to \parskip = zero + % trivlist will set \parsep to \parskip (which itself is set to zero above) % \leftmargin will be set to zero by trivlist \rightmargin\z@ \parindent \z@% becomes \itemindent. Default zero, but perhaps overwritten. \trivlist\item\relax - \ifsphinxverbatimwithminipage\spx@inframedtrue\fi - % use a minipage if we are already inside a framed environment - \ifspx@inframed\noindent\begin{minipage}{\linewidth}\fi - \MakeFramed {% adapted over from framed.sty's snugshade environment + \ifspx@inframed\setbox\sphinxVerbatim@ContentsBox\vbox\bgroup + \@setminipage\hsize\linewidth + % use bulk of minipage paragraph shape restores (this is needed + % in indented contexts, at least for some) + \textwidth\hsize \columnwidth\hsize \@totalleftmargin\z@ + \leftskip\z@skip \rightskip\z@skip \@rightskip\z@skip + \else + \ifsphinxverbatimwithminipage\noindent\begin{minipage}{\linewidth}\fi + \MakeFramed {% adapted over from framed.sty's snugshade environment \advance\hsize-\width\@totalleftmargin\z@\linewidth\hsize\@setminipage }% + \fi % For grid placement from \strut's in \FancyVerbFormatLine \lineskip\z@skip % active comma should not be overwritten by \@noligs @@ -1279,8 +1288,35 @@ } {% \endOriginalVerbatim - \par\unskip\@minipagefalse\endMakeFramed % from framed.sty snugshade - \ifspx@inframed\end{minipage}\fi + \ifspx@inframed + \egroup % finish \sphinxVerbatim@ContentsBox vbox + \ifdim\dimexpr\ht\sphinxVerbatim@ContentsBox+ + \dp\sphinxVerbatim@ContentsBox+ + \ht\sphinxVerbatim@TitleBox+ + \dp\sphinxVerbatim@TitleBox>.9\textheight + % long contents: do not \MakeFramed. Do make a caption (either before or + % after) if title exists. Continuation hints across pagebreaks dropped. + \spx@opt@verbatimwithframefalse + \def\sphinxVerbatim@Title{\noindent\box\sphinxVerbatim@TitleBox\par}% + \sphinxVerbatim@Before + \noindent\unvbox\sphinxVerbatim@ContentsBox\par + \sphinxVerbatim@After + \else + % short enough contents: use \MakeFramed. As it is nested, this requires + % minipage encapsulation. May move to next page as it is non-breakable. + \noindent\begin{minipage}{\linewidth}% + \MakeFramed {% Use it now with the fetched contents + \advance\hsize-\width\@totalleftmargin\z@\linewidth\hsize\@setminipage + }% + \unvbox\sphinxVerbatim@ContentsBox + % some of this may be superfluous: + \par\unskip\@minipagefalse\endMakeFramed + \end{minipage}% + \fi + \else % non-nested \MakeFramed + \par\unskip\@minipagefalse\endMakeFramed % from framed.sty snugshade + \ifsphinxverbatimwithminipage\end{minipage}\fi + \fi \endtrivlist } \newenvironment {sphinxVerbatimNoFrame} From e6ba2fe6d71f5fabec4ab504c82e415a7758889f Mon Sep 17 00:00:00 2001 From: jfbu Date: Sun, 10 Nov 2019 13:45:22 +0100 Subject: [PATCH 12/23] Update CHANGES --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index d6f0caefb..1d27e38c4 100644 --- a/CHANGES +++ b/CHANGES @@ -50,6 +50,8 @@ Bugs fixed * #6712: Allow not to install sphinx.testing as runtime (mainly for ALT Linux) * #6741: html: search result was broken with empty :confval:`html_file_suffix` * #6001: LaTeX does not wrap long code lines at backslash character +* #6804: LaTeX: PDF build breaks if admonition of danger type contains + code-block long enough not to fit on one page Testing -------- From 0a437bc9d65edf842949b6f5971b326a2784144d Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 Nov 2019 16:10:06 +0000 Subject: [PATCH 13/23] Give a warning when extensions are explicitly not parallel safe --- sphinx/application.py | 23 ++++++++++++++--------- tests/test_application.py | 2 ++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/sphinx/application.py b/sphinx/application.py index 1c7a05357..9ea424dbf 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -1190,28 +1190,33 @@ class Sphinx: ``typ`` is a type of processing; ``'read'`` or ``'write'``. """ + if typ == 'read': attrname = 'parallel_read_safe' - message = __("the %s extension does not declare if it is safe " - "for parallel reading, assuming it isn't - please " - "ask the extension author to check and make it " - "explicit") + message_none = __("the %s extension does not declare if it is safe " + "for parallel reading, assuming it isn't - please " + "ask the extension author to check and make it " + "explicit") + message_false = "the %s extension is not safe for parallel reading" elif typ == 'write': attrname = 'parallel_write_safe' - message = __("the %s extension does not declare if it is safe " - "for parallel writing, assuming it isn't - please " - "ask the extension author to check and make it " - "explicit") + message_none = __("the %s extension does not declare if it is safe " + "for parallel writing, assuming it isn't - please " + "ask the extension author to check and make it " + "explicit") + message_false = "the %s extension is not safe for parallel writing" else: raise ValueError('parallel type %s is not supported' % typ) for ext in self.extensions.values(): allowed = getattr(ext, attrname, None) if allowed is None: - logger.warning(message, ext.name) + logger.warning(message_none, ext.name) logger.warning(__('doing serial %s'), typ) return False elif not allowed: + logger.warning(message_false, ext.name) + logger.warning(__('doing serial %s'), typ) return False return True diff --git a/tests/test_application.py b/tests/test_application.py index f10592b51..ffe30a91b 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -105,6 +105,8 @@ def test_add_is_parallel_allowed(app, status, warning): app.setup_extension('read_serial') assert app.is_parallel_allowed('read') is False + assert ("the read_serial extension is not safe for parallel reading") in warning.getvalue() + warning.truncate(0) # reset warnings assert app.is_parallel_allowed('write') is True assert warning.getvalue() == '' app.extensions.pop('read_serial') From fa1cca1feea915bc05443ff16e6cad9b58260907 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 Nov 2019 14:29:20 +0000 Subject: [PATCH 14/23] Improve variable names and code style --- sphinx/application.py | 25 ++++++++++++------------- tests/test_application.py | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/sphinx/application.py b/sphinx/application.py index 9ea424dbf..641ce9893 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -1190,32 +1190,31 @@ class Sphinx: ``typ`` is a type of processing; ``'read'`` or ``'write'``. """ - if typ == 'read': attrname = 'parallel_read_safe' - message_none = __("the %s extension does not declare if it is safe " - "for parallel reading, assuming it isn't - please " - "ask the extension author to check and make it " - "explicit") - message_false = "the %s extension is not safe for parallel reading" + message_not_declared = __("the %s extension does not declare if it " + "is safe for parallel reading, assuming " + "it isn't - please ask the extension author " + "to check and make it explicit") + message_not_safe = __("the %s extension is not safe for parallel reading") elif typ == 'write': attrname = 'parallel_write_safe' - message_none = __("the %s extension does not declare if it is safe " - "for parallel writing, assuming it isn't - please " - "ask the extension author to check and make it " - "explicit") - message_false = "the %s extension is not safe for parallel writing" + message_not_declared = __("the %s extension does not declare if it " + "is safe for parallel writing, assuming " + "it isn't - please ask the extension author " + "to check and make it explicit") + message_not_safe = __("the %s extension is not safe for parallel writing") else: raise ValueError('parallel type %s is not supported' % typ) for ext in self.extensions.values(): allowed = getattr(ext, attrname, None) if allowed is None: - logger.warning(message_none, ext.name) + logger.warning(message_not_declared, ext.name) logger.warning(__('doing serial %s'), typ) return False elif not allowed: - logger.warning(message_false, ext.name) + logger.warning(message_not_safe, ext.name) logger.warning(__('doing serial %s'), typ) return False diff --git a/tests/test_application.py b/tests/test_application.py index ffe30a91b..a268f492f 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -105,7 +105,7 @@ def test_add_is_parallel_allowed(app, status, warning): app.setup_extension('read_serial') assert app.is_parallel_allowed('read') is False - assert ("the read_serial extension is not safe for parallel reading") in warning.getvalue() + assert "the read_serial extension is not safe for parallel reading" in warning.getvalue() warning.truncate(0) # reset warnings assert app.is_parallel_allowed('write') is True assert warning.getvalue() == '' From c7fd4cc82701baf26e1f2a4cc5812b97e91374d3 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 12 Nov 2019 23:55:32 +0900 Subject: [PATCH 15/23] Update CHANGES for PR #6812 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index d6f0caefb..de7e1ee30 100644 --- a/CHANGES +++ b/CHANGES @@ -30,6 +30,7 @@ Features added * #1331: Add new config variable: :confval:`user_agent` * #6000: LaTeX: have backslash also be an inline literal word wrap break character +* #6812: Improve a warning message when extensions are not parallel safe Bugs fixed ---------- From ff2f4a0e5478cb69acf197475e850b67ca391963 Mon Sep 17 00:00:00 2001 From: Andriy Orehov Date: Thu, 14 Nov 2019 13:49:42 +0200 Subject: [PATCH 16/23] add urls to Code and Issue tracker for PyPi --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 0a75278b4..202523fe0 100644 --- a/setup.py +++ b/setup.py @@ -176,6 +176,10 @@ setup( description='Python documentation generator', long_description=long_desc, long_description_content_type='text/x-rst', + project_urls={ + "Code": "https://github.com/sphinx-doc/sphinx", + "Issue tracker": "https://github.com/sphinx-doc/sphinx/issues", + }, zip_safe=False, classifiers=[ 'Development Status :: 5 - Production/Stable', From ea3ba6c210be04a3908868b1351e3e1dcfb95f75 Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Fri, 15 Nov 2019 14:27:30 +1100 Subject: [PATCH 17/23] Fetch inventories concurrently --- AUTHORS | 2 +- CHANGES | 1 + sphinx/ext/intersphinx.py | 41 +++++++++++++++++++++++++-------------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/AUTHORS b/AUTHORS index 1cc77b640..8505b644d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -36,7 +36,7 @@ Other contributors, listed alphabetically, are: * Hernan Grecco -- search improvements * Horst Gutmann -- internationalization support * Martin Hans -- autodoc improvements -* Zac Hatfield-Dodds -- doctest reporting improvements +* Zac Hatfield-Dodds -- doctest reporting improvements, intersphinx performance * Doug Hellmann -- graphviz improvements * Tim Hoffmann -- theme improvements * Antti Kaihola -- doctest extension (skipif option) diff --git a/CHANGES b/CHANGES index afdd8d436..adc9c6f09 100644 --- a/CHANGES +++ b/CHANGES @@ -32,6 +32,7 @@ Features added * #6000: LaTeX: have backslash also be an inline literal word wrap break character * #6812: Improve a warning message when extensions are not parallel safe +* #6818: Improve Intersphinx performance for multiple remote inventories. Bugs fixed ---------- diff --git a/sphinx/ext/intersphinx.py b/sphinx/ext/intersphinx.py index 4389c5a44..7745d43d4 100644 --- a/sphinx/ext/intersphinx.py +++ b/sphinx/ext/intersphinx.py @@ -23,6 +23,7 @@ :license: BSD, see LICENSE for details. """ +import concurrent.futures import functools import posixpath import sys @@ -187,21 +188,18 @@ def fetch_inventory(app: Sphinx, uri: str, inv: Any) -> Any: return invdata -def load_mappings(app: Sphinx) -> None: - """Load all intersphinx mappings into the environment.""" - now = int(time.time()) +def fetch_inventory_group( + name: str, uri: str, invs: Any, cache: Any, app: Any, now: float +) -> bool: cache_time = now - app.config.intersphinx_cache_limit * 86400 - inventories = InventoryAdapter(app.builder.env) - update = False - for key, (name, (uri, invs)) in app.config.intersphinx_mapping.items(): - failures = [] + failures = [] + try: for inv in invs: if not inv: inv = posixpath.join(uri, INVENTORY_FILENAME) # decide whether the inventory must be read: always read local # files; remote ones only if the cache time is expired - if '://' not in inv or uri not in inventories.cache \ - or inventories.cache[uri][1] < cache_time: + if '://' not in inv or uri not in cache or cache[uri][1] < cache_time: safe_inv_url = _get_safe_url(inv) logger.info(__('loading intersphinx inventory from %s...'), safe_inv_url) try: @@ -209,12 +207,11 @@ def load_mappings(app: Sphinx) -> None: except Exception as err: failures.append(err.args) continue - if invdata: - inventories.cache[uri] = (name, now, invdata) - update = True - break - + cache[uri] = (name, now, invdata) + return True + return False + finally: if failures == []: pass elif len(failures) < len(invs): @@ -227,7 +224,21 @@ def load_mappings(app: Sphinx) -> None: logger.warning(__("failed to reach any of the inventories " "with the following issues:") + "\n" + issues) - if update: + +def load_mappings(app: Sphinx) -> None: + """Load all intersphinx mappings into the environment.""" + now = int(time.time()) + inventories = InventoryAdapter(app.builder.env) + + with concurrent.futures.ThreadPoolExecutor() as pool: + futures = [] + for name, (uri, invs) in app.config.intersphinx_mapping.values(): + futures.append(pool.submit( + fetch_inventory_group, name, uri, invs, inventories.cache, app, now + )) + updated = [f.result() for f in concurrent.futures.as_completed(futures)] + + if any(updated): inventories.clear() # Duplicate values in different inventories will shadow each From 53ead2a0d9cbd0ab63eca2272ff51a913a9cc153 Mon Sep 17 00:00:00 2001 From: jfbu Date: Mon, 11 Nov 2019 10:11:34 +0100 Subject: [PATCH 18/23] LaTeX: fix #6809 --- CHANGES | 2 ++ sphinx/texinputs/sphinx.sty | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 68b6b1b19..0b0e0c2b0 100644 --- a/CHANGES +++ b/CHANGES @@ -54,6 +54,8 @@ Bugs fixed * #6001: LaTeX does not wrap long code lines at backslash character * #6804: LaTeX: PDF build breaks if admonition of danger type contains code-block long enough not to fit on one page +* #6809: LaTeX: code-block in a danger type admonition can easily spill over + bottom of page Testing -------- diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index 2ae4f5e1c..0acdac989 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -1290,12 +1290,26 @@ \endOriginalVerbatim \ifspx@inframed \egroup % finish \sphinxVerbatim@ContentsBox vbox + \nobreak % update page totals \ifdim\dimexpr\ht\sphinxVerbatim@ContentsBox+ \dp\sphinxVerbatim@ContentsBox+ \ht\sphinxVerbatim@TitleBox+ - \dp\sphinxVerbatim@TitleBox>.9\textheight + \dp\sphinxVerbatim@TitleBox+ + 2\fboxsep+2\fboxrule+ + % try to account for external frame parameters + \FrameSep+\FrameRule+ + % Usage here of 2 baseline distances is empirical. + % In border case where code-block fits barely in remaining space, + % it gets framed and looks good but the outer frame may continue + % on top of next page and give (if no contents after code-block) + % an empty framed line, as testing showed. + 2\baselineskip+ + % now add all to accumulated page totals and compare to \pagegoal + \pagetotal+\pagedepth>\pagegoal % long contents: do not \MakeFramed. Do make a caption (either before or % after) if title exists. Continuation hints across pagebreaks dropped. + % FIXME? a bottom caption may end up isolated at top of next page + % (no problem with a top caption, which is default) \spx@opt@verbatimwithframefalse \def\sphinxVerbatim@Title{\noindent\box\sphinxVerbatim@TitleBox\par}% \sphinxVerbatim@Before From b6cb5dd2191f93b56febd40f0366108ef85f1759 Mon Sep 17 00:00:00 2001 From: jfbu Date: Mon, 11 Nov 2019 11:05:37 +0100 Subject: [PATCH 19/23] Trim possibly erroneous comment from Sphinx LaTeX code --- sphinx/texinputs/sphinx.sty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty index 0acdac989..5d9adb5bb 100644 --- a/sphinx/texinputs/sphinx.sty +++ b/sphinx/texinputs/sphinx.sty @@ -1317,7 +1317,7 @@ \sphinxVerbatim@After \else % short enough contents: use \MakeFramed. As it is nested, this requires - % minipage encapsulation. May move to next page as it is non-breakable. + % minipage encapsulation. \noindent\begin{minipage}{\linewidth}% \MakeFramed {% Use it now with the fetched contents \advance\hsize-\width\@totalleftmargin\z@\linewidth\hsize\@setminipage From 75b6d66f1d6477e430b21f202d69a07b9d8f67e9 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 16 Nov 2019 16:46:21 +0900 Subject: [PATCH 20/23] Fix #6738: Use get_encode_func in todo extension --- sphinx/ext/todo.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sphinx/ext/todo.py b/sphinx/ext/todo.py index 5da0342f8..d2a8a666d 100644 --- a/sphinx/ext/todo.py +++ b/sphinx/ext/todo.py @@ -30,7 +30,7 @@ from sphinx.locale import _, __ from sphinx.util import logging from sphinx.util.docutils import SphinxDirective from sphinx.util.nodes import make_refnode -from sphinx.util.texescape import tex_escape_map +from sphinx.util.texescape import get_escape_func from sphinx.writers.html import HTMLTranslator from sphinx.writers.latex import LaTeXTranslator @@ -299,10 +299,11 @@ def depart_todo_node(self: HTMLTranslator, node: todo_node) -> None: def latex_visit_todo_node(self: LaTeXTranslator, node: todo_node) -> None: if self.config.todo_include_todos: + escape = get_escape_func(self.config.latex_engine) self.body.append('\n\\begin{sphinxadmonition}{note}{') self.body.append(self.hypertarget_to(node)) title_node = cast(nodes.title, node[0]) - self.body.append('%s:}' % title_node.astext().translate(tex_escape_map)) + self.body.append('%s:}' % escape(title_node.astext())) node.pop(0) else: raise nodes.SkipNode From 660e746cf801b9fce867a35a1ad3b65ae9ef43c5 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 16 Nov 2019 16:29:14 +0900 Subject: [PATCH 21/23] Fix #6738: latex: literal_block does not support raw unicode characters --- sphinx/highlighting.py | 11 +++++++---- sphinx/util/texescape.py | 28 ++++++++++++++++++++++++++-- sphinx/writers/latex.py | 3 ++- tests/test_markup.py | 18 ++++++++++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py index 977b71f28..0a6cf8d6e 100644 --- a/sphinx/highlighting.py +++ b/sphinx/highlighting.py @@ -28,7 +28,7 @@ from sphinx.ext import doctest from sphinx.locale import __ from sphinx.pygments_styles import SphinxStyle, NoneStyle from sphinx.util import logging -from sphinx.util.texescape import tex_hl_escape_map_new +from sphinx.util.texescape import get_hlescape_func, tex_hl_escape_map_new if False: # For type annotation @@ -68,9 +68,11 @@ class PygmentsBridge: html_formatter = HtmlFormatter latex_formatter = LatexFormatter - def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=None): - # type: (str, str, bool) -> None + def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=None, + latex_engine=None): + # type: (str, str, bool, str) -> None self.dest = dest + self.latex_engine = latex_engine style = self.get_style(stylename) self.formatter_args = {'style': style} # type: Dict[str, Any] @@ -192,7 +194,8 @@ class PygmentsBridge: if self.dest == 'html': return hlsource else: - return hlsource.translate(tex_hl_escape_map_new) + escape = get_hlescape_func(self.latex_engine) + return escape(hlsource) def get_stylesheet(self): # type: () -> str diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py index 4e7055119..c3231d3d3 100644 --- a/sphinx/util/texescape.py +++ b/sphinx/util/texescape.py @@ -32,7 +32,6 @@ tex_replacements = [ ('¶', r'\P{}'), ('§', r'\S{}'), ('€', r'\texteuro{}'), - ('∞', r'\(\infty\)'), ('±', r'\(\pm\)'), ('→', r'\(\rightarrow\)'), ('‣', r'\(\rightarrow\)'), @@ -53,6 +52,8 @@ tex_replacements = [ # A map Unicode characters to LaTeX representation # (for LaTeX engines which don't support unicode) unicode_tex_replacements = [ + # map special Unicode characters to TeX commands + ('∞', r'\(\infty\)'), # superscript ('⁰', r'\(\sp{\text{0}}\)'), ('¹', r'\(\sp{\text{1}}\)'), @@ -80,7 +81,8 @@ unicode_tex_replacements = [ tex_escape_map = {} # type: Dict[int, str] tex_escape_map_without_unicode = {} # type: Dict[int, str] tex_replace_map = {} -tex_hl_escape_map_new = {} +tex_hl_escape_map_new = {} # type: Dict[int, str] +tex_hl_escape_map_new_without_unicode = {} # type: Dict[int, str] def get_escape_func(latex_engine: str) -> Callable[[str], str]: @@ -101,6 +103,24 @@ def escape_for_unicode_latex_engine(s: str) -> str: return s.translate(tex_escape_map_without_unicode) +def get_hlescape_func(latex_engine: str) -> Callable[[str], str]: + """Get hlescape() function for given latex_engine.""" + if latex_engine in ('lualatex', 'xelatex'): + return hlescape_for_unicode_latex_engine + else: + return hlescape + + +def hlescape(s: str) -> str: + """Escape text for LaTeX highlighter.""" + return s.translate(tex_hl_escape_map_new) + + +def hlescape_for_unicode_latex_engine(s: str) -> str: + """Escape text for unicode supporting LaTeX engine.""" + return s.translate(tex_hl_escape_map_new_without_unicode) + + def escape_abbr(text: str) -> str: """Adjust spacing after abbreviations. Works with @ letter or other.""" return re.sub(r'\.(?=\s|$)', r'.\@{}', text) @@ -120,3 +140,7 @@ def init() -> None: if a in '[]{}\\': continue tex_hl_escape_map_new[ord(a)] = b + tex_hl_escape_map_new_without_unicode[ord(a)] = b + + for a, b in unicode_tex_replacements: + tex_hl_escape_map_new[ord(a)] = b diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 29367a79e..5facdc40c 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -653,7 +653,8 @@ class LaTeXTranslator(SphinxTranslator): self.elements['classoptions'] += ',' + \ self.elements['extraclassoptions'] - self.highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style) + self.highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style, + latex_engine=self.config.latex_engine) self.context = [] # type: List[Any] self.descstack = [] # type: List[str] self.table = None # type: Table diff --git a/tests/test_markup.py b/tests/test_markup.py index b8c9b66d9..94d1af951 100644 --- a/tests/test_markup.py +++ b/tests/test_markup.py @@ -312,6 +312,24 @@ def test_inline(get_verifier, type, rst, html_expected, latex_expected): verifier(rst, html_expected, latex_expected) +@pytest.mark.sphinx(confoverrides={'latex_engine': 'xelatex'}) +@pytest.mark.parametrize('type,rst,html_expected,latex_expected', [ + ( + # in verbatim code fragments + 'verify', + '::\n\n @Γ\\∞${}', + None, + ('\\begin{sphinxVerbatim}[commandchars=\\\\\\{\\}]\n' + '@Γ\\PYGZbs{}∞\\PYGZdl{}\\PYGZob{}\\PYGZcb{}\n' + '\\end{sphinxVerbatim}'), + ), +]) +def test_inline_for_unicode_latex_engine(get_verifier, type, rst, + html_expected, latex_expected): + verifier = get_verifier(type) + verifier(rst, html_expected, latex_expected) + + def test_samp_role(parse): # no braces text = ':samp:`a{b}c`' From 38926cf708fac6770c67423eb87fc74dc8f02896 Mon Sep 17 00:00:00 2001 From: jfbu Date: Fri, 15 Nov 2019 19:35:01 +0100 Subject: [PATCH 22/23] LaTeX: escape fewer Unicode chars if engine is lualatex or xelatex --- CHANGES | 3 ++- sphinx/util/texescape.py | 32 +++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/CHANGES b/CHANGES index 9d3b51cc4..250e9589c 100644 --- a/CHANGES +++ b/CHANGES @@ -43,7 +43,8 @@ Bugs fixed .. _latex3/latex2e#173: https://github.com/latex3/latex2e/issues/173 * #6618: LaTeX: Avoid section names at the end of a page * #6738: LaTeX: Do not replace unicode characters by LaTeX macros on unicode - supported LaTeX engines + supported LaTeX engines: ¶, §, €, ∞, ±, →, ‣, –, superscript and subscript + digits go through "as is" (as default OpenType font supports them) * #6704: linkcheck: Be defensive and handle newly defined HTTP error code * #6655: image URLs containing ``data:`` causes gettext builder crashed * #6584: i18n: Error when compiling message catalogs on Hindi diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py index c3231d3d3..e3c45adb4 100644 --- a/sphinx/util/texescape.py +++ b/sphinx/util/texescape.py @@ -20,29 +20,28 @@ tex_replacements = [ ('_', r'\_'), ('{', r'\{'), ('}', r'\}'), - ('[', r'{[}'), - (']', r'{]}'), - ('`', r'{}`'), ('\\', r'\textbackslash{}'), ('~', r'\textasciitilde{}'), + ('^', r'\textasciicircum{}'), + # map chars to avoid mis-interpretation in LaTeX + ('[', r'{[}'), + (']', r'{]}'), + # map chars to avoid TeX ligatures + # 1. ' - and , not here for some legacy reason + # 2. no effect with lualatex (done otherwise: #5790) + ('`', r'{}`'), ('<', r'\textless{}'), ('>', r'\textgreater{}'), - ('^', r'\textasciicircum{}'), + # map char for some unknown reason. TODO: remove this? + ('|', r'\textbar{}'), # map special Unicode characters to TeX commands - ('¶', r'\P{}'), - ('§', r'\S{}'), - ('€', r'\texteuro{}'), - ('±', r'\(\pm\)'), - ('→', r'\(\rightarrow\)'), - ('‣', r'\(\rightarrow\)'), ('✓', r'\(\checkmark\)'), ('✔', r'\(\pmb{\checkmark}\)'), # used to separate -- in options ('', r'{}'), # map some special Unicode characters to similar ASCII ones + # (even for Unicode LaTeX as may not be supported by OpenType font) ('⎽', r'\_'), - ('–', r'\textendash{}'), - ('|', r'\textbar{}'), ('ℯ', r'e'), ('ⅈ', r'i'), # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc @@ -52,8 +51,15 @@ tex_replacements = [ # A map Unicode characters to LaTeX representation # (for LaTeX engines which don't support unicode) unicode_tex_replacements = [ - # map special Unicode characters to TeX commands + # map some more common Unicode characters to TeX commands + ('¶', r'\P{}'), + ('§', r'\S{}'), + ('€', r'\texteuro{}'), ('∞', r'\(\infty\)'), + ('±', r'\(\pm\)'), + ('→', r'\(\rightarrow\)'), + ('‣', r'\(\rightarrow\)'), + ('–', r'\textendash{}'), # superscript ('⁰', r'\(\sp{\text{0}}\)'), ('¹', r'\(\sp{\text{1}}\)'), From ff079de0100c2dd684d7a3dd0cb7577b6f39e435 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 8 Nov 2019 00:07:37 +0900 Subject: [PATCH 23/23] Close #2546: apidoc: .so file support --- CHANGES | 1 + sphinx/ext/apidoc.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 9d3b51cc4..25d56ed6f 100644 --- a/CHANGES +++ b/CHANGES @@ -33,6 +33,7 @@ Features added character * #6812: Improve a warning message when extensions are not parallel safe * #6818: Improve Intersphinx performance for multiple remote inventories. +* #2546: apidoc: .so file support Bugs fixed ---------- diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py index b1d19e5cb..16eea9c11 100644 --- a/sphinx/ext/apidoc.py +++ b/sphinx/ext/apidoc.py @@ -21,6 +21,7 @@ import os import sys import warnings from fnmatch import fnmatch +from importlib.machinery import EXTENSION_SUFFIXES from os import path from typing import Any, List, Tuple @@ -45,7 +46,7 @@ else: ] INITPY = '__init__.py' -PY_SUFFIXES = {'.py', '.pyx'} +PY_SUFFIXES = ('.py', '.pyx') + tuple(EXTENSION_SUFFIXES) template_dir = path.join(package_dir, 'templates', 'apidoc') @@ -232,7 +233,7 @@ def recurse_tree(rootpath: str, excludes: List[str], opts: Any, for root, subs, files in os.walk(rootpath, followlinks=followlinks): # document only Python module files (that aren't excluded) py_files = sorted(f for f in files - if path.splitext(f)[1] in PY_SUFFIXES and + if f.endswith(PY_SUFFIXES) and not is_excluded(path.join(root, f), excludes)) is_pkg = INITPY in py_files is_namespace = INITPY not in py_files and implicit_namespaces @@ -270,7 +271,7 @@ def recurse_tree(rootpath: str, excludes: List[str], opts: Any, assert root == rootpath and root_package is None for py_file in py_files: if not is_skipped_module(path.join(rootpath, py_file), opts, excludes): - module = path.splitext(py_file)[0] + module = py_file.split('.')[0] create_module_file(root_package, module, opts, user_template_dir) toplevels.append(module)