From 4c38038f647b48466a3a12216c57b9dfb81dece6 Mon Sep 17 00:00:00 2001 From: jfbu Date: Sat, 5 Nov 2016 18:43:03 +0100 Subject: [PATCH 01/33] latex multilingual and contentsname setup to after getting user setup --- sphinx/writers/latex.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 84697a4f9..3b2a43fd5 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -437,6 +437,9 @@ class LaTeXTranslator(nodes.NodeVisitor): # disable fncychap in Japanese documents self.elements['fncychap'] = '' + # set 'babel' to improbable value to detect if user has used it + self.elements['babel'] = '3.1415' + if getattr(builder, 'usepackages', None): def declare_package(packagename, options=None): if options: @@ -462,12 +465,17 @@ class LaTeXTranslator(nodes.NodeVisitor): if tocdepth >= SECNUMDEPTH: # Increase secnumdepth if tocdepth is depther than default SECNUMDEPTH self.elements['secnumdepth'] = '\\setcounter{secnumdepth}{%d}' % tocdepth - if getattr(document.settings, 'contentsname', None): - self.elements['contentsname'] = \ - self.babel_renewcommand('\\contentsname', document.settings.contentsname) # allow the user to override them all self.check_latex_elements() self.elements.update(builder.config.latex_elements) + + if self.elements['babel'] != '3.1415': + self.elements['multilingual'] = self.elements['babel'] + + if getattr(document.settings, 'contentsname', None): + self.elements['contentsname'] = \ + self.babel_renewcommand('\\contentsname', document.settings.contentsname) + if self.elements['maxlistdepth']: self.elements['sphinxpkgoptions'] += (',maxlistdepth=%s' % self.elements['maxlistdepth']) From b35539bd2e83c61436ca201e79af48fe037669e1 Mon Sep 17 00:00:00 2001 From: jfbu Date: Sat, 5 Nov 2016 19:25:20 +0100 Subject: [PATCH 02/33] reset ``'babel'`` to empty if it was for ``babel_renewcommand`` ok --- sphinx/writers/latex.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 3b2a43fd5..d224ae323 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -438,7 +438,10 @@ class LaTeXTranslator(nodes.NodeVisitor): self.elements['fncychap'] = '' # set 'babel' to improbable value to detect if user has used it - self.elements['babel'] = '3.1415' + if self.elements['babel']: + self.elements['babel'] = '3.1415' + else: + self.elements['babel'] = '2.7182' if getattr(builder, 'usepackages', None): def declare_package(packagename, options=None): @@ -470,7 +473,10 @@ class LaTeXTranslator(nodes.NodeVisitor): self.elements.update(builder.config.latex_elements) if self.elements['babel'] != '3.1415': - self.elements['multilingual'] = self.elements['babel'] + if self.elements['babel'] != '2.7182': + self.elements['multilingual'] = self.elements['babel'] + else: + self.elements['babel'] = '' if getattr(document.settings, 'contentsname', None): self.elements['contentsname'] = \ From 5663f750fe9e1f2cd8e766d3df3766462633f285 Mon Sep 17 00:00:00 2001 From: jfbu Date: Thu, 17 Nov 2016 10:00:01 +0100 Subject: [PATCH 03/33] refactor multilingual (load user settings on top) --- sphinx/writers/latex.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index d224ae323..214f14fe9 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -109,6 +109,7 @@ ADDITIONAL_SETTINGS = { 'xelatex': { 'latex_engine': 'xelatex', 'polyglossia': '\\usepackage{polyglossia}', + 'babel': '', 'fontenc': '\\usepackage{fontspec}', 'fontpkg': '', 'utf8extra': ('\\catcode`^^^^00a0\\active\\protected\\def^^^^00a0' @@ -372,6 +373,11 @@ class LaTeXTranslator(nodes.NodeVisitor): # sort out some elements self.elements = DEFAULT_SETTINGS.copy() self.elements.update(ADDITIONAL_SETTINGS.get(builder.config.latex_engine, {})) + # allow the user to override them all + self.check_latex_elements() + self.elements.update(builder.config.latex_elements) + + # but some have other interface in config file self.elements.update({ 'wrapperclass': self.format_docclass(document.settings.docclass), # if empty, the title is set to the first section title @@ -398,7 +404,8 @@ class LaTeXTranslator(nodes.NodeVisitor): self.elements['logo'] = '\\sphinxincludegraphics{%s}\\par' % \ path.basename(builder.config.latex_logo) - if builder.config.language: + if builder.config.language \ + and 'fncychap' not in builder.config.latex_elements: # use Sonny style if any language specified self.elements['fncychap'] = '\\usepackage[Sonny]{fncychap}' @@ -414,17 +421,16 @@ class LaTeXTranslator(nodes.NodeVisitor): self.elements['classoptions'] += ',' + self.babel.get_language() # set up multilingual module... - if self.elements['polyglossia']: - self.elements['babel'] = '' # disable babel - self.elements['multilingual'] = '%s\n\\setmainlanguage{%s}' % \ - (self.elements['polyglossia'], self.babel.get_language()) - elif self.elements['babel']: + # 'babel' key is public and user setting must be obeyed + if self.elements['babel']: + # this branch is not taken for xelatex with writer default settings self.elements['multilingual'] = self.elements['babel'] if builder.config.language: self.elements['shorthandoff'] = self.babel.get_shorthandoff() # Times fonts don't work with Cyrillic languages - if self.babel.uses_cyrillic(): + if self.babel.uses_cyrillic() \ + and 'fontpkg' not in builder.config.latex_elements: self.elements['fontpkg'] = '' # pTeX (Japanese TeX) for support @@ -436,12 +442,9 @@ class LaTeXTranslator(nodes.NodeVisitor): self.elements['multilingual'] = '' # disable fncychap in Japanese documents self.elements['fncychap'] = '' - - # set 'babel' to improbable value to detect if user has used it - if self.elements['babel']: - self.elements['babel'] = '3.1415' - else: - self.elements['babel'] = '2.7182' + elif self.elements['polyglossia']: + self.elements['multilingual'] = '%s\n\\setmainlanguage{%s}' % \ + (self.elements['polyglossia'], self.babel.get_language()) if getattr(builder, 'usepackages', None): def declare_package(packagename, options=None): @@ -468,15 +471,6 @@ class LaTeXTranslator(nodes.NodeVisitor): if tocdepth >= SECNUMDEPTH: # Increase secnumdepth if tocdepth is depther than default SECNUMDEPTH self.elements['secnumdepth'] = '\\setcounter{secnumdepth}{%d}' % tocdepth - # allow the user to override them all - self.check_latex_elements() - self.elements.update(builder.config.latex_elements) - - if self.elements['babel'] != '3.1415': - if self.elements['babel'] != '2.7182': - self.elements['multilingual'] = self.elements['babel'] - else: - self.elements['babel'] = '' if getattr(document.settings, 'contentsname', None): self.elements['contentsname'] = \ From 8e1db118c0da8a7d6872baf933a2439353b80044 Mon Sep 17 00:00:00 2001 From: jfbu Date: Thu, 17 Nov 2016 10:01:46 +0100 Subject: [PATCH 04/33] polyglossia honors ``\addto\captions`` like babel --- sphinx/writers/latex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 214f14fe9..8d3eef097 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -590,7 +590,7 @@ class LaTeXTranslator(nodes.NodeVisitor): return self.idescape(ref).replace('-', '\\string-') def babel_renewcommand(self, command, definition): - if self.elements['babel']: + if self.elements['multilingual']: prefix = '\\addto\\captions%s{' % self.babel.get_language() suffix = '}' else: # babel is disabled (mainly for Japanese environment) From 308d3aee89046d1dc893a04d3cd6f97d4a1454f9 Mon Sep 17 00:00:00 2001 From: jfbu Date: Thu, 17 Nov 2016 10:12:53 +0100 Subject: [PATCH 05/33] make shorthandoff extra safe also for turkish (perhaps xelatex+babel+turkish may not use active ``=`` either now or in future) --- sphinx/writers/latex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 8d3eef097..454c30e9c 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -175,7 +175,7 @@ class ExtBabel(Babel): 'italian'): return '\\if\\catcode`\\"\\active\\shorthandoff{"}\\fi' elif shortlang in ('tr', 'turkish'): - return '\\shorthandoff{=}' + return '\\if\\catcode`\\=\\active\\shorthandoff{=}\\fi' return '' def uses_cyrillic(self): From c5ce00b448a92a70188366c435a3cb687a9d77b3 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 17 Nov 2016 20:28:34 +0900 Subject: [PATCH 06/33] Remove meaningless commas --- sphinx/builders/applehelp.py | 4 ++-- sphinx/ext/inheritance_diagram.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sphinx/builders/applehelp.py b/sphinx/builders/applehelp.py index 7db086953..d199f33b7 100644 --- a/sphinx/builders/applehelp.py +++ b/sphinx/builders/applehelp.py @@ -286,10 +286,10 @@ def setup(app): app.add_config_value('applehelp_title', lambda self: self.project + ' Help', 'applehelp') app.add_config_value('applehelp_codesign_identity', lambda self: environ.get('CODE_SIGN_IDENTITY', None), - 'applehelp'), + 'applehelp') app.add_config_value('applehelp_codesign_flags', lambda self: shlex.split(environ.get('OTHER_CODE_SIGN_FLAGS', '')), - 'applehelp'), + 'applehelp') app.add_config_value('applehelp_indexer_path', '/usr/bin/hiutil', 'applehelp') app.add_config_value('applehelp_codesign_path', '/usr/bin/codesign', 'applehelp') app.add_config_value('applehelp_disable_external_tools', False, None) diff --git a/sphinx/ext/inheritance_diagram.py b/sphinx/ext/inheritance_diagram.py index 11af67dc5..f37786a0e 100644 --- a/sphinx/ext/inheritance_diagram.py +++ b/sphinx/ext/inheritance_diagram.py @@ -420,7 +420,7 @@ def setup(app): man=(skip, None), texinfo=(texinfo_visit_inheritance_diagram, None)) app.add_directive('inheritance-diagram', InheritanceDiagram) - app.add_config_value('inheritance_graph_attrs', {}, False), - app.add_config_value('inheritance_node_attrs', {}, False), - app.add_config_value('inheritance_edge_attrs', {}, False), + app.add_config_value('inheritance_graph_attrs', {}, False) + app.add_config_value('inheritance_node_attrs', {}, False) + app.add_config_value('inheritance_edge_attrs', {}, False) return {'version': sphinx.__display_version__, 'parallel_read_safe': True} From 6c6ea35524886480bd9511b0e0002a96167e3732 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 17 Nov 2016 20:29:55 +0900 Subject: [PATCH 07/33] Use boolean value to Sphinx.info() --- sphinx/application.py | 6 +++--- sphinx/environment/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sphinx/application.py b/sphinx/application.py index 93f12f3b6..f8074048b 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -463,9 +463,9 @@ class Sphinx(object): l = 0 for item in iterable: if l == 0: - self.info(bold(summary), nonl=1) + self.info(bold(summary), nonl=True) l = 1 - self.info(colorfunc(stringify_func(item)) + ' ', nonl=1) + self.info(colorfunc(stringify_func(item)) + ' ', nonl=True) yield item if l == 1: self.info() @@ -488,7 +488,7 @@ class Sphinx(object): s += '\n' else: s = term_width_line(s) - self.info(s, nonl=1) + self.info(s, nonl=True) yield item if l > 0: self.info() diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index d750b0284..f5cd1550b 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -494,7 +494,7 @@ class BuildEnvironment(object): # this cache also needs to be updated every time self._nitpick_ignore = set(self.config.nitpick_ignore) - app.info(bold('updating environment: '), nonl=1) + app.info(bold('updating environment: '), nonl=True) added, changed, removed = self.get_outdated_files(config_changed) From bc329dcc711f98fef371d6961f9d4faed0a35a94 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 17 Nov 2016 20:32:45 +0900 Subject: [PATCH 08/33] Remove return syntax if retval is unexpected --- sphinx/application.py | 2 +- sphinx/builders/epub.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/application.py b/sphinx/application.py index f8074048b..cee780c9c 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -301,7 +301,7 @@ class Sphinx(object): self.info('not yet created') else: self.info('failed: %s' % err) - return self._init_env(freshenv=True) + self._init_env(freshenv=True) def _init_builder(self, buildername): if buildername is None: diff --git a/sphinx/builders/epub.py b/sphinx/builders/epub.py index b4b657468..0e1c844fe 100644 --- a/sphinx/builders/epub.py +++ b/sphinx/builders/epub.py @@ -413,7 +413,7 @@ class EpubBuilder(StandaloneHTMLBuilder): """ self.fix_ids(doctree) self.add_visible_links(doctree, self.config.epub_show_urls) - return StandaloneHTMLBuilder.write_doc(self, docname, doctree) + StandaloneHTMLBuilder.write_doc(self, docname, doctree) def fix_genindex(self, tree): """Fix href attributes for genindex pages.""" From 8006d31b32916bdc57dda1faa42605088a4a0c6d Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 16 Nov 2016 23:23:18 +0900 Subject: [PATCH 09/33] Fix #3095: Add tls_verify and tls_cacerts to support self-signed servers --- CHANGES | 3 ++ doc/config.rst | 15 +++++++++ sphinx/builders/linkcheck.py | 11 +++---- sphinx/config.py | 3 ++ sphinx/ext/intersphinx.py | 8 ++--- sphinx/util/requests.py | 60 +++++++++++++++++++++++++++++++++++- 6 files changed, 89 insertions(+), 11 deletions(-) diff --git a/CHANGES b/CHANGES index 0a0606a5a..e26fba64a 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,9 @@ Incompatible changes Features added -------------- +* #3095: Add :confval:`tls_verify` and :confval:`tls_cacerts` to support + self-signed HTTPS servers in linkcheck and intersphinx + Bugs fixed ---------- diff --git a/doc/config.rst b/doc/config.rst index 34900535f..7f1f73186 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -318,6 +318,21 @@ General configuration .. versionadded:: 1.3 +.. confval:: tls_verify + + If true, Sphinx verifies server certifications. Default is ``True``. + + .. versionadded:: 1.5 + +.. confval:: tls_cacerts + + A path to a certification file of CA or a path to directory which + contains the certificates. This also allows a dictionary mapping + hostname to the path to certificate file. + The certificates are used to verify server certifications. + + .. versionadded:: 1.5 + Project information ------------------- diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py index f49f4f9a3..5a1943f48 100644 --- a/sphinx/builders/linkcheck.py +++ b/sphinx/builders/linkcheck.py @@ -32,10 +32,10 @@ except ImportError: pass from sphinx.builders import Builder -from sphinx.util import encode_uri +from sphinx.util import encode_uri, requests from sphinx.util.console import purple, red, darkgreen, darkgray, \ darkred, turquoise -from sphinx.util.requests import requests, useragent_header, is_ssl_error +from sphinx.util.requests import is_ssl_error class AnchorCheckParser(HTMLParser): @@ -87,7 +87,6 @@ class CheckExternalLinksBuilder(Builder): self.good = set() self.broken = {} self.redirected = {} - self.headers = dict(useragent_header) # set a timeout for non-responding servers socket.setdefaulttimeout(5.0) # create output file @@ -131,7 +130,7 @@ class CheckExternalLinksBuilder(Builder): try: if anchor and self.app.config.linkcheck_anchors: # Read the whole document and see if #anchor exists - response = requests.get(req_url, stream=True, headers=self.headers, + response = requests.get(req_url, stream=True, config=self.app.config, **kwargs) found = check_anchor(response, unquote(anchor)) @@ -141,12 +140,12 @@ class CheckExternalLinksBuilder(Builder): try: # try a HEAD request first, which should be easier on # the server and the network - response = requests.head(req_url, headers=self.headers, **kwargs) + response = requests.head(req_url, config=self.app.config, **kwargs) response.raise_for_status() except HTTPError as err: # retry with GET request if that fails, some servers # don't like HEAD requests. - response = requests.get(req_url, stream=True, headers=self.headers, + response = requests.get(req_url, stream=True, config=self.app.config, **kwargs) response.raise_for_status() except HTTPError as err: diff --git a/sphinx/config.py b/sphinx/config.py index 5741d66bf..61516a1ff 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -112,6 +112,9 @@ class Config(object): 'code-block': l_('Listing %s')}, 'env'), + tls_verify = (True, 'env'), + tls_cacerts = (None, 'env'), + # pre-initialized confval for HTML builder html_translator_class = (None, 'html', string_classes), ) diff --git a/sphinx/ext/intersphinx.py b/sphinx/ext/intersphinx.py index 4ef7e4b9b..b7cc849a4 100644 --- a/sphinx/ext/intersphinx.py +++ b/sphinx/ext/intersphinx.py @@ -41,7 +41,7 @@ from docutils.utils import relative_path import sphinx from sphinx.locale import _ from sphinx.builders.html import INVENTORY_FILENAME -from sphinx.util.requests import requests, useragent_header +from sphinx.util import requests UTF8StreamReader = codecs.lookup('utf-8')[2] @@ -145,7 +145,7 @@ def _strip_basic_auth(url): return urlunsplit(frags) -def _read_from_url(url, timeout=None): +def _read_from_url(url, config=None): """Reads data from *url* with an HTTP *GET*. This function supports fetching from resources which use basic HTTP auth as @@ -161,7 +161,7 @@ def _read_from_url(url, timeout=None): :return: data read from resource described by *url* :rtype: ``file``-like object """ - r = requests.get(url, stream=True, timeout=timeout, headers=dict(useragent_header)) + r = requests.get(url, stream=True, config=config, timeout=config.intersphinx_timeout) r.raise_for_status() r.raw.url = r.url return r.raw @@ -202,7 +202,7 @@ def fetch_inventory(app, uri, inv): uri = _strip_basic_auth(uri) try: if '://' in inv: - f = _read_from_url(inv, timeout=app.config.intersphinx_timeout) + f = _read_from_url(inv, config=app.config) else: f = open(path.join(app.srcdir, inv), 'rb') except Exception as err: diff --git a/sphinx/util/requests.py b/sphinx/util/requests.py index 36ac1e0e7..e2ac94e80 100644 --- a/sphinx/util/requests.py +++ b/sphinx/util/requests.py @@ -14,7 +14,10 @@ from __future__ import absolute_import import requests import warnings import pkg_resources -from requests.packages.urllib3.exceptions import SSLError + +from six import string_types +from six.moves.urllib.parse import urlsplit +from requests.packages.urllib3.exceptions import SSLError, InsecureRequestWarning # try to load requests[security] try: @@ -45,6 +48,7 @@ useragent_header = [('User-Agent', def is_ssl_error(exc): + """Check an exception is SSLError.""" if isinstance(exc, SSLError): return True else: @@ -53,3 +57,57 @@ def is_ssl_error(exc): return True else: return False + + +def _get_tls_cacert(url, config): + """Get addiotinal CA cert for a specific URL. + + This also returns ``False`` if verification is disabled. + And returns ``True`` if additional CA cert not found. + """ + if not config.tls_verify: + return False + + certs = getattr(config, 'tls_cacerts', None) + if not certs: + return True + elif isinstance(certs, (string_types, tuple)): + return certs + else: + hostname = urlsplit(url)[1] + if '@' in hostname: + hostname = hostname.split('@')[1] + + return certs.get(hostname, True) + + +def get(url, **kwargs): + """Sends a GET request like requests.get(). + + This sets up User-Agent header and TLS verification automatically.""" + kwargs.setdefault('headers', dict(useragent_header)) + config = kwargs.pop('config', None) + if config: + kwargs.setdefault('verify', _get_tls_cacert(url, config)) + + with warnings.catch_warnings(): + if not kwargs.get('verify'): + # ignore InsecureRequestWarning if verify=False + warnings.filterwarnings("ignore", category=InsecureRequestWarning) + return requests.get(url, **kwargs) + + +def head(url, **kwargs): + """Sends a HEAD request like requests.head(). + + This sets up User-Agent header and TLS verification automatically.""" + kwargs.setdefault('headers', dict(useragent_header)) + config = kwargs.pop('config', None) + if config: + kwargs.setdefault('verify', _get_tls_cacert(url, config)) + + with warnings.catch_warnings(): + if not kwargs.get('verify'): + # ignore InsecureRequestWarning if verify=False + warnings.filterwarnings("ignore", category=InsecureRequestWarning) + return requests.get(url, **kwargs) From 7a29b741a348b534a53a2983f0f240dba95a87ab Mon Sep 17 00:00:00 2001 From: jfbu Date: Thu, 17 Nov 2016 19:28:02 +0100 Subject: [PATCH 10/33] Update CHANGES for PR#3124 --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index 0a0606a5a..5c7fd36f6 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,9 @@ Features added Bugs fixed ---------- +* #3069: Even if ``'babel'`` key is set to empty string, LaTeX output contains + one ``\addto\captions...`` + Release 1.5 beta1 (released Nov 6, 2016) ======================================== From 12507384ae16e933032d3429cca9fddad0ea6d7a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 18 Nov 2016 11:38:48 +0900 Subject: [PATCH 11/33] Fix #3155: Fix JavaScript for `html_sourcelink_suffix` fails with IE and Opera --- CHANGES | 1 + sphinx/themes/basic/static/searchtools.js_t | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 5c7fd36f6..ffc18f18e 100644 --- a/CHANGES +++ b/CHANGES @@ -12,6 +12,7 @@ Bugs fixed * #3069: Even if ``'babel'`` key is set to empty string, LaTeX output contains one ``\addto\captions...`` +* #3155: Fix JavaScript for `html_sourcelink_suffix` fails with IE and Opera Release 1.5 beta1 (released Nov 6, 2016) ======================================== diff --git a/sphinx/themes/basic/static/searchtools.js_t b/sphinx/themes/basic/static/searchtools.js_t index f521c3794..3d434853f 100644 --- a/sphinx/themes/basic/static/searchtools.js_t +++ b/sphinx/themes/basic/static/searchtools.js_t @@ -261,7 +261,7 @@ var Search = { }); } else if (DOCUMENTATION_OPTIONS.HAS_SOURCE) { var suffix = DOCUMENTATION_OPTIONS.SOURCELINK_SUFFIX; - $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + (item[5].endsWith(suffix) ? '' : suffix), + $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + (item[5].slice(-suffix.length) === suffix ? '' : suffix), dataType: "text", complete: function(jqxhr, textstatus) { var data = jqxhr.responseText; From 8a1703c94463c4de53e31a304b6e25586d771b52 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 18 Nov 2016 12:11:01 +0900 Subject: [PATCH 12/33] Fix #2986: ``themes/basic/defindex.html`` is now deprecated --- CHANGES | 2 ++ sphinx/builders/html.py | 1 + sphinx/themes/basic/defindex.html | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index ffc18f18e..d154fc65a 100644 --- a/CHANGES +++ b/CHANGES @@ -4,6 +4,8 @@ Release 1.5 beta2 (in development) Incompatible changes -------------------- +* #2986: ``themes/basic/defindex.html`` is now deprecated + Features added -------------- diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index e13d752d7..50a5caef6 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -353,6 +353,7 @@ class StandaloneHTMLBuilder(Builder): parents = [], logo = logo, favicon = favicon, + warn = self.warn ) if self.theme: self.globalcontext.update( diff --git a/sphinx/themes/basic/defindex.html b/sphinx/themes/basic/defindex.html index 33becfa0d..190680724 100644 --- a/sphinx/themes/basic/defindex.html +++ b/sphinx/themes/basic/defindex.html @@ -6,7 +6,7 @@ :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. -#} +#}{{ warn('Now base template defindex.html is deprecated.') }} {%- extends "layout.html" %} {% set title = _('Overview') %} {% block body %} From 5a606ee96a9c14924c78a1e3072c72997baf267b Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 18 Nov 2016 13:49:31 +0900 Subject: [PATCH 13/33] Fix HTML helper method should set up in handle_page() --- sphinx/builders/html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 50a5caef6..4fbbff153 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -353,7 +353,6 @@ class StandaloneHTMLBuilder(Builder): parents = [], logo = logo, favicon = favicon, - warn = self.warn ) if self.theme: self.globalcontext.update( @@ -774,6 +773,7 @@ class StandaloneHTMLBuilder(Builder): def handle_page(self, pagename, addctx, templatename='page.html', outfilename=None, event_arg=None): ctx = self.globalcontext.copy() + ctx['warn'] = self.warn # current_page_name is backwards compatibility ctx['pagename'] = ctx['current_page_name'] = pagename default_baseuri = self.get_target_uri(pagename) From 9569c6dbff8e9bb2cc032ec4bc7ebdd50c4b570e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 17 Nov 2016 12:42:43 +0900 Subject: [PATCH 14/33] Add ``config-inited`` event --- sphinx/application.py | 6 +++++- sphinx/config.py | 9 +++++++++ sphinx/domains/std.py | 16 +++++++--------- sphinx/util/texescape.py | 10 ++++++++++ sphinx/writers/html.py | 2 +- sphinx/writers/latex.py | 23 +++++++++-------------- tests/roots/test-numfig/index.rst | 2 -- tests/test_build_html.py | 10 ---------- 8 files changed, 41 insertions(+), 37 deletions(-) diff --git a/sphinx/application.py b/sphinx/application.py index cee780c9c..c17cbb617 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -29,7 +29,7 @@ from docutils.parsers.rst import convert_directive_function, \ import sphinx from sphinx import package_dir, locale from sphinx.roles import XRefRole -from sphinx.config import Config +from sphinx.config import Config, convert_numfig_format from sphinx.errors import SphinxError, SphinxWarning, ExtensionError, \ VersionRequirementError, ConfigError from sphinx.domains import ObjType @@ -236,6 +236,7 @@ class Sphinx(object): self._init_i18n() # check all configuration values for permissible types self.config.check_types(self.warn) + self._post_init_config() # set up source_parsers self._init_source_parsers() # set up the build environment @@ -270,6 +271,9 @@ class Sphinx(object): else: self.info('not available for built-in messages') + def _post_init_config(self): + convert_numfig_format(self.config) + def _init_source_parsers(self): for suffix, parser in iteritems(self._additional_source_parsers): if suffix not in self.config.source_suffix: diff --git a/sphinx/config.py b/sphinx/config.py index 5741d66bf..1f1d6ec30 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -286,3 +286,12 @@ class Config(object): def __contains__(self, name): return name in self.values + + +def convert_numfig_format(config): + """ Convert numfi_format to new one. """ + + numfig_format = {} + for figtype, fmt in iteritems(config.numfig_format): + numfig_format[figtype] = fmt.replace('%s', '{number}') + config.numfig_format = numfig_format diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py index b7f2597d4..f83afdab1 100644 --- a/sphinx/domains/std.py +++ b/sphinx/domains/std.py @@ -673,20 +673,18 @@ class StandardDomain(Domain): else: title = env.config.numfig_format.get(figtype, '') - if figname is None and '%{name}' in title: + # convert old styled numfig_format to new style + title = title.replace('%s', '{number}') + + if figname is None and '{name}' in title: env.warn_node('the link has no caption: %s' % title, node) return contnode else: fignum = '.'.join(map(str, fignumber)) - if '{name}' in title or 'number' in title: - # new style format (cf. "Fig.%{number}") - if figname: - newtitle = title.format(name=figname, number=fignum) - else: - newtitle = title.format(number=fignum) + if figname: + newtitle = title.format(name=figname, number=fignum) else: - # old style format (cf. "Fig.%s") - newtitle = title % fignum + newtitle = title.format(number=fignum) except KeyError as exc: env.warn_node('invalid numfig_format: %s (%r)' % (title, exc), node) return contnode diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py index 0a3192f6a..861026b55 100644 --- a/sphinx/util/texescape.py +++ b/sphinx/util/texescape.py @@ -118,10 +118,17 @@ tex_replacements = [ ('Ω', r'\(\Omega\)'), ('Ω', r'\(\Omega\)'), ] +tex_pyformats = [ + # map special chars of str.format() to escaped one + ('{', '{{'), + ('}', '}}'), +] + tex_escape_map = {} tex_replace_map = {} tex_hl_escape_map_new = {} +tex_pyformat_map = {} def init(): @@ -133,3 +140,6 @@ def init(): if a in '[]{}\\': continue tex_hl_escape_map_new[ord(a)] = b + + for a, b in tex_pyformats: + tex_pyformat_map[ord(a)] = b diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py index ba2b758d8..7dd286cf7 100644 --- a/sphinx/writers/html.py +++ b/sphinx/writers/html.py @@ -291,7 +291,7 @@ class HTMLTranslator(BaseTranslator): self.builder.warn(msg) else: numbers = self.builder.fignumbers[key][figure_id] - self.body.append(prefix % '.'.join(map(str, numbers)) + ' ') + self.body.append(prefix.format(number='.'.join(map(str, numbers))) + ' ') self.body.append('') figtype = self.builder.env.domains['std'].get_figtype(node) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index ca0f1a7dd..0a158297b 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -29,7 +29,7 @@ from sphinx.util import split_into from sphinx.util.i18n import format_date from sphinx.util.nodes import clean_astext, traverse_parent from sphinx.util.template import LaTeXRenderer -from sphinx.util.texescape import tex_escape_map, tex_replace_map +from sphinx.util.texescape import tex_escape_map, tex_replace_map, tex_pyformat_map from sphinx.util.smartypants import educate_quotes_latex @@ -610,7 +610,7 @@ class LaTeXTranslator(nodes.NodeVisitor): def generate_numfig_format(self, builder): ret = [] - figure = self.builder.config.numfig_format['figure'].split('%s', 1) + figure = self.builder.config.numfig_format['figure'].split('{number}', 1) if len(figure) == 1: ret.append('\\def\\fnum@figure{%s}\n' % escape_abbr(text_type(figure[0]).translate(tex_escape_map))) @@ -623,7 +623,7 @@ class LaTeXTranslator(nodes.NodeVisitor): escape_abbr(text_type(figure[1]).translate(tex_escape_map))) ret.append('\\makeatother\n') - table = self.builder.config.numfig_format['table'].split('%s', 1) + table = self.builder.config.numfig_format['table'].split('{number}', 1) if len(table) == 1: ret.append('\\def\\fnum@table{%s}\n' % escape_abbr(text_type(table[0]).translate(tex_escape_map))) @@ -636,7 +636,7 @@ class LaTeXTranslator(nodes.NodeVisitor): escape_abbr(text_type(table[1]).translate(tex_escape_map))) ret.append('\\makeatother\n') - codeblock = self.builder.config.numfig_format['code-block'].split('%s', 1) + codeblock = self.builder.config.numfig_format['code-block'].split('{number}', 1) if len(codeblock) == 1: pass # FIXME else: @@ -1789,16 +1789,11 @@ class LaTeXTranslator(nodes.NodeVisitor): else: id = node.get('refuri', '')[1:].replace('#', ':') - title = node.get('title', '%s') - title = text_type(title).translate(tex_escape_map).replace('\\%s', '%s') - if '\\{name\\}' in title or '\\{number\\}' in title: - # new style format (cf. "Fig.%{number}") - title = title.replace('\\{name\\}', '{name}').replace('\\{number\\}', '{number}') - text = escape_abbr(title).format(name='\\nameref{%s}' % self.idescape(id), - number='\\ref{%s}' % self.idescape(id)) - else: - # old style format (cf. "Fig.%{number}") - text = escape_abbr(title) % ('\\ref{%s}' % self.idescape(id)) + title = node.get('title', '{number}').replace('%s', '{number}') + title = text_type(title).translate(tex_escape_map).translate(tex_pyformat_map) + title = title.replace('\\{{name\\}}', '{name}').replace('\\{{number\\}}', '{number}') + text = escape_abbr(title).format(name='\\nameref{%s}' % self.idescape(id), + number='\\ref{%s}' % self.idescape(id)) hyperref = '\\hyperref[%s]{%s}' % (self.idescape(id), text) self.body.append(hyperref) diff --git a/tests/roots/test-numfig/index.rst b/tests/roots/test-numfig/index.rst index 939903839..499a1b7dc 100644 --- a/tests/roots/test-numfig/index.rst +++ b/tests/roots/test-numfig/index.rst @@ -53,7 +53,5 @@ test-tocdepth * Section.1 is :numref:`foo` * Section.2.1 is :numref:`bar_a` * Unnumbered section is :numref:`index` -* Invalid numfig_format 01: :numref:`invalid ` -* Invalid numfig_format 02: :numref:`Fig %s %s ` * Fig.1 is :numref:`Fig.{number} {name} ` * Section.1 is :numref:`Sect.{number} {name} ` diff --git a/tests/test_build_html.py b/tests/test_build_html.py index d8aff88ab..176590f75 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -506,8 +506,6 @@ def test_numfig_disabled(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' not in warnings - assert 'index.rst:56: WARNING: invalid numfig_format: invalid' not in warnings - assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' not in warnings expects = { 'index.html': [ @@ -570,8 +568,6 @@ def test_numfig_without_numbered_toctree(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' in warnings - assert 'index.rst:56: WARNING: invalid numfig_format: invalid' in warnings - assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' in warnings expects = { 'index.html': [ @@ -670,8 +666,6 @@ def test_numfig_with_numbered_toctree(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' in warnings - assert 'index.rst:56: WARNING: invalid numfig_format: invalid' in warnings - assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' in warnings expects = { 'index.html': [ @@ -774,8 +768,6 @@ def test_numfig_with_prefix(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' in warnings - assert 'index.rst:56: WARNING: invalid numfig_format: invalid' in warnings - assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' in warnings expects = { 'index.html': [ @@ -874,8 +866,6 @@ def test_numfig_with_secnum_depth(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' in warnings - assert 'index.rst:56: WARNING: invalid numfig_format: invalid' in warnings - assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' in warnings expects = { 'index.html': [ From 987ebe4e1798ce6d23d4a27005ec87724f62ebd4 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 19 Nov 2016 13:00:11 +0900 Subject: [PATCH 15/33] Update CHANGES for PR#3124 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index d154fc65a..fba9c997a 100644 --- a/CHANGES +++ b/CHANGES @@ -14,6 +14,7 @@ Bugs fixed * #3069: Even if ``'babel'`` key is set to empty string, LaTeX output contains one ``\addto\captions...`` +* #3123: user ``'babel'`` key setting is not obeyed anymore * #3155: Fix JavaScript for `html_sourcelink_suffix` fails with IE and Opera Release 1.5 beta1 (released Nov 6, 2016) From d51ab948c0f3e645849173e20c29d13a04ecefa3 Mon Sep 17 00:00:00 2001 From: jfbu Date: Sat, 19 Nov 2016 09:47:16 +0100 Subject: [PATCH 16/33] Add missing trailing space in French translation --- sphinx/locale/fr/LC_MESSAGES/sphinx.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/locale/fr/LC_MESSAGES/sphinx.po b/sphinx/locale/fr/LC_MESSAGES/sphinx.po index c476663be..2778dffec 100644 --- a/sphinx/locale/fr/LC_MESSAGES/sphinx.po +++ b/sphinx/locale/fr/LC_MESSAGES/sphinx.po @@ -112,7 +112,7 @@ msgstr "Auteur du module : " #: sphinx/directives/other.py:153 msgid "Code author: " -msgstr "Auteur du code :" +msgstr "Auteur du code : " #: sphinx/directives/other.py:155 msgid "Author: " From 873fab53b4e4e995b679e79cae516eda98f75b4e Mon Sep 17 00:00:00 2001 From: Yoshiki Shibukawa Date: Wed, 16 Nov 2016 11:59:04 +0900 Subject: [PATCH 17/33] fix #3150 --- .gitignore | 1 + CHANGES | 1 + sphinx/search/__init__.py | 4 +- sphinx/search/jssplitter.py | 110 +++++++++++++++ sphinx/themes/basic/static/searchtools.js_t | 10 +- utils/jssplitter_generator.py | 142 ++++++++++++++++++++ 6 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 sphinx/search/jssplitter.py create mode 100644 utils/jssplitter_generator.py diff --git a/.gitignore b/.gitignore index be28908ec..9d163ff7b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ Sphinx.egg-info/ doc/_build/ tests/.coverage tests/build/ +utils/regression_test.js diff --git a/CHANGES b/CHANGES index 1120e0d7b..49b0ca941 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,7 @@ Bugs fixed * #3068: Allow the '=' character in the -D option of sphinx-build.py * #3074: ``add_source_parser()`` crashes in debug mode * #3135: ``sphinx.ext.autodoc`` crashes with plain Callable +* #3150: Fix query word splitter in JavaScript. It behaves as same as Python's regular expression. Release 1.4.8 (released Oct 1, 2016) ==================================== diff --git a/sphinx/search/__init__.py b/sphinx/search/__init__.py index 1fedc5352..7f8803fd2 100644 --- a/sphinx/search/__init__.py +++ b/sphinx/search/__init__.py @@ -17,7 +17,7 @@ from os import path from sphinx.util import jsdump, rpartition from sphinx.util.pycompat import htmlescape - +from sphinx.search.jssplitter import splitter_code class SearchLanguage(object): """ @@ -241,6 +241,7 @@ class IndexBuilder(object): self.js_scorer_code = fp.read().decode('utf-8') else: self.js_scorer_code = u'' + self.js_splitter_code = splitter_code def load(self, stream, format): """Reconstruct from frozen data.""" @@ -381,6 +382,7 @@ class IndexBuilder(object): search_language_stemming_code = self.lang.js_stemmer_code, search_language_stop_words = jsdump.dumps(sorted(self.lang.stopwords)), search_scorer_tool = self.js_scorer_code, + search_word_splitter_code = self.js_splitter_code, ) def get_js_stemmer_rawcode(self): diff --git a/sphinx/search/jssplitter.py b/sphinx/search/jssplitter.py new file mode 100644 index 000000000..a3bd8b767 --- /dev/null +++ b/sphinx/search/jssplitter.py @@ -0,0 +1,110 @@ + +"""# -*- coding: utf-8 -*- + sphinx.search.jssplitter + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Provides Python compatible word splitter to JavaScript + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. + + DO NOT EDIT. This is generated by utils/jssplitter_generator.py +""" + +splitter_code = """ + +var splitChars = (function() { + var result = {}; + var singles = [96, 180, 187, 191, 215, 247, 749, 885, 903, 907, 909, 930, 1014, 1648, + 1748, 1809, 2416, 2473, 2481, 2526, 2601, 2609, 2612, 2615, 2653, 2702, + 2706, 2729, 2737, 2740, 2857, 2865, 2868, 2910, 2928, 2948, 2961, 2971, + 2973, 3085, 3089, 3113, 3124, 3213, 3217, 3241, 3252, 3295, 3341, 3345, + 3369, 3506, 3516, 3633, 3715, 3721, 3736, 3744, 3748, 3750, 3756, 3761, + 3781, 3912, 4239, 4347, 4681, 4695, 4697, 4745, 4785, 4799, 4801, 4823, + 4881, 5760, 5901, 5997, 6313, 7405, 8024, 8026, 8028, 8030, 8117, 8125, + 8133, 8181, 8468, 8485, 8487, 8489, 8494, 8527, 11311, 11359, 11687, 11695, + 11703, 11711, 11719, 11727, 11735, 12448, 12539, 43010, 43014, 43019, 43587, + 43696, 43713, 64286, 64297, 64311, 64317, 64319, 64322, 64325, 65141]; + var i, j, start, end; + for (i = 0; i < singles.length; i++) { + result[singles[i]] = true; + } + var ranges = [[0, 47], [58, 64], [91, 94], [123, 169], [171, 177], [182, 184], [706, 709], + [722, 735], [741, 747], [751, 879], [888, 889], [894, 901], [1154, 1161], + [1318, 1328], [1367, 1368], [1370, 1376], [1416, 1487], [1515, 1519], [1523, 1568], + [1611, 1631], [1642, 1645], [1750, 1764], [1767, 1773], [1789, 1790], [1792, 1807], + [1840, 1868], [1958, 1968], [1970, 1983], [2027, 2035], [2038, 2041], [2043, 2047], + [2070, 2073], [2075, 2083], [2085, 2087], [2089, 2307], [2362, 2364], [2366, 2383], + [2385, 2391], [2402, 2405], [2419, 2424], [2432, 2436], [2445, 2446], [2449, 2450], + [2483, 2485], [2490, 2492], [2494, 2509], [2511, 2523], [2530, 2533], [2546, 2547], + [2554, 2564], [2571, 2574], [2577, 2578], [2618, 2648], [2655, 2661], [2672, 2673], + [2677, 2692], [2746, 2748], [2750, 2767], [2769, 2783], [2786, 2789], [2800, 2820], + [2829, 2830], [2833, 2834], [2874, 2876], [2878, 2907], [2914, 2917], [2930, 2946], + [2955, 2957], [2966, 2968], [2976, 2978], [2981, 2983], [2987, 2989], [3002, 3023], + [3025, 3045], [3059, 3076], [3130, 3132], [3134, 3159], [3162, 3167], [3170, 3173], + [3184, 3191], [3199, 3204], [3258, 3260], [3262, 3293], [3298, 3301], [3312, 3332], + [3386, 3388], [3390, 3423], [3426, 3429], [3446, 3449], [3456, 3460], [3479, 3481], + [3518, 3519], [3527, 3584], [3636, 3647], [3655, 3663], [3674, 3712], [3717, 3718], + [3723, 3724], [3726, 3731], [3752, 3753], [3764, 3772], [3774, 3775], [3783, 3791], + [3802, 3803], [3806, 3839], [3841, 3871], [3892, 3903], [3949, 3975], [3980, 4095], + [4139, 4158], [4170, 4175], [4182, 4185], [4190, 4192], [4194, 4196], [4199, 4205], + [4209, 4212], [4226, 4237], [4250, 4255], [4294, 4303], [4349, 4351], [4686, 4687], + [4702, 4703], [4750, 4751], [4790, 4791], [4806, 4807], [4886, 4887], [4955, 4968], + [4989, 4991], [5008, 5023], [5109, 5120], [5741, 5742], [5787, 5791], [5867, 5869], + [5873, 5887], [5906, 5919], [5938, 5951], [5970, 5983], [6001, 6015], [6068, 6102], + [6104, 6107], [6109, 6111], [6122, 6127], [6138, 6159], [6170, 6175], [6264, 6271], + [6315, 6319], [6390, 6399], [6429, 6469], [6510, 6511], [6517, 6527], [6572, 6592], + [6600, 6607], [6619, 6655], [6679, 6687], [6741, 6783], [6794, 6799], [6810, 6822], + [6824, 6916], [6964, 6980], [6988, 6991], [7002, 7042], [7073, 7085], [7098, 7167], + [7204, 7231], [7242, 7244], [7294, 7400], [7410, 7423], [7616, 7679], [7958, 7959], + [7966, 7967], [8006, 8007], [8014, 8015], [8062, 8063], [8127, 8129], [8141, 8143], + [8148, 8149], [8156, 8159], [8173, 8177], [8189, 8303], [8306, 8307], [8314, 8318], + [8330, 8335], [8341, 8449], [8451, 8454], [8456, 8457], [8470, 8472], [8478, 8483], + [8506, 8507], [8512, 8516], [8522, 8525], [8586, 9311], [9372, 9449], [9472, 10101], + [10132, 11263], [11493, 11498], [11503, 11516], [11518, 11519], [11558, 11567], + [11622, 11630], [11632, 11647], [11671, 11679], [11743, 11822], [11824, 12292], + [12296, 12320], [12330, 12336], [12342, 12343], [12349, 12352], [12439, 12444], + [12544, 12548], [12590, 12592], [12687, 12689], [12694, 12703], [12728, 12783], + [12800, 12831], [12842, 12880], [12896, 12927], [12938, 12976], [12992, 13311], + [19894, 19967], [40908, 40959], [42125, 42191], [42238, 42239], [42509, 42511], + [42540, 42559], [42592, 42593], [42607, 42622], [42648, 42655], [42736, 42774], + [42784, 42785], [42889, 42890], [42893, 43002], [43043, 43055], [43062, 43071], + [43124, 43137], [43188, 43215], [43226, 43249], [43256, 43258], [43260, 43263], + [43302, 43311], [43335, 43359], [43389, 43395], [43443, 43470], [43482, 43519], + [43561, 43583], [43596, 43599], [43610, 43615], [43639, 43641], [43643, 43647], + [43698, 43700], [43703, 43704], [43710, 43711], [43715, 43738], [43742, 43967], + [44003, 44015], [44026, 44031], [55204, 55215], [55239, 55242], [55292, 55295], + [57344, 63743], [64046, 64047], [64110, 64111], [64218, 64255], [64263, 64274], + [64280, 64284], [64434, 64466], [64830, 64847], [64912, 64913], [64968, 65007], + [65020, 65135], [65277, 65295], [65306, 65312], [65339, 65344], [65371, 65381], + [65471, 65473], [65480, 65481], [65488, 65489], [65496, 65497]]; + for (i = 0; i < ranges.length; i++) { + start = ranges[i][0]; + end = ranges[i][1]; + for (j = start; j <= end; j++) { + result[j] = true; + } + } + return result; +})(); + +function splitQuery(query) { + var result = []; + var start = -1; + for (var i = 0; i < query.length; i++) { + if (splitChars[query.charCodeAt(i)]) { + if (start !== -1) { + result.push(query.slice(start, i)); + start = -1; + } + } else if (start === -1) { + start = i; + } + } + if (start !== -1) { + result.push(query.slice(start)); + } + return result; +} + +""" diff --git a/sphinx/themes/basic/static/searchtools.js_t b/sphinx/themes/basic/static/searchtools.js_t index 45ff1e4de..6d1dbc1b6 100644 --- a/sphinx/themes/basic/static/searchtools.js_t +++ b/sphinx/themes/basic/static/searchtools.js_t @@ -47,6 +47,14 @@ var Scorer = { }; {% endif %} +{% if search_word_splitter_code %} +{{ search_word_splitter_code }} +{% else %} +function splitQuery(query) { + return query.split(/\s+/); +} +{% endif %} + /** * Search Module */ @@ -145,7 +153,7 @@ var Search = { var searchterms = []; var excluded = []; var hlterms = []; - var tmp = query.split(/\W+/); + var tmp = splitQuery(query); var objectterms = []; for (i = 0; i < tmp.length; i++) { if (tmp[i] !== "") { diff --git a/utils/jssplitter_generator.py b/utils/jssplitter_generator.py new file mode 100644 index 000000000..6eef86cc3 --- /dev/null +++ b/utils/jssplitter_generator.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +import re +import json +import subprocess +import sys +import six + +# find char codes they are matched with Python's \\w(?u) + +match = re.compile(r'\w(?u)') +begin = -1 + +ranges = [] +singles = [] + +for i in range(65536): + # 0xd800-0xdfff is surrogate pair area. skip this. + if not match.match(six.unichr(i)) and not (0xd800 <= i <= 0xdfff): + if begin == -1: + begin = i + elif begin != -1: + if begin + 1 == i: + singles.append(begin) + else: + ranges.append((begin, i - 1)) + begin = -1 + + +# fold json within almost 80 chars per line +def fold(jsonData, splitter): + code = json.dumps(jsonData) + lines = [] + while True: + if len(code) < 71: + lines.append(' ' + code) + break + index = code.index(splitter, 70) + lines.append(' ' + code[:index+len(splitter)]) + code = code[index+len(splitter):] + lines[0] = lines[0][8:] + return '\n'.join(lines) + + +# JavaScript code +js_src = ''' +var splitChars = (function() { + var result = {}; + var singles = %s; + var i, j, start, end; + for (i = 0; i < singles.length; i++) { + result[singles[i]] = true; + } + var ranges = %s; + for (i = 0; i < ranges.length; i++) { + start = ranges[i][0]; + end = ranges[i][1]; + for (j = start; j <= end; j++) { + result[j] = true; + } + } + return result; +})(); + +function splitQuery(query) { + var result = []; + var start = -1; + for (var i = 0; i < query.length; i++) { + if (splitChars[query.charCodeAt(i)]) { + if (start !== -1) { + result.push(query.slice(start, i)); + start = -1; + } + } else if (start === -1) { + start = i; + } + } + if (start !== -1) { + result.push(query.slice(start)); + } + return result; +} +''' % (fold(singles, ','), fold(ranges, '],')) + +js_test_src = u''' +// This is regression test for https://github.com/sphinx-doc/sphinx/issues/3150 +// generated by compat_regexp_generator.py +// it needs node.js for testing +var assert = require('assert'); + +%s + +console.log("test splitting English words") +assert.deepEqual(['Hello', 'World'], splitQuery(' Hello World ')); +console.log(' ... ok\\n') + +console.log("test splitting special characters") +assert.deepEqual(['Pin', 'Code'], splitQuery('Pin-Code')); +console.log(' ... ok\\n') + +console.log("test splitting Chinese characters") +assert.deepEqual(['Hello', 'from', '中国', '上海'], splitQuery('Hello from 中国 上海')); +console.log(' ... ok\\n') + +console.log("test splitting Emoji(surrogate pair) characters. It should keep emojis.") +assert.deepEqual(['😁😁'], splitQuery('😁😁')); +console.log(' ... ok\\n') + +console.log("test splitting umlauts. It should keep umlauts.") +assert.deepEqual( + ['Löschen', 'Prüfung', 'Abändern', 'ærlig', 'spørsmål'], + splitQuery('Löschen Prüfung Abändern ærlig spørsmål')); +console.log(' ... ok\\n') + +''' % js_src + +python_src = ''' +"""# -*- coding: utf-8 -*- + sphinx.search.jssplitter + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Provides Python compatible word splitter to JavaScript + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. + + DO NOT EDIT. This is generated by utils/jssplitter_generator.py +""" + +splitter_code = """ +%s +""" +''' % js_src + +with open('../sphinx/search/jssplitter.py', 'w') as f: + f.write(python_src) + +with open('./regression_test.js', 'w') as f: + f.write(js_test_src.encode('utf-8')) + +print("starting test...") +result = subprocess.call(['node', './regression_test.js']) +sys.exit(result) From b84035416f2e5baeeabcb28cd594ae5f09ed6f0c Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 20 Nov 2016 12:45:31 +0900 Subject: [PATCH 18/33] Revert "Add ``config-inited`` event" This reverts commit 9569c6dbff8e9bb2cc032ec4bc7ebdd50c4b570e. --- sphinx/application.py | 6 +----- sphinx/config.py | 9 --------- sphinx/domains/std.py | 16 +++++++++------- sphinx/util/texescape.py | 10 ---------- sphinx/writers/html.py | 2 +- sphinx/writers/latex.py | 23 ++++++++++++++--------- tests/roots/test-numfig/index.rst | 2 ++ tests/test_build_html.py | 10 ++++++++++ 8 files changed, 37 insertions(+), 41 deletions(-) diff --git a/sphinx/application.py b/sphinx/application.py index c17cbb617..cee780c9c 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -29,7 +29,7 @@ from docutils.parsers.rst import convert_directive_function, \ import sphinx from sphinx import package_dir, locale from sphinx.roles import XRefRole -from sphinx.config import Config, convert_numfig_format +from sphinx.config import Config from sphinx.errors import SphinxError, SphinxWarning, ExtensionError, \ VersionRequirementError, ConfigError from sphinx.domains import ObjType @@ -236,7 +236,6 @@ class Sphinx(object): self._init_i18n() # check all configuration values for permissible types self.config.check_types(self.warn) - self._post_init_config() # set up source_parsers self._init_source_parsers() # set up the build environment @@ -271,9 +270,6 @@ class Sphinx(object): else: self.info('not available for built-in messages') - def _post_init_config(self): - convert_numfig_format(self.config) - def _init_source_parsers(self): for suffix, parser in iteritems(self._additional_source_parsers): if suffix not in self.config.source_suffix: diff --git a/sphinx/config.py b/sphinx/config.py index a5761e349..61516a1ff 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -289,12 +289,3 @@ class Config(object): def __contains__(self, name): return name in self.values - - -def convert_numfig_format(config): - """ Convert numfi_format to new one. """ - - numfig_format = {} - for figtype, fmt in iteritems(config.numfig_format): - numfig_format[figtype] = fmt.replace('%s', '{number}') - config.numfig_format = numfig_format diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py index f83afdab1..b7f2597d4 100644 --- a/sphinx/domains/std.py +++ b/sphinx/domains/std.py @@ -673,18 +673,20 @@ class StandardDomain(Domain): else: title = env.config.numfig_format.get(figtype, '') - # convert old styled numfig_format to new style - title = title.replace('%s', '{number}') - - if figname is None and '{name}' in title: + if figname is None and '%{name}' in title: env.warn_node('the link has no caption: %s' % title, node) return contnode else: fignum = '.'.join(map(str, fignumber)) - if figname: - newtitle = title.format(name=figname, number=fignum) + if '{name}' in title or 'number' in title: + # new style format (cf. "Fig.%{number}") + if figname: + newtitle = title.format(name=figname, number=fignum) + else: + newtitle = title.format(number=fignum) else: - newtitle = title.format(number=fignum) + # old style format (cf. "Fig.%s") + newtitle = title % fignum except KeyError as exc: env.warn_node('invalid numfig_format: %s (%r)' % (title, exc), node) return contnode diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py index 861026b55..0a3192f6a 100644 --- a/sphinx/util/texescape.py +++ b/sphinx/util/texescape.py @@ -118,17 +118,10 @@ tex_replacements = [ ('Ω', r'\(\Omega\)'), ('Ω', r'\(\Omega\)'), ] -tex_pyformats = [ - # map special chars of str.format() to escaped one - ('{', '{{'), - ('}', '}}'), -] - tex_escape_map = {} tex_replace_map = {} tex_hl_escape_map_new = {} -tex_pyformat_map = {} def init(): @@ -140,6 +133,3 @@ def init(): if a in '[]{}\\': continue tex_hl_escape_map_new[ord(a)] = b - - for a, b in tex_pyformats: - tex_pyformat_map[ord(a)] = b diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py index 7dd286cf7..ba2b758d8 100644 --- a/sphinx/writers/html.py +++ b/sphinx/writers/html.py @@ -291,7 +291,7 @@ class HTMLTranslator(BaseTranslator): self.builder.warn(msg) else: numbers = self.builder.fignumbers[key][figure_id] - self.body.append(prefix.format(number='.'.join(map(str, numbers))) + ' ') + self.body.append(prefix % '.'.join(map(str, numbers)) + ' ') self.body.append('') figtype = self.builder.env.domains['std'].get_figtype(node) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 0a158297b..ca0f1a7dd 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -29,7 +29,7 @@ from sphinx.util import split_into from sphinx.util.i18n import format_date from sphinx.util.nodes import clean_astext, traverse_parent from sphinx.util.template import LaTeXRenderer -from sphinx.util.texescape import tex_escape_map, tex_replace_map, tex_pyformat_map +from sphinx.util.texescape import tex_escape_map, tex_replace_map from sphinx.util.smartypants import educate_quotes_latex @@ -610,7 +610,7 @@ class LaTeXTranslator(nodes.NodeVisitor): def generate_numfig_format(self, builder): ret = [] - figure = self.builder.config.numfig_format['figure'].split('{number}', 1) + figure = self.builder.config.numfig_format['figure'].split('%s', 1) if len(figure) == 1: ret.append('\\def\\fnum@figure{%s}\n' % escape_abbr(text_type(figure[0]).translate(tex_escape_map))) @@ -623,7 +623,7 @@ class LaTeXTranslator(nodes.NodeVisitor): escape_abbr(text_type(figure[1]).translate(tex_escape_map))) ret.append('\\makeatother\n') - table = self.builder.config.numfig_format['table'].split('{number}', 1) + table = self.builder.config.numfig_format['table'].split('%s', 1) if len(table) == 1: ret.append('\\def\\fnum@table{%s}\n' % escape_abbr(text_type(table[0]).translate(tex_escape_map))) @@ -636,7 +636,7 @@ class LaTeXTranslator(nodes.NodeVisitor): escape_abbr(text_type(table[1]).translate(tex_escape_map))) ret.append('\\makeatother\n') - codeblock = self.builder.config.numfig_format['code-block'].split('{number}', 1) + codeblock = self.builder.config.numfig_format['code-block'].split('%s', 1) if len(codeblock) == 1: pass # FIXME else: @@ -1789,11 +1789,16 @@ class LaTeXTranslator(nodes.NodeVisitor): else: id = node.get('refuri', '')[1:].replace('#', ':') - title = node.get('title', '{number}').replace('%s', '{number}') - title = text_type(title).translate(tex_escape_map).translate(tex_pyformat_map) - title = title.replace('\\{{name\\}}', '{name}').replace('\\{{number\\}}', '{number}') - text = escape_abbr(title).format(name='\\nameref{%s}' % self.idescape(id), - number='\\ref{%s}' % self.idescape(id)) + title = node.get('title', '%s') + title = text_type(title).translate(tex_escape_map).replace('\\%s', '%s') + if '\\{name\\}' in title or '\\{number\\}' in title: + # new style format (cf. "Fig.%{number}") + title = title.replace('\\{name\\}', '{name}').replace('\\{number\\}', '{number}') + text = escape_abbr(title).format(name='\\nameref{%s}' % self.idescape(id), + number='\\ref{%s}' % self.idescape(id)) + else: + # old style format (cf. "Fig.%{number}") + text = escape_abbr(title) % ('\\ref{%s}' % self.idescape(id)) hyperref = '\\hyperref[%s]{%s}' % (self.idescape(id), text) self.body.append(hyperref) diff --git a/tests/roots/test-numfig/index.rst b/tests/roots/test-numfig/index.rst index 499a1b7dc..939903839 100644 --- a/tests/roots/test-numfig/index.rst +++ b/tests/roots/test-numfig/index.rst @@ -53,5 +53,7 @@ test-tocdepth * Section.1 is :numref:`foo` * Section.2.1 is :numref:`bar_a` * Unnumbered section is :numref:`index` +* Invalid numfig_format 01: :numref:`invalid ` +* Invalid numfig_format 02: :numref:`Fig %s %s ` * Fig.1 is :numref:`Fig.{number} {name} ` * Section.1 is :numref:`Sect.{number} {name} ` diff --git a/tests/test_build_html.py b/tests/test_build_html.py index 176590f75..d8aff88ab 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -506,6 +506,8 @@ def test_numfig_disabled(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' not in warnings + assert 'index.rst:56: WARNING: invalid numfig_format: invalid' not in warnings + assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' not in warnings expects = { 'index.html': [ @@ -568,6 +570,8 @@ def test_numfig_without_numbered_toctree(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' in warnings + assert 'index.rst:56: WARNING: invalid numfig_format: invalid' in warnings + assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' in warnings expects = { 'index.html': [ @@ -666,6 +670,8 @@ def test_numfig_with_numbered_toctree(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' in warnings + assert 'index.rst:56: WARNING: invalid numfig_format: invalid' in warnings + assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' in warnings expects = { 'index.html': [ @@ -768,6 +774,8 @@ def test_numfig_with_prefix(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' in warnings + assert 'index.rst:56: WARNING: invalid numfig_format: invalid' in warnings + assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' in warnings expects = { 'index.html': [ @@ -866,6 +874,8 @@ def test_numfig_with_secnum_depth(app, status, warning): warnings = warning.getvalue() assert 'index.rst:47: WARNING: numfig is disabled. :numref: is ignored.' not in warnings assert 'index.rst:55: WARNING: no number is assigned for section: index' in warnings + assert 'index.rst:56: WARNING: invalid numfig_format: invalid' in warnings + assert 'index.rst:57: WARNING: invalid numfig_format: Fig %s %s' in warnings expects = { 'index.html': [ From 7c19a2f36a6931fd8c2bb36fddff4519f064b8e2 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 20 Nov 2016 12:46:05 +0900 Subject: [PATCH 19/33] Fix #3118: Update document for numfig_format and numref --- doc/config.rst | 6 +----- doc/markup/inline.rst | 6 +++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/config.rst b/doc/config.rst index 7f1f73186..a3f45abe9 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -297,8 +297,7 @@ General configuration A dictionary mapping ``'figure'``, ``'table'``, ``'code-block'`` and ``'section'`` to strings that are used for format of figure numbers. - As a special character, `%s` and `{number}` will be replaced to figure - number. `{name}` will be replaced to figure caption. + As a special character, `%s` will be replaced to figure number. Default is to use ``'Fig. %s'`` for ``'figure'``, ``'Table %s'`` for ``'table'``, ``'Listing %s'`` for ``'code-block'`` and ``'Section'`` for @@ -306,9 +305,6 @@ General configuration .. versionadded:: 1.3 - .. versionchanged:: 1.5 - Support format of section. Allow to refer the caption of figures. - .. confval:: numfig_secnum_depth The scope of figure numbers, that is, the numfig feature numbers figures diff --git a/doc/markup/inline.rst b/doc/markup/inline.rst index bd02dfa08..a59585bab 100644 --- a/doc/markup/inline.rst +++ b/doc/markup/inline.rst @@ -214,6 +214,7 @@ Cross-referencing figures by figure number .. versionchanged:: 1.5 `numref` role can also refer sections. + And `numref` allows `{name}` for the link text. .. rst:role:: numref @@ -223,7 +224,10 @@ Cross-referencing figures by figure number If an explicit link text is given (like usual: ``:numref:`Image of Sphinx (Fig. %s) ```), the link caption will be the title of the reference. - The format of link text is same as :confval:`numfig_format`. + As a special character, `%s` and `{number}` will be replaced to figure + number. `{name}` will be replaced to figure caption. + If no explicit link text is given, the value of :confval:`numfig_format` is + used to default value of link text. If :confval:`numfig` is ``False``, figures are not numbered. so this role inserts not a reference but labels or link text. From a13edb86318b21e224cf1365fc9c5edb05df18dc Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 20 Nov 2016 17:32:05 +0900 Subject: [PATCH 20/33] refs #3093: gettext build broken on substituted images. --- CHANGES | 1 + sphinx/transforms.py | 4 ++-- sphinx/util/nodes.py | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 49b0ca941..ef2997d10 100644 --- a/CHANGES +++ b/CHANGES @@ -11,6 +11,7 @@ Bugs fixed * #3074: ``add_source_parser()`` crashes in debug mode * #3135: ``sphinx.ext.autodoc`` crashes with plain Callable * #3150: Fix query word splitter in JavaScript. It behaves as same as Python's regular expression. +* #3093: gettext build broken on substituted images. Release 1.4.8 (released Oct 1, 2016) ==================================== diff --git a/sphinx/transforms.py b/sphinx/transforms.py index cb4a5779b..ba15ae614 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -166,7 +166,7 @@ class ApplySourceWorkaround(Transform): def apply(self): for n in self.document.traverse(): - if isinstance(n, nodes.TextElement): + if isinstance(n, (nodes.TextElement, nodes.image)): apply_source_workaround(n) @@ -192,7 +192,7 @@ class ExtraTranslatableNodes(Transform): """ make nodes translatable """ - default_priority = 10 + default_priority = 9 def apply(self): targets = self.document.settings.env.config.gettext_additional_targets diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index e4a2fd73b..e51ad2972 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -46,6 +46,8 @@ def apply_source_workaround(node): node.source = definition_list_item.source node.line = definition_list_item.line - 1 node.rawsource = node.astext() # set 'classifier1' (or 'classifier2') + if isinstance(node, nodes.image) and node.source is None: + node.source, node.line = node.parent.source, node.parent.line if isinstance(node, nodes.term): # strip classifier from rawsource of term for classifier in reversed(node.parent.traverse(nodes.classifier)): From 76eb8ab56d49c6fbe28e03934fdd60ad92459452 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 20 Nov 2016 17:44:10 +0900 Subject: [PATCH 21/33] * #3093: gettext build broken on image node under ``note`` directive. --- CHANGES | 1 + sphinx/util/nodes.py | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index ef2997d10..0ae8cb26d 100644 --- a/CHANGES +++ b/CHANGES @@ -12,6 +12,7 @@ Bugs fixed * #3135: ``sphinx.ext.autodoc`` crashes with plain Callable * #3150: Fix query word splitter in JavaScript. It behaves as same as Python's regular expression. * #3093: gettext build broken on substituted images. +* #3093: gettext build broken on image node under ``note`` directive. Release 1.4.8 (released Oct 1, 2016) ==================================== diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index e51ad2972..b3f7efa8e 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -71,6 +71,7 @@ def apply_source_workaround(node): nodes.title, nodes.rubric, nodes.line, + nodes.image, ))): node.source = find_source_node(node) node.line = 0 # need fix docutils to get `node.line` From 3566352bdbd21a2f6668d5f3da815ed6f8de3f50 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 20 Nov 2016 17:48:07 +0900 Subject: [PATCH 22/33] refs #3093: add tests --- tests/roots/test-intl/conf.py | 2 +- tests/roots/test-intl/figure.txt | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/roots/test-intl/conf.py b/tests/roots/test-intl/conf.py index aafd9ba79..e465e75dc 100644 --- a/tests/roots/test-intl/conf.py +++ b/tests/roots/test-intl/conf.py @@ -6,5 +6,5 @@ keep_warnings = True templates_path = ['_templates'] html_additional_pages = {'index': 'index.html'} release = version = '2013.120' -gettext_additional_targets = ['index'] +gettext_additional_targets = ['index', 'image'] exclude_patterns = ['_build'] diff --git a/tests/roots/test-intl/figure.txt b/tests/roots/test-intl/figure.txt index b639a4ac1..e6ebfd7ce 100644 --- a/tests/roots/test-intl/figure.txt +++ b/tests/roots/test-intl/figure.txt @@ -34,3 +34,20 @@ image url and alt .. figure:: img.png :alt: img + +image on substitution +--------------------- + +.. |sub image| image:: i18n.png + +image under note +----------------- + +.. note:: + + .. image:: i18n.png + :alt: i18n + + .. figure:: img.png + :alt: img + From 3c89ddd99f7d703a975272040dfbfead85db500b Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 20 Nov 2016 17:55:28 +0900 Subject: [PATCH 23/33] refs #3093: revert non-related changed line --- sphinx/transforms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/transforms.py b/sphinx/transforms.py index ba15ae614..3281541c0 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -192,7 +192,7 @@ class ExtraTranslatableNodes(Transform): """ make nodes translatable """ - default_priority = 9 + default_priority = 10 def apply(self): targets = self.document.settings.env.config.gettext_additional_targets From e712401dd1b3df2ae2e839f7c467c6393244dc04 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 20 Nov 2016 18:00:23 +0900 Subject: [PATCH 24/33] #3093: fix test --- tests/roots/test-intl/conf.py | 2 +- tests/roots/test-intl/figure.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/roots/test-intl/conf.py b/tests/roots/test-intl/conf.py index e465e75dc..aafd9ba79 100644 --- a/tests/roots/test-intl/conf.py +++ b/tests/roots/test-intl/conf.py @@ -6,5 +6,5 @@ keep_warnings = True templates_path = ['_templates'] html_additional_pages = {'index': 'index.html'} release = version = '2013.120' -gettext_additional_targets = ['index', 'image'] +gettext_additional_targets = ['index'] exclude_patterns = ['_build'] diff --git a/tests/roots/test-intl/figure.txt b/tests/roots/test-intl/figure.txt index e6ebfd7ce..633e12ee8 100644 --- a/tests/roots/test-intl/figure.txt +++ b/tests/roots/test-intl/figure.txt @@ -46,8 +46,8 @@ image under note .. note:: .. image:: i18n.png - :alt: i18n + :alt: i18n under note .. figure:: img.png - :alt: img + :alt: img under note From 6b0d3e7400ab7b50ce840a8eec9507d15cf74b14 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 20 Nov 2016 18:17:16 +0900 Subject: [PATCH 25/33] #3093: fix test again --- tests/roots/test-intl/figure.po | 7 +++++++ tests/test_intl.py | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/tests/roots/test-intl/figure.po b/tests/roots/test-intl/figure.po index cbd9c8837..449b15e3f 100644 --- a/tests/roots/test-intl/figure.po +++ b/tests/roots/test-intl/figure.po @@ -50,3 +50,10 @@ msgid "" msgstr "" ".. image:: img.png\n" " :alt: I18N -> IMG" + +msgid "image on substitution" +msgstr "IMAGE ON SUBSTITUTION" + +msgid "image under note" +msgstr "IMAGE UNDER NOTE" + diff --git a/tests/test_intl.py b/tests/test_intl.py index b31f1678b..e1c0784d8 100644 --- a/tests/test_intl.py +++ b/tests/test_intl.py @@ -261,6 +261,18 @@ def test_text_builder(app, status, warning): u"[image: i18n][image]\n" u"\n" u" [image: img][image]\n" + u"\n" + u"\n" + u"IMAGE ON SUBSTITUTION\n" + u"=====================\n" + u"\n" + u"\n" + u"IMAGE UNDER NOTE\n" + u"================\n" + u"\n" + u"Note: [image: i18n under note][image]\n" + u"\n" + u" [image: img under note][image]\n" ) yield assert_equal, result, expect From 8f79c11b04c6ea675ead6a019013e1d208efec97 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 20 Nov 2016 18:35:12 +0900 Subject: [PATCH 26/33] #3093: fix test again --- tests/test_intl.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_intl.py b/tests/test_intl.py index e1c0784d8..4fac62c21 100644 --- a/tests/test_intl.py +++ b/tests/test_intl.py @@ -261,18 +261,18 @@ def test_text_builder(app, status, warning): u"[image: i18n][image]\n" u"\n" u" [image: img][image]\n" - u"\n" - u"\n" - u"IMAGE ON SUBSTITUTION\n" - u"=====================\n" - u"\n" - u"\n" - u"IMAGE UNDER NOTE\n" - u"================\n" - u"\n" - u"Note: [image: i18n under note][image]\n" - u"\n" - u" [image: img under note][image]\n" + u"\n" + u"\n" + u"IMAGE ON SUBSTITUTION\n" + u"=====================\n" + u"\n" + u"\n" + u"IMAGE UNDER NOTE\n" + u"================\n" + u"\n" + u"Note: [image: i18n under note][image]\n" + u"\n" + u" [image: img under note][image]\n" ) yield assert_equal, result, expect From 9690be4943887d70c4e4eb1cd309656f8831fb52 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 21 Nov 2016 20:06:23 +0900 Subject: [PATCH 27/33] Fix imgmath: crashes on showing error messages if image generation failed --- CHANGES | 1 + sphinx/ext/imgmath.py | 5 +++-- sphinx/ext/pngmath.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index 0ae8cb26d..c8e0bb941 100644 --- a/CHANGES +++ b/CHANGES @@ -13,6 +13,7 @@ Bugs fixed * #3150: Fix query word splitter in JavaScript. It behaves as same as Python's regular expression. * #3093: gettext build broken on substituted images. * #3093: gettext build broken on image node under ``note`` directive. +* imgmath: crashes on showing error messages if image generation failed Release 1.4.8 (released Oct 1, 2016) ==================================== diff --git a/sphinx/ext/imgmath.py b/sphinx/ext/imgmath.py index f8e402be3..9b75800af 100644 --- a/sphinx/ext/imgmath.py +++ b/sphinx/ext/imgmath.py @@ -245,10 +245,11 @@ def html_visit_displaymath(self, node): try: fname, depth = render_math(self, latex) except MathExtError as exc: - sm = nodes.system_message(str(exc), type='WARNING', level=2, + msg = text_type(exc) + sm = nodes.system_message(msg, type='WARNING', level=2, backrefs=[], source=node['latex']) sm.walkabout(self) - self.builder.warn('inline latex %r: ' % node['latex'] + str(exc)) + self.builder.warn('inline latex %r: ' % node['latex'] + msg) raise nodes.SkipNode self.body.append(self.starttag(node, 'div', CLASS='math')) self.body.append('

') diff --git a/sphinx/ext/pngmath.py b/sphinx/ext/pngmath.py index d7660550e..655eb562f 100644 --- a/sphinx/ext/pngmath.py +++ b/sphinx/ext/pngmath.py @@ -218,10 +218,11 @@ def html_visit_displaymath(self, node): try: fname, depth = render_math(self, latex) except MathExtError as exc: - sm = nodes.system_message(str(exc), type='WARNING', level=2, + msg = text_type(exc) + sm = nodes.system_message(msg, type='WARNING', level=2, backrefs=[], source=node['latex']) sm.walkabout(self) - self.builder.warn('inline latex %r: ' % node['latex'] + str(exc)) + self.builder.warn('inline latex %r: ' % node['latex'] + msg) raise nodes.SkipNode self.body.append(self.starttag(node, 'div', CLASS='math')) self.body.append('

') From 736c18d49861231e80726765136c10732e37f78a Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Wed, 23 Nov 2016 00:40:24 +0900 Subject: [PATCH 28/33] add MoinMoin to EXAMPLES --- EXAMPLES | 1 + 1 file changed, 1 insertion(+) diff --git a/EXAMPLES b/EXAMPLES index e84e0db26..d6770436b 100644 --- a/EXAMPLES +++ b/EXAMPLES @@ -135,6 +135,7 @@ Documentation using another builtin theme * jsFiddle: http://doc.jsfiddle.net/ (nature) * libLAS: http://www.liblas.org/ (nature) * Linguistica: http://linguistica-uchicago.github.io/lxa5/ (sphinx_rtd_theme) +* MoinMoin: https://moin-20.readthedocs.io/en/latest/ (sphinx_rtd_theme) * MPipe: http://vmlaker.github.io/mpipe/ (sphinx13) * pip: https://pip.pypa.io/en/latest/ (sphinx_rtd_theme) * Pyramid web framework: From 43197de29998ceee85cfe1f7471f97799717a78d Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 23 Nov 2016 00:42:15 +0900 Subject: [PATCH 29/33] Fix #3117: LaTeX writer crashes if admonition is placed before first section title --- CHANGES | 1 + sphinx/environment/managers/toctree.py | 580 +++++++++++++++++++++++++ sphinx/writers/latex.py | 54 +-- tests/roots/test-latex-title/conf.py | 8 + tests/roots/test-latex-title/index.rst | 12 + tests/test_build_latex.py | 20 + 6 files changed, 648 insertions(+), 27 deletions(-) create mode 100644 sphinx/environment/managers/toctree.py create mode 100644 tests/roots/test-latex-title/conf.py create mode 100644 tests/roots/test-latex-title/index.rst diff --git a/CHANGES b/CHANGES index c8e0bb941..9c6ca126b 100644 --- a/CHANGES +++ b/CHANGES @@ -14,6 +14,7 @@ Bugs fixed * #3093: gettext build broken on substituted images. * #3093: gettext build broken on image node under ``note`` directive. * imgmath: crashes on showing error messages if image generation failed +* #3117: LaTeX writer crashes if admonition is placed before first section title Release 1.4.8 (released Oct 1, 2016) ==================================== diff --git a/sphinx/environment/managers/toctree.py b/sphinx/environment/managers/toctree.py new file mode 100644 index 000000000..26c8f385d --- /dev/null +++ b/sphinx/environment/managers/toctree.py @@ -0,0 +1,580 @@ +# -*- coding: utf-8 -*- +""" + sphinx.environment.managers.toctree + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Toctree manager for sphinx.environment. + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from six import iteritems + +from docutils import nodes + +from sphinx import addnodes +from sphinx.util import url_re +from sphinx.util.nodes import clean_astext, process_only_nodes +from sphinx.transforms import SphinxContentsFilter +from sphinx.environment.managers import EnvironmentManager + +if False: + # For type annotation + from typing import Any, Tuple # NOQA + from sphinx.builders import Builder # NOQA + from sphinx.environment import BuildEnvironment # NOQA + + +class Toctree(EnvironmentManager): + name = 'toctree' + + def __init__(self, env): + # type: (BuildEnvironment) -> None + super(Toctree, self).__init__(env) + + self.tocs = env.tocs + self.toc_num_entries = env.toc_num_entries + self.toc_secnumbers = env.toc_secnumbers + self.toc_fignumbers = env.toc_fignumbers + self.toctree_includes = env.toctree_includes + self.files_to_rebuild = env.files_to_rebuild + self.glob_toctrees = env.glob_toctrees + self.numbered_toctrees = env.numbered_toctrees + + def clear_doc(self, docname): + # type: (unicode) -> None + self.tocs.pop(docname, None) + self.toc_secnumbers.pop(docname, None) + self.toc_fignumbers.pop(docname, None) + self.toc_num_entries.pop(docname, None) + self.toctree_includes.pop(docname, None) + self.glob_toctrees.discard(docname) + self.numbered_toctrees.discard(docname) + + for subfn, fnset in list(self.files_to_rebuild.items()): + fnset.discard(docname) + if not fnset: + del self.files_to_rebuild[subfn] + + def merge_other(self, docnames, other): + # type: (List[unicode], BuildEnvironment) -> None + for docname in docnames: + self.tocs[docname] = other.tocs[docname] + self.toc_num_entries[docname] = other.toc_num_entries[docname] + if docname in other.toctree_includes: + self.toctree_includes[docname] = other.toctree_includes[docname] + if docname in other.glob_toctrees: + self.glob_toctrees.add(docname) + if docname in other.numbered_toctrees: + self.numbered_toctrees.add(docname) + + for subfn, fnset in other.files_to_rebuild.items(): + self.files_to_rebuild.setdefault(subfn, set()).update(fnset & docnames) + + def process_doc(self, docname, doctree): + # type: (unicode, nodes.Node) -> None + """Build a TOC from the doctree and store it in the inventory.""" + numentries = [0] # nonlocal again... + + def traverse_in_section(node, cls): + """Like traverse(), but stay within the same section.""" + result = [] + if isinstance(node, cls): + result.append(node) + for child in node.children: + if isinstance(child, nodes.section): + continue + result.extend(traverse_in_section(child, cls)) + return result + + def build_toc(node, depth=1): + entries = [] + for sectionnode in node: + # find all toctree nodes in this section and add them + # to the toc (just copying the toctree node which is then + # resolved in self.get_and_resolve_doctree) + if isinstance(sectionnode, addnodes.only): + onlynode = addnodes.only(expr=sectionnode['expr']) + blist = build_toc(sectionnode, depth) + if blist: + onlynode += blist.children + entries.append(onlynode) + continue + if not isinstance(sectionnode, nodes.section): + for toctreenode in traverse_in_section(sectionnode, + addnodes.toctree): + item = toctreenode.copy() + entries.append(item) + # important: do the inventory stuff + self.note_toctree(docname, toctreenode) + continue + title = sectionnode[0] + # copy the contents of the section title, but without references + # and unnecessary stuff + visitor = SphinxContentsFilter(doctree) + title.walkabout(visitor) + nodetext = visitor.get_entry_text() + if not numentries[0]: + # for the very first toc entry, don't add an anchor + # as it is the file's title anyway + anchorname = '' + else: + anchorname = '#' + sectionnode['ids'][0] + numentries[0] += 1 + # make these nodes: + # list_item -> compact_paragraph -> reference + reference = nodes.reference( + '', '', internal=True, refuri=docname, + anchorname=anchorname, *nodetext) + para = addnodes.compact_paragraph('', '', reference) + item = nodes.list_item('', para) + sub_item = build_toc(sectionnode, depth + 1) + item += sub_item + entries.append(item) + if entries: + return nodes.bullet_list('', *entries) + return [] + toc = build_toc(doctree) + if toc: + self.tocs[docname] = toc + else: + self.tocs[docname] = nodes.bullet_list('') + self.toc_num_entries[docname] = numentries[0] + + def note_toctree(self, docname, toctreenode): + # type: (unicode, addnodes.toctree) -> None + """Note a TOC tree directive in a document and gather information about + file relations from it. + """ + if toctreenode['glob']: + self.glob_toctrees.add(docname) + if toctreenode.get('numbered'): + self.numbered_toctrees.add(docname) + includefiles = toctreenode['includefiles'] + for includefile in includefiles: + # note that if the included file is rebuilt, this one must be + # too (since the TOC of the included file could have changed) + self.files_to_rebuild.setdefault(includefile, set()).add(docname) + self.toctree_includes.setdefault(docname, []).extend(includefiles) + + def get_toc_for(self, docname, builder): + # type: (unicode, Builder) -> None + """Return a TOC nodetree -- for use on the same page only!""" + tocdepth = self.env.metadata[docname].get('tocdepth', 0) + try: + toc = self.tocs[docname].deepcopy() + self._toctree_prune(toc, 2, tocdepth) + except KeyError: + # the document does not exist anymore: return a dummy node that + # renders to nothing + return nodes.paragraph() + process_only_nodes(toc, builder.tags, warn_node=self.env.warn_node) + for node in toc.traverse(nodes.reference): + node['refuri'] = node['anchorname'] or '#' + return toc + + def get_toctree_for(self, docname, builder, collapse, **kwds): + # type: (unicode, Builder, bool, Any) -> nodes.Node + """Return the global TOC nodetree.""" + doctree = self.env.get_doctree(self.env.config.master_doc) + toctrees = [] + if 'includehidden' not in kwds: + kwds['includehidden'] = True + if 'maxdepth' not in kwds: + kwds['maxdepth'] = 0 + kwds['collapse'] = collapse + for toctreenode in doctree.traverse(addnodes.toctree): + toctree = self.env.resolve_toctree(docname, builder, toctreenode, + prune=True, **kwds) + if toctree: + toctrees.append(toctree) + if not toctrees: + return None + result = toctrees[0] + for toctree in toctrees[1:]: + result.extend(toctree.children) + return result + + def resolve_toctree(self, docname, builder, toctree, prune=True, maxdepth=0, + titles_only=False, collapse=False, includehidden=False): + # type: (unicode, Builder, addnodes.toctree, bool, int, bool, bool, bool) -> nodes.Node + """Resolve a *toctree* node into individual bullet lists with titles + as items, returning None (if no containing titles are found) or + a new node. + + If *prune* is True, the tree is pruned to *maxdepth*, or if that is 0, + to the value of the *maxdepth* option on the *toctree* node. + If *titles_only* is True, only toplevel document titles will be in the + resulting tree. + If *collapse* is True, all branches not containing docname will + be collapsed. + """ + if toctree.get('hidden', False) and not includehidden: + return None + + # For reading the following two helper function, it is useful to keep + # in mind the node structure of a toctree (using HTML-like node names + # for brevity): + # + #

+ # + # The transformation is made in two passes in order to avoid + # interactions between marking and pruning the tree (see bug #1046). + + toctree_ancestors = self.get_toctree_ancestors(docname) + + def _toctree_add_classes(node, depth): + """Add 'toctree-l%d' and 'current' classes to the toctree.""" + for subnode in node.children: + if isinstance(subnode, (addnodes.compact_paragraph, + nodes.list_item)): + # for

and

  • , indicate the depth level and recurse + subnode['classes'].append('toctree-l%d' % (depth-1)) + _toctree_add_classes(subnode, depth) + elif isinstance(subnode, nodes.bullet_list): + # for