Merge branch '1.5-release'

.gitignore

@@ -20,3 +20,4 @@ Sphinx.egg-info/
 doc/_build/
 tests/.coverage
 tests/build/
+utils/regression_test.js

CHANGES

@@ -18,12 +18,22 @@ Release 1.5 beta2 (in development)
 Incompatible changes
 --------------------
 
+* #2986: ``themes/basic/defindex.html`` is now deprecated
+
 Features added
 --------------
 
+* #3095: Add :confval:`tls_verify` and :confval:`tls_cacerts` to support
+  self-signed HTTPS servers in linkcheck and intersphinx
+
 Bugs fixed
 ----------
 
+* #3069: Even if ``'babel'`` key is set to empty string, LaTeX output contains
+  one ``\addto\captions...``
+* #3123: user ``'babel'`` key setting is not obeyed anymore
+* #3155: Fix JavaScript for `html_sourcelink_suffix` fails with IE and Opera
+
 Release 1.5 beta1 (released Nov 6, 2016)
 ========================================
 
@@ -281,6 +291,12 @@ Bugs fixed
 * #3068: Allow the '=' character in the -D option of sphinx-build.py
 * #3074: ``add_source_parser()`` crashes in debug mode
 * #3135: ``sphinx.ext.autodoc`` crashes with plain Callable
+* #3150: Fix query word splitter in JavaScript. It behaves as same as Python's regular expression.
+* #3093: gettext build broken on substituted images.
+* #3093: gettext build broken on image node under ``note`` directive.
+* imgmath: crashes on showing error messages if image generation failed
+* #3117: LaTeX writer crashes if admonition is placed before first section title
+* #3164: Change search order of ``sphinx.ext.inheritance_diagram``
 
 Release 1.4.8 (released Oct 1, 2016)
 ====================================

EXAMPLES

@@ -135,6 +135,7 @@ Documentation using another builtin theme
 * jsFiddle: http://doc.jsfiddle.net/ (nature)
 * libLAS: http://www.liblas.org/ (nature)
 * Linguistica: http://linguistica-uchicago.github.io/lxa5/ (sphinx_rtd_theme)
+* MoinMoin: https://moin-20.readthedocs.io/en/latest/ (sphinx_rtd_theme)
 * MPipe: http://vmlaker.github.io/mpipe/ (sphinx13)
 * pip: https://pip.pypa.io/en/latest/ (sphinx_rtd_theme)
 * Pyramid web framework:

doc/config.rst

@@ -297,8 +297,7 @@ General configuration
 
    A dictionary mapping ``'figure'``, ``'table'``, ``'code-block'`` and
    ``'section'`` to strings that are used for format of figure numbers.
-   As a special character, `%s` and `{number}` will be replaced to figure
+   As a special character, `%s` will be replaced to figure number.
-   number.  `{name}` will be replaced to figure caption.
 
    Default is to use ``'Fig. %s'`` for ``'figure'``, ``'Table %s'`` for
    ``'table'``, ``'Listing %s'`` for ``'code-block'`` and ``'Section'`` for
@@ -306,9 +305,6 @@ General configuration
 
    .. versionadded:: 1.3
 
-   .. versionchanged:: 1.5
-      Support format of section. Allow to refer the caption of figures.
-
 .. confval:: numfig_secnum_depth
 
    The scope of figure numbers, that is, the numfig feature numbers figures
@@ -318,6 +314,21 @@ General configuration
 
    .. versionadded:: 1.3
 
+.. confval:: tls_verify
+
+   If true, Sphinx verifies server certifications.  Default is ``True``.
+
+   .. versionadded:: 1.5
+
+.. confval:: tls_cacerts
+
+   A path to a certification file of CA or a path to directory which
+   contains the certificates.  This also allows a dictionary mapping
+   hostname to the path to certificate file.
+   The certificates are used to verify server certifications.
+
+   .. versionadded:: 1.5
+
 Project information
 -------------------
 

doc/markup/inline.rst

@@ -214,6 +214,7 @@ Cross-referencing figures by figure number
 
 .. versionchanged:: 1.5
    `numref` role can also refer sections.
+   And `numref` allows `{name}` for the link text.
 
 .. rst:role:: numref
 
@@ -223,7 +224,10 @@ Cross-referencing figures by figure number
 
    If an explicit link text is given (like usual: ``:numref:`Image of Sphinx (Fig.
    %s) <my-figure>```), the link caption will be the title of the reference.
-   The format of link text is same as :confval:`numfig_format`.
+   As a special character, `%s` and `{number}` will be replaced to figure
+   number.  `{name}` will be replaced to figure caption.
+   If no explicit link text is given, the value of :confval:`numfig_format` is
+   used to default value of link text.
 
    If :confval:`numfig` is ``False``, figures are not numbered.
    so this role inserts not a reference but labels or link text.

sphinx/application.py

@@ -315,7 +315,7 @@ class Sphinx(object):
                     self.info('not yet created')
                 else:
                     self.info('failed: %s' % err)
-                return self._init_env(freshenv=True)
+                self._init_env(freshenv=True)
 
     def _init_builder(self, buildername):
         # type: (unicode) -> None
@@ -488,9 +488,9 @@ class Sphinx(object):
         l = 0
         for item in iterable:
             if l == 0:
-                self.info(bold(summary), nonl=1)
+                self.info(bold(summary), nonl=True)
                 l = 1
-            self.info(colorfunc(stringify_func(item)) + ' ', nonl=1)
+            self.info(colorfunc(stringify_func(item)) + ' ', nonl=True)
             yield item
         if l == 1:
             self.info()
@@ -514,7 +514,7 @@ class Sphinx(object):
                 s += '\n'
             else:
                 s = term_width_line(s)
-            self.info(s, nonl=1)
+            self.info(s, nonl=True)
             yield item
         if l > 0:
             self.info()

sphinx/builders/applehelp.py

@@ -294,10 +294,10 @@ def setup(app):
     app.add_config_value('applehelp_title', lambda self: self.project + ' Help', 'applehelp')
     app.add_config_value('applehelp_codesign_identity',
                          lambda self: environ.get('CODE_SIGN_IDENTITY', None),
-                         'applehelp'),
+                         'applehelp')
     app.add_config_value('applehelp_codesign_flags',
                          lambda self: shlex.split(environ.get('OTHER_CODE_SIGN_FLAGS', '')),
-                         'applehelp'),
+                         'applehelp')
     app.add_config_value('applehelp_indexer_path', '/usr/bin/hiutil', 'applehelp')
     app.add_config_value('applehelp_codesign_path', '/usr/bin/codesign', 'applehelp')
     app.add_config_value('applehelp_disable_external_tools', False, None)

sphinx/builders/epub.py

@@ -432,7 +432,7 @@ class EpubBuilder(StandaloneHTMLBuilder):
         """
         self.fix_ids(doctree)
         self.add_visible_links(doctree, self.config.epub_show_urls)
-        return StandaloneHTMLBuilder.write_doc(self, docname, doctree)
+        StandaloneHTMLBuilder.write_doc(self, docname, doctree)
 
     def fix_genindex(self, tree):
         # type: (nodes.Node) -> None

sphinx/builders/html.py

@@ -817,6 +817,7 @@ class StandaloneHTMLBuilder(Builder):
                     outfilename=None, event_arg=None):
         # type: (unicode, Dict, unicode, unicode, Any) -> None
         ctx = self.globalcontext.copy()
+        ctx['warn'] = self.warn
         # current_page_name is backwards compatibility
         ctx['pagename'] = ctx['current_page_name'] = pagename
         default_baseuri = self.get_target_uri(pagename)

sphinx/builders/linkcheck.py

@@ -33,11 +33,11 @@ except ImportError:
         pass
 
 from sphinx.builders import Builder
-from sphinx.util import encode_uri
+from sphinx.util import encode_uri, requests
 from sphinx.util.console import (  # type: ignore
     purple, red, darkgreen, darkgray, darkred, turquoise
 )
-from sphinx.util.requests import requests, useragent_header, is_ssl_error
+from sphinx.util.requests import is_ssl_error
 
 if False:
     # For type annotation
@@ -98,7 +98,6 @@ class CheckExternalLinksBuilder(Builder):
         self.good = set()       # type: Set[unicode]
         self.broken = {}        # type: Dict[unicode, unicode]
         self.redirected = {}    # type: Dict[unicode, Tuple[unicode, int]]
-        self.headers = dict(useragent_header)
         # set a timeout for non-responding servers
         socket.setdefaulttimeout(5.0)
         # create output file
@@ -144,7 +143,7 @@ class CheckExternalLinksBuilder(Builder):
             try:
                 if anchor and self.app.config.linkcheck_anchors:
                     # Read the whole document and see if #anchor exists
-                    response = requests.get(req_url, stream=True, headers=self.headers,
+                    response = requests.get(req_url, stream=True, config=self.app.config,
                                             **kwargs)
                     found = check_anchor(response, unquote(anchor))
 
@@ -154,12 +153,12 @@ class CheckExternalLinksBuilder(Builder):
                     try:
                         # try a HEAD request first, which should be easier on
                         # the server and the network
-                        response = requests.head(req_url, headers=self.headers, **kwargs)
+                        response = requests.head(req_url, config=self.app.config, **kwargs)
                         response.raise_for_status()
                     except HTTPError as err:
                         # retry with GET request if that fails, some servers
                         # don't like HEAD requests.
-                        response = requests.get(req_url, stream=True, headers=self.headers,
+                        response = requests.get(req_url, stream=True, config=self.app.config,
                                                 **kwargs)
                         response.raise_for_status()
             except HTTPError as err:

sphinx/config.py

@@ -119,6 +119,9 @@ class Config(object):
                           'code-block': l_('Listing %s')},
                          'env'),
 
+        tls_verify = (True, 'env'),
+        tls_cacerts = (None, 'env'),
+
         # pre-initialized confval for HTML builder
         html_translator_class = (None, 'html', string_classes),
     )  # type: Dict[unicode, Tuple]

sphinx/environment/__init__.py

@@ -545,7 +545,7 @@ class BuildEnvironment(object):
         # this cache also needs to be updated every time
         self._nitpick_ignore = set(self.config.nitpick_ignore)
 
-        app.info(bold('updating environment: '), nonl=1)
+        app.info(bold('updating environment: '), nonl=True)
 
         added, changed, removed = self.get_outdated_files(config_changed)
 

sphinx/ext/imgmath.py

@@ -258,10 +258,11 @@ def html_visit_displaymath(self, node):
     try:
         fname, depth = render_math(self, latex)
     except MathExtError as exc:
-        sm = nodes.system_message(str(exc), type='WARNING', level=2,
+        msg = text_type(exc)
+        sm = nodes.system_message(msg, type='WARNING', level=2,
                                   backrefs=[], source=node['latex'])
         sm.walkabout(self)
-        self.builder.warn('inline latex %r: ' % node['latex'] + str(exc))
+        self.builder.warn('inline latex %r: ' % node['latex'] + msg)
         raise nodes.SkipNode
     self.body.append(self.starttag(node, 'div', CLASS='math'))
     self.body.append('<p>')

sphinx/ext/inheritance_diagram.py

@@ -64,9 +64,63 @@ if False:
     from sphinx.environment import BuildEnvironment  # NOQA
 
 
-class_sig_re = re.compile(r'''^([\w.]*\.)?    # module names
+module_sig_re = re.compile(r'''^(?:([\w.]*)\.)?  # module names
-                          (\w+)  \s* $        # class/final module name
+                           (\w+)  \s* $          # class/final module name
-                          ''', re.VERBOSE)
+                           ''', re.VERBOSE)
+
+
+def try_import(objname):
+    # type: (unicode) -> Any
+    """Import a object or module using *name* and *currentmodule*.
+    *name* should be a relative name from *currentmodule* or
+    a fully-qualified name.
+
+    Returns imported object or module.  If failed, returns None value.
+    """
+    try:
+        __import__(objname)
+        return sys.modules.get(objname)
+    except ImportError:
+        modname, attrname = module_sig_re.match(objname).groups()
+        if modname is None:
+            return None
+        try:
+            __import__(modname)
+            return getattr(sys.modules.get(modname), attrname, None)
+        except ImportError:
+            return None
+
+
+def import_classes(name, currmodule):
+    # type: (unicode, unicode) -> Any
+    """Import a class using its fully-qualified *name*."""
+    target = None
+
+    # import class or module using currmodule
+    if currmodule:
+        target = try_import(currmodule + '.' + name)
+
+    # import class or module without currmodule
+    if target is None:
+        target = try_import(name)
+
+    if target is None:
+        raise InheritanceException(
+            'Could not import class or module %r specified for '
+            'inheritance diagram' % name)
+
+    if inspect.isclass(target):
+        # If imported object is a class, just return it
+        return [target]
+    elif inspect.ismodule(target):
+        # If imported object is a module, return classes defined on it
+        classes = []
+        for cls in target.__dict__.values():
+            if inspect.isclass(cls) and cls.__module__ == target.__name__:
+                classes.append(cls)
+        return classes
+    raise InheritanceException('%r specified for inheritance diagram is '
+                               'not a class or module' % name)
 
 
 class InheritanceException(Exception):
@@ -95,58 +149,12 @@ class InheritanceGraph(object):
             raise InheritanceException('No classes found for '
                                        'inheritance diagram')
 
-    def _import_class_or_module(self, name, currmodule):
-        # type: (unicode, str) -> Any
-        """Import a class using its fully-qualified *name*."""
-        try:
-            path, base = class_sig_re.match(name).groups()  # type: ignore
-        except (AttributeError, ValueError):
-            raise InheritanceException('Invalid class or module %r specified '
-                                       'for inheritance diagram' % name)
-
-        fullname = (path or '') + base
-        path = (path and path.rstrip('.') or '')
-
-        # two possibilities: either it is a module, then import it
-        try:
-            __import__(fullname)
-            todoc = sys.modules[fullname]
-        except ImportError:
-            # else it is a class, then import the module
-            if not path:
-                if currmodule:
-                    # try the current module
-                    path = currmodule
-                else:
-                    raise InheritanceException(
-                        'Could not import class %r specified for '
-                        'inheritance diagram' % base)
-            try:
-                __import__(path)
-                todoc = getattr(sys.modules[path], base)
-            except (ImportError, AttributeError):
-                raise InheritanceException(
-                    'Could not import class or module %r specified for '
-                    'inheritance diagram' % (path + '.' + base))
-
-        # If a class, just return it
-        if inspect.isclass(todoc):
-            return [todoc]
-        elif inspect.ismodule(todoc):
-            classes = []
-            for cls in todoc.__dict__.values():  # type: ignore
-                if inspect.isclass(cls) and cls.__module__ == todoc.__name__:
-                    classes.append(cls)
-            return classes
-        raise InheritanceException('%r specified for inheritance diagram is '
-                                   'not a class or module' % name)
-
     def _import_classes(self, class_names, currmodule):
         # type: (unicode, str) -> List[Any]
         """Import a list of classes."""
         classes = []  # type: List[Any]
         for name in class_names:
-            classes.extend(self._import_class_or_module(name, currmodule))
+            classes.extend(import_classes(name, currmodule))
         return classes
 
     def _class_info(self, classes, show_builtins, private_bases, parts):
@@ -443,7 +451,7 @@ def setup(app):
         man=(skip, None),
         texinfo=(texinfo_visit_inheritance_diagram, None))
     app.add_directive('inheritance-diagram', InheritanceDiagram)
-    app.add_config_value('inheritance_graph_attrs', {}, False),
+    app.add_config_value('inheritance_graph_attrs', {}, False)
-    app.add_config_value('inheritance_node_attrs', {}, False),
+    app.add_config_value('inheritance_node_attrs', {}, False)
-    app.add_config_value('inheritance_edge_attrs', {}, False),
+    app.add_config_value('inheritance_edge_attrs', {}, False)
     return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

sphinx/ext/intersphinx.py

@@ -42,12 +42,13 @@ from docutils.utils import relative_path
 import sphinx
 from sphinx.locale import _
 from sphinx.builders.html import INVENTORY_FILENAME
-from sphinx.util.requests import requests, useragent_header
+from sphinx.util import requests
 
 if False:
     # For type annotation
     from typing import Any, Callable, Dict, IO, Iterator, Tuple, Union  # NOQA
     from sphinx.application import Sphinx  # NOQA
+    from sphinx.config import Config  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA
 
     if PY3:
@@ -163,8 +164,8 @@ def _strip_basic_auth(url):
     return urlunsplit(frags)
 
 
-def _read_from_url(url, timeout=None):
+def _read_from_url(url, config=None):
-    # type: (unicode, int) -> IO
+    # type: (unicode, Config) -> IO
     """Reads data from *url* with an HTTP *GET*.
 
     This function supports fetching from resources which use basic HTTP auth as
@@ -180,7 +181,7 @@ def _read_from_url(url, timeout=None):
     :return: data read from resource described by *url*
     :rtype: ``file``-like object
     """
-    r = requests.get(url, stream=True, timeout=timeout, headers=dict(useragent_header))
+    r = requests.get(url, stream=True, config=config, timeout=config.intersphinx_timeout)
     r.raise_for_status()
     r.raw.url = r.url
     return r.raw
@@ -223,7 +224,7 @@ def fetch_inventory(app, uri, inv):
         uri = _strip_basic_auth(uri)
     try:
         if '://' in inv:
-            f = _read_from_url(inv, timeout=app.config.intersphinx_timeout)
+            f = _read_from_url(inv, config=app.config)
         else:
             f = open(path.join(app.srcdir, inv), 'rb')
     except Exception as err:

sphinx/ext/pngmath.py

@@ -230,10 +230,11 @@ def html_visit_displaymath(self, node):
     try:
         fname, depth = render_math(self, latex)
     except MathExtError as exc:
-        sm = nodes.system_message(str(exc), type='WARNING', level=2,
+        msg = text_type(exc)
+        sm = nodes.system_message(msg, type='WARNING', level=2,
                                   backrefs=[], source=node['latex'])
         sm.walkabout(self)
-        self.builder.warn('inline latex %r: ' % node['latex'] + str(exc))
+        self.builder.warn('inline latex %r: ' % node['latex'] + msg)
         raise nodes.SkipNode
     self.body.append(self.starttag(node, 'div', CLASS='math'))
     self.body.append('<p>')

sphinx/locale/fr/LC_MESSAGES/sphinx.po

@@ -112,7 +112,7 @@ msgstr "Auteur du module : "
 
 #: sphinx/directives/other.py:153
 msgid "Code author: "
-msgstr "Auteur du code :"
+msgstr "Auteur du code : "
 
 #: sphinx/directives/other.py:155
 msgid "Author: "

sphinx/search/__init__.py

@@ -19,6 +19,7 @@ from docutils.nodes import raw, comment, title, Text, NodeVisitor, SkipNode
 import sphinx
 from sphinx.util import jsdump, rpartition
 from sphinx.util.pycompat import htmlescape
+from sphinx.search.jssplitter import splitter_code
 
 if False:
     # For type annotation
@@ -280,6 +281,7 @@ class IndexBuilder(object):
                 self.js_scorer_code = fp.read().decode('utf-8')
         else:
             self.js_scorer_code = u''
+        self.js_splitter_code = splitter_code
 
     def load(self, stream, format):
         # type: (IO, Any) -> None
@@ -439,6 +441,7 @@ class IndexBuilder(object):
             search_language_stemming_code = self.lang.js_stemmer_code,
             search_language_stop_words = jsdump.dumps(sorted(self.lang.stopwords)),
             search_scorer_tool = self.js_scorer_code,
+            search_word_splitter_code = self.js_splitter_code,
         )
 
     def get_js_stemmer_rawcode(self):

sphinx/search/jssplitter.py

@@ -0,0 +1,110 @@
+
+"""# -*- coding: utf-8 -*-
+    sphinx.search.jssplitter
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Provides Python compatible word splitter to JavaScript
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+
+    DO NOT EDIT. This is generated by utils/jssplitter_generator.py
+"""
+
+splitter_code = """
+
+var splitChars = (function() {
+    var result = {};
+    var singles = [96, 180, 187, 191, 215, 247, 749, 885, 903, 907, 909, 930, 1014, 1648,
+         1748, 1809, 2416, 2473, 2481, 2526, 2601, 2609, 2612, 2615, 2653, 2702,
+         2706, 2729, 2737, 2740, 2857, 2865, 2868, 2910, 2928, 2948, 2961, 2971,
+         2973, 3085, 3089, 3113, 3124, 3213, 3217, 3241, 3252, 3295, 3341, 3345,
+         3369, 3506, 3516, 3633, 3715, 3721, 3736, 3744, 3748, 3750, 3756, 3761,
+         3781, 3912, 4239, 4347, 4681, 4695, 4697, 4745, 4785, 4799, 4801, 4823,
+         4881, 5760, 5901, 5997, 6313, 7405, 8024, 8026, 8028, 8030, 8117, 8125,
+         8133, 8181, 8468, 8485, 8487, 8489, 8494, 8527, 11311, 11359, 11687, 11695,
+         11703, 11711, 11719, 11727, 11735, 12448, 12539, 43010, 43014, 43019, 43587,
+         43696, 43713, 64286, 64297, 64311, 64317, 64319, 64322, 64325, 65141];
+    var i, j, start, end;
+    for (i = 0; i < singles.length; i++) {
+        result[singles[i]] = true;
+    }
+    var ranges = [[0, 47], [58, 64], [91, 94], [123, 169], [171, 177], [182, 184], [706, 709],
+         [722, 735], [741, 747], [751, 879], [888, 889], [894, 901], [1154, 1161],
+         [1318, 1328], [1367, 1368], [1370, 1376], [1416, 1487], [1515, 1519], [1523, 1568],
+         [1611, 1631], [1642, 1645], [1750, 1764], [1767, 1773], [1789, 1790], [1792, 1807],
+         [1840, 1868], [1958, 1968], [1970, 1983], [2027, 2035], [2038, 2041], [2043, 2047],
+         [2070, 2073], [2075, 2083], [2085, 2087], [2089, 2307], [2362, 2364], [2366, 2383],
+         [2385, 2391], [2402, 2405], [2419, 2424], [2432, 2436], [2445, 2446], [2449, 2450],
+         [2483, 2485], [2490, 2492], [2494, 2509], [2511, 2523], [2530, 2533], [2546, 2547],
+         [2554, 2564], [2571, 2574], [2577, 2578], [2618, 2648], [2655, 2661], [2672, 2673],
+         [2677, 2692], [2746, 2748], [2750, 2767], [2769, 2783], [2786, 2789], [2800, 2820],
+         [2829, 2830], [2833, 2834], [2874, 2876], [2878, 2907], [2914, 2917], [2930, 2946],
+         [2955, 2957], [2966, 2968], [2976, 2978], [2981, 2983], [2987, 2989], [3002, 3023],
+         [3025, 3045], [3059, 3076], [3130, 3132], [3134, 3159], [3162, 3167], [3170, 3173],
+         [3184, 3191], [3199, 3204], [3258, 3260], [3262, 3293], [3298, 3301], [3312, 3332],
+         [3386, 3388], [3390, 3423], [3426, 3429], [3446, 3449], [3456, 3460], [3479, 3481],
+         [3518, 3519], [3527, 3584], [3636, 3647], [3655, 3663], [3674, 3712], [3717, 3718],
+         [3723, 3724], [3726, 3731], [3752, 3753], [3764, 3772], [3774, 3775], [3783, 3791],
+         [3802, 3803], [3806, 3839], [3841, 3871], [3892, 3903], [3949, 3975], [3980, 4095],
+         [4139, 4158], [4170, 4175], [4182, 4185], [4190, 4192], [4194, 4196], [4199, 4205],
+         [4209, 4212], [4226, 4237], [4250, 4255], [4294, 4303], [4349, 4351], [4686, 4687],
+         [4702, 4703], [4750, 4751], [4790, 4791], [4806, 4807], [4886, 4887], [4955, 4968],
+         [4989, 4991], [5008, 5023], [5109, 5120], [5741, 5742], [5787, 5791], [5867, 5869],
+         [5873, 5887], [5906, 5919], [5938, 5951], [5970, 5983], [6001, 6015], [6068, 6102],
+         [6104, 6107], [6109, 6111], [6122, 6127], [6138, 6159], [6170, 6175], [6264, 6271],
+         [6315, 6319], [6390, 6399], [6429, 6469], [6510, 6511], [6517, 6527], [6572, 6592],
+         [6600, 6607], [6619, 6655], [6679, 6687], [6741, 6783], [6794, 6799], [6810, 6822],
+         [6824, 6916], [6964, 6980], [6988, 6991], [7002, 7042], [7073, 7085], [7098, 7167],
+         [7204, 7231], [7242, 7244], [7294, 7400], [7410, 7423], [7616, 7679], [7958, 7959],
+         [7966, 7967], [8006, 8007], [8014, 8015], [8062, 8063], [8127, 8129], [8141, 8143],
+         [8148, 8149], [8156, 8159], [8173, 8177], [8189, 8303], [8306, 8307], [8314, 8318],
+         [8330, 8335], [8341, 8449], [8451, 8454], [8456, 8457], [8470, 8472], [8478, 8483],
+         [8506, 8507], [8512, 8516], [8522, 8525], [8586, 9311], [9372, 9449], [9472, 10101],
+         [10132, 11263], [11493, 11498], [11503, 11516], [11518, 11519], [11558, 11567],
+         [11622, 11630], [11632, 11647], [11671, 11679], [11743, 11822], [11824, 12292],
+         [12296, 12320], [12330, 12336], [12342, 12343], [12349, 12352], [12439, 12444],
+         [12544, 12548], [12590, 12592], [12687, 12689], [12694, 12703], [12728, 12783],
+         [12800, 12831], [12842, 12880], [12896, 12927], [12938, 12976], [12992, 13311],
+         [19894, 19967], [40908, 40959], [42125, 42191], [42238, 42239], [42509, 42511],
+         [42540, 42559], [42592, 42593], [42607, 42622], [42648, 42655], [42736, 42774],
+         [42784, 42785], [42889, 42890], [42893, 43002], [43043, 43055], [43062, 43071],
+         [43124, 43137], [43188, 43215], [43226, 43249], [43256, 43258], [43260, 43263],
+         [43302, 43311], [43335, 43359], [43389, 43395], [43443, 43470], [43482, 43519],
+         [43561, 43583], [43596, 43599], [43610, 43615], [43639, 43641], [43643, 43647],
+         [43698, 43700], [43703, 43704], [43710, 43711], [43715, 43738], [43742, 43967],
+         [44003, 44015], [44026, 44031], [55204, 55215], [55239, 55242], [55292, 55295],
+         [57344, 63743], [64046, 64047], [64110, 64111], [64218, 64255], [64263, 64274],
+         [64280, 64284], [64434, 64466], [64830, 64847], [64912, 64913], [64968, 65007],
+         [65020, 65135], [65277, 65295], [65306, 65312], [65339, 65344], [65371, 65381],
+         [65471, 65473], [65480, 65481], [65488, 65489], [65496, 65497]];
+    for (i = 0; i < ranges.length; i++) {
+        start = ranges[i][0];
+        end = ranges[i][1];
+        for (j = start; j <= end; j++) {
+            result[j] = true;
+        }
+    }
+    return result;
+})();
+
+function splitQuery(query) {
+    var result = [];
+    var start = -1;
+    for (var i = 0; i < query.length; i++) {
+        if (splitChars[query.charCodeAt(i)]) {
+            if (start !== -1) {
+                result.push(query.slice(start, i));
+                start = -1;
+            }
+        } else if (start === -1) {
+            start = i;
+        }
+    }
+    if (start !== -1) {
+        result.push(query.slice(start));
+    }
+    return result;
+}
+
+"""

sphinx/themes/basic/defindex.html

@@ -6,7 +6,7 @@
 
     :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
-#}
+#}{{ warn('Now base template defindex.html is deprecated.') }}
 {%- extends "layout.html" %}
 {% set title = _('Overview') %}
 {% block body %}

sphinx/themes/basic/static/searchtools.js_t

@@ -47,6 +47,14 @@ var Scorer = {
 };
 {% endif %}
 
+{% if search_word_splitter_code %}
+{{ search_word_splitter_code }}
+{% else %}
+function splitQuery(query) {
+    return query.split(/\s+/);
+}
+{% endif %}
+
 /**
  * Search Module
  */
@@ -145,7 +153,7 @@ var Search = {
     var searchterms = [];
     var excluded = [];
     var hlterms = [];
-    var tmp = query.split(/\W+/);
+    var tmp = splitQuery(query);
     var objectterms = [];
     for (i = 0; i < tmp.length; i++) {
       if (tmp[i] !== "") {
@@ -261,7 +269,7 @@ var Search = {
           });
         } else if (DOCUMENTATION_OPTIONS.HAS_SOURCE) {
           var suffix = DOCUMENTATION_OPTIONS.SOURCELINK_SUFFIX;
-          $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + (item[5].endsWith(suffix) ? '' : suffix),
+          $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + (item[5].slice(-suffix.length) === suffix ? '' : suffix),
                   dataType: "text",
                   complete: function(jqxhr, textstatus) {
                     var data = jqxhr.responseText;

sphinx/transforms/__init__.py

@@ -135,7 +135,7 @@ class CitationReferences(Transform):
         # type: () -> None
         for citnode in self.document.traverse(nodes.citation_reference):
             cittext = citnode.astext()
-            refnode = addnodes.pending_xref(cittext, refdomain='std', reftype='citation',
+            refnode = addnodes.pending_xref(cittext, reftype='citation',
                                             reftarget=cittext, refwarn=True,
                                             ids=citnode["ids"])
             refnode.source = citnode.source or citnode.parent.source
@@ -162,7 +162,7 @@ class ApplySourceWorkaround(Transform):
     def apply(self):
         # type: () -> None
         for n in self.document.traverse():
-            if isinstance(n, nodes.TextElement):
+            if isinstance(n, (nodes.TextElement, nodes.image)):
                 apply_source_workaround(n)
 
 

sphinx/util/nodes.py

@@ -57,6 +57,8 @@ def apply_source_workaround(node):
         node.source = definition_list_item.source
         node.line = definition_list_item.line - 1
         node.rawsource = node.astext()  # set 'classifier1' (or 'classifier2')
+    if isinstance(node, nodes.image) and node.source is None:
+        node.source, node.line = node.parent.source, node.parent.line
     if isinstance(node, nodes.term):
         # strip classifier from rawsource of term
         for classifier in reversed(node.parent.traverse(nodes.classifier)):
@@ -80,6 +82,7 @@ def apply_source_workaround(node):
             nodes.title,
             nodes.rubric,
             nodes.line,
+            nodes.image,
     ))):
         node.source = find_source_node(node)
         node.line = 0  # need fix docutils to get `node.line`

sphinx/util/requests.py

@@ -14,7 +14,10 @@ from __future__ import absolute_import
 import requests
 import warnings
 import pkg_resources
-from requests.packages.urllib3.exceptions import SSLError
+
+from six import string_types
+from six.moves.urllib.parse import urlsplit
+from requests.packages.urllib3.exceptions import SSLError, InsecureRequestWarning
 
 # try to load requests[security]
 try:
@@ -45,6 +48,7 @@ useragent_header = [('User-Agent',
 
 
 def is_ssl_error(exc):
+    """Check an exception is SSLError."""
     if isinstance(exc, SSLError):
         return True
     else:
@@ -53,3 +57,57 @@ def is_ssl_error(exc):
             return True
         else:
             return False
+
+
+def _get_tls_cacert(url, config):
+    """Get addiotinal CA cert for a specific URL.
+
+    This also returns ``False`` if verification is disabled.
+    And returns ``True`` if additional CA cert not found.
+    """
+    if not config.tls_verify:
+        return False
+
+    certs = getattr(config, 'tls_cacerts', None)
+    if not certs:
+        return True
+    elif isinstance(certs, (string_types, tuple)):
+        return certs
+    else:
+        hostname = urlsplit(url)[1]
+        if '@' in hostname:
+            hostname = hostname.split('@')[1]
+
+        return certs.get(hostname, True)
+
+
+def get(url, **kwargs):
+    """Sends a GET request like requests.get().
+
+    This sets up User-Agent header and TLS verification automatically."""
+    kwargs.setdefault('headers', dict(useragent_header))
+    config = kwargs.pop('config', None)
+    if config:
+        kwargs.setdefault('verify', _get_tls_cacert(url, config))
+
+    with warnings.catch_warnings():
+        if not kwargs.get('verify'):
+            # ignore InsecureRequestWarning if verify=False
+            warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+        return requests.get(url, **kwargs)
+
+
+def head(url, **kwargs):
+    """Sends a HEAD request like requests.head().
+
+    This sets up User-Agent header and TLS verification automatically."""
+    kwargs.setdefault('headers', dict(useragent_header))
+    config = kwargs.pop('config', None)
+    if config:
+        kwargs.setdefault('verify', _get_tls_cacert(url, config))
+
+    with warnings.catch_warnings():
+        if not kwargs.get('verify'):
+            # ignore InsecureRequestWarning if verify=False
+            warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+        return requests.get(url, **kwargs)

sphinx/writers/latex.py

@@ -113,6 +113,7 @@ ADDITIONAL_SETTINGS = {
     'xelatex': {
         'latex_engine': 'xelatex',
         'polyglossia':  '\\usepackage{polyglossia}',
+        'babel':        '',
         'fontenc':      '\\usepackage{fontspec}',
         'fontpkg':      '',
         'utf8extra':   ('\\catcode`^^^^00a0\\active\\protected\\def^^^^00a0'
@@ -182,7 +183,7 @@ class ExtBabel(Babel):
                          'italian'):
             return '\\if\\catcode`\\"\\active\\shorthandoff{"}\\fi'
         elif shortlang in ('tr', 'turkish'):
-            return '\\shorthandoff{=}'
+            return '\\if\\catcode`\\=\\active\\shorthandoff{=}\\fi'
         return ''
 
     def uses_cyrillic(self):
@@ -396,6 +397,11 @@ class LaTeXTranslator(nodes.NodeVisitor):
         # sort out some elements
         self.elements = DEFAULT_SETTINGS.copy()
         self.elements.update(ADDITIONAL_SETTINGS.get(builder.config.latex_engine, {}))
+        # allow the user to override them all
+        self.check_latex_elements()
+        self.elements.update(builder.config.latex_elements)
+
+        # but some have other interface in config file
         self.elements.update({
             'wrapperclass': self.format_docclass(document.settings.docclass),
             # if empty, the title is set to the first section title
@@ -422,7 +428,8 @@ class LaTeXTranslator(nodes.NodeVisitor):
             self.elements['logo'] = '\\sphinxincludegraphics{%s}\\par' % \
                                     path.basename(builder.config.latex_logo)
 
-        if builder.config.language:
+        if builder.config.language \
+           and 'fncychap' not in builder.config.latex_elements:
             # use Sonny style if any language specified
             self.elements['fncychap'] = '\\usepackage[Sonny]{fncychap}'
 
@@ -438,17 +445,16 @@ class LaTeXTranslator(nodes.NodeVisitor):
         self.elements['classoptions'] += ',' + self.babel.get_language()
 
         # set up multilingual module...
-        if self.elements['polyglossia']:
+        # 'babel' key is public and user setting must be obeyed
-            self.elements['babel'] = ''  # disable babel
+        if self.elements['babel']:
-            self.elements['multilingual'] = '%s\n\\setmainlanguage{%s}' % \
+            # this branch is not taken for xelatex with writer default settings
-                (self.elements['polyglossia'], self.babel.get_language())
-        elif self.elements['babel']:
             self.elements['multilingual'] = self.elements['babel']
             if builder.config.language:
                 self.elements['shorthandoff'] = self.babel.get_shorthandoff()
 
                 # Times fonts don't work with Cyrillic languages
-                if self.babel.uses_cyrillic():
+                if self.babel.uses_cyrillic() \
+                   and 'fontpkg' not in builder.config.latex_elements:
                     self.elements['fontpkg'] = ''
 
                 # pTeX (Japanese TeX) for support
@@ -460,6 +466,9 @@ class LaTeXTranslator(nodes.NodeVisitor):
                     self.elements['multilingual'] = ''
                     # disable fncychap in Japanese documents
                     self.elements['fncychap'] = ''
+        elif self.elements['polyglossia']:
+            self.elements['multilingual'] = '%s\n\\setmainlanguage{%s}' % \
+                (self.elements['polyglossia'], self.babel.get_language())
 
         if getattr(builder, 'usepackages', None):
             def declare_package(packagename, options=None):
@@ -487,12 +496,11 @@ class LaTeXTranslator(nodes.NodeVisitor):
             if tocdepth >= SECNUMDEPTH:
                 # Increase secnumdepth if tocdepth is depther than default SECNUMDEPTH
                 self.elements['secnumdepth'] = '\\setcounter{secnumdepth}{%d}' % tocdepth
+
         if getattr(document.settings, 'contentsname', None):
             self.elements['contentsname'] = \
                 self.babel_renewcommand('\\contentsname', document.settings.contentsname)
-        # allow the user to override them all
+
-        self.check_latex_elements()
-        self.elements.update(builder.config.latex_elements)
         if self.elements['maxlistdepth']:
             self.elements['sphinxpkgoptions'] += (',maxlistdepth=%s' %
                                                   self.elements['maxlistdepth'])
@@ -622,7 +630,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
 
     def babel_renewcommand(self, command, definition):
         # type: (unicode, unicode) -> unicode
-        if self.elements['babel']:
+        if self.elements['multilingual']:
             prefix = '\\addto\\captions%s{' % self.babel.get_language()
             suffix = '}'
         else:  # babel is disabled (mainly for Japanese environment)
@@ -882,36 +890,36 @@ class LaTeXTranslator(nodes.NodeVisitor):
         if isinstance(parent, addnodes.seealso):
             # the environment already handles this
             raise nodes.SkipNode
-        elif self.this_is_the_title:
-            if len(node.children) != 1 and not isinstance(node.children[0],
-                                                          nodes.Text):
-                self.builder.warn('document title is not a single Text node',
-                                  (self.curfilestack[-1], node.line))
-            if not self.elements['title']:
-                # text needs to be escaped since it is inserted into
-                # the output literally
-                self.elements['title'] = node.astext().translate(tex_escape_map)
-            self.this_is_the_title = 0
-            raise nodes.SkipNode
         elif isinstance(parent, nodes.section):
-            short = ''
+            if self.this_is_the_title:
-            if node.traverse(nodes.image):
+                if len(node.children) != 1 and not isinstance(node.children[0],
-                short = ('[%s]' %
+                                                              nodes.Text):
-                         u' '.join(clean_astext(node).split()).translate(tex_escape_map))
+                    self.builder.warn('document title is not a single Text node',
+                                      (self.curfilestack[-1], node.line))
+                if not self.elements['title']:
+                    # text needs to be escaped since it is inserted into
+                    # the output literally
+                    self.elements['title'] = node.astext().translate(tex_escape_map)
+                self.this_is_the_title = 0
+                raise nodes.SkipNode
+            else:
+                short = ''
+                if node.traverse(nodes.image):
+                    short = ('[%s]' %
+                             u' '.join(clean_astext(node).split()).translate(tex_escape_map))
 
-            try:
+                try:
-                self.body.append(r'\%s%s{' % (self.sectionnames[self.sectionlevel], short))
+                    self.body.append(r'\%s%s{' % (self.sectionnames[self.sectionlevel], short))
-            except IndexError:
+                except IndexError:
-                # just use "subparagraph", it's not numbered anyway
+                    # just use "subparagraph", it's not numbered anyway
-                self.body.append(r'\%s%s{' % (self.sectionnames[-1], short))
+                    self.body.append(r'\%s%s{' % (self.sectionnames[-1], short))
-            self.context.append('}\n')
+                self.context.append('}\n')
-
-            self.restrict_footnote(node)
-            if self.next_section_ids:
-                for id in self.next_section_ids:
-                    self.context[-1] += self.hypertarget(id, anchor=False)
-                self.next_section_ids.clear()
 
+                self.restrict_footnote(node)
+                if self.next_section_ids:
+                    for id in self.next_section_ids:
+                        self.context[-1] += self.hypertarget(id, anchor=False)
+                    self.next_section_ids.clear()
         elif isinstance(parent, nodes.topic):
             self.body.append(r'\sphinxstyletopictitle{')
             self.context.append('}\n')

tests/roots/test-ext-inheritance_diagram/example/__init__.py

@@ -0,0 +1 @@
+# example.py

tests/roots/test-ext-inheritance_diagram/example/sphinx.py

@@ -0,0 +1,5 @@
+# example.sphinx
+
+
+class DummyClass(object):
+    pass

tests/roots/test-intl/figure.po

@@ -50,3 +50,10 @@ msgid ""
 msgstr ""
 ".. image:: img.png\n"
 "   :alt: I18N -> IMG"
+
+msgid "image on substitution"
+msgstr "IMAGE ON SUBSTITUTION"
+
+msgid "image under note"
+msgstr "IMAGE UNDER NOTE"
+

tests/roots/test-intl/figure.txt

@@ -34,3 +34,20 @@ image url and alt
 .. figure:: img.png
    :alt: img
 
+
+image on substitution
+---------------------
+
+.. |sub image| image:: i18n.png
+
+image under note
+-----------------
+
+.. note::
+
+   .. image:: i18n.png
+      :alt: i18n under note
+
+   .. figure:: img.png
+      :alt: img under note
+

tests/roots/test-latex-title/conf.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+
+master_doc = 'index'
+
+# set empty string to the third column to use the first section title to document title
+latex_documents = [
+    (master_doc, 'test.tex', '', 'Sphinx', 'report')
+]

tests/roots/test-latex-title/index.rst

@@ -0,0 +1,12 @@
+.. admonition:: Notice
+
+   This generates nodes.title node before first section title.
+
+test-latex-title
+================
+
+.. toctree::
+   :numbered:
+
+   foo
+   bar

tests/test_build_latex.py

@@ -149,6 +149,26 @@ def test_latex_warnings(app, status, warning):
         '--- Got:\n' + warnings
 
 
+@with_app(buildername='latex', testroot='basic')
+def test_latex_title(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'test.tex').text(encoding='utf8')
+    print(result)
+    print(status.getvalue())
+    print(warning.getvalue())
+    assert '\\title{The basic Sphinx documentation for testing}' in result
+
+
+@with_app(buildername='latex', testroot='latex-title')
+def test_latex_title_after_admonitions(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'test.tex').text(encoding='utf8')
+    print(result)
+    print(status.getvalue())
+    print(warning.getvalue())
+    assert '\\title{test-latex-title}' in result
+
+
 @with_app(buildername='latex', testroot='numfig',
           confoverrides={'numfig': True})
 def test_numref(app, status, warning):

tests/test_ext_inheritance_diagram.py

@@ -9,9 +9,10 @@
     :license: BSD, see LICENSE for details.
 """
 
-import re
+import sys
-from util import with_app
+from util import with_app, rootdir, raises
 from test_ext_graphviz import skip_if_graphviz_not_found
+from sphinx.ext.inheritance_diagram import InheritanceException, import_classes
 
 
 @with_app('html', testroot='ext-inheritance_diagram')
@@ -40,3 +41,48 @@ def test_inheritance_diagram_latex(app, status, warning):
                '\\\\includegraphics{inheritance-\\w+.pdf}\n'
                '\\\\caption{Test Foo!}\\\\label{index:id1}\\\\end{figure}')
     assert re.search(pattern, content, re.M)
+
+
+def test_import_classes():
+    from sphinx.application import Sphinx, TemplateBridge
+    from sphinx.util.i18n import CatalogInfo
+
+    try:
+        sys.path.append(rootdir / 'roots/test-ext-inheritance_diagram')
+        from example.sphinx import DummyClass
+
+        # got exception for unknown class or module
+        raises(InheritanceException, import_classes, 'unknown', None)
+        raises(InheritanceException, import_classes, 'unknown.Unknown', None)
+
+        # a module having no classes
+        classes = import_classes('sphinx', None)
+        assert classes == []
+
+        classes = import_classes('sphinx', 'foo')
+        assert classes == []
+
+        # all of classes in the module
+        classes = import_classes('sphinx.application', None)
+        assert set(classes) == set([Sphinx, TemplateBridge])
+
+        # specified class in the module
+        classes = import_classes('sphinx.application.Sphinx', None)
+        assert classes == [Sphinx]
+
+        # specified class in current module
+        classes = import_classes('Sphinx', 'sphinx.application')
+        assert classes == [Sphinx]
+
+        # relative module name to current module
+        classes = import_classes('i18n.CatalogInfo', 'sphinx.util')
+        assert classes == [CatalogInfo]
+
+        # got exception for functions
+        raises(InheritanceException, import_classes, 'encode_uri', 'sphinx.util')
+
+        # import submodule on current module (refs: #3164)
+        classes = import_classes('sphinx', 'example')
+        assert classes == [DummyClass]
+    finally:
+        sys.path.pop()

tests/test_intl.py

@@ -266,6 +266,18 @@ def test_text_builder(app, status, warning):
               u"[image: i18n][image]\n"
               u"\n"
               u"   [image: img][image]\n"
+              u"\n"
+              u"\n"
+              u"IMAGE ON SUBSTITUTION\n"
+              u"=====================\n"
+              u"\n"
+              u"\n"
+              u"IMAGE UNDER NOTE\n"
+              u"================\n"
+              u"\n"
+              u"Note: [image: i18n under note][image]\n"
+              u"\n"
+              u"     [image: img under note][image]\n"
               )
     yield assert_equal, result, expect
 

utils/jssplitter_generator.py

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+import re
+import json
+import subprocess
+import sys
+import six
+
+# find char codes they are matched with Python's \\w(?u)
+
+match = re.compile(r'\w(?u)')
+begin = -1
+
+ranges = []
+singles = []
+
+for i in range(65536):
+    # 0xd800-0xdfff is surrogate pair area. skip this.
+    if not match.match(six.unichr(i)) and not (0xd800 <= i <= 0xdfff):
+        if begin == -1:
+            begin = i
+    elif begin != -1:
+        if begin + 1 == i:
+            singles.append(begin)
+        else:
+            ranges.append((begin, i - 1))
+        begin = -1
+
+
+# fold json within almost 80 chars per line
+def fold(jsonData, splitter):
+    code = json.dumps(jsonData)
+    lines = []
+    while True:
+        if len(code) < 71:
+            lines.append('        ' + code)
+            break
+        index = code.index(splitter, 70)
+        lines.append('        ' + code[:index+len(splitter)])
+        code = code[index+len(splitter):]
+    lines[0] = lines[0][8:]
+    return '\n'.join(lines)
+
+
+# JavaScript code
+js_src = '''
+var splitChars = (function() {
+    var result = {};
+    var singles = %s;
+    var i, j, start, end;
+    for (i = 0; i < singles.length; i++) {
+        result[singles[i]] = true;
+    }
+    var ranges = %s;
+    for (i = 0; i < ranges.length; i++) {
+        start = ranges[i][0];
+        end = ranges[i][1];
+        for (j = start; j <= end; j++) {
+            result[j] = true;
+        }
+    }
+    return result;
+})();
+
+function splitQuery(query) {
+    var result = [];
+    var start = -1;
+    for (var i = 0; i < query.length; i++) {
+        if (splitChars[query.charCodeAt(i)]) {
+            if (start !== -1) {
+                result.push(query.slice(start, i));
+                start = -1;
+            }
+        } else if (start === -1) {
+            start = i;
+        }
+    }
+    if (start !== -1) {
+        result.push(query.slice(start));
+    }
+    return result;
+}
+''' % (fold(singles, ','), fold(ranges, '],'))
+
+js_test_src = u'''
+// This is regression test for https://github.com/sphinx-doc/sphinx/issues/3150
+// generated by compat_regexp_generator.py
+// it needs node.js for testing
+var assert = require('assert');
+
+%s
+
+console.log("test splitting English words")
+assert.deepEqual(['Hello', 'World'], splitQuery('   Hello    World   '));
+console.log('   ... ok\\n')
+
+console.log("test splitting special characters")
+assert.deepEqual(['Pin', 'Code'], splitQuery('Pin-Code'));
+console.log('   ... ok\\n')
+
+console.log("test splitting Chinese characters")
+assert.deepEqual(['Hello', 'from', '中国', '上海'], splitQuery('Hello from 中国 上海'));
+console.log('   ... ok\\n')
+
+console.log("test splitting Emoji(surrogate pair) characters. It should keep emojis.")
+assert.deepEqual(['😁😁'], splitQuery('😁😁'));
+console.log('   ... ok\\n')
+
+console.log("test splitting umlauts. It should keep umlauts.")
+assert.deepEqual(
+    ['Löschen', 'Prüfung', 'Abändern', 'ærlig', 'spørsmål'],
+    splitQuery('Löschen Prüfung Abändern ærlig spørsmål'));
+console.log('   ... ok\\n')
+
+''' % js_src
+
+python_src = '''
+"""# -*- coding: utf-8 -*-
+    sphinx.search.jssplitter
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Provides Python compatible word splitter to JavaScript
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+
+    DO NOT EDIT. This is generated by utils/jssplitter_generator.py
+"""
+
+splitter_code = """
+%s
+"""
+''' % js_src
+
+with open('../sphinx/search/jssplitter.py', 'w') as f:
+    f.write(python_src)
+
+with open('./regression_test.js', 'w') as f:
+    f.write(js_test_src.encode('utf-8'))
+
+print("starting test...")
+result = subprocess.call(['node', './regression_test.js'])
+sys.exit(result)