Merge branch '1.7'

CHANGES

@@ -116,6 +116,9 @@ Bugs fixed
 * #4837: latex with class memoir Error: Font command ``\sf`` is not supported
 * #4803: latex: too slow in proportion to number of auto numbered footnotes
 * #4838: htmlhelp: The entries in .hhp file is not ordered
+* toctree directive tries to glob for URL having query_string
+* #4871: html search: Upper characters problem in German
+* #4717: latex: Compilation for German docs failed with LuaLaTeX and XeLaTeX
 
 Testing
 --------
@@ -226,6 +229,8 @@ Incompatible changes
   (refs: #4295)
 * #4246: Limit width of text body for all themes. Conifigurable via theme
   options ``body_min_width`` and ``body_max_width``.
+* #4771: apidoc: The ``exclude_patterns`` arguments are ignored if they are
+  placed just after command line options
 
 1.7.0b2
 

sphinx/directives/other.py

@@ -7,6 +7,8 @@
     :license: BSD, see LICENSE for details.
 """
 
+import re
+
 from docutils import nodes
 from docutils.parsers.rst import directives
 from docutils.parsers.rst.directives.admonitions import BaseAdmonition
@@ -42,6 +44,8 @@ locale.versionlabels = DeprecatedDict(
     RemovedInSphinx30Warning
 )
 
+glob_re = re.compile('.*[*?\[].*')
+
 
 def int_or_nothing(argument):
     # type: (unicode) -> int
@@ -73,29 +77,50 @@ class TocTree(SphinxDirective):
 
     def run(self):
         # type: () -> List[nodes.Node]
-        suffixes = self.config.source_suffix
+        subnode = addnodes.toctree()
-        glob = 'glob' in self.options
+        subnode['parent'] = self.env.docname
 
-        ret = []
         # (title, ref) pairs, where ref may be a document, or an external link,
         # and title may be None if the document's title is to be used
-        entries = []        # type: List[Tuple[unicode, unicode]]
+        subnode['entries'] = []
-        includefiles = []
+        subnode['includefiles'] = []
+        subnode['maxdepth'] = self.options.get('maxdepth', -1)
+        subnode['caption'] = self.options.get('caption')
+        subnode['glob'] = 'glob' in self.options
+        subnode['hidden'] = 'hidden' in self.options
+        subnode['includehidden'] = 'includehidden' in self.options
+        subnode['numbered'] = self.options.get('numbered', 0)
+        subnode['titlesonly'] = 'titlesonly' in self.options
+        set_source_info(self, subnode)
+        wrappernode = nodes.compound(classes=['toctree-wrapper'])
+        wrappernode.append(subnode)
+        self.add_name(wrappernode)
+
+        ret = self.parse_content(subnode)
+        ret.append(wrappernode)
+        return ret
+
+    def parse_content(self, toctree):
+        suffixes = self.config.source_suffix
+
+        # glob target documents
         all_docnames = self.env.found_docs.copy()
-        # don't add the currently visited file in catch-all patterns
+        all_docnames.remove(self.env.docname)  # remove current document
-        all_docnames.remove(self.env.docname)
+
+        ret = []
         for entry in self.content:
             if not entry:
                 continue
             # look for explicit titles ("Some Title <document>")
             explicit = explicit_title_re.match(entry)
-            if glob and ('*' in entry or '?' in entry or '[' in entry) and not explicit:
+            if (toctree['glob'] and glob_re.match(entry) and
+                    not explicit and not url_re.match(entry)):
                 patname = docname_join(self.env.docname, entry)
-                docnames = sorted(patfilter(all_docnames, patname))  # type: ignore
+                docnames = sorted(patfilter(all_docnames, patname))
                 for docname in docnames:
                     all_docnames.remove(docname)  # don't include it again
-                    entries.append((None, docname))
+                    toctree['entries'].append((None, docname))
-                    includefiles.append(docname)
+                    toctree['includefiles'].append(docname)
                 if not docnames:
                     ret.append(self.state.document.reporter.warning(
                         'toctree glob pattern %r didn\'t match any documents'
@@ -116,7 +141,7 @@ class TocTree(SphinxDirective):
                 # absolutize filenames
                 docname = docname_join(self.env.docname, docname)
                 if url_re.match(ref) or ref == 'self':
-                    entries.append((title, ref))
+                    toctree['entries'].append((title, ref))
                 elif docname not in self.env.found_docs:
                     ret.append(self.state.document.reporter.warning(
                         'toctree contains reference to nonexisting '
@@ -124,28 +149,13 @@ class TocTree(SphinxDirective):
                     self.env.note_reread()
                 else:
                     all_docnames.discard(docname)
-                    entries.append((title, docname))
+                    toctree['entries'].append((title, docname))
-                    includefiles.append(docname)
+                    toctree['includefiles'].append(docname)
-        subnode = addnodes.toctree()
+
-        subnode['parent'] = self.env.docname
         # entries contains all entries (self references, external links etc.)
         if 'reversed' in self.options:
-            entries.reverse()
+            toctree['entries'] = list(reversed(toctree['entries']))
-        subnode['entries'] = entries
+
-        # includefiles only entries that are documents
-        subnode['includefiles'] = includefiles
-        subnode['maxdepth'] = self.options.get('maxdepth', -1)
-        subnode['caption'] = self.options.get('caption')
-        subnode['glob'] = glob
-        subnode['hidden'] = 'hidden' in self.options
-        subnode['includehidden'] = 'includehidden' in self.options
-        subnode['numbered'] = self.options.get('numbered', 0)
-        subnode['titlesonly'] = 'titlesonly' in self.options
-        set_source_info(self, subnode)
-        wrappernode = nodes.compound(classes=['toctree-wrapper'])
-        wrappernode.append(subnode)
-        self.add_name(wrappernode)
-        ret.append(wrappernode)
         return ret
 
 

sphinx/locale/zh_CN/LC_MESSAGES/sphinx.po

@@ -425,7 +425,7 @@ msgstr "行规范 %r：未能从包含文件 %r 中拉取行"
 
 #: sphinx/directives/other.py:157
 msgid "Section author: "
-msgstr "节作者："
+msgstr "章节作者："
 
 #: sphinx/directives/other.py:159
 msgid "Module author: "
@@ -878,7 +878,7 @@ msgstr "格式转换程序退出并报错：\n[stderr]\n%s\n[stdout]\n%s"
 
 #: sphinx/ext/imgmath.py:338 sphinx/ext/jsmath.py:41 sphinx/ext/mathjax.py:42
 msgid "Permalink to this equation"
-msgstr "永久链接至公式"
+msgstr "公式的永久链接"
 
 #: sphinx/ext/intersphinx.py:339
 #, python-format
@@ -1047,7 +1047,7 @@ msgstr "搜索"
 
 #: sphinx/themes/agogo/layout.html:54 sphinx/themes/basic/searchbox.html:16
 msgid "Go"
-msgstr "转向"
+msgstr "搜"
 
 #: sphinx/themes/agogo/layout.html:81 sphinx/themes/basic/sourcelink.html:15
 msgid "Show Source"
@@ -1250,13 +1250,13 @@ msgstr "其他更改"
 #: sphinx/writers/html.py:410 sphinx/writers/html5.py:351
 #: sphinx/writers/html5.py:356
 msgid "Permalink to this headline"
-msgstr "永久链接至标题"
+msgstr "标题的永久链接"
 
 #: sphinx/themes/basic/static/doctools.js_t:199 sphinx/writers/html.py:126
 #: sphinx/writers/html.py:137 sphinx/writers/html5.py:95
 #: sphinx/writers/html5.py:106
 msgid "Permalink to this definition"
-msgstr "永久链接至目标"
+msgstr "定义的永久链接"
 
 #: sphinx/themes/basic/static/doctools.js_t:232
 msgid "Hide Search Matches"
@@ -1313,19 +1313,19 @@ msgstr "当添加指令类时，不应该给定额外参数"
 
 #: sphinx/writers/html.py:414 sphinx/writers/html5.py:360
 msgid "Permalink to this table"
-msgstr "永久链接至表格"
+msgstr "表格的永久链接"
 
 #: sphinx/writers/html.py:466 sphinx/writers/html5.py:412
 msgid "Permalink to this code"
-msgstr "永久链接至代码"
+msgstr "代码的永久链接"
 
 #: sphinx/writers/html.py:470 sphinx/writers/html5.py:416
 msgid "Permalink to this image"
-msgstr "永久链接至图片"
+msgstr "图片的永久链接"
 
 #: sphinx/writers/html.py:472 sphinx/writers/html5.py:418
 msgid "Permalink to this toctree"
-msgstr "永久链接至目录树"
+msgstr "目录的永久链接"
 
 #: sphinx/writers/latex.py:554
 msgid "Release"

sphinx/locale/zh_TW/LC_MESSAGES/sphinx.po

@@ -1046,7 +1046,7 @@ msgstr "搜尋"
 
 #: sphinx/themes/agogo/layout.html:54 sphinx/themes/basic/searchbox.html:16
 msgid "Go"
-msgstr "前往"
+msgstr "搜"
 
 #: sphinx/themes/agogo/layout.html:81 sphinx/themes/basic/sourcelink.html:15
 msgid "Show Source"

sphinx/search/de.py

@@ -318,4 +318,4 @@ class SearchGerman(SearchLanguage):
 
     def stem(self, word):
         # type: (unicode) -> unicode
-        return self.stemmer.stemWord(word)
+        return self.stemmer.stemWord(word.lower())

sphinx/writers/latex.py

@@ -189,10 +189,11 @@ class LaTeXWriter(writers.Writer):
 # Helper classes
 
 class ExtBabel(Babel):
-    def __init__(self, language_code):
+    def __init__(self, language_code, use_polyglossia=False):
-        # type: (unicode) -> None
+        # type: (unicode, bool) -> None
         super(ExtBabel, self).__init__(language_code or '')
         self.language_code = language_code
+        self.use_polyglossia = use_polyglossia
 
     def get_shorthandoff(self):
         # type: () -> unicode
@@ -220,11 +221,28 @@ class ExtBabel(Babel):
     def get_language(self):
         # type: () -> unicode
         language = super(ExtBabel, self).get_language()
-        if not language:
+        if language == 'ngerman' and self.use_polyglossia:
+            # polyglossia calls new orthography (Neue Rechtschreibung) as
+            # german (with new spelling option).
+            return 'german'
+        elif not language:
             return 'english'  # fallback to english
         else:
             return language
 
+    def get_mainlanguage_options(self):
+        # type: () -> unicode
+        """Return options for polyglossia's ``\setmainlanguage``."""
+        language = super(ExtBabel, self).get_language()
+        if self.use_polyglossia is False:
+            return None
+        elif language == 'ngerman':
+            return 'spelling=new'
+        elif language == 'german':
+            return 'spelling=old'
+        else:
+            return None
+
 
 class Table(object):
     """A table data"""
@@ -516,7 +534,8 @@ class LaTeXTranslator(nodes.NodeVisitor):
                                          '\\sffamily}\n\\ChTitleVar{\\Large'
                                          '\\normalfont\\sffamily}')
 
-        self.babel = ExtBabel(builder.config.language)
+        self.babel = ExtBabel(builder.config.language,
+                              not self.elements['babel'])
         if builder.config.language and not self.babel.is_supported_language():
             # emit warning if specified language is invalid
             # (only emitting, nothing changed to processing)
@@ -550,8 +569,15 @@ class LaTeXTranslator(nodes.NodeVisitor):
                     # disable fncychap in Japanese documents
                     self.elements['fncychap'] = ''
         elif self.elements['polyglossia']:
-            self.elements['multilingual'] = '%s\n\\setmainlanguage{%s}' % \
+            options = self.babel.get_mainlanguage_options()
-                (self.elements['polyglossia'], self.babel.get_language())
+            if options:
+                mainlanguage = r'\setmainlanguage[%s]{%s}' % (options,
+                                                              self.babel.get_language())
+            else:
+                mainlanguage = r'\setmainlanguage{%s}' % self.babel.get_language()
+
+            self.elements['multilingual'] = '%s\n%s' % (self.elements['polyglossia'],
+                                                        mainlanguage)
 
         if getattr(builder, 'usepackages', None):
             def declare_package(packagename, options=None):

tests/roots/test-toctree-glob/index.rst

@@ -15,7 +15,7 @@ normal order
    hyperref <https://sphinx-doc.org/?q=sphinx>
 
 reversed order
--------------
+--------------
 
 .. toctree::
    :glob:

tests/test_build_latex.py

@@ -534,6 +534,50 @@ def test_babel_with_unknown_language(app, status, warning):
     assert "WARNING: no Babel option known for language 'unknown'" in warning.getvalue()
 
 
+@pytest.mark.sphinx(
+    'latex', testroot='latex-babel',
+    confoverrides={'language': 'de', 'latex_engine': 'lualatex'})
+def test_polyglossia_with_language_de(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'Python.tex').text(encoding='utf8')
+    print(result)
+    print(status.getvalue())
+    print(warning.getvalue())
+    assert '\\documentclass[letterpaper,10pt,german]{sphinxmanual}' in result
+    assert '\\usepackage{polyglossia}' in result
+    assert '\\setmainlanguage[spelling=new]{german}' in result
+    assert '\\usepackage{times}' not in result
+    assert '\\usepackage[Sonny]{fncychap}' in result
+    assert ('\\addto\\captionsgerman{\\renewcommand{\\contentsname}{Table of content}}\n'
+            in result)
+    assert '\\addto\\captionsgerman{\\renewcommand{\\figurename}{Fig.}}\n' in result
+    assert '\\addto\\captionsgerman{\\renewcommand{\\tablename}{Table.}}\n' in result
+    assert '\\def\\pageautorefname{Seite}\n' in result
+    assert '\\shorthandoff' not in result
+
+
+@pytest.mark.sphinx(
+    'latex', testroot='latex-babel',
+    confoverrides={'language': 'de-1901', 'latex_engine': 'lualatex'})
+def test_polyglossia_with_language_de_1901(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'Python.tex').text(encoding='utf8')
+    print(result)
+    print(status.getvalue())
+    print(warning.getvalue())
+    assert '\\documentclass[letterpaper,10pt,german]{sphinxmanual}' in result
+    assert '\\usepackage{polyglossia}' in result
+    assert '\\setmainlanguage[spelling=old]{german}' in result
+    assert '\\usepackage{times}' not in result
+    assert '\\usepackage[Sonny]{fncychap}' in result
+    assert ('\\addto\\captionsgerman{\\renewcommand{\\contentsname}{Table of content}}\n'
+            in result)
+    assert '\\addto\\captionsgerman{\\renewcommand{\\figurename}{Fig.}}\n' in result
+    assert '\\addto\\captionsgerman{\\renewcommand{\\tablename}{Table.}}\n' in result
+    assert '\\def\\pageautorefname{page}\n' in result
+    assert '\\shorthandoff' not in result
+
+
 @pytest.mark.sphinx('latex')
 def test_footnote(app, status, warning):
     app.builder.build_all()

tests/test_directive_other.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""
+    test_directive_other
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Test the other directives.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import pytest
+from docutils import nodes
+from docutils.core import publish_doctree
+
+from sphinx import addnodes
+from sphinx.io import SphinxStandaloneReader
+from sphinx.parsers import RSTParser
+from sphinx.testing.util import assert_node
+
+
+def parse(app, docname, text):
+    app.env.temp_data['docname'] = docname
+    return publish_doctree(text, app.srcdir / docname + '.rst',
+                           reader=SphinxStandaloneReader(app),
+                           parser=RSTParser(),
+                           settings_overrides={'env': app.env,
+                                               'gettext_compact': True})
+
+
+@pytest.mark.sphinx(testroot='toctree-glob')
+def test_toctree(app):
+    text = (".. toctree::\n"
+            "\n"
+            "   foo\n"
+            "   bar/index\n"
+            "   baz\n")
+
+    app.env.find_files(app.config, app.builder)
+    doctree = parse(app, 'index', text)
+    assert_node(doctree, [nodes.document, nodes.compound, addnodes.toctree])
+    assert_node(doctree[0][0],
+                entries=[(None, 'foo'), (None, 'bar/index'), (None, 'baz')],
+                includefiles=['foo', 'bar/index', 'baz'])
+
+
+@pytest.mark.sphinx(testroot='toctree-glob')
+def test_relative_toctree(app):
+    text = (".. toctree::\n"
+            "\n"
+            "   bar_1\n"
+            "   bar_2\n"
+            "   bar_3\n"
+            "   ../quux\n")
+
+    app.env.find_files(app.config, app.builder)
+    doctree = parse(app, 'bar/index', text)
+    assert_node(doctree, [nodes.document, nodes.compound, addnodes.toctree])
+    assert_node(doctree[0][0],
+                entries=[(None, 'bar/bar_1'), (None, 'bar/bar_2'), (None, 'bar/bar_3'),
+                         (None, 'quux')],
+                includefiles=['bar/bar_1', 'bar/bar_2', 'bar/bar_3', 'quux'])
+
+
+@pytest.mark.sphinx(testroot='toctree-glob')
+def test_toctree_urls_and_titles(app):
+    text = (".. toctree::\n"
+            "\n"
+            "   Sphinx <https://www.sphinx-doc.org/>\n"
+            "   https://readthedocs.org/\n"
+            "   The BAR <bar/index>\n")
+
+    app.env.find_files(app.config, app.builder)
+    doctree = parse(app, 'index', text)
+    assert_node(doctree, [nodes.document, nodes.compound, addnodes.toctree])
+    assert_node(doctree[0][0],
+                entries=[('Sphinx', 'https://www.sphinx-doc.org/'),
+                         (None, 'https://readthedocs.org/'),
+                         ('The BAR', 'bar/index')],
+                includefiles=['bar/index'])
+
+
+@pytest.mark.sphinx(testroot='toctree-glob')
+def test_toctree_glob(app):
+    text = (".. toctree::\n"
+            "   :glob:\n"
+            "\n"
+            "   *\n")
+
+    app.env.find_files(app.config, app.builder)
+    doctree = parse(app, 'index', text)
+    assert_node(doctree, [nodes.document, nodes.compound, addnodes.toctree])
+    assert_node(doctree[0][0],
+                entries=[(None, 'baz'), (None, 'foo'), (None, 'quux')],
+                includefiles=['baz', 'foo', 'quux'])
+
+    # give both docname and glob (case1)
+    text = (".. toctree::\n"
+            "   :glob:\n"
+            "\n"
+            "   foo\n"
+            "   *\n")
+
+    app.env.find_files(app.config, app.builder)
+    doctree = parse(app, 'index', text)
+    assert_node(doctree, [nodes.document, nodes.compound, addnodes.toctree])
+    assert_node(doctree[0][0],
+                entries=[(None, 'foo'), (None, 'baz'), (None, 'quux')],
+                includefiles=['foo', 'baz', 'quux'])
+
+    # give both docname and glob (case2)
+    text = (".. toctree::\n"
+            "   :glob:\n"
+            "\n"
+            "   *\n"
+            "   foo\n")
+
+    app.env.find_files(app.config, app.builder)
+    doctree = parse(app, 'index', text)
+    assert_node(doctree, [nodes.document, nodes.compound, addnodes.toctree])
+    assert_node(doctree[0][0],
+                entries=[(None, 'baz'), (None, 'foo'), (None, 'quux'), (None, 'foo')],
+                includefiles=['baz', 'foo', 'quux', 'foo'])
+
+
+@pytest.mark.sphinx(testroot='toctree-glob')
+def test_toctree_glob_and_url(app):
+    text = (".. toctree::\n"
+            "   :glob:\n"
+            "\n"
+            "   https://example.com/?q=sphinx\n")
+
+    app.env.find_files(app.config, app.builder)
+    doctree = parse(app, 'index', text)
+    assert_node(doctree, [nodes.document, nodes.compound, addnodes.toctree])
+    assert_node(doctree[0][0],
+                entries=[(None, 'https://example.com/?q=sphinx')],
+                includefiles=[])
+
+
+@pytest.mark.sphinx(testroot='toctree-glob')
+def test_toctree_twice(app):
+    text = (".. toctree::\n"
+            "\n"
+            "   foo\n"
+            "   foo\n")
+
+    app.env.find_files(app.config, app.builder)
+    doctree = parse(app, 'index', text)
+    assert_node(doctree, [nodes.document, nodes.compound, addnodes.toctree])
+    assert_node(doctree[0][0],
+                entries=[(None, 'foo'), (None, 'foo')],
+                includefiles=['foo', 'foo'])