Merge branch '1.8'

CHANGES

@@ -87,7 +87,7 @@ Bugs fixed
 Testing
 --------
 
-Release 1.8.2 (in development)
+Release 1.8.3 (in development)
 ==============================
 
 Dependencies
@@ -105,6 +105,11 @@ Features added
 Bugs fixed
 ----------
 
+* #5460: html search does not work with some 3rd party themes
+* #5520: LaTeX, caption package incompatibility since Sphinx 1.6
+* #5614: autodoc: incremental build is broken when builtin modules are imported
+* #5627: qthelp: index.html missing in QtHelp
+
 Testing
 --------
 

doc/conf.py

@@ -44,6 +44,7 @@ epub_post_files = [('usage/installation.xhtml', 'Installing Sphinx'),
 epub_exclude_files = ['_static/opensearch.xml', '_static/doctools.js',
                       '_static/jquery.js', '_static/searchtools.js',
                       '_static/underscore.js', '_static/basic.css',
+                      '_static/language_data.js',
                       'search.html', '_static/websupport.js']
 epub_fix_images = False
 epub_max_image_width = 0

doc/templating.rst

@@ -200,6 +200,10 @@ Overriding works like this::
 
       {% set script_files = script_files + ["_static/myscript.js"] %}
 
+   .. deprecated:: 1.8.0
+
+      Please use ``.Sphinx.add_js_file()`` instead.
+
 Helper Functions
 ~~~~~~~~~~~~~~~~
 

sphinx/builders/html.py

@@ -359,6 +359,7 @@ class StandaloneHTMLBuilder(Builder):
         self.add_js_file('jquery.js')
         self.add_js_file('underscore.js')
         self.add_js_file('doctools.js')
+        self.add_js_file('language_data.js')
 
         for filename, attrs in self.app.registry.js_files:
             self.add_js_file(filename, **attrs)

sphinx/builders/qthelp.py

@@ -260,7 +260,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
             resourcedir = root.startswith((staticdir, imagesdir))
             for fn in sorted(files):
                 if (resourcedir and not fn.endswith('.js')) or fn.endswith('.html'):
-                    filename = posixpath.join(root, fn)[olen:]
+                    filename = path.join(root, fn)[olen:]
                     project_files.append(filename)
 
         return project_files

sphinx/templates/latex/longtable.tex_t

@@ -8,6 +8,7 @@
 <%- endif -%>
 <%= table.get_colspec() %>
 <%- if table.caption -%>
+\sphinxthelongtablecaptionisattop
 \caption{<%= ''.join(table.caption) %>\strut}<%= labels %>\\*[\sphinxlongtablecapskipadjust]
 \hline
 <% elif labels -%>

sphinx/templates/latex/tabular.tex_t

@@ -12,8 +12,9 @@
 <%- endif %>
 <% if table.caption -%>
 \sphinxcapstartof{table}
+\sphinxthecaptionisattop
 \sphinxcaption{<%= ''.join(table.caption) %>}<%= labels %>
-\sphinxaftercaption
+\sphinxaftertopcaption
 <% elif labels -%>
 \phantomsection<%= labels %>\nobreak
 <% endif -%>

sphinx/templates/latex/tabulary.tex_t

@@ -12,8 +12,9 @@
 <%- endif %>
 <% if table.caption -%>
 \sphinxcapstartof{table}
+\sphinxthecaptionisattop
 \sphinxcaption{<%= ''.join(table.caption) %>}<%= labels %>
-\sphinxaftercaption
+\sphinxaftertopcaption
 <% elif labels -%>
 \phantomsection<%= labels %>\nobreak
 <% endif -%>

sphinx/texinputs/sphinx.sty

@@ -6,7 +6,7 @@
 %
 
 \NeedsTeXFormat{LaTeX2e}[1995/12/01]
-\ProvidesPackage{sphinx}[2018/11/06 v1.8.2 LaTeX package (Sphinx markup)]
+\ProvidesPackage{sphinx}[2018/11/18 v1.8.3 LaTeX package (Sphinx markup)]
 
 % provides \ltx@ifundefined
 % (many packages load ltxcmds: graphicx does for pdftex and lualatex but
@@ -82,11 +82,26 @@
 % User interface to set-up whitespace before and after tables:
 \newcommand*\sphinxtablepre {0pt}%
 \newcommand*\sphinxtablepost{\medskipamount}%
+% Space from caption baseline to top of table or frame of literal-block
 \newcommand*\sphinxbelowcaptionspace{.5\sphinxbaselineskip}%
 % as one can not use \baselineskip from inside longtable (it is zero there)
 % we need \sphinxbaselineskip, which defaults to \baselineskip
 \def\sphinxbaselineskip{\baselineskip}%
-% These commands are inserted by the table templates
+% The following is to ensure that, whether tabular(y) or longtable:
+% - if a caption is on top of table:
+%   a) the space between its last baseline and the top rule of table is
+%      exactly \sphinxbelowcaptionspace
+%   b) the space from last baseline of previous text to first baseline of
+%      caption is exactly \parskip+\baselineskip+ height of a strut.
+%   c) the caption text will wrap at width \LTcapwidth (4in)
+% - make sure this works also if "caption" package is loaded by user
+%   (with its width or margin option taking place of \LTcapwidth role)
+% TODO: obtain same for caption of literal block: a) & c) DONE, b) TO BE DONE
+%
+% To modify space below such top caption, adjust \sphinxbelowcaptionspace
+% To add or remove space above such top caption, adjust \sphinxtablepre:
+%   notice that \abovecaptionskip, \belowcaptionskip, \LTpre are **ignored**
+% A. Table with longtable
 \def\sphinxatlongtablestart
    {\par
     \vskip\parskip
@@ -95,48 +110,76 @@
     \LTpre\z@skip\LTpost\z@skip % set to zero longtable's own skips
     \edef\sphinxbaselineskip{\dimexpr\the\dimexpr\baselineskip\relax\relax}%
    }%
-\def\sphinxatlongtableend{\prevdepth\z@\vskip\sphinxtablepost\relax}%
+% Compatibility with caption package
+\def\sphinxthelongtablecaptionisattop{%
+    \spx@ifcaptionpackage{\noalign{\vskip-\belowcaptionskip}}{}%
+}%
+% Achieves exactly \sphinxbelowcaptionspace below longtable caption
 \def\sphinxlongtablecapskipadjust
-   {\dimexpr-\dp\strutbox-\sphinxbaselineskip+\sphinxbelowcaptionspace\relax}%
-% Now for tables not using longtable
-\def\sphinxattablestart
-   {\par
-    \vskip\dimexpr\sphinxtablepre\relax
-   }%
+   {\dimexpr-\dp\strutbox
+            -\spx@ifcaptionpackage{\abovecaptionskip}{\sphinxbaselineskip}%
+            +\sphinxbelowcaptionspace\relax}%
+\def\sphinxatlongtableend{\prevdepth\z@\vskip\sphinxtablepost\relax}%
+% B. Table with tabular or tabulary
+\def\sphinxattablestart{\par\vskip\dimexpr\sphinxtablepre\relax}%
 \let\sphinxattableend\sphinxatlongtableend
-% longtable's wraps captions to a maximal width of \LTcapwidth
-% so we do the same for all tables
+% This is used by tabular and tabulary templates
 \newcommand*\sphinxcapstartof[1]{%
    \vskip\parskip
    \vbox{}% force baselineskip for good positioning by capstart of hyperanchor
+   % hyperref puts the anchor 6pt above this baseline; in case of caption
+   % this baseline will be \ht\strutbox above first baseline of caption
    \def\@captype{#1}%
    \capstart
-% move back vertically to compensate space inserted by next paragraph
+% move back vertically, as tabular (or its caption) will compensate
    \vskip-\baselineskip\vskip-\parskip
 }%
-% use \LTcapwidth (default is 4in) to wrap caption (if line width is bigger)
-\newcommand\sphinxcaption[2][\LTcapwidth]{%
+\def\sphinxthecaptionisattop{% locate it after \sphinxcapstartof
+    \spx@ifcaptionpackage
+      {\caption@setposition{t}%
+       \vskip\baselineskip\vskip\parskip % undo those from \sphinxcapstartof
+       \vskip-\belowcaptionskip          % anticipate caption package skip
+       % caption package uses a \vbox, not a \vtop, so "single line" case
+       % gives different result from "multi-line" without this:
+       \nointerlineskip
+      }%
+      {}%
+}%
+\def\sphinxthecaptionisatbottom{% (not finalized; for template usage)
+    \spx@ifcaptionpackage{\caption@setposition{b}}{}%
+}%
+% The aim of \sphinxcaption is to apply to tabular(y) the maximal width
+% of caption as done by longtable
+\def\sphinxtablecapwidth{\LTcapwidth}%
+\newcommand\sphinxcaption{\@dblarg\spx@caption}%
+\long\def\spx@caption[#1]#2{%
    \noindent\hb@xt@\linewidth{\hss
-      \vtop{\@tempdima\dimexpr#1\relax
+      \vtop{\@tempdima\dimexpr\sphinxtablecapwidth\relax
 % don't exceed linewidth for the caption width
             \ifdim\@tempdima>\linewidth\hsize\linewidth\else\hsize\@tempdima\fi
-% longtable ignores \abovecaptionskip/\belowcaptionskip, so add hooks here
-% to uniformize control of caption distance to tables
-            \abovecaptionskip\sphinxabovecaptionskip
-            \belowcaptionskip\sphinxbelowcaptionskip
-            \caption[{#2}]%
+% longtable ignores \abovecaptionskip/\belowcaptionskip, so do the same here
+            \abovecaptionskip\sphinxabovecaptionskip % \z@skip
+            \belowcaptionskip\sphinxbelowcaptionskip % \z@skip
+            \caption[{#1}]%
                {\strut\ignorespaces#2\ifhmode\unskip\@finalstrut\strutbox\fi}%
            }\hss}%
    \par\prevdepth\dp\strutbox
 }%
-\def\spx@abovecaptionskip{\abovecaptionskip}
-\newcommand*\sphinxabovecaptionskip{\z@skip}
-\newcommand*\sphinxbelowcaptionskip{\z@skip}
-
-\newcommand\sphinxaftercaption
-{% this default definition serves with a caption *above* a table, to make sure
- % its last baseline is \sphinxbelowcaptionspace above table top
- \nobreak
+\def\sphinxabovecaptionskip{\z@skip}% Do not use! Flagged for removal
+\def\sphinxbelowcaptionskip{\z@skip}% Do not use! Flagged for removal
+% This wrapper of \abovecaptionskip is used in sphinxVerbatim for top
+% caption, and with another value in sphinxVerbatimintable
+% TODO: To unify space above caption of a code-block with the one above
+%       caption of a table/longtable, \abovecaptionskip must not be used
+%       This auxiliary will get renamed and receive a different meaning
+%       in future.
+\def\spx@abovecaptionskip{\abovecaptionskip}%
+% Achieve \sphinxbelowcaptionspace below a caption located above a tabular
+% or a tabulary
+\newcommand\sphinxaftertopcaption
+{%
+   \spx@ifcaptionpackage
+     {\par\prevdepth\dp\strutbox\nobreak\vskip-\abovecaptionskip}{\nobreak}%
    \vskip\dimexpr\sphinxbelowcaptionspace\relax
    \vskip-\baselineskip\vskip-\parskip
 }%
@@ -683,8 +726,22 @@
   \sphinxsetvskipsforfigintablecaption
   \begin{minipage}{#1}%
 }{\end{minipage}}
-% store original \caption macro for use with figures in longtable and tabulary
-\AtBeginDocument{\let\spx@originalcaption\caption}
+% store the original \caption macro for usage with figures inside longtable
+% and tabulary cells. Make sure we get the final \caption in presence of
+% caption package, whether the latter was loaded before or after sphinx.
+\AtBeginDocument{%
+    \let\spx@originalcaption\caption
+    \@ifpackageloaded{caption}
+      {\let\spx@ifcaptionpackage\@firstoftwo
+       \caption@AtBeginDocument*{\let\spx@originalcaption\caption}%
+% in presence of caption package, drop our own \sphinxcaption whose aim was to
+% ensure same width of caption to all kinds of tables (tabular(y), longtable),
+% because caption package has its own width (or margin) option
+       \def\sphinxcaption{\caption}%
+      }%
+      {\let\spx@ifcaptionpackage\@secondoftwo}%
+}
+% tabulary expands twice contents, we need to prevent double counter stepping
 \newcommand*\sphinxfigcaption
   {\ifx\equation$%$% this is trick to identify tabulary first pass
        \firstchoice@false\else\firstchoice@true\fi
@@ -1049,17 +1106,28 @@
        \vskip\spx@abovecaptionskip
        \def\sphinxVerbatim@Before
            {\sphinxVerbatim@Title\nointerlineskip
-            \kern\dimexpr-\dp\strutbox+\sphinxbelowcaptionspace\relax}%
+            \kern\dimexpr-\dp\strutbox+\sphinxbelowcaptionspace
+                 % if no frame (code-blocks inside table cells), remove
+                 % the "verbatimsep" whitespace from the top (better visually)
+                 \ifspx@opt@verbatimwithframe\else-\sphinxverbatimsep\fi
+                 % caption package adds \abovecaptionskip vspace, remove it
+                 \spx@ifcaptionpackage{-\abovecaptionskip}{}\relax}%
      \else
        \vskip\sphinxverbatimsmallskipamount
        \def\sphinxVerbatim@After
-          {\nointerlineskip\kern\dp\strutbox\sphinxVerbatim@Title}%
+          {\nointerlineskip\kern\dimexpr\dp\strutbox
+            \ifspx@opt@verbatimwithframe\else-\sphinxverbatimsep\fi
+            \spx@ifcaptionpackage{-\abovecaptionskip}{}\relax
+           \sphinxVerbatim@Title}%
      \fi
      \def\@captype{literalblock}%
      \capstart
      % \sphinxVerbatimTitle must reset color
      \setbox\sphinxVerbatim@TitleBox
             \hbox{\begin{minipage}{\linewidth}%
+     % caption package may detect wrongly if top or bottom, so we help it
+                    \spx@ifcaptionpackage
+                      {\caption@setposition{\spx@opt@literalblockcappos}}{}%
                     \sphinxVerbatimTitle
                   \end{minipage}}%
   \fi

sphinx/themes/basic/static/documentation_options.js_t

@@ -7,18 +7,4 @@ var DOCUMENTATION_OPTIONS = {
     HAS_SOURCE: {{ has_source|lower }},
     SOURCELINK_SUFFIX: '{{ sourcelink_suffix }}',
     NAVIGATION_WITH_KEYS: {{ 'true' if theme_navigation_with_keys|tobool else 'false'}},
-    SEARCH_LANGUAGE_STOP_WORDS: {{ search_language_stop_words }}
 };
-
-
-{% if search_language_stemming_code %}
-/* Non-minified version JS is _stemmer.js if file is provided */ {% endif -%}
-{{ search_language_stemming_code|safe }}
-
-{% if search_scorer_tool %}
-{{ search_scorer_tool|safe }}
-{% endif -%}
-
-{% if search_word_splitter_code %}
-{{ search_word_splitter_code }}
-{% endif -%}

sphinx/themes/basic/static/language_data.js_t

@@ -0,0 +1,25 @@
+/*
+ * language_data.js
+ * ~~~~~~~~~~~~~~~~
+ *
+ * This script contains the language-specific data used by searchtools.js,
+ * namely the list of stopwords, stemmer, scorer and splitter.
+ *
+ * :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ */
+
+var stopwords = {{ search_language_stop_words }};
+
+{% if search_language_stemming_code %}
+/* Non-minified version JS is _stemmer.js if file is provided */ {% endif -%}
+{{ search_language_stemming_code|safe }}
+
+{% if search_scorer_tool %}
+{{ search_scorer_tool|safe }}
+{% endif -%}
+
+{% if search_word_splitter_code %}
+{{ search_word_splitter_code }}
+{% endif -%}

sphinx/themes/basic/static/searchtools.js

@@ -146,7 +146,6 @@ var Search = {
    */
   query : function(query) {
     var i;
-    var stopwords = DOCUMENTATION_OPTIONS.SEARCH_LANGUAGE_STOP_WORDS;
 
     // stem the searchterms and add them to the correct list
     var stemmer = new Stemmer();

sphinx/util/__init__.py

@@ -301,7 +301,9 @@ def get_module_source(modname):
             raise PycodeError('error getting filename for %r' % filename, err)
     if filename is None and loader:
         try:
-            return 'string', loader.get_source(modname)
+            filename = loader.get_source(modname)
+            if filename:
+                return 'string', filename
         except Exception as err:
             raise PycodeError('error getting source for %r' % modname, err)
     if filename is None:

tests/roots/test-latex-table/expects/longtable_having_caption.tex

@@ -1,6 +1,7 @@
 \label{\detokenize{longtable:longtable-having-caption}}
 
 \begin{savenotes}\sphinxatlongtablestart\begin{longtable}{|l|l|}
+\sphinxthelongtablecaptionisattop
 \caption{caption for longtable\strut}\label{\detokenize{longtable:id1}}\\*[\sphinxlongtablecapskipadjust]
 \hline
 \sphinxstyletheadfamily 

tests/roots/test-latex-table/expects/table_having_caption.tex

@@ -3,8 +3,9 @@
 \begin{savenotes}\sphinxattablestart
 \centering
 \sphinxcapstartof{table}
+\sphinxthecaptionisattop
 \sphinxcaption{caption for table}\label{\detokenize{tabular:id1}}
-\sphinxaftercaption
+\sphinxaftertopcaption
 \begin{tabulary}{\linewidth}[t]{|T|T|}
 \hline
 \sphinxstyletheadfamily 

tests/test_application.py

@@ -59,6 +59,8 @@ def test_extension_in_blacklist(app, status, warning):
 
 
 @pytest.mark.sphinx(testroot='add_source_parser')
+@pytest.mark.filterwarnings('ignore:The config variable "source_parsers"')
+@pytest.mark.filterwarnings('ignore:app.add_source_parser\\(\\) does not support suffix')
 def test_add_source_parser(app, status, warning):
     assert set(app.config.source_suffix) == set(['.rst', '.md', '.test'])
 

tests/test_autodoc.py

@@ -1483,6 +1483,7 @@ def test_partialmethod(app):
 
 
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
+@pytest.mark.filterwarnings('ignore:autodoc_default_flags is now deprecated.')
 def test_merge_autodoc_default_flags1(app):
     app.config.autodoc_default_flags = ['members', 'undoc-members']
     merge_autodoc_default_flags(app, app.config)
@@ -1491,6 +1492,7 @@ def test_merge_autodoc_default_flags1(app):
 
 
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
+@pytest.mark.filterwarnings('ignore:autodoc_default_flags is now deprecated.')
 def test_merge_autodoc_default_flags2(app):
     app.config.autodoc_default_flags = ['members', 'undoc-members']
     app.config.autodoc_default_options = {'members': 'this,that,order',

tests/test_quickstart.py

@@ -218,7 +218,7 @@ def test_generated_files_eol(tempdir):
     qs.generate(d)
 
     def assert_eol(filename, eol):
-        content = filename.bytes().decode('unicode-escape')
+        content = filename.bytes().decode()
         assert all([l[-len(eol):] == eol for l in content.splitlines(True)])
 
     assert_eol(tempdir / 'make.bat', '\r\n')

tests/test_util.py

@@ -15,9 +15,14 @@ import tempfile
 import pytest
 from mock import patch
 
+from six import PY2
+
+import sphinx
+from sphinx.errors import PycodeError
 from sphinx.testing.util import strip_escseq
 from sphinx.util import (
-    display_chunk, encode_uri, ensuredir, parselinenos, status_iterator, xmlname_checker
+    display_chunk, encode_uri, ensuredir, get_module_source, parselinenos, status_iterator,
+    xmlname_checker
 )
 from sphinx.util import logging
 
@@ -59,6 +64,19 @@ def test_display_chunk():
     assert display_chunk(('hello', 'sphinx', 'world')) == 'hello .. world'
 
 
+def test_get_module_source():
+    if PY2:
+        assert get_module_source('sphinx') == ('file', sphinx.__file__.replace('.pyc', '.py'))
+    else:
+        assert get_module_source('sphinx') == ('file', sphinx.__file__)
+
+    # failed to obtain source information from builtin modules
+    with pytest.raises(PycodeError):
+        get_module_source('builtins')
+    with pytest.raises(PycodeError):
+        get_module_source('itertools')
+
+
 @pytest.mark.sphinx('dummy')
 @patch('sphinx.util.console._tw', 40)  # terminal width = 40
 def test_status_iterator(app, status, warning):

tests/test_util_i18n.py

@@ -144,6 +144,7 @@ def test_get_catalogs_from_multiple_locale_dirs(tempdir):
     assert domains == ['test1', 'test1', 'test2']
 
 
+@pytest.mark.filterwarnings('ignore:gettext_compact argument')
 def test_get_catalogs_with_compact(tempdir):
     (tempdir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs()
     (tempdir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#')

tests/test_util_images.py

@@ -34,6 +34,7 @@ def test_get_image_size(testroot):
     assert get_image_size(testroot / TXT_FILENAME) is None
 
 
+@pytest.mark.filterwarnings('ignore:The content argument')
 def test_guess_mimetype(testroot):
     # guess by filename
     assert guess_mimetype('img.png') == 'image/png'