Merge branch 'master' into add_override_option

.codecov.yml

@@ -2,9 +2,7 @@ coverage:
   status:
     project:
       default:
-        # allowed to drop X% and still result in a "success" commit status
+        enabled: no
-        threshold: 0.05
     patch:
       default:
-        # allowed to drop X% and still result in a "success" commit status
+        enabled: no
-        threshold: 0.05

CHANGES

@@ -49,6 +49,8 @@ Features added
 * Add ``app.add_message_catalog()`` and ``sphinx.locale.get_translations()`` to
   support translation for 3rd party extensions
 * helper function ``warning()`` for HTML themes is added
+* Add ``Domain.enumerable_nodes`` to manage own enumerable nodes for domains
+  (experimental)
 * Add a new keyword argument ``override`` to Application APIs
 
 Bugs fixed
@@ -64,7 +66,7 @@ Features removed
 
 * ``sphinx.ext.pngmath`` extension
 
-Release 1.7.2 (in development)
+Release 1.7.3 (in development)
 ==============================
 
 Dependencies
@@ -82,6 +84,20 @@ Features added
 Bugs fixed
 ----------
 
+Testing
+--------
+
+Release 1.7.2 (released Mar 21, 2018)
+=====================================
+
+Incompatible changes
+--------------------
+* #4520: apidoc: folders with an empty __init__.py are no longer excluded from
+  TOC
+
+Bugs fixed
+----------
+
 * #4669: sphinx.build_main and sphinx.make_main throw NameError
 * #4685: autosummary emits meaningless warnings
 * autodoc: crashed when invalid options given
@@ -96,9 +112,11 @@ Bugs fixed
 * #4720: message when an image is mismatched for builder is not clear
 * #4655, #4684: Incomplete localization strings in Polish and Chinese
 * #2286: Sphinx crashes when error is happens in rendering HTML pages
-
+* #4688: Error to download remote images having long URL
-Testing
+* #4754: sphinx/pycode/__init__.py raises AttributeError
---------
+* #1435: qthelp builder should htmlescape keywords
+* epub: Fix docTitle elements of toc.ncx is not escaped
+* #4520: apidoc: Subpackage not in toc (introduced in 1.6.6) now fixed
 
 Release 1.7.1 (released Feb 23, 2018)
 =====================================

doc/_themes/sphinx13/layout.html

@@ -13,6 +13,11 @@
 {% block sidebar1 %}{{ sidebar() }}{% endblock %}
 {% block sidebar2 %}{% endblock %}
 
+{% block linktags %}
+{{ super() }}
+    <link rel="canonical" href="http://www.sphinx-doc.org/en/master/{{ pagename }}{{ file_suffix }}" />
+{% endblock %}
+
 {% block extrahead %}
     <link href='https://fonts.googleapis.com/css?family=Open+Sans:300,400,700'
           rel='stylesheet' type='text/css' />

doc/man/sphinx-autogen.rst

@@ -73,7 +73,7 @@ If you run the following:
 
 .. code-block:: bash
 
-    $ sphinx-autodoc doc/index.rst
+    $ PYTHONPATH=. sphinx-autodoc doc/index.rst
 
 then the following stub files will be created in ``docs``::
 

doc/setuptools.rst

@@ -62,7 +62,7 @@ Options for setuptools integration
    A boolean that determines whether the saved environment should be discarded
    on build. Default is false.
 
-   This can also be set by passing the `-E` flag to ``setup.py``.
+   This can also be set by passing the `-E` flag to ``setup.py``:
 
    .. code-block:: bash
 

sphinx/builders/_epub_base.py

@@ -673,7 +673,7 @@ class EpubBuilder(StandaloneHTMLBuilder):
         """
         metadata = {}  # type: Dict[unicode, Any]
         metadata['uid'] = self.config.epub_uid
-        metadata['title'] = self.config.epub_title
+        metadata['title'] = self.esc(self.config.epub_title)
         metadata['level'] = level
         metadata['navpoints'] = navpoints
         return metadata

sphinx/builders/qthelp.py

@@ -19,6 +19,7 @@ from docutils import nodes
 from six import text_type
 
 from sphinx import addnodes
+from sphinx import package_dir
 from sphinx.builders.html import StandaloneHTMLBuilder
 from sphinx.config import string_classes
 from sphinx.environment.adapters.indexentries import IndexEntries
@@ -26,6 +27,7 @@ from sphinx.locale import __
 from sphinx.util import force_decode, logging
 from sphinx.util.osutil import make_filename
 from sphinx.util.pycompat import htmlescape
+from sphinx.util.template import SphinxRenderer
 
 if False:
     # For type annotation
@@ -40,67 +42,13 @@ _idpattern = re.compile(
     r'(?P<title>.+) (\((class in )?(?P<id>[\w\.]+)( (?P<descr>\w+))?\))$')
 
 
-# Qt Help Collection Project (.qhcp).
-# Is the input file for the help collection generator.
-# It contains references to compressed help files which should be
-# included in the collection.
-# It may contain various other information for customizing Qt Assistant.
-collection_template = u'''\
-<?xml version="1.0" encoding="utf-8" ?>
-<QHelpCollectionProject version="1.0">
-    <assistant>
-        <title>%(title)s</title>
-        <homePage>%(homepage)s</homePage>
-        <startPage>%(startpage)s</startPage>
-    </assistant>
-    <docFiles>
-        <generate>
-            <file>
-                <input>%(outname)s.qhp</input>
-                <output>%(outname)s.qch</output>
-            </file>
-        </generate>
-        <register>
-            <file>%(outname)s.qch</file>
-        </register>
-    </docFiles>
-</QHelpCollectionProject>
-'''
-
-# Qt Help Project (.qhp)
-# This is the input file for the help generator.
-# It contains the table of contents, indices and references to the
-# actual documentation files (*.html).
-# In addition it defines a unique namespace for the documentation.
-project_template = u'''\
-<?xml version="1.0" encoding="utf-8" ?>
-<QtHelpProject version="1.0">
-    <namespace>%(namespace)s</namespace>
-    <virtualFolder>doc</virtualFolder>
-    <customFilter name="%(project)s %(version)s">
-        <filterAttribute>%(outname)s</filterAttribute>
-        <filterAttribute>%(version)s</filterAttribute>
-    </customFilter>
-    <filterSection>
-        <filterAttribute>%(outname)s</filterAttribute>
-        <filterAttribute>%(version)s</filterAttribute>
-        <toc>
-            <section title="%(title)s" ref="%(masterdoc)s.html">
-%(sections)s
-            </section>
-        </toc>
-        <keywords>
-%(keywords)s
-        </keywords>
-        <files>
-%(files)s
-        </files>
-    </filterSection>
-</QtHelpProject>
-'''
-
 section_template = '<section title="%(title)s" ref="%(ref)s"/>'
-file_template = ' ' * 12 + '<file>%(filename)s</file>'
+
+
+def render_file(filename, **kwargs):
+    # type: (unicode, Any) -> unicode
+    pathname = os.path.join(package_dir, 'templates', 'qthelp', filename)
+    return SphinxRenderer.render_from_file(pathname, kwargs)
 
 
 class QtHelpBuilder(StandaloneHTMLBuilder):
@@ -184,24 +132,6 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
                 keywords.extend(self.build_keywords(title, refs, subitems))
         keywords = u'\n'.join(keywords)  # type: ignore
 
-        # files
-        if not outdir.endswith(os.sep):
-            outdir += os.sep
-        olen = len(outdir)
-        projectfiles = []
-        staticdir = path.join(outdir, '_static')
-        imagesdir = path.join(outdir, self.imagedir)
-        for root, dirs, files in os.walk(outdir):
-            resourcedir = root.startswith(staticdir) or \
-                root.startswith(imagesdir)
-            for fn in sorted(files):
-                if (resourcedir and not fn.endswith('.js')) or \
-                   fn.endswith('.html'):
-                    filename = path.join(root, fn)[olen:]
-                    projectfiles.append(file_template %
-                                        {'filename': htmlescape(filename)})
-        projectfiles = '\n'.join(projectfiles)  # type: ignore
-
         # it seems that the "namespace" may not contain non-alphanumeric
         # characters, and more than one successive dot, or leading/trailing
         # dots, are also forbidden
@@ -216,16 +146,13 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
 
         # write the project file
         with codecs.open(path.join(outdir, outname + '.qhp'), 'w', 'utf-8') as f:  # type: ignore  # NOQA
-            f.write(project_template % {
+            body = render_file('project.qhp', outname=outname,
-                'outname': htmlescape(outname),
+                               title=self.config.html_title, version=self.config.version,
-                'title': htmlescape(self.config.html_title),
+                               project=self.config.project, namespace=nspace,
-                'version': htmlescape(self.config.version),
+                               master_doc=self.config.master_doc,
-                'project': htmlescape(self.config.project),
+                               sections=sections, keywords=keywords,
-                'namespace': htmlescape(nspace),
+                               files=self.get_project_files(outdir))
-                'masterdoc': htmlescape(self.config.master_doc),
+            f.write(body)
-                'sections': sections,
-                'keywords': keywords,
-                'files': projectfiles})
 
         homepage = 'qthelp://' + posixpath.join(
             nspace, 'doc', self.get_target_uri(self.config.master_doc))
@@ -233,11 +160,10 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
 
         logger.info(__('writing collection project file...'))
         with codecs.open(path.join(outdir, outname + '.qhcp'), 'w', 'utf-8') as f:  # type: ignore  # NOQA
-            f.write(collection_template % {
+            body = render_file('project.qhcp', outname=outname,
-                'outname': htmlescape(outname),
+                               title=self.config.html_short_title,
-                'title': htmlescape(self.config.html_short_title),
+                               homepage=homepage, startpage=startpage)
-                'homepage': htmlescape(homepage),
+            f.write(body)
-                'startpage': htmlescape(startpage)})
 
     def isdocnode(self, node):
         # type: (nodes.Node) -> bool
@@ -299,11 +225,12 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
         else:
             id = None
 
+        nameattr = htmlescape(name, quote=True)
+        refattr = htmlescape(ref[1], quote=True)
         if id:
-            item = ' ' * 12 + '<keyword name="%s" id="%s" ref="%s"/>' % (
+            item = ' ' * 12 + '<keyword name="%s" id="%s" ref="%s"/>' % (nameattr, id, refattr)
-                name, id, ref[1])
         else:
-            item = ' ' * 12 + '<keyword name="%s" ref="%s"/>' % (name, ref[1])
+            item = ' ' * 12 + '<keyword name="%s" ref="%s"/>' % (nameattr, refattr)
         item.encode('ascii', 'xmlcharrefreplace')
         return item
 
@@ -311,7 +238,6 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
         # type: (unicode, List[Any], Any) -> List[unicode]
         keywords = []  # type: List[unicode]
 
-        title = htmlescape(title)
         # if len(refs) == 0: # XXX
         #     write_param('See Also', title)
         if len(refs) == 1:
@@ -331,6 +257,23 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
 
         return keywords
 
+    def get_project_files(self, outdir):
+        # type: (unicode) -> List[unicode]
+        if not outdir.endswith(os.sep):
+            outdir += os.sep
+        olen = len(outdir)
+        project_files = []
+        staticdir = path.join(outdir, '_static')
+        imagesdir = path.join(outdir, self.imagedir)
+        for root, dirs, files in os.walk(outdir):
+            resourcedir = root.startswith((staticdir, imagesdir))
+            for fn in sorted(files):
+                if (resourcedir and not fn.endswith('.js')) or fn.endswith('.html'):
+                    filename = path.join(root, fn)[olen:]
+                    project_files.append(filename)
+
+        return project_files
+
 
 def setup(app):
     # type: (Sphinx) -> Dict[unicode, Any]

sphinx/domains/__init__.py

@@ -150,6 +150,8 @@ class Domain(object):
     indices = []            # type: List[Type[Index]]
     #: role name -> a warning message if reference is missing
     dangling_warnings = {}  # type: Dict[unicode, unicode]
+    #: node_class -> (enum_node_type, title_getter)
+    enumerable_nodes = {}   # type: Dict[nodes.Node, Tuple[unicode, Callable]]
 
     #: data value for a fresh environment
     initial_data = {}       # type: Dict
@@ -333,6 +335,12 @@ class Domain(object):
             return type.lname
         return _('%s %s') % (self.label, type.lname)
 
+    def get_enumerable_node_type(self, node):
+        # type: (nodes.Node) -> unicode
+        """Get type of enumerable nodes (experimental)."""
+        enum_node_type, _ = self.enumerable_nodes.get(node.__class__, (None, None))
+        return enum_node_type
+
     def get_full_qualified_name(self, node):
         # type: (nodes.Node) -> unicode
         """Return full qualified name for given node."""

sphinx/domains/std.py

@@ -11,6 +11,7 @@
 
 import re
 import unicodedata
+import warnings
 from copy import copy
 
 from docutils import nodes
@@ -19,6 +20,7 @@ from docutils.statemachine import ViewList
 from six import iteritems
 
 from sphinx import addnodes
+from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.directives import ObjectDescription
 from sphinx.domains import Domain, ObjType
 from sphinx.locale import _, __
@@ -726,7 +728,7 @@ class StandardDomain(Domain):
             return None
 
         target_node = env.get_doctree(docname).ids.get(labelid)
-        figtype = self.get_figtype(target_node)
+        figtype = self.get_enumerable_node_type(target_node)
         if figtype is None:
             return None
 
@@ -926,9 +928,9 @@ class StandardDomain(Domain):
 
         return None
 
-    def get_figtype(self, node):
+    def get_enumerable_node_type(self, node):
         # type: (nodes.Node) -> unicode
-        """Get figure type of nodes."""
+        """Get type of enumerable nodes."""
         def has_child(node, cls):
             # type: (nodes.Node, Type) -> bool
             return any(isinstance(child, cls) for child in node)
@@ -944,6 +946,17 @@ class StandardDomain(Domain):
             figtype, _ = self.enumerable_nodes.get(node.__class__, (None, None))
             return figtype
 
+    def get_figtype(self, node):
+        # type: (nodes.Node) -> unicode
+        """Get figure type of nodes.
+
+        .. deprecated:: 1.8
+        """
+        warnings.warn('StandardDomain.get_figtype() is deprecated. '
+                      'Please use get_enumerable_node_type() instead.',
+                      RemovedInSphinx30Warning)
+        return self.get_enumerable_node_type(node)
+
     def get_fignumber(self, env, builder, figtype, docname, target_node):
         # type: (BuildEnvironment, Builder, unicode, unicode, nodes.Node) -> Tuple[int, ...]
         if figtype == 'section':

sphinx/environment/collectors/toctree.py

@@ -223,6 +223,15 @@ class TocTreeCollector(EnvironmentCollector):
         env.toc_fignumbers = {}
         fignum_counter = {}  # type: Dict[unicode, Dict[Tuple[int, ...], int]]
 
+        def get_figtype(node):
+            # type: (nodes.Node) -> unicode
+            for domain in env.domains.values():
+                figtype = domain.get_enumerable_node_type(node)
+                if figtype:
+                    return figtype
+
+            return None
+
         def get_section_number(docname, section):
             # type: (unicode, nodes.Node) -> Tuple[int, ...]
             anchorname = '#' + section['ids'][0]
@@ -270,7 +279,7 @@ class TocTreeCollector(EnvironmentCollector):
 
                     continue
 
-                figtype = env.get_domain('std').get_figtype(subnode)
+                figtype = get_figtype(subnode)
                 if figtype and subnode['ids']:
                     register_fignumber(docname, secnum, figtype, subnode)
 

sphinx/ext/apidoc.py

@@ -197,16 +197,16 @@ def shall_skip(module, opts, excludes=[]):
     if not opts.implicit_namespaces and not path.exists(module):
         return True
 
-    # skip it if there is nothing (or just \n or \r\n) in the file
+    # Are we a package (here defined as __init__.py, not the folder in itself)
-    if path.exists(module) and path.getsize(module) <= 2:
+    if os.path.basename(module) == INITPY:
-        if os.path.basename(module) == '__init__.py':
+        # Yes, check if we have any non-excluded modules at all here
-            # We only want to skip packages if they do not contain any
+        all_skipped = True
-            # .py files other than __init__.py.
+        basemodule = path.dirname(module)
-            basemodule = path.dirname(module)
+        for module in glob.glob(path.join(basemodule, '*.py')):
-            for module in glob.glob(path.join(basemodule, '*.py')):
+            if not is_excluded(path.join(basemodule, module), excludes):
-                if not is_excluded(path.join(basemodule, module), excludes):
+                # There's a non-excluded module here, we won't skip
-                    return True
+                all_skipped = False
-        else:
+        if all_skipped:
             return True
 
     # skip if it has a "private" name and this is selected

sphinx/ext/mathbase.py

@@ -62,6 +62,9 @@ class MathDomain(Domain):
     dangling_warnings = {
         'eq': 'equation not found: %(target)s',
     }
+    enumerable_nodes = {  # node_class -> (figtype, title_getter)
+        displaymath: ('displaymath', None),
+    }  # type: Dict[nodes.Node, Tuple[unicode, Callable]]
 
     def clear_doc(self, docname):
         # type: (unicode) -> None
@@ -378,12 +381,12 @@ def setup_math(app, htmlinlinevisitors, htmldisplayvisitors):
                  man=(man_visit_math, None),
                  texinfo=(texinfo_visit_math, None),
                  html=htmlinlinevisitors)
-    app.add_enumerable_node(displaymath, 'displaymath',
+    app.add_node(displaymath,
-                            latex=(latex_visit_displaymath, None),
+                 latex=(latex_visit_displaymath, None),
-                            text=(text_visit_displaymath, None),
+                 text=(text_visit_displaymath, None),
-                            man=(man_visit_displaymath, man_depart_displaymath),
+                 man=(man_visit_displaymath, man_depart_displaymath),
-                            texinfo=(texinfo_visit_displaymath, texinfo_depart_displaymath),
+                 texinfo=(texinfo_visit_displaymath, texinfo_depart_displaymath),
-                            html=htmldisplayvisitors)
+                 html=htmldisplayvisitors)
     app.add_node(eqref, latex=(latex_visit_eqref, None))
     app.add_role('math', math_role)
     app.add_role('eq', EqXRefRole(warn_dangling=True))

sphinx/pycode/__init__.py

@@ -120,23 +120,3 @@ class ModuleAnalyzer(object):
             self.parse()
 
         return self.tags
-
-
-if __name__ == '__main__':
-    import time
-    import pprint
-    x0 = time.time()
-    # ma = ModuleAnalyzer.for_file(__file__.rstrip('c'), 'sphinx.builders.html')
-    ma = ModuleAnalyzer.for_file('sphinx/environment.py',
-                                 'sphinx.environment')
-    ma.tokenize()  # type: ignore
-    x1 = time.time()
-    ma.parse()
-    x2 = time.time()
-    # for (ns, name), doc in iteritems(ma.find_attr_docs()):
-    #     print '>>', ns, name
-    #     print '\n'.join(doc)
-    pprint.pprint(ma.find_tags())
-    x3 = time.time()
-    # print nodes.nice_repr(ma.parsetree, number2name)
-    print("tokenizing %.4f, parsing %.4f, finding %.4f" % (x1 - x0, x2 - x1, x3 - x2))

sphinx/templates/qthelp/project.qhcp

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<QHelpCollectionProject version="1.0">
+    <assistant>
+        <title>{{ title|e }}</title>
+        <homePage>{{ homepage|e }}</homePage>
+        <startPage>{{ startpage|e }}</startPage>
+    </assistant>
+    <docFiles>
+        <generate>
+            <file>
+                <input>{{ outname|e }}.qhp</input>
+                <output>{{ outname|e }}.qch</output>
+            </file>
+        </generate>
+        <register>
+            <file>{{ outname|e }}.qch</file>
+        </register>
+    </docFiles>
+</QHelpCollectionProject>

sphinx/templates/qthelp/project.qhp

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<QtHelpProject version="1.0">
+    <namespace>{{ namespace|e }}</namespace>
+    <virtualFolder>doc</virtualFolder>
+    <customFilter name="{{ project|e }} {{ version|e }}">
+        <filterAttribute>{{ outname|e }}</filterAttribute>
+        <filterAttribute>{{ version|e }}</filterAttribute>
+    </customFilter>
+    <filterSection>
+        <filterAttribute>{{ outname|e }}</filterAttribute>
+        <filterAttribute>{{ version|e }}</filterAttribute>
+        <toc>
+            <section title="{{ title|e }}" ref="{{ master_doc|e }}.html">
+{{ sections }}
+            </section>
+        </toc>
+        <keywords>
+{{ keywords }}
+        </keywords>
+        <files>
+            {%- for filename in files %}
+            <file>{{ filename|e }}</file>
+            {%- endfor %}
+        </files>
+    </filterSection>
+</QtHelpProject>

sphinx/transforms/post_transforms/images.py

@@ -31,6 +31,8 @@ if False:
 
 logger = logging.getLogger(__name__)
 
+MAX_FILENAME_LEN = 32
+
 
 class BaseImageConverter(SphinxTransform):
     def apply(self):
@@ -67,16 +69,21 @@ class ImageDownloader(BaseImageConverter):
 
     def handle(self, node):
         # type: (nodes.Node) -> None
-        basename = os.path.basename(node['uri'])
-        if '?' in basename:
-            basename = basename.split('?')[0]
-        if basename == '':
-            basename = sha1(node['uri'].encode("utf-8")).hexdigest()
-        dirname = node['uri'].replace('://', '/').translate({ord("?"): u"/",
-                                                             ord("&"): u"/"})
-        ensuredir(os.path.join(self.imagedir, dirname))
-        path = os.path.join(self.imagedir, dirname, basename)
         try:
+            basename = os.path.basename(node['uri'])
+            if '?' in basename:
+                basename = basename.split('?')[0]
+            if basename == '' or len(basename) > MAX_FILENAME_LEN:
+                filename, ext = os.path.splitext(node['uri'])
+                basename = sha1(filename.encode("utf-8")).hexdigest() + ext
+
+            dirname = node['uri'].replace('://', '/').translate({ord("?"): u"/",
+                                                                 ord("&"): u"/"})
+            if len(dirname) > MAX_FILENAME_LEN:
+                dirname = sha1(dirname.encode('utf-8')).hexdigest()
+            ensuredir(os.path.join(self.imagedir, dirname))
+            path = os.path.join(self.imagedir, dirname, basename)
+
             headers = {}
             if os.path.exists(path):
                 timestamp = ceil(os.stat(path).st_mtime)  # type: float

sphinx/writers/html.py

@@ -342,7 +342,7 @@ class HTMLTranslator(BaseTranslator):
                     self.body.append(prefix % '.'.join(map(str, numbers)) + ' ')
                     self.body.append('</span>')
 
-        figtype = self.builder.env.domains['std'].get_figtype(node)  # type: ignore
+        figtype = self.builder.env.domains['std'].get_enumerable_node_type(node)
         if figtype:
             if len(node['ids']) == 0:
                 msg = __('Any IDs not assigned for %s node') % node.tagname

sphinx/writers/html5.py

@@ -310,7 +310,7 @@ class HTML5Translator(BaseTranslator):
                     self.body.append(prefix % '.'.join(map(str, numbers)) + ' ')
                     self.body.append('</span>')
 
-        figtype = self.builder.env.domains['std'].get_figtype(node)  # type: ignore
+        figtype = self.builder.env.domains['std'].get_enumerable_node_type(node)
         if figtype:
             if len(node['ids']) == 0:
                 msg = __('Any IDs not assigned for %s node') % node.tagname

sphinx/writers/latex.py

@@ -1995,7 +1995,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 return
             else:
                 domain = self.builder.env.get_domain('std')
-                figtype = domain.get_figtype(next)
+                figtype = domain.get_enumerable_node_type(next)
                 if figtype and domain.get_numfig_title(next):
                     ids = set()
                     # labels for figures go in the figure body, not before

tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py

@@ -0,0 +1 @@
+"foo"

tests/roots/test-need-escaped/bar.rst

@@ -0,0 +1,2 @@
+bar
+===

tests/roots/test-need-escaped/baz.rst

@@ -0,0 +1,2 @@
+baz
+===

tests/roots/test-need-escaped/conf.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+master_doc = 'index'
+project = 'need <b>"escaped"</b> project'
+smartquotes = False

tests/roots/test-need-escaped/foo.rst

@@ -0,0 +1,15 @@
+<foo>
+=====
+
+.. toctree::
+
+   quux
+
+foo "1"
+-------
+
+foo.1-1
+^^^^^^^
+
+foo.2
+-----

tests/roots/test-need-escaped/index.rst

@@ -0,0 +1,30 @@
+.. Sphinx Tests documentation master file, created by sphinx-quickstart on Wed Jun  4 23:49:58 2008.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Sphinx Tests's documentation!
+========================================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+   :numbered:
+   :caption: Table of Contents
+   :name: mastertoc
+
+   foo
+   bar
+   http://sphinx-doc.org/
+   baz
+   qux
+
+.. index::
+   pair: "subsection"; <subsection>
+
+----------
+subsection
+----------
+
+subsubsection
+-------------

tests/roots/test-need-escaped/quux.rst

@@ -0,0 +1,2 @@
+quux
+====

tests/roots/test-need-escaped/qux.rst

@@ -0,0 +1 @@
+qux.rst has no section title

tests/test_autodoc.py

@@ -11,13 +11,15 @@
 """
 
 import sys
+from warnings import catch_warnings
 
 import pytest
 from docutils.statemachine import ViewList
 from six import PY3
 
-from sphinx.ext.autodoc import AutoDirective, add_documenter, \
+from sphinx.ext.autodoc import (
-    ModuleLevelDocumenter, FunctionDocumenter, cut_lines, between, ALL
+    AutoDirective, ModuleLevelDocumenter, FunctionDocumenter, cut_lines, between, ALL
+)
 from sphinx.testing.util import SphinxTestApp, Struct  # NOQA
 from sphinx.util import logging
 
@@ -550,7 +552,7 @@ def test_new_documenter():
         def document_members(self, all_members=False):
             return
 
-    add_documenter(MyDocumenter)
+    app.add_autodocumenter(MyDocumenter)
 
     def assert_result_contains(item, objtype, name, **kw):
         app._warning.truncate(0)
@@ -591,12 +593,13 @@ def test_attrgetter_using():
             assert fullname not in documented_members, \
                 '%r was not hooked by special_attrgetter function' % fullname
 
-    options.members = ALL
+    with catch_warnings(record=True):
-    options.inherited_members = False
+        options.members = ALL
-    assert_getter_works('class', 'target.Class', Class, ['meth'])
+        options.inherited_members = False
+        assert_getter_works('class', 'target.Class', Class, ['meth'])
 
-    options.inherited_members = True
+        options.inherited_members = True
-    assert_getter_works('class', 'target.Class', Class, ['meth', 'inheritedmeth'])
+        assert_getter_works('class', 'target.Class', Class, ['meth', 'inheritedmeth'])
 
 
 @pytest.mark.usefixtures('setup_test')

tests/test_build_epub.py

@@ -19,17 +19,17 @@ import pytest
 # check given command is runnable
 def runnable(command):
     try:
-        p = Popen(command, stdout=PIPE)
+        p = Popen(command, stdout=PIPE, stderr=PIPE)
     except OSError:
         # command not found
         return False
     else:
         p.communicate()
-        return p.returncode
+        return p.returncode == 0
 
 
 class EPUBElementTree(object):
-    """Test helper for content.opf and tox.ncx"""
+    """Test helper for content.opf and toc.ncx"""
     namespaces = {
         'idpf': 'http://www.idpf.org/2007/opf',
         'dc': 'http://purl.org/dc/elements/1.1/',
@@ -226,6 +226,62 @@ def test_nested_toc(app):
     assert navinfo(grandchild[0]) == ('foo.xhtml#foo-1-1', 'foo.1-1')
 
 
+@pytest.mark.sphinx('epub', testroot='need-escaped')
+def test_escaped_toc(app):
+    app.build()
+
+    # toc.ncx
+    toc = EPUBElementTree.fromstring((app.outdir / 'toc.ncx').bytes())
+    assert toc.find("./ncx:docTitle/ncx:text").text == ('need <b>"escaped"</b> '
+                                                        'project  documentation')
+
+    # toc.ncx / navPoint
+    def navinfo(elem):
+        label = elem.find("./ncx:navLabel/ncx:text")
+        content = elem.find("./ncx:content")
+        return (elem.get('id'), elem.get('playOrder'),
+                content.get('src'), label.text)
+
+    navpoints = toc.findall("./ncx:navMap/ncx:navPoint")
+    assert len(navpoints) == 4
+    assert navinfo(navpoints[0]) == ('navPoint1', '1', 'index.xhtml',
+                                     u"Welcome to Sphinx Tests's documentation!")
+    assert navpoints[0].findall("./ncx:navPoint") == []
+
+    # toc.ncx / nested navPoints
+    assert navinfo(navpoints[1]) == ('navPoint2', '2', 'foo.xhtml', '<foo>')
+    navchildren = navpoints[1].findall("./ncx:navPoint")
+    assert len(navchildren) == 4
+    assert navinfo(navchildren[0]) == ('navPoint3', '2', 'foo.xhtml', '<foo>')
+    assert navinfo(navchildren[1]) == ('navPoint4', '3', 'quux.xhtml', 'quux')
+    assert navinfo(navchildren[2]) == ('navPoint5', '4', 'foo.xhtml#foo-1', u'foo “1”')
+    assert navinfo(navchildren[3]) == ('navPoint8', '6', 'foo.xhtml#foo-2', 'foo.2')
+
+    # nav.xhtml / nav
+    def navinfo(elem):
+        anchor = elem.find("./xhtml:a")
+        return (anchor.get('href'), anchor.text)
+
+    nav = EPUBElementTree.fromstring((app.outdir / 'nav.xhtml').bytes())
+    toc = nav.findall("./xhtml:body/xhtml:nav/xhtml:ol/xhtml:li")
+    assert len(toc) == 4
+    assert navinfo(toc[0]) == ('index.xhtml',
+                               "Welcome to Sphinx Tests's documentation!")
+    assert toc[0].findall("./xhtml:ol") == []
+
+    # nav.xhtml / nested toc
+    assert navinfo(toc[1]) == ('foo.xhtml', '<foo>')
+    tocchildren = toc[1].findall("./xhtml:ol/xhtml:li")
+    assert len(tocchildren) == 3
+    assert navinfo(tocchildren[0]) == ('quux.xhtml', 'quux')
+    assert navinfo(tocchildren[1]) == ('foo.xhtml#foo-1', u'foo “1”')
+    assert navinfo(tocchildren[2]) == ('foo.xhtml#foo-2', 'foo.2')
+
+    grandchild = tocchildren[1].findall("./xhtml:ol/xhtml:li")
+    assert len(grandchild) == 1
+    assert navinfo(grandchild[0]) == ('foo.xhtml#foo-1-1', 'foo.1-1')
+
+
 @pytest.mark.sphinx('epub', testroot='basic')
 def test_epub_writing_mode(app):
     # horizontal (default)
@@ -266,7 +322,7 @@ def test_run_epubcheck(app):
     app.build()
 
     epubcheck = os.environ.get('EPUBCHECK_PATH', '/usr/share/java/epubcheck.jar')
-    if runnable('java') and os.path.exists(epubcheck):
+    if runnable(['java', '-version']) and os.path.exists(epubcheck):
         p = Popen(['java', '-jar', epubcheck, app.outdir / 'SphinxTests.epub'],
                   stdout=PIPE, stderr=PIPE)
         stdout, stderr = p.communicate()

tests/test_build_qthelp.py

@@ -13,16 +13,107 @@
 
 import pytest
 
+from sphinx.testing.util import etree_parse
+
+
+@pytest.mark.sphinx('qthelp', testroot='basic')
+def test_qthelp_basic(app, status, warning):
+    app.builder.build_all()
+
+    qhp = (app.outdir / 'Python.qhp').text()
+    assert '<customFilter name="Python ">' in qhp
+    assert '<filterAttribute>Python</filterAttribute>' in qhp
+    assert '<filterAttribute></filterAttribute>' in qhp
+    assert '<section title="Python  documentation" ref="index.html">' in qhp
+    assert '<file>genindex.html</file>' in qhp
+    assert '<file>index.html</file>' in qhp
+    assert '<file>_static/basic.css</file>' in qhp
+    assert '<file>_static/down.png</file>' in qhp
+
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<title>Python  documentation</title>' in qhcp
+    assert '<homePage>qthelp://org.sphinx.python/doc/index.html</homePage>' in qhcp
+    assert '<startPage>qthelp://org.sphinx.python/doc/index.html</startPage>' in qhcp
+    assert '<input>Python.qhp</input>' in qhcp
+    assert '<output>Python.qch</output>' in qhcp
+    assert '<file>Python.qch</file>' in qhcp
+
+
+@pytest.mark.sphinx('qthelp', testroot='need-escaped')
+def test_qthelp_escaped(app, status, warning):
+    app.builder.build_all()
+
+    et = etree_parse(app.outdir / 'needbescapedbproject.qhp')
+    customFilter = et.find('.//customFilter')
+    assert len(customFilter) == 2
+    assert customFilter.attrib == {'name': 'need <b>"escaped"</b> project '}
+    assert customFilter[0].text == 'needbescapedbproject'
+    assert customFilter[1].text is None
+
+    toc = et.find('.//toc')
+    assert len(toc) == 1
+    assert toc[0].attrib == {'title': 'need <b>"escaped"</b> project  documentation',
+                             'ref': 'index.html'}
+    assert len(toc[0]) == 4
+    assert toc[0][0].attrib == {'title': '<foo>', 'ref': 'foo.html'}
+    assert toc[0][0][0].attrib == {'title': 'quux', 'ref': 'quux.html'}
+    assert toc[0][0][1].attrib == {'title': 'foo "1"', 'ref': 'foo.html#foo-1'}
+    assert toc[0][0][1][0].attrib == {'title': 'foo.1-1', 'ref': 'foo.html#foo-1-1'}
+    assert toc[0][0][2].attrib == {'title': 'foo.2', 'ref': 'foo.html#foo-2'}
+    assert toc[0][1].attrib == {'title': 'bar', 'ref': 'bar.html'}
+    assert toc[0][2].attrib == {'title': 'http://sphinx-doc.org/',
+                                'ref': 'http://sphinx-doc.org/'}
+    assert toc[0][3].attrib == {'title': 'baz', 'ref': 'baz.html'}
+
+    keywords = et.find('.//keywords')
+    assert len(keywords) == 2
+    assert keywords[0].attrib == {'name': '<subsection>', 'ref': 'index.html#index-0'}
+    assert keywords[1].attrib == {'name': '"subsection"', 'ref': 'index.html#index-0'}
+
 
 @pytest.mark.sphinx('qthelp', testroot='basic')
 def test_qthelp_namespace(app, status, warning):
     # default namespace
     app.builder.build_all()
+
     qhp = (app.outdir / 'Python.qhp').text()
     assert '<namespace>org.sphinx.python</namespace>' in qhp
 
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<homePage>qthelp://org.sphinx.python/doc/index.html</homePage>' in qhcp
+    assert '<startPage>qthelp://org.sphinx.python/doc/index.html</startPage>' in qhcp
+
     # give a namespace
     app.config.qthelp_namespace = 'org.sphinx-doc.sphinx'
     app.builder.build_all()
+
     qhp = (app.outdir / 'Python.qhp').text()
     assert '<namespace>org.sphinxdoc.sphinx</namespace>' in qhp
+
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<homePage>qthelp://org.sphinxdoc.sphinx/doc/index.html</homePage>' in qhcp
+    assert '<startPage>qthelp://org.sphinxdoc.sphinx/doc/index.html</startPage>' in qhcp
+
+
+@pytest.mark.sphinx('qthelp', testroot='basic')
+def test_qthelp_title(app, status, warning):
+    # default title
+    app.builder.build_all()
+
+    qhp = (app.outdir / 'Python.qhp').text()
+    assert '<section title="Python  documentation" ref="index.html">' in qhp
+
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<title>Python  documentation</title>' in qhcp
+
+    # give a title
+    app.config.html_title = 'Sphinx <b>"full"</b> title'
+    app.config.html_short_title = 'Sphinx <b>"short"</b> title'
+    app.builder.build_all()
+
+    qhp = (app.outdir / 'Python.qhp').text()
+    assert ('<section title="Sphinx &lt;b&gt;&#34;full&#34;&lt;/b&gt; title" ref="index.html">'
+            in qhp)
+
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<title>Sphinx &lt;b&gt;&#34;short&#34;&lt;/b&gt; title</title>' in qhcp

tests/test_ext_apidoc.py

@@ -211,7 +211,7 @@ def test_trailing_underscore(make_app, apidoc):
 
 @pytest.mark.apidoc(
     coderoot='test-apidoc-pep420/a',
-    excludes=["b/c/d.py", "b/e/f.py"],
+    excludes=["b/c/d.py", "b/e/f.py", "b/e/__init__.py"],
     options=["--implicit-namespaces", "--separate"],
 )
 def test_excludes(apidoc):
@@ -223,6 +223,45 @@ def test_excludes(apidoc):
     assert (outdir / 'a.b.x.y.rst').isfile()
 
 
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=["b/e"],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_subpackage_should_be_skipped(apidoc):
+    """Subpackage exclusion should work."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert not (outdir / 'a.b.e.f.rst').isfile()  # skipped because 'b/e' subpackage is skipped
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=["b/e/f.py"],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_module_should_be_skipped(apidoc):
+    """Module exclusion should work."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert not (outdir / 'a.b.e.f.rst').isfile()  # skipped because of empty after excludes
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=[],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_module_should_not_be_skipped(apidoc):
+    """Module should be included if no excludes are used."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert (outdir / 'a.b.e.f.rst').isfile()  # skipped because of empty after excludes
+
+
 @pytest.mark.apidoc(
     coderoot='test-root',
     options=[
@@ -339,3 +378,29 @@ def extract_toc(path):
     toctree = rst[start_idx + len(toctree_start):end_idx]
 
     return toctree
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-subpackage-in-toc',
+    options=['--separate']
+)
+
+
+def test_subpackage_in_toc(make_app, apidoc):
+    """Make sure that empty subpackages with non-empty subpackages in them
+       are not skipped (issue #4520)
+    """
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+
+    assert (outdir / 'parent.rst').isfile()
+    with open(outdir / 'parent.rst') as f:
+        parent = f.read()
+    assert 'parent.child' in parent
+
+    assert (outdir / 'parent.child.rst').isfile()
+    with open(outdir / 'parent.child.rst') as f:
+        parent_child = f.read()
+    assert 'parent.child.foo' in parent_child
+
+    assert (outdir / 'parent.child.foo.rst').isfile()

tox.ini

@@ -5,7 +5,7 @@ envlist = docs,flake8,mypy,coverage,py{27,34,35,36,py},du{11,12,13,14}
 [testenv]
 usedevelop = True
 passenv =
-    https_proxy http_proxy no_proxy PERL PERL5LIB PYTEST_ADDOPTS
+    https_proxy http_proxy no_proxy PERL PERL5LIB PYTEST_ADDOPTS EPUBCHECK_PATH
 description =
     py{27,34,35,36,py}: Run unit tests against {envname}.
     du{11,12,13,14}: Run unit tests with the given version of docutils.