Merge branch '1.8' into 4034_fix_download_url2

CHANGES

@@ -20,6 +20,8 @@ Deprecated
 Features added
 --------------
 
+* #5388: Ensure frozen object descriptions are reproducible
+
 Bugs fixed
 ----------
 
@@ -31,6 +33,9 @@ Bugs fixed
 * #5348: download reference to remote file is not displayed
 * #5282: html theme: ``pygments_style`` of theme was overrided by ``conf.py``
   by default
+* #4379: toctree shows confusible warning when document is excluded
+* #2401: autodoc: ``:members:`` causes ``:special-members:`` not to be shown
+* autodoc: ImportError is replaced by AttributeError for deeper module
 * #2720, #4034: Incorrect links with ``:download:``, duplicate names, and
   parallel builds
 
@@ -286,8 +291,8 @@ Documentation
 * #5083: Fix wrong make.bat option for internationalization.
 * #5115: napoleon: add admonitions added by #4613 to the docs.
 
-Release 1.7.9 (in development)
+Release 1.7.10 (in development)
-==============================
+===============================
 
 Dependencies
 ------------
@@ -307,6 +312,19 @@ Bugs fixed
 Testing
 --------
 
+Release 1.7.9 (released Sep 05, 2018)
+=====================================
+
+Features added
+--------------
+
+* #5359: Make generated texinfo files reproducible by sorting the anchors
+
+Bugs fixed
+----------
+
+* #5361: crashed on incremental build if document uses include directive
+
 Release 1.7.8 (released Aug 29, 2018)
 =====================================
 

doc/usage/extensions/autosummary.rst

@@ -208,6 +208,8 @@ The following variables available in the templates:
    List containing names of all inherited members of class.  Only available for
    classes.
 
+   .. versionadded:: 1.8.0
+
 .. data:: functions
 
    List containing names of "public" functions in the module.  Here, "public"

sphinx/directives/other.py

@@ -22,7 +22,7 @@ from sphinx.domains.changeset import VersionChange  # NOQA  # for compatibility
 from sphinx.locale import _
 from sphinx.util import url_re, docname_join
 from sphinx.util.docutils import SphinxDirective
-from sphinx.util.matching import patfilter
+from sphinx.util.matching import Matcher, patfilter
 from sphinx.util.nodes import explicit_title_re, set_source_info, \
     process_index_entry
 
@@ -96,6 +96,7 @@ class TocTree(SphinxDirective):
         all_docnames.remove(self.env.docname)  # remove current document
 
         ret = []
+        excluded = Matcher(self.config.exclude_patterns)
         for entry in self.content:
             if not entry:
                 continue
@@ -131,9 +132,13 @@ class TocTree(SphinxDirective):
                 if url_re.match(ref) or ref == 'self':
                     toctree['entries'].append((title, ref))
                 elif docname not in self.env.found_docs:
-                    ret.append(self.state.document.reporter.warning(
+                    if excluded(self.env.doc2path(docname, None)):
-                        'toctree contains reference to nonexisting '
+                        message = 'toctree contains reference to excluded document %r'
-                        'document %r' % docname, line=self.lineno))
+                    else:
+                        message = 'toctree contains reference to nonexisting document %r'
+
+                    ret.append(self.state.document.reporter.warning(message % docname,
+                                                                    line=self.lineno))
                     self.env.note_reread()
                 else:
                     all_docnames.discard(docname)

sphinx/environment/__init__.py

@@ -19,7 +19,7 @@ from os import path
 
 from docutils.utils import get_source_line
 from six import BytesIO, next
-from six.moves import cPickle as pickle, reduce
+from six.moves import cPickle as pickle
 
 from sphinx import addnodes
 from sphinx.deprecation import RemovedInSphinx20Warning, RemovedInSphinx30Warning
@@ -67,7 +67,7 @@ default_settings = {
 # or changed to properly invalidate pickle files.
 #
 # NOTE: increase base version by 2 to have distinct numbers for Py2 and 3
-ENV_VERSION = 53 + (sys.version_info[0] - 2)
+ENV_VERSION = 54 + (sys.version_info[0] - 2)
 
 # config status
 CONFIG_OK = 1
@@ -725,7 +725,7 @@ class BuildEnvironment(object):
     def check_consistency(self):
         # type: () -> None
         """Do consistency checks."""
-        included = reduce(lambda x, y: x | y, self.included.values(), set())  # type: Set[unicode]  # NOQA
+        included = set().union(*self.included.values())  # type: ignore
         for docname in sorted(self.all_docs):
             if docname not in self.files_to_rebuild:
                 if docname == self.config.master_doc:

sphinx/environment/adapters/toctree.py

@@ -15,6 +15,7 @@ from six import iteritems
 from sphinx import addnodes
 from sphinx.locale import __
 from sphinx.util import url_re, logging
+from sphinx.util.matching import Matcher
 from sphinx.util.nodes import clean_astext, process_only_nodes
 
 if False:
@@ -83,6 +84,7 @@ class TocTree(object):
         # interactions between marking and pruning the tree (see bug #1046).
 
         toctree_ancestors = self.get_toctree_ancestors(docname)
+        excluded = Matcher(self.env.config.exclude_patterns)
 
         def _toctree_add_classes(node, depth):
             # type: (nodes.Node, int) -> None
@@ -172,8 +174,12 @@ class TocTree(object):
                                        ref, location=toctreenode)
                 except KeyError:
                     # this is raised if the included file does not exist
-                    logger.warning(__('toctree contains reference to nonexisting document %r'),
+                    if excluded(self.env.doc2path(ref, None)):
-                                   ref, location=toctreenode)
+                        message = __('toctree contains reference to excluded document %r')
+                    else:
+                        message = __('toctree contains reference to nonexisting document %r')
+
+                    logger.warning(message, ref, location=toctreenode)
                 else:
                     # if titles_only is given, only keep the main title and
                     # sub-toctrees

sphinx/ext/autodoc/__init__.py

@@ -110,6 +110,20 @@ def bool_option(arg):
     return True
 
 
+def merge_special_members_option(options):
+    # type: (Dict) -> None
+    """Merge :special-members: option to :members: option."""
+    if 'special-members' in options and options['special-members'] is not ALL:
+        if options.get('members') is ALL:
+            pass
+        elif options.get('members'):
+            for member in options['special-members']:
+                if member not in options['members']:
+                    options['members'].append(member)
+        else:
+            options['members'] = options['special-members']
+
+
 class AutodocReporter(object):
     """
     A reporter replacement that assigns the correct source name
@@ -823,6 +837,11 @@ class ModuleDocumenter(Documenter):
         'imported-members': bool_option, 'ignore-module-all': bool_option
     }  # type: Dict[unicode, Callable]
 
+    def __init__(self, *args):
+        # type: (Any) -> None
+        super(ModuleDocumenter, self).__init__(*args)
+        merge_special_members_option(self.options)
+
     @classmethod
     def can_document_member(cls, member, membername, isattr, parent):
         # type: (Any, unicode, bool, Any) -> bool
@@ -1075,6 +1094,11 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
         'private-members': bool_option, 'special-members': members_option,
     }  # type: Dict[unicode, Callable]
 
+    def __init__(self, *args):
+        # type: (Any) -> None
+        super(ClassDocumenter, self).__init__(*args)
+        merge_special_members_option(self.options)
+
     @classmethod
     def can_document_member(cls, member, membername, isattr, parent):
         # type: (Any, unicode, bool, Any) -> bool

sphinx/ext/autodoc/importer.py

@@ -168,13 +168,15 @@ def import_object(modname, objpath, objtype='', attrgetter=safe_getattr, warning
 
     try:
         module = None
+        exc_on_importing = None
         objpath = list(objpath)
         while module is None:
             try:
                 module = import_module(modname, warningiserror=warningiserror)
                 logger.debug('[autodoc] import %s => %r', modname, module)
-            except ImportError:
+            except ImportError as exc:
                 logger.debug('[autodoc] import %s => failed', modname)
+                exc_on_importing = exc
                 if '.' in modname:
                     # retry with parent module
                     modname, name = modname.rsplit('.', 1)
@@ -193,6 +195,10 @@ def import_object(modname, objpath, objtype='', attrgetter=safe_getattr, warning
             object_name = attrname
         return [module, parent, object_name, obj]
     except (AttributeError, ImportError) as exc:
+        if isinstance(exc, AttributeError) and exc_on_importing:
+            # restore ImportError
+            exc = exc_on_importing
+
         if objpath:
             errmsg = ('autodoc: failed to import %s %r from module %r' %
                       (objtype, '.'.join(objpath), modname))

sphinx/ext/autosummary/__init__.py

@@ -81,6 +81,7 @@ from sphinx.util import import_object, rst, logging
 from sphinx.util.docutils import (
     NullReporter, SphinxDirective, new_document, switch_source_input
 )
+from sphinx.util.matching import Matcher
 
 if False:
     # For type annotation
@@ -261,10 +262,15 @@ class Autosummary(SphinxDirective):
 
             tree_prefix = self.options['toctree'].strip()
             docnames = []
+            excluded = Matcher(self.config.exclude_patterns)
             for name, sig, summary, real_name in items:
                 docname = posixpath.join(tree_prefix, real_name)
                 docname = posixpath.normpath(posixpath.join(dirname, docname))
                 if docname not in self.env.found_docs:
+                    if excluded(self.env.doc2path(docname, None)):
+                        self.warn('toctree references excluded document %r'
+                                  % docname)
+                    else:
                         self.warn('toctree references unknown document %r'
                                   % docname)
                 docnames.append(docname)

sphinx/ext/autosummary/generate.py

@@ -43,11 +43,12 @@ from sphinx.util.rst import escape as rst_escape
 
 if False:
     # For type annotation
-    from typing import Any, Callable, Dict, Tuple, List  # NOQA
+    from typing import Any, Callable, Dict, List, Tuple, Type  # NOQA
     from jinja2 import BaseLoader  # NOQA
     from sphinx import addnodes  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA
+    from sphinx.ext.autodoc import Documenter  # NOQA
 
 
 class DummyApplication(object):
@@ -69,7 +70,7 @@ def setup_documenters(app):
         ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter,
         FunctionDocumenter, MethodDocumenter, AttributeDocumenter,
         InstanceAttributeDocumenter
-    ]
+    ]  # type: List[Type[Documenter]]
     for documenter in documenters:
         app.registry.add_documenter(documenter.objtype, documenter)
 

sphinx/writers/texinfo.py

@@ -612,7 +612,7 @@ class TexinfoTranslator(nodes.NodeVisitor):
         node_name = node['node_name']
         pointers = tuple([node_name] + self.rellinks[node_name])
         self.body.append('\n@node %s,%s,%s,%s\n' % pointers)  # type: ignore
-        for id in self.next_section_ids:
+        for id in sorted(self.next_section_ids):
             self.add_anchor(id, node)
 
         self.next_section_ids.clear()

tests/py35/test_autodoc_py35.py

@@ -18,7 +18,7 @@ import six
 from docutils.statemachine import ViewList
 from six import StringIO
 
-from sphinx.ext.autodoc import add_documenter, FunctionDocumenter, ALL  # NOQA
+from sphinx.ext.autodoc import add_documenter, FunctionDocumenter, ALL, Options  # NOQA
 from sphinx.testing.util import SphinxTestApp, Struct
 from sphinx.util import logging
 
@@ -49,7 +49,7 @@ def setup_test():
     global options, directive
     global processed_docstrings, processed_signatures
 
-    options = Struct(
+    options = Options(
         inherited_members = False,
         undoc_members = False,
         private_members = False,

tests/test_autodoc.py

@@ -21,7 +21,7 @@ from six import PY3
 
 from sphinx.ext.autodoc import (
     AutoDirective, ModuleLevelDocumenter, cut_lines, between, ALL,
-    merge_autodoc_default_flags
+    merge_autodoc_default_flags, Options
 )
 from sphinx.ext.autodoc.directive import DocumenterBridge, process_documenter_options
 from sphinx.testing.util import SphinxTestApp, Struct  # NOQA
@@ -79,7 +79,7 @@ def setup_test():
     global options, directive
     global processed_docstrings, processed_signatures
 
-    options = Struct(
+    options = Options(
         inherited_members = False,
         undoc_members = False,
         private_members = False,
@@ -757,6 +757,29 @@ def test_autodoc_imported_members(app):
 
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
 def test_autodoc_special_members(app):
+    # specific special methods
+    options = {"undoc-members": None,
+               "special-members": "__init__,__special1__"}
+    actual = do_autodoc(app, 'class', 'target.Class', options)
+    assert list(filter(lambda l: '::' in l, actual)) == [
+        '.. py:class:: Class(arg)',
+        '   .. py:method:: Class.__init__(arg)',
+        '   .. py:method:: Class.__special1__()',
+    ]
+
+    # combination with specific members
+    options = {"members": "attr,docattr",
+               "undoc-members": None,
+               "special-members": "__init__,__special1__"}
+    actual = do_autodoc(app, 'class', 'target.Class', options)
+    assert list(filter(lambda l: '::' in l, actual)) == [
+        '.. py:class:: Class(arg)',
+        '   .. py:method:: Class.__init__(arg)',
+        '   .. py:method:: Class.__special1__()',
+        '   .. py:attribute:: Class.attr',
+        '   .. py:attribute:: Class.docattr',
+    ]
+
     # all special methods
     options = {"members": None,
                "undoc-members": None,
@@ -786,33 +809,6 @@ def test_autodoc_special_members(app):
         '   .. py:method:: Class.undocmeth()'
     ]
 
-    # specific special methods
-    options = {"members": None,
-               "undoc-members": None,
-               "special-members": "__init__,__special1__"}
-    actual = do_autodoc(app, 'class', 'target.Class', options)
-    assert list(filter(lambda l: '::' in l, actual)) == [
-        '.. py:class:: Class(arg)',
-        '   .. py:method:: Class.__init__(arg)',
-        '   .. py:method:: Class.__special1__()',
-        '   .. py:attribute:: Class.attr',
-        '   .. py:attribute:: Class.descr',
-        '   .. py:attribute:: Class.docattr',
-        '   .. py:method:: Class.excludemeth()',
-        '   .. py:attribute:: Class.inst_attr_comment',
-        '   .. py:attribute:: Class.inst_attr_inline',
-        '   .. py:attribute:: Class.inst_attr_string',
-        '   .. py:attribute:: Class.mdocattr',
-        '   .. py:method:: Class.meth()',
-        '   .. py:classmethod:: Class.moore(a, e, f) -> happiness',
-        '   .. py:attribute:: Class.prop',
-        ROGER_METHOD,
-        '   .. py:attribute:: Class.skipattr',
-        '   .. py:method:: Class.skipmeth()',
-        '   .. py:attribute:: Class.udocattr',
-        '   .. py:method:: Class.undocmeth()'
-    ]
-
 
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
 def test_autodoc_ignore_module_all(app):
@@ -1551,9 +1547,7 @@ def test_autodoc_default_options_with_values(app):
     assert '   .. py:attribute:: EnumCls.val4' not in actual
 
     # with :special-members:
-    # Note that :members: must be *on* for :special-members: to work.
     app.config.autodoc_default_options = {
-        'members': None,
         'special-members': '__init__,__iter__',
     }
     actual = do_autodoc(app, 'class', 'target.CustomIter')

tests/test_build_texinfo.py

@@ -50,6 +50,10 @@ def test_texinfo_warnings(app, status, warning):
 def test_texinfo(app, status, warning):
     TexinfoTranslator.ignore_missing_images = True
     app.builder.build_all()
+    result = (app.outdir / 'SphinxTests.texi').text(encoding='utf8')
+    assert ('@anchor{markup doc}@anchor{12}'
+            '@anchor{markup id1}@anchor{13}'
+            '@anchor{markup testing-various-markup}@anchor{14}' in result)
     # now, try to run makeinfo over it
     cwd = os.getcwd()
     os.chdir(app.outdir)