Merge commit '4277eb13315d5649401190df86874b3832ddfa3e'

CHANGES

@@ -126,10 +126,12 @@ Features added
 * ``VerbatimHighlightColor`` is a new
   :ref:`LaTeX 'sphinxsetup' <latexsphinxsetup>` key (refs: #4285)
 * Easier customizability of LaTeX macros involved in rendering of code-blocks
+* Show traceback if conf.py raises an exception (refs: #4369)
 
 Bugs fixed
 ----------
 
+* #4334: sphinx-apidoc: Don't generate references to non-existing files in TOC
 * #4206: latex: reST label between paragraphs loses paragraph break
 * #4231: html: Apply fixFirefoxAnchorBug only under Firefox
 * #4221: napoleon depends on autodoc, but users need to load it manually
@@ -146,6 +148,8 @@ Bugs fixed
 * #4315: For PDF 'howto' documents, ``latex_toplevel_sectioning='part'`` generates
   ``\chapter`` commands 
 * #4214: Two todolist directives break sphinx-1.6.5
+* Fix links to external option docs with intersphinx (refs: #3769)
+* #4091: Private members not documented without :undoc-members:
 
 Testing
 --------

sphinx/config.py

@@ -10,6 +10,7 @@
 """
 
 import re
+import traceback
 from os import path, getenv
 
 from six import PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types
@@ -35,6 +36,7 @@ copyright_year_re = re.compile(r'^((\d{4}-)?)(\d{4})(?=[ ,])')
 CONFIG_SYNTAX_ERROR = "There is a syntax error in your configuration file: %s"
 if PY3:
     CONFIG_SYNTAX_ERROR += "\nDid you change the syntax from 2.x to 3.x?"
+CONFIG_ERROR = "There is a programable error in your configuration file:\n\n%s"
 CONFIG_EXIT_ERROR = "The configuration file (or one of the modules it imports) " \
                     "called sys.exit()"
 CONFIG_ENUM_WARNING = "The config value `{name}` has to be a one of {candidates}, " \
@@ -155,6 +157,8 @@ class Config(object):
                     raise ConfigError(CONFIG_SYNTAX_ERROR % err)
                 except SystemExit:
                     raise ConfigError(CONFIG_EXIT_ERROR)
+                except Exception:
+                    raise ConfigError(CONFIG_ERROR % traceback.format_exc())
 
         self._raw_config = config
         # these two must be preinitialized because extensions can add their

sphinx/domains/std.py

@@ -959,12 +959,18 @@ class StandardDomain(Domain):
 
     def get_full_qualified_name(self, node):
         # type: (nodes.Node) -> unicode
-        progname = node.get('std:program')
-        target = node.get('reftarget')
-        if progname is None or target is None:
-            return None
+        if node.get('reftype') == 'option':
+            progname = node.get('std:program')
+            command = ws_re.split(node.get('reftarget'))
+            if progname:
+                command.insert(0, progname)
+            option = command.pop()
+            if command:
+                return '.'.join(['-'.join(command), option])
+            else:
+                return None
         else:
-            return '.'.join([progname, target])
+            return None
 
 
 def setup(app):

sphinx/ext/apidoc.py

@@ -117,7 +117,11 @@ def create_package_file(root, master_package, subroot, py_files, opts, subs, is_
         text += '\n'
 
     # build a list of directories that are szvpackages (contain an INITPY file)
-    subs = [sub for sub in subs if path.isfile(path.join(root, sub, INITPY))]
+    # and also checks the INITPY file is not empty, or there are other python
+    # source files in that folder.
+    # (depending on settings - but shall_skip() takes care of that)
+    subs = [sub for sub in subs if not
+            shall_skip(path.join(root, sub, INITPY), opts)]
     # if there are some package directories, add a TOC for theses subpackages
     if subs:
         text += format_heading(2, 'Subpackages')

sphinx/ext/autodoc/__init__.py

@@ -715,8 +715,7 @@ class Documenter(object):
             elif (namespace, membername) in attr_docs:
                 if want_all and membername.startswith('_'):
                     # ignore members whose name starts with _ by default
-                    keep = self.options.private_members and \
-                        (has_doc or self.options.undoc_members)
+                    keep = self.options.private_members
                 else:
                     # keep documented attributes
                     keep = True

sphinx/ext/intersphinx.py

@@ -304,6 +304,7 @@ def missing_reference(app, env, node, contnode):
             in_set = setname
             to_try.append((inventories.named_inventory[setname], newtarget))
             if domain:
+                node['reftarget'] = newtarget
                 full_qualified_name = env.get_domain(domain).get_full_qualified_name(node)
                 if full_qualified_name:
                     to_try.append((inventories.named_inventory[setname], full_qualified_name))

tests/roots/test-apidoc-toc/mypackage/main.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+import os
+
+import mod_resource
+
+import mod_something
+
+
+if __name__ == "__main__":
+    print("Hello, world! -> something returns: {}".format(mod_something.something()))
+
+    res_path = \
+        os.path.join(os.path.dirname(mod_resource.__file__), 'resource.txt')
+    with open(res_path) as f:
+        text = f.read()
+    print("From mod_resource:resource.txt -> {}".format(text))

tests/roots/test-apidoc-toc/mypackage/no_init/foo.py

@@ -0,0 +1 @@
+MESSAGE="There's no __init__.py in this folder, hence we should be left out"

tests/roots/test-apidoc-toc/mypackage/resource/resource.txt

@@ -0,0 +1 @@
+This is a text resource to be included in this otherwise empty module. No python contents here.

tests/roots/test-apidoc-toc/mypackage/something/__init__.py

@@ -0,0 +1 @@
+"Subpackage Something"

tests/test_ext_apidoc.py

@@ -188,3 +188,80 @@ def test_extension_parsed(make_app, apidoc):
     with open(outdir / 'conf.py') as f:
         rst = f.read()
         assert "sphinx.ext.mathjax" in rst
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-toc',
+    options=["--implicit-namespaces"],
+)
+def test_toc_all_references_should_exist_pep420_enabled(make_app, apidoc):
+    """All references in toc should exist. This test doesn't say if
+       directories with empty __init__.py and and nothing else should be
+       skipped, just ensures consistency between what's referenced in the toc
+       and what is created. This is the variant with pep420 enabled.
+    """
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+
+    toc = extract_toc(outdir / 'mypackage.rst')
+
+    refs = [l.strip() for l in toc.splitlines() if l.strip()]
+    found_refs = []
+    missing_files = []
+    for ref in refs:
+        if ref and ref[0] in (':', '#'):
+            continue
+        found_refs.append(ref)
+        filename = "{}.rst".format(ref)
+        if not (outdir / filename).isfile():
+            missing_files.append(filename)
+
+    assert len(missing_files) == 0, \
+        'File(s) referenced in TOC not found: {}\n' \
+        'TOC:\n{}'.format(", ".join(missing_files), toc)
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-toc',
+)
+def test_toc_all_references_should_exist_pep420_disabled(make_app, apidoc):
+    """All references in toc should exist. This test doesn't say if
+       directories with empty __init__.py and and nothing else should be
+       skipped, just ensures consistency between what's referenced in the toc
+       and what is created. This is the variant with pep420 disabled.
+    """
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+
+    toc = extract_toc(outdir / 'mypackage.rst')
+
+    refs = [l.strip() for l in toc.splitlines() if l.strip()]
+    found_refs = []
+    missing_files = []
+    for ref in refs:
+        if ref and ref[0] in (':', '#'):
+            continue
+        filename = "{}.rst".format(ref)
+        found_refs.append(ref)
+        if not (outdir / filename).isfile():
+            missing_files.append(filename)
+
+    assert len(missing_files) == 0, \
+        'File(s) referenced in TOC not found: {}\n' \
+        'TOC:\n{}'.format(", ".join(missing_files), toc)
+
+
+def extract_toc(path):
+    """Helper: Extract toc section from package rst file"""
+    with open(path) as f:
+        rst = f.read()
+
+    # Read out the part containing the toctree
+    toctree_start = "\n.. toctree::\n"
+    toctree_end = "\nSubmodules"
+
+    start_idx = rst.index(toctree_start)
+    end_idx = rst.index(toctree_end, start_idx)
+    toctree = rst[start_idx + len(toctree_start):end_idx]
+
+    return toctree

tests/test_ext_intersphinx.py

@@ -194,7 +194,7 @@ def test_missing_reference_stddomain(tempdir, app, status, warning):
     inv_file = tempdir / 'inventory'
     inv_file.write_bytes(inventory_v2)
     app.config.intersphinx_mapping = {
-        'https://docs.python.org/': inv_file,
+        'cmd': ('https://docs.python.org/', inv_file),
     }
     app.config.intersphinx_cache_limit = 0
 
@@ -213,6 +213,12 @@ def test_missing_reference_stddomain(tempdir, app, status, warning):
     rn = missing_reference(app, app.env, node, contnode)
     assert rn.astext() == 'ls -l'
 
+    # refers inventory by name
+    kwargs = {}
+    node, contnode = fake_node('std', 'option', 'cmd:ls -l', '-l', **kwargs)
+    rn = missing_reference(app, app.env, node, contnode)
+    assert rn.astext() == '-l'
+
 
 @pytest.mark.sphinx('html', testroot='ext-intersphinx-cppdomain')
 def test_missing_reference_cppdomain(tempdir, app, status, warning):