Closes #4520 - acidic: Subpackage not in toc. The rule of skipping folders with only an empty __init__.py has been removed. The reason for this is that it was never working consistently in the first place and made the code unnecessary hard to reason about. Tests for the TOC generation have been added, as well as tests for the exclude mechanism since they are coupled. One test (test_ext_apidoc.py::test_exclude) has also been modified to reflect the new behaviour.

CHANGES

@@ -6,6 +6,7 @@ Dependencies
 
 Incompatible changes
 --------------------
+* apidoc: As a consequence of a bug fix (#4520#) and cleaning up the code, folders with an empty __init__.py are no longer excluded from TOC.
 
 Deprecated
 ----------
@@ -34,6 +35,7 @@ Bugs fixed
 * #4754: sphinx/pycode/__init__.py raises AttributeError
 * #1435: qthelp builder should htmlescape keywords
 * epub: Fix docTitle elements of toc.ncx is not escaped
+* #4520#: apidoc: Subpackage not in toc (introduced in 1.6.6) now fixed.
 
 Testing
 --------
@@ -41,6 +43,9 @@ Testing
 Release 1.7.1 (released Feb 23, 2018)
 =====================================
 
+Dependencies
+------------
+
 Deprecated
 ----------
 

sphinx/ext/apidoc.py

@@ -194,16 +194,15 @@ def shall_skip(module, opts, excludes=[]):
     if not opts.implicit_namespaces and not path.exists(module):
         return True
 
-    # skip it if there is nothing (or just \n or \r\n) in the file
-    if path.exists(module) and path.getsize(module) <= 2:
-        if os.path.basename(module) == '__init__.py':
-            # We only want to skip packages if they do not contain any
-            # .py files other than __init__.py.
-            basemodule = path.dirname(module)
-            for module in glob.glob(path.join(basemodule, '*.py')):
-                if not is_excluded(path.join(basemodule, module), excludes):
-                    return True
-        else:
+    # Are we a package (here defined as __init__.py, not the folder in itself)
+    if os.path.basename(module) == INITPY:
+        # Yes, check if we have any non-excluded modules at all here
+        basemodule = path.dirname(module)
+        for module in glob.glob(path.join(basemodule, '*.py')):
+            if not is_excluded(path.join(basemodule, module), excludes):
+                # There's a non-excluded module here, we won't skip
+                all_skipped = False
+        if all_skipped:
             return True
 
     # skip if it has a "private" name and this is selected

tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py

@@ -0,0 +1 @@
+"foo"

tests/test_ext_apidoc.py

@@ -211,7 +211,7 @@ def test_trailing_underscore(make_app, apidoc):
 
 @pytest.mark.apidoc(
     coderoot='test-apidoc-pep420/a',
-    excludes=["b/c/d.py", "b/e/f.py"],
+    excludes=["b/c/d.py", "b/e/f.py", "b/e/__init__.py"],
     options=["--implicit-namespaces", "--separate"],
 )
 def test_excludes(apidoc):
@@ -223,6 +223,45 @@ def test_excludes(apidoc):
     assert (outdir / 'a.b.x.y.rst').isfile()
 
 
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=["b/e"],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_subpackage_should_be_skipped(apidoc):
+    """Subpackage exclusion should work."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert not (outdir / 'a.b.e.f.rst').isfile()  # skipped because 'b/e' subpackage is skipped
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=["b/e/f.py"],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_module_should_be_skipped(apidoc):
+    """Module exclusion should work."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert not (outdir / 'a.b.e.f.rst').isfile()  # skipped because of empty after excludes
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=[],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_module_should_not_be_skipped(apidoc):
+    """Module should be included if no excludes are used."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert (outdir / 'a.b.e.f.rst').isfile()  # skipped because of empty after excludes
+
+
 @pytest.mark.apidoc(
     coderoot='test-root',
     options=[
@@ -339,3 +378,29 @@ def extract_toc(path):
     toctree = rst[start_idx + len(toctree_start):end_idx]
 
     return toctree
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-subpackage-in-toc',
+    options=['--separate']
+)
+
+
+def test_subpackage_in_toc(make_app, apidoc):
+    """Make sure that empty subpackages with non-empty subpackages in them
+       are not skipped (issue #4520)
+    """
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+
+    assert (outdir / 'parent.rst').isfile()
+    with open(outdir / 'parent.rst') as f:
+        parent = f.read()
+    assert 'parent.child' in parent
+
+    assert (outdir / 'parent.child.rst').isfile()
+    with open(outdir / 'parent.child.rst') as f:
+        parent_child = f.read()
+    assert 'parent.child.foo' in parent_child
+
+    assert (outdir / 'parent.child.foo.rst').isfile()