diff --git a/CHANGES b/CHANGES
index b226db3a3..34d5a5bd8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,7 @@ Dependencies
 
 Incompatible changes
 --------------------
+* apidoc: As a consequence of a bug fix (#4520#) and cleaning up the code, folders with an empty __init__.py are no longer excluded from TOC.
 
 Deprecated
 ----------
@@ -34,6 +35,7 @@ Bugs fixed
 * #4754: sphinx/pycode/__init__.py raises AttributeError
 * #1435: qthelp builder should htmlescape keywords
 * epub: Fix docTitle elements of toc.ncx is not escaped
+* #4520#: apidoc: Subpackage not in toc (introduced in 1.6.6) now fixed.
 
 Testing
 --------
@@ -41,6 +43,9 @@ Testing
 Release 1.7.1 (released Feb 23, 2018)
 =====================================
 
+Dependencies
+------------
+
 Deprecated
 ----------
 
diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py
index 6704b150d..3ea563d50 100644
--- a/sphinx/ext/apidoc.py
+++ b/sphinx/ext/apidoc.py
@@ -194,16 +194,15 @@ def shall_skip(module, opts, excludes=[]):
     if not opts.implicit_namespaces and not path.exists(module):
         return True
 
-    # skip it if there is nothing (or just \n or \r\n) in the file
-    if path.exists(module) and path.getsize(module) <= 2:
-        if os.path.basename(module) == '__init__.py':
-            # We only want to skip packages if they do not contain any
-            # .py files other than __init__.py.
-            basemodule = path.dirname(module)
-            for module in glob.glob(path.join(basemodule, '*.py')):
-                if not is_excluded(path.join(basemodule, module), excludes):
-                    return True
-        else:
+    # Are we a package (here defined as __init__.py, not the folder in itself)
+    if os.path.basename(module) == INITPY:
+        # Yes, check if we have any non-excluded modules at all here
+        basemodule = path.dirname(module)
+        for module in glob.glob(path.join(basemodule, '*.py')):
+            if not is_excluded(path.join(basemodule, module), excludes):
+                # There's a non-excluded module here, we won't skip
+                all_skipped = False
+        if all_skipped:
             return True
 
     # skip if it has a "private" name and this is selected
diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/__init__.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/child/__init__.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py
new file mode 100644
index 000000000..810c96eee
--- /dev/null
+++ b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py
@@ -0,0 +1 @@
+"foo"
diff --git a/tests/test_ext_apidoc.py b/tests/test_ext_apidoc.py
index 836c20b04..d3d61d1e0 100644
--- a/tests/test_ext_apidoc.py
+++ b/tests/test_ext_apidoc.py
@@ -211,7 +211,7 @@ def test_trailing_underscore(make_app, apidoc):
 
 @pytest.mark.apidoc(
     coderoot='test-apidoc-pep420/a',
-    excludes=["b/c/d.py", "b/e/f.py"],
+    excludes=["b/c/d.py", "b/e/f.py", "b/e/__init__.py"],
     options=["--implicit-namespaces", "--separate"],
 )
 def test_excludes(apidoc):
@@ -223,6 +223,45 @@ def test_excludes(apidoc):
     assert (outdir / 'a.b.x.y.rst').isfile()
 
 
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=["b/e"],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_subpackage_should_be_skipped(apidoc):
+    """Subpackage exclusion should work."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert not (outdir / 'a.b.e.f.rst').isfile()  # skipped because 'b/e' subpackage is skipped
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=["b/e/f.py"],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_module_should_be_skipped(apidoc):
+    """Module exclusion should work."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert not (outdir / 'a.b.e.f.rst').isfile()  # skipped because of empty after excludes
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=[],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_module_should_not_be_skipped(apidoc):
+    """Module should be included if no excludes are used."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert (outdir / 'a.b.e.f.rst').isfile()  # skipped because of empty after excludes
+
+
 @pytest.mark.apidoc(
     coderoot='test-root',
     options=[
@@ -339,3 +378,29 @@ def extract_toc(path):
     toctree = rst[start_idx + len(toctree_start):end_idx]
 
     return toctree
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-subpackage-in-toc',
+    options=['--separate']
+)
+
+
+def test_subpackage_in_toc(make_app, apidoc):
+    """Make sure that empty subpackages with non-empty subpackages in them
+       are not skipped (issue #4520)
+    """
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+
+    assert (outdir / 'parent.rst').isfile()
+    with open(outdir / 'parent.rst') as f:
+        parent = f.read()
+    assert 'parent.child' in parent
+
+    assert (outdir / 'parent.child.rst').isfile()
+    with open(outdir / 'parent.child.rst') as f:
+        parent_child = f.read()
+    assert 'parent.child.foo' in parent_child
+
+    assert (outdir / 'parent.child.foo.rst').isfile()