diff --git a/tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py
deleted file mode 100644
index 9e3f9f195..000000000
--- a/tests/roots/test-ext-autosummary-recursive/package/package/module_importfail.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import sys
-
-# Fail module import in a catastrophic way
-sys.exit(1)
diff --git a/tests/roots/test-ext-autosummary-recursive/package/package/package/__init__.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tests/roots/test-ext-autosummary-recursive/package/package/package/module.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/module.py
deleted file mode 100644
index 5506d0bc9..000000000
--- a/tests/roots/test-ext-autosummary-recursive/package/package/package/module.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from os import *  # NOQA
-
-
-class Foo:
-    def __init__(self):
-        pass
-
-    def bar(self):
-        pass
-
-    @property
-    def baz(self):
-        pass
diff --git a/tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py b/tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py
deleted file mode 100644
index 9e3f9f195..000000000
--- a/tests/roots/test-ext-autosummary-recursive/package/package/package/module_importfail.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import sys
-
-# Fail module import in a catastrophic way
-sys.exit(1)
diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py
index 3bcbe56bc..0890a1e5d 100644
--- a/tests/test_ext_autosummary.py
+++ b/tests/test_ext_autosummary.py
@@ -273,29 +273,19 @@ def test_autosummary_recursive(app, status, warning):
     content = (generated / 'package.rst').read_text()
     assert 'package.module' in content
     assert 'package.package' in content
+    assert 'package.module_importfail' in content
 
     # Recursively generate modules of top-level package
     generated /= toctree
     assert (generated / 'package.module.rst').exists()
+    assert (generated / 'package.module_importfail.rst').exists() is False
     assert (generated / 'package.package.rst').exists()
     content = (generated / 'package.package.rst').read_text()
     assert 'package.package.module' in content
-    assert 'package.package.package' in content
 
     # Recursively generate modules of sub-package
     generated /= toctree
     assert (generated / 'package.package.module.rst').exists()
-    assert (generated / 'package.package.package.rst').exists()
-    content = (generated / 'package.package.package.rst').read_text()
-    assert 'package.package.package.module' in content
-    assert 'package.package.package.package' not in content
-
-    # Last sub-package has no sub-packages
-    generated /= toctree
-    assert (generated / 'package.package.package.module.rst').exists()
-    assert not (generated / 'package.package.package.package.rst').exists()
-    if toctree:
-        assert not (generated / toctree).exists()
 
     # autosummary without :recursive: option
     generated = app.srcdir / 'generated'