diff --git a/sphinx/testing/util.py b/sphinx/testing/util.py
index 42d8023f6..e15a43b4b 100644
--- a/sphinx/testing/util.py
+++ b/sphinx/testing/util.py
@@ -177,4 +177,3 @@ def _clean_up_global_state() -> None:
 
     # clean up autodoc global state
     sphinx.pycode.ModuleAnalyzer.cache.clear()
-    sys.modules.pop('autodoc_fodder', None)
diff --git a/tests/test_builders/test_build_html.py b/tests/test_builders/test_build_html.py
index 1638969f8..65ab6a54e 100644
--- a/tests/test_builders/test_build_html.py
+++ b/tests/test_builders/test_build_html.py
@@ -3,6 +3,7 @@
 import os
 import posixpath
 import re
+import sys
 
 import pytest
 
@@ -80,6 +81,7 @@ def test_html_warnings(app, warning):
         + f'--- Expected (regex):\n{warnings_exp}\n'
         + f'--- Got:\n{warnings}'
     )
+    sys.modules.pop('autodoc_fodder', None)
 
 
 def test_html4_error(make_app, tmp_path):
diff --git a/tests/test_builders/test_build_latex.py b/tests/test_builders/test_build_latex.py
index 544a7a37a..75e649f4d 100644
--- a/tests/test_builders/test_build_latex.py
+++ b/tests/test_builders/test_build_latex.py
@@ -3,6 +3,7 @@
 import os
 import re
 import subprocess
+import sys
 from itertools import chain, product
 from pathlib import Path
 from shutil import copyfile
@@ -183,6 +184,7 @@ def test_latex_warnings(app, status, warning):
         + f'--- Expected (regex):\n{warnings_exp}\n'
         + f'--- Got:\n{warnings}'
     )
+    sys.modules.pop('autodoc_fodder', None)
 
 
 @pytest.mark.sphinx('latex', testroot='basic')
diff --git a/tests/test_builders/test_build_texinfo.py b/tests/test_builders/test_build_texinfo.py
index 7759d9520..7edf82dab 100644
--- a/tests/test_builders/test_build_texinfo.py
+++ b/tests/test_builders/test_build_texinfo.py
@@ -3,6 +3,7 @@
 import os
 import re
 import subprocess
+import sys
 from pathlib import Path
 from subprocess import CalledProcessError
 from unittest.mock import Mock
@@ -36,6 +37,7 @@ def test_texinfo_warnings(app, status, warning):
         + f'--- Expected (regex):\n{warnings_exp}\n'
         + f'--- Got:\n{warnings}'
     )
+    sys.modules.pop('autodoc_fodder', None)
 
 
 @pytest.mark.sphinx('texinfo')
diff --git a/tests/test_builders/test_builder.py b/tests/test_builders/test_builder.py
index 1ff8aea05..ee946a5d5 100644
--- a/tests/test_builders/test_builder.py
+++ b/tests/test_builders/test_builder.py
@@ -1,4 +1,7 @@
 """Test the Builder class."""
+
+import sys
+
 import pytest
 
 
@@ -37,3 +40,5 @@ def test_incremental_reading_for_missing_files(app):
     # "index" is listed up to updated because it contains references
     # to nonexisting downloadable or image files
     assert set(updated) == {'index'}
+
+    sys.modules.pop('autodoc_fodder', None)