Leverage `importlib.reload` for reloading modules (#11679)

Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com>

CHANGES

@@ -4,6 +4,11 @@ Release 7.2.6 (in development)
 Bugs fixed
 ----------
 
+* #11679: Add the :envvar:`!SPHINX_AUTODOC_RELOAD_MODULES` environment variable,
+  which if set reloads modules when using autodoc with ``TYPE_CHECKING = True``.
+  Patch by Matt Wozniski and Adam Turner.
+* #11679: Use :py:func:`importlib.reload` to reload modules in autodoc.
+  Patch by Matt Wozniski and Adam Turner.
 
 Release 7.2.5 (released Aug 30, 2023)
 =====================================

doc/conf.py

@@ -6,6 +6,8 @@ import time
 
 import sphinx
 
+os.environ['SPHINX_AUTODOC_RELOAD_MODULES'] = '1'
+
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo',
               'sphinx.ext.autosummary', 'sphinx.ext.extlinks',
               'sphinx.ext.intersphinx',

sphinx/ext/autodoc/importer.py

@@ -2,7 +2,9 @@
 
 from __future__ import annotations
 
+import contextlib
 import importlib
+import os
 import sys
 import traceback
 import typing
@@ -21,6 +23,8 @@ from sphinx.util.inspect import (
 )
 
 if TYPE_CHECKING:
+    from types import ModuleType
+
     from sphinx.ext.autodoc import ObjectMember
 
 logger = logging.getLogger(__name__)
@@ -69,6 +73,19 @@ def import_module(modname: str, warningiserror: bool = False) -> Any:
         raise ImportError(exc, traceback.format_exc()) from exc
 
 
+def _reload_module(module: ModuleType, warningiserror: bool = False) -> Any:
+    """
+    Call importlib.reload(module), convert exceptions to ImportError
+    """
+    try:
+        with logging.skip_warningiserror(not warningiserror):
+            return importlib.reload(module)
+    except BaseException as exc:
+        # Importing modules may cause any side effects, including
+        # SystemExit, so we need to catch all errors.
+        raise ImportError(exc, traceback.format_exc()) from exc
+
+
 def import_object(modname: str, objpath: list[str], objtype: str = '',
                   attrgetter: Callable[[Any, str], Any] = safe_getattr,
                   warningiserror: bool = False) -> Any:
@@ -83,23 +100,20 @@ def import_object(modname: str, objpath: list[str], objtype: str = '',
         objpath = list(objpath)
         while module is None:
             try:
-                orig_modules = frozenset(sys.modules)
-                try:
-                    # try importing with ``typing.TYPE_CHECKING == True``
-                    typing.TYPE_CHECKING = True
-                    module = import_module(modname, warningiserror=warningiserror)
-                except ImportError:
-                    # if that fails (e.g. circular import), retry with
-                    # ``typing.TYPE_CHECKING == False`` after reverting
-                    # changes made to ``sys.modules`` by the failed try
-                    for m in [m for m in sys.modules if m not in orig_modules]:
-                        sys.modules.pop(m)
-
-                    typing.TYPE_CHECKING = False
-                    module = import_module(modname, warningiserror=warningiserror)
-                finally:
-                    # ensure ``typing.TYPE_CHECKING == False``
-                    typing.TYPE_CHECKING = False
+                original_module_names = frozenset(sys.modules)
+                module = import_module(modname, warningiserror=warningiserror)
+                if os.environ.get('SPHINX_AUTODOC_RELOAD_MODULES'):
+                    new_modules = [m for m in sys.modules if m not in original_module_names]
+                    # Try reloading modules with ``typing.TYPE_CHECKING == True``.
+                    try:
+                        typing.TYPE_CHECKING = True
+                        # Ignore failures; we've already successfully loaded these modules
+                        with contextlib.suppress(ImportError, KeyError):
+                            for m in new_modules:
+                                _reload_module(sys.modules[m])
+                    finally:
+                        typing.TYPE_CHECKING = False
+                    module = sys.modules[modname]
                 logger.debug('[autodoc] import %s => %r', modname, module)
             except ImportError as exc:
                 logger.debug('[autodoc] import %s => failed', modname)

tests/conftest.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 import docutils
@@ -24,6 +25,8 @@ pytest_plugins = 'sphinx.testing.fixtures'
 # Exclude 'roots' dirs for pytest test collector
 collect_ignore = ['roots']
 
+os.environ['SPHINX_AUTODOC_RELOAD_MODULES'] = '1'
+
 
 @pytest.fixture(scope='session')
 def rootdir():