Viewcode: Fix issue with import paths that differ from the directory structure (#13195)

Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com>

AUTHORS.rst

@@ -28,7 +28,7 @@ Contributors
 * Antonio Valentino -- qthelp builder, docstring inheritance
 * Antti Kaihola -- doctest extension (skipif option)
 * Barry Warsaw -- setup command improvements
-* Ben Egan -- Napoleon improvements
+* Ben Egan -- Napoleon improvements & viewcode improvements
 * Benjamin Peterson -- unittests
 * Blaise Laflamme -- pyramid theme
 * Brecht Machiels -- builder entry-points

CHANGES.rst

@@ -45,6 +45,9 @@ Bugs fixed
   term indices before accessing them.
 * #11233: linkcheck: match redirect URIs against :confval:`linkcheck_ignore` by
   overriding session-level ``requests.get_redirect_target``.
+* #13195: viewcode: Fix issue where import paths differ from the directory
+  structure.
+  Patch by Ben Egan and Adam Turner.
 
 Testing
 -------

sphinx/ext/viewcode.py

@@ -2,11 +2,11 @@
 
 from __future__ import annotations
 
+import importlib.util
 import operator
 import os.path
 import posixpath
 import traceback
-from importlib import import_module
 from typing import TYPE_CHECKING, Any, cast
 
 from docutils import nodes
@@ -48,12 +48,30 @@ class viewcode_anchor(Element):
 
 
 def _get_full_modname(modname: str, attribute: str) -> str | None:
+    if modname is None:
+        # Prevents a TypeError: if the last getattr() call will return None
+        # then it's better to return it directly
+        return None
+
     try:
-        if modname is None:
-            # Prevents a TypeError: if the last getattr() call will return None
-            # then it's better to return it directly
+        # Attempt to find full path of module
+        module_path = modname.split('.')
+        num_parts = len(module_path)
+        for i in range(num_parts, 0, -1):
+            mod_root = '.'.join(module_path[:i])
+            module_spec = importlib.util.find_spec(mod_root)
+            if module_spec is not None:
+                break
+        else:
             return None
-        module = import_module(modname)
+        # Load and execute the module
+        module = importlib.util.module_from_spec(module_spec)
+        if module_spec.loader is None:
+            return None
+        module_spec.loader.exec_module(module)
+        if i != num_parts:
+            for mod in module_path[i:]:
+                module = getattr(module, mod)
 
         # Allow an attribute to have multiple parts and incidentally allow
         # repeated .s in the attribute.

tests/roots/test-ext-viewcode-find-package/conf.py

@@ -0,0 +1,24 @@
+import os
+import sys
+
+source_dir = os.path.abspath('.')
+if source_dir not in sys.path:
+    sys.path.insert(0, source_dir)
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
+exclude_patterns = ['_build']
+
+
+if 'test_linkcode' in tags:  # NoQA: F821 (tags is injected into conf.py)
+    extensions.remove('sphinx.ext.viewcode')
+    extensions.append('sphinx.ext.linkcode')
+
+    def linkcode_resolve(domain, info):
+        if domain == 'py':
+            fn = info['module'].replace('.', '/')
+            return 'http://foobar/source/%s.py' % fn
+        elif domain == 'js':
+            return 'http://foobar/js/' + info['fullname']
+        elif domain in {'c', 'cpp'}:
+            return 'http://foobar/%s/%s' % (domain, ''.join(info['names']))
+        else:
+            raise AssertionError

tests/roots/test-ext-viewcode-find-package/index.rst

@@ -0,0 +1,10 @@
+viewcode
+========
+
+.. currentmodule:: main_package.subpackage.submodule
+
+.. autofunction:: func1
+
+.. autoclass:: Class1
+
+.. autoclass:: Class3

tests/roots/test-ext-viewcode-find-package/main_package/__init__.py

@@ -0,0 +1 @@
+from main_package import subpackage

tests/roots/test-ext-viewcode-find-package/main_package/subpackage/__init__.py

@@ -0,0 +1,3 @@
+from main_package.subpackage._subpackage2 import submodule
+
+__all__ = ['submodule']

tests/roots/test-ext-viewcode-find-package/main_package/subpackage/_subpackage2/__init__.py

@@ -0,0 +1 @@
+

tests/roots/test-ext-viewcode-find-package/main_package/subpackage/_subpackage2/submodule.py

@@ -0,0 +1,32 @@
+"""
+submodule
+"""
+# raise RuntimeError('This module should not get imported')
+
+
+def decorator(f):
+    return f
+
+
+@decorator
+def func1(a, b):
+    """
+    this is func1
+    """
+    return a, b
+
+
+@decorator
+class Class1:
+    """
+    this is Class1
+    """
+
+
+class Class3:
+    """
+    this is Class3
+    """
+
+    class_attr = 42
+    """this is the class attribute class_attr"""

tests/test_extensions/test_ext_viewcode.py

@@ -162,3 +162,24 @@ def test_local_source_files(app):
         'This is the class attribute class_attr',
     ):
         assert result.count(needle) == 1
+
+
+@pytest.mark.sphinx('html', testroot='ext-viewcode-find-package', freshenv=True)
+def test_find_local_package_import_path(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'index.html').read_text(encoding='utf8')
+
+    count_func1 = result.count(
+        'href="_modules/main_package/subpackage/_subpackage2/submodule.html#func1"'
+    )
+    assert count_func1 == 1
+
+    count_class1 = result.count(
+        'href="_modules/main_package/subpackage/_subpackage2/submodule.html#Class1"'
+    )
+    assert count_class1 == 1
+
+    count_class3 = result.count(
+        'href="_modules/main_package/subpackage/_subpackage2/submodule.html#Class3"'
+    )
+    assert count_class3 == 1