ext: enhance autodoc_mock_imports

The autodoc_mock_imports option requires to explicitly declare *every*
external module and sub-module that are imported by the documented code.
This is not practical as the list can become very large and must be
maintained as the code changes.

Also, the mocking is minimal which causes errors when compiling the
docs. For example, if you declare:

    autodoc_mock_imports = ['django.template']

And try to document a module:

    .. automodule:: my.lib.util

Which contains this code:

    from django.template import Library
    register = Library()

The following error occurs:

    File ".../my/lib/util.py" line 2
        register = Library()
    TypeError: 'object' object is not callable

Other similar errors can occur such as "TypeError: 'object' object has
no len".

To address these limitations, only require to declare the top-level
module that should be mocked:

    autodoc_mock_imports = ['django']

Will mock "django" but also any sub-module: "django.template",
"django.contrib", etc.

Also, make the mocked modules yield more complete dummy objects to avoid
these TypeError problems.

Behind the scenes, it uses the python import hooks mechanism specified
in PEP 302.

Signed-off-by: Robin Jarry <robin@jarry.cc>

doc/ext/autodoc.rst

@@ -371,7 +371,14 @@ There are also new config values that you can set:
 
    This value contains a list of modules to be mocked up. This is useful when
    some external dependencies are not met at build time and break the building
-   process.
+   process. You may only specify the root package of the dependencies
+   themselves and ommit the sub-modules:
+
+   .. code-block:: python
+
+      autodoc_mock_imports = ["django"]
+
+   Will mock all imports under the ``django`` package.
 
    .. versionadded:: 1.3
 

sphinx/ext/autodoc.py

@@ -16,7 +16,7 @@ import sys
 import inspect
 import traceback
 import warnings
-from types import FunctionType, BuiltinFunctionType, MethodType
+from types import FunctionType, BuiltinFunctionType, MethodType, ModuleType
 
 from six import PY2, iterkeys, iteritems, itervalues, text_type, class_types, \
     string_types, StringIO
@@ -41,7 +41,6 @@ from sphinx.util.docstrings import prepare_docstring
 if False:
     # For type annotation
     from typing import Any, Callable, Dict, Iterator, List, Sequence, Set, Tuple, Type, Union  # NOQA
-    from types import ModuleType  # NOQA
     from docutils.utils import Reporter  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -107,49 +106,102 @@ class Options(dict):
             return None
 
 
-class _MockModule(object):
+class _MockObject(object):
     """Used by autodoc_mock_imports."""
-    __file__ = '/dev/null'
-    __path__ = '/dev/null'
 
     def __init__(self, *args, **kwargs):
         # type: (Any, Any) -> None
-        self.__all__ = []  # type: List[str]
+        pass
 
-    def __call__(self, *args, **kwargs):
-        # type: (Any, Any) -> _MockModule
+    def __len__(self):
+        # type: () -> int
+        return 0
+
+    def __contains__(self, key):
+        # type: (str) -> bool
+        return False
+
+    def __iter__(self):
+        # type: () -> None
+        pass
+
+    def __getitem__(self, key):
+        # type: (str) -> _MockObject
+        return self
+
+    def __getattr__(self, key):
+        # type: (str) -> _MockObject
+        return self
+
+    def __call__(self, *args, **kw):
+        # type: (Any, Any) -> Any
         if args and type(args[0]) in [FunctionType, MethodType]:
             # Appears to be a decorator, pass through unchanged
             return args[0]
-        return _MockModule()
+        return self
 
-    def _append_submodule(self, submod):
-        # type: (str) -> None
-        self.__all__.append(submod)
 
-    @classmethod
-    def __getattr__(cls, name):
-        # type: (unicode) -> Any
-        if name[0] == name[0].upper():
-            # Not very good, we assume Uppercase names are classes...
-            mocktype = type(name, (), {})  # type: ignore
-            mocktype.__module__ = __name__
-            return mocktype
+class _MockModule(ModuleType):
+    """Used by autodoc_mock_imports."""
+    __file__ = '/dev/null'
+
+    def __init__(self, name, loader):
+        # type: (str, _MockImporter) -> None
+        self.__name__ = self.__package__ = name
+        self.__loader__ = loader
+        self.__all__ = []  # type: List[str]
+        self.__path__ = []  # type: List[str]
+
+    def __getattr__(self, name):
+        # type: (str) -> _MockObject
+        o = _MockObject()
+        o.__module__ = self.__name__
+        return o
+
+
+class _MockImporter(object):
+
+    def __init__(self, names):
+        # type: (List[str]) -> None
+        self.base_packages = set()  # type: Set[str]
+        for n in names:
+            # Convert module names:
+            #     ['a.b.c', 'd.e']
+            # to a set of base packages:
+            #     set(['a', 'd'])
+            self.base_packages.add(n.split('.')[0])
+        self.mocked_modules = []  # type: List[str]
+        self.orig_meta_path = sys.meta_path
+        # enable hook by adding itself to meta_path
+        sys.meta_path = sys.meta_path + [self]
+
+    def disable(self):
+        # restore original meta_path to disable import hook
+        sys.meta_path = self.orig_meta_path
+        # remove mocked modules from sys.modules to avoid side effects after
+        # running auto-documenter
+        for m in self.mocked_modules:
+            if m in sys.modules:
+                del sys.modules[m]
+
+    def find_module(self, name, path=None):
+        # type: (str, str) -> Any
+        base_package = name.split('.')[0]
+        if base_package in self.base_packages:
+            return self
+        return None
+
+    def load_module(self, name):
+        # type: (str) -> ModuleType
+        if name in sys.modules:
+            # module has already been imported, return it
+            return sys.modules[name]
         else:
-            return _MockModule()
-
-
-def mock_import(modname):
-    # type: (str) -> None
-    if '.' in modname:
-        pkg, _n, mods = modname.rpartition('.')
-        mock_import(pkg)
-        if isinstance(sys.modules[pkg], _MockModule):
-            sys.modules[pkg]._append_submodule(mods)  # type: ignore
-
-    if modname not in sys.modules:
-        mod = _MockModule()
-        sys.modules[modname] = mod  # type: ignore
+            logger.debug('[autodoc] adding a mock module %s!', name)
+            module = _MockModule(name, self)
+            sys.modules[name] = module
+            self.mocked_modules.append(name)
+            return module
 
 
 ALL = object()
@@ -587,11 +639,11 @@ class Documenter(object):
         if self.objpath:
             logger.debug('[autodoc] from %s import %s',
                          self.modname, '.'.join(self.objpath))
+        # always enable mock import hook
+        # it will do nothing if autodoc_mock_imports is empty
+        import_hook = _MockImporter(self.env.config.autodoc_mock_imports)
         try:
             logger.debug('[autodoc] import %s', self.modname)
-            for modname in self.env.config.autodoc_mock_imports:
-                logger.debug('[autodoc] adding a mock module %s!', modname)
-                mock_import(modname)
             with warnings.catch_warnings():
                 warnings.filterwarnings("ignore", category=ImportWarning)
                 __import__(self.modname)
@@ -628,6 +680,8 @@ class Documenter(object):
             self.directive.warn(errmsg)
             self.env.note_reread()
             return False
+        finally:
+            import_hook.disable()
 
     def get_real_modname(self):
         # type: () -> str