Merge branch '2.0' into 7019_gettext_absolute_paths

CHANGES

@@ -35,6 +35,10 @@ Features added
   images (imagesize-1.2.0 or above is required)
 * #6994: imgconverter: Support illustrator file (.ai) to .png conversion
 * autodoc: Support Positional-Only Argument separator (PEP-570 compliant)
+* #2755: autodoc: Add new event: :event:`autodoc-before-process-signature`
+* #2755: autodoc: Support type_comment style (ex. ``# type: (str) -> str``)
+  annotation (python3.8+ or `typed_ast <https://github.com/python/typed_ast>`_
+  is required)
 * SphinxTranslator now calls visitor/departure method for super node class if
   visitor/departure method for original node class not found
 
@@ -48,6 +52,7 @@ Bugs fixed
 * #6986: apidoc: misdetects module name for .so file inside module
 * #6999: napoleon: fails to parse tilde in :exc: role
 * #7019: gettext: Absolute path used in message catalogs
+* #7023: autodoc: nested partial functions are not listed
 
 Testing
 --------

doc/usage/extensions/autodoc.rst

@@ -494,6 +494,17 @@ autodoc provides the following additional events:
       auto directive
    :param lines: the lines of the docstring, see above
 
+.. event:: autodoc-before-process-signature (app, obj, bound_method)
+
+   .. versionadded:: 2.4
+
+   Emitted before autodoc formats a signature for an object. The event handler
+   can modify an object to change its signature.
+
+   :param app: the Sphinx application object
+   :param obj: the object itself
+   :param bound_method: a boolean indicates an object is bound method or not
+
 .. event:: autodoc-process-signature (app, what, name, obj, options, signature, return_annotation)
 
    .. versionadded:: 0.5

setup.py

@@ -42,7 +42,7 @@ extras_require = {
         'sphinxcontrib-websupport',
     ],
     'test': [
-        'pytest',
+        'pytest < 5.3.3',
         'pytest-cov',
         'html5lib',
         'flake8>=3.5.0',

sphinx/ext/autodoc/__init__.py

@@ -983,8 +983,11 @@ class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # typ
                     not inspect.isbuiltin(self.object) and
                     not inspect.isclass(self.object) and
                     hasattr(self.object, '__call__')):
+                self.env.app.emit('autodoc-before-process-signature',
+                                  self.object.__call__, False)
                 sig = inspect.signature(self.object.__call__)
             else:
+                self.env.app.emit('autodoc-before-process-signature', self.object, False)
                 sig = inspect.signature(self.object)
             args = stringify_signature(sig, **kwargs)
         except TypeError:
@@ -996,9 +999,13 @@ class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # typ
             # typing) we try to use the constructor signature as function
             # signature without the first argument.
             try:
+                self.env.app.emit('autodoc-before-process-signature',
+                                  self.object.__new__, True)
                 sig = inspect.signature(self.object.__new__, bound_method=True)
                 args = stringify_signature(sig, show_return_annotation=False, **kwargs)
             except TypeError:
+                self.env.app.emit('autodoc-before-process-signature',
+                                  self.object.__init__, True)
                 sig = inspect.signature(self.object.__init__, bound_method=True)
                 args = stringify_signature(sig, show_return_annotation=False, **kwargs)
 
@@ -1081,6 +1088,7 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
                 not(inspect.ismethod(initmeth) or inspect.isfunction(initmeth)):
             return None
         try:
+            self.env.app.emit('autodoc-before-process-signature', initmeth, True)
             sig = inspect.signature(initmeth, bound_method=True)
             return stringify_signature(sig, show_return_annotation=False, **kwargs)
         except TypeError:
@@ -1284,8 +1292,10 @@ class MethodDocumenter(DocstringSignatureMixin, ClassLevelDocumenter):  # type:
             # can never get arguments of a C function or method
             return None
         if inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name):
+            self.env.app.emit('autodoc-before-process-signature', self.object, False)
             sig = inspect.signature(self.object, bound_method=False)
         else:
+            self.env.app.emit('autodoc-before-process-signature', self.object, True)
             sig = inspect.signature(self.object, bound_method=True)
         args = stringify_signature(sig, **kwargs)
 
@@ -1558,10 +1568,12 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_config_value('autodoc_typehints', "signature", True, ENUM("signature", "none"))
     app.add_config_value('autodoc_warningiserror', True, True)
     app.add_config_value('autodoc_inherit_docstrings', True, True)
+    app.add_event('autodoc-before-process-signature')
     app.add_event('autodoc-process-docstring')
     app.add_event('autodoc-process-signature')
     app.add_event('autodoc-skip-member')
 
     app.connect('config-inited', merge_autodoc_default_flags)
+    app.setup_extension('sphinx.ext.autodoc.type_comment')
 
     return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

sphinx/ext/autodoc/type_comment.py

@@ -0,0 +1,74 @@
+"""
+    sphinx.ext.autodoc.type_comment
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Update annotations info of living objects using type_comments.
+
+    :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import ast
+from inspect import getsource
+from typing import Any, Dict
+from typing import cast
+
+import sphinx
+from sphinx.application import Sphinx
+from sphinx.pycode.ast import parse as ast_parse
+from sphinx.pycode.ast import unparse as ast_unparse
+from sphinx.util import inspect
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_type_comment(obj: Any) -> ast.FunctionDef:
+    """Get type_comment'ed FunctionDef object from living object.
+
+    This tries to parse original code for living object and returns
+    AST node for given *obj*.  It requires py38+ or typed_ast module.
+    """
+    try:
+        source = getsource(obj)
+        if source.startswith((' ', r'\t')):
+            # subject is placed inside class or block.  To read its docstring,
+            # this adds if-block before the declaration.
+            module = ast_parse('if True:\n' + source)
+            subject = cast(ast.FunctionDef, module.body[0].body[0])  # type: ignore
+        else:
+            module = ast_parse(source)
+            subject = cast(ast.FunctionDef, module.body[0])  # type: ignore
+
+        if getattr(subject, "type_comment", None):
+            return ast_parse(subject.type_comment, mode='func_type')  # type: ignore
+        else:
+            return None
+    except (OSError, TypeError):  # failed to load source code
+        return None
+    except SyntaxError:  # failed to parse type_comments
+        return None
+
+
+def update_annotations_using_type_comments(app: Sphinx, obj: Any, bound_method: bool) -> None:
+    """Update annotations info of *obj* using type_comments."""
+    try:
+        function = get_type_comment(obj)
+        if function and hasattr(function, 'argtypes'):
+            if function.argtypes != [ast.Ellipsis]:  # type: ignore
+                sig = inspect.signature(obj, bound_method)
+                for i, param in enumerate(sig.parameters.values()):
+                    if param.name not in obj.__annotations__:
+                        annotation = ast_unparse(function.argtypes[i])  # type: ignore
+                        obj.__annotations__[param.name] = annotation
+
+            if 'return' not in obj.__annotations__:
+                obj.__annotations__['return'] = ast_unparse(function.returns)  # type: ignore
+    except NotImplementedError as exc:  # failed to ast.unparse()
+        logger.warning("Failed to parse type_comment for %r: %s", obj, exc)
+
+
+def setup(app: Sphinx) -> Dict[str, Any]:
+    app.connect('autodoc-before-process-signature', update_annotations_using_type_comments)
+
+    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

sphinx/pycode/ast.py

@@ -0,0 +1,80 @@
+"""
+    sphinx.pycode.ast
+    ~~~~~~~~~~~~~~~~~
+
+    Helpers for AST (Abstract Syntax Tree).
+
+    :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys
+
+if sys.version_info > (3, 8):
+    import ast
+else:
+    try:
+        # use typed_ast module if installed
+        from typed_ast import ast3 as ast
+    except ImportError:
+        import ast  # type: ignore
+
+
+def parse(code: str, mode: str = 'exec') -> "ast.AST":
+    """Parse the *code* using built-in ast or typed_ast.
+
+    This enables "type_comments" feature if possible.
+    """
+    try:
+        # type_comments parameter is available on py38+
+        return ast.parse(code, mode=mode, type_comments=True)  # type: ignore
+    except TypeError:
+        # fallback to ast module.
+        # typed_ast is used to parse type_comments if installed.
+        return ast.parse(code, mode=mode)
+
+
+def unparse(node: ast.AST) -> str:
+    """Unparse an AST to string."""
+    if node is None:
+        return None
+    elif isinstance(node, ast.Attribute):
+        return "%s.%s" % (unparse(node.value), node.attr)
+    elif isinstance(node, ast.Bytes):
+        return repr(node.s)
+    elif isinstance(node, ast.Call):
+        args = ([unparse(e) for e in node.args] +
+                ["%s=%s" % (k.arg, unparse(k.value)) for k in node.keywords])
+        return "%s(%s)" % (unparse(node.func), ", ".join(args))
+    elif isinstance(node, ast.Dict):
+        keys = (unparse(k) for k in node.keys)
+        values = (unparse(v) for v in node.values)
+        items = (k + ": " + v for k, v in zip(keys, values))
+        return "{" + ", ".join(items) + "}"
+    elif isinstance(node, ast.Ellipsis):
+        return "..."
+    elif isinstance(node, ast.Index):
+        return unparse(node.value)
+    elif isinstance(node, ast.Lambda):
+        return "<function <lambda>>"  # TODO
+    elif isinstance(node, ast.List):
+        return "[" + ", ".join(unparse(e) for e in node.elts) + "]"
+    elif isinstance(node, ast.Name):
+        return node.id
+    elif isinstance(node, ast.NameConstant):
+        return repr(node.value)
+    elif isinstance(node, ast.Num):
+        return repr(node.n)
+    elif isinstance(node, ast.Set):
+        return "{" + ", ".join(unparse(e) for e in node.elts) + "}"
+    elif isinstance(node, ast.Str):
+        return repr(node.s)
+    elif isinstance(node, ast.Subscript):
+        return "%s[%s]" % (unparse(node.value), unparse(node.slice))
+    elif isinstance(node, ast.Tuple):
+        return ", ".join(unparse(e) for e in node.elts)
+    elif sys.version_info > (3, 6) and isinstance(node, ast.Constant):
+        # this branch should be placed at last
+        return repr(node.value)
+    else:
+        raise NotImplementedError('Unable to parse %s object' % type(node).__name__)

sphinx/util/inspect.py

@@ -121,6 +121,17 @@ def isenumattribute(x: Any) -> bool:
     return isinstance(x, enum.Enum)
 
 
+def unpartial(obj: Any) -> Any:
+    """Get an original object from partial object.
+
+    This returns given object itself if not partial.
+    """
+    while ispartial(obj):
+        obj = obj.func
+
+    return obj
+
+
 def ispartial(obj: Any) -> bool:
     """Check if the object is partial."""
     return isinstance(obj, (partial, partialmethod))
@@ -197,24 +208,21 @@ def isattributedescriptor(obj: Any) -> bool:
 
 def isfunction(obj: Any) -> bool:
     """Check if the object is function."""
-    return inspect.isfunction(obj) or ispartial(obj) and inspect.isfunction(obj.func)
+    return inspect.isfunction(unpartial(obj))
 
 
 def isbuiltin(obj: Any) -> bool:
     """Check if the object is builtin."""
-    return inspect.isbuiltin(obj) or ispartial(obj) and inspect.isbuiltin(obj.func)
+    return inspect.isbuiltin(unpartial(obj))
 
 
 def iscoroutinefunction(obj: Any) -> bool:
     """Check if the object is coroutine-function."""
+    obj = unpartial(obj)
     if hasattr(obj, '__code__') and inspect.iscoroutinefunction(obj):
         # check obj.__code__ because iscoroutinefunction() crashes for custom method-like
         # objects (see https://github.com/sphinx-doc/sphinx/issues/6605)
         return True
-    elif (ispartial(obj) and hasattr(obj.func, '__code__') and
-          inspect.iscoroutinefunction(obj.func)):
-        # partialed
-        return True
     else:
         return False
 

tests/roots/test-ext-autodoc/target/partialfunction.py

@@ -1,11 +1,12 @@
 from functools import partial
 
 
-def func1():
+def func1(a, b, c):
     """docstring of func1"""
     pass
 
 
-func2 = partial(func1)
+func2 = partial(func1, 1)
-func3 = partial(func1)
+func3 = partial(func2, 2)
 func3.__doc__ = "docstring of func3"
+func4 = partial(func3, 3)

tests/roots/test-ext-autodoc/target/partialmethod.py

@@ -14,5 +14,5 @@ class Cell(object):
     #: Make a cell alive.
     set_alive = partialmethod(set_state, True)
 
+    # a partialmethod with no docstring
     set_dead = partialmethod(set_state, False)
-    """Make a cell dead."""

tests/roots/test-ext-autodoc/target/typehints.py

@@ -2,9 +2,23 @@ def incr(a: int, b: int = 1) -> int:
     return a + b
 
 
+def decr(a, b = 1):
+    # type: (int, int) -> int
+    return a - b
+
+
 class Math:
     def __init__(self, s: str, o: object = None) -> None:
         pass
 
     def incr(self, a: int, b: int = 1) -> int:
         return a + b
+
+    def decr(self, a, b = 1):
+        # type: (int, int) -> int
+        return a - b
+
+
+def complex_func(arg1, arg2, arg3=None, *args, **kwargs):
+    # type: (str, List[int], Tuple[int, Union[str, Unknown]], *str, **str) -> None
+    pass

tests/test_autodoc.py

@@ -1241,19 +1241,25 @@ def test_partialfunction():
         '.. py:module:: target.partialfunction',
         '',
         '',
-        '.. py:function:: func1()',
+        '.. py:function:: func1(a, b, c)',
         '   :module: target.partialfunction',
         '',
         '   docstring of func1',
         '   ',
         '',
-        '.. py:function:: func2()',
+        '.. py:function:: func2(b, c)',
         '   :module: target.partialfunction',
         '',
         '   docstring of func1',
         '   ',
         '',
-        '.. py:function:: func3()',
+        '.. py:function:: func3(c)',
+        '   :module: target.partialfunction',
+        '',
+        '   docstring of func3',
+        '   ',
+        '',
+        '.. py:function:: func4()',
         '   :module: target.partialfunction',
         '',
         '   docstring of func3',
@@ -1324,12 +1330,6 @@ def test_partialmethod(app):
         '      Make a cell alive.',
         '      ',
         '   ',
-        '   .. py:method:: Cell.set_dead()',
-        '      :module: target.partialmethod',
-        '   ',
-        '      Make a cell dead.',
-        '      ',
-        '   ',
         '   .. py:method:: Cell.set_state(state)',
         '      :module: target.partialmethod',
         '   ',
@@ -1342,6 +1342,41 @@ def test_partialmethod(app):
     assert list(actual) == expected
 
 
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_partialmethod_undoc_members(app):
+    expected = [
+        '',
+        '.. py:class:: Cell',
+        '   :module: target.partialmethod',
+        '',
+        '   An example for partialmethod.',
+        '   ',
+        '   refs: https://docs.python.jp/3/library/functools.html#functools.partialmethod',
+        '   ',
+        '   ',
+        '   .. py:method:: Cell.set_alive()',
+        '      :module: target.partialmethod',
+        '   ',
+        '      Make a cell alive.',
+        '      ',
+        '   ',
+        '   .. py:method:: Cell.set_dead()',
+        '      :module: target.partialmethod',
+        '   ',
+        '   ',
+        '   .. py:method:: Cell.set_state(state)',
+        '      :module: target.partialmethod',
+        '   ',
+        '      Update state of cell to *state*.',
+        '      ',
+    ]
+
+    options = {"members": None,
+               "undoc-members": None}
+    actual = do_autodoc(app, 'class', 'target.partialmethod.Cell', options)
+    assert list(actual) == expected
+
+
 @pytest.mark.sphinx('html', testroot='pycode-egg')
 def test_autodoc_for_egged_code(app):
     options = {"members": None,

tests/test_ext_autodoc_configs.py

@@ -479,10 +479,23 @@ def test_autodoc_typehints_signature(app):
         '   :module: target.typehints',
         '',
         '   ',
+        '   .. py:method:: Math.decr(a: int, b: int = 1) -> int',
+        '      :module: target.typehints',
+        '   ',
+        '   ',
         '   .. py:method:: Math.incr(a: int, b: int = 1) -> int',
         '      :module: target.typehints',
         '   ',
         '',
+        '.. py:function:: complex_func(arg1: str, arg2: List[int], arg3: Tuple[int, '
+        'Union[str, Unknown]] = None, *args: str, **kwargs: str) -> None',
+        '   :module: target.typehints',
+        '',
+        '',
+        '.. py:function:: decr(a: int, b: int = 1) -> int',
+        '   :module: target.typehints',
+        '',
+        '',
         '.. py:function:: incr(a: int, b: int = 1) -> int',
         '   :module: target.typehints',
         ''
@@ -505,10 +518,22 @@ def test_autodoc_typehints_none(app):
         '   :module: target.typehints',
         '',
         '   ',
+        '   .. py:method:: Math.decr(a, b=1)',
+        '      :module: target.typehints',
+        '   ',
+        '   ',
         '   .. py:method:: Math.incr(a, b=1)',
         '      :module: target.typehints',
         '   ',
         '',
+        '.. py:function:: complex_func(arg1, arg2, arg3=None, *args, **kwargs)',
+        '   :module: target.typehints',
+        '',
+        '',
+        '.. py:function:: decr(a, b=1)',
+        '   :module: target.typehints',
+        '',
+        '',
         '.. py:function:: incr(a, b=1)',
         '   :module: target.typehints',
         ''

tests/test_pycode_ast.py

@@ -0,0 +1,40 @@
+"""
+    test_pycode_ast
+    ~~~~~~~~~~~~~~~
+
+    Test pycode.ast
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import pytest
+
+from sphinx.pycode import ast
+
+
+@pytest.mark.parametrize('source,expected', [
+    ("os.path", "os.path"),                     # Attribute
+    ("b'bytes'", "b'bytes'"),                   # Bytes
+    ("object()", "object()"),                   # Call
+    ("1234", "1234"),                           # Constant
+    ("{'key1': 'value1', 'key2': 'value2'}",
+     "{'key1': 'value1', 'key2': 'value2'}"),   # Dict
+    ("...", "..."),                             # Ellipsis
+    ("Tuple[int, int]", "Tuple[int, int]"),     # Index, Subscript
+    ("lambda x, y: x + y",
+     "<function <lambda>>"),                    # Lambda
+    ("[1, 2, 3]", "[1, 2, 3]"),                 # List
+    ("sys", "sys"),                             # Name, NameConstant
+    ("1234", "1234"),                           # Num
+    ("{1, 2, 3}", "{1, 2, 3}"),                 # Set
+    ("'str'", "'str'"),                         # Str
+    ("(1, 2, 3)", "1, 2, 3"),                   # Tuple
+])
+def test_unparse(source, expected):
+    module = ast.parse(source)
+    assert ast.unparse(module.body[0].value) == expected
+
+
+def test_unparse_None():
+    assert ast.unparse(None) is None

tests/test_util_inspect.py

@@ -511,3 +511,15 @@ def test_isproperty(app):
     assert inspect.isproperty(Base.meth) is False       # method of class
     assert inspect.isproperty(Base().meth) is False     # method of instance
     assert inspect.isproperty(func) is False            # function
+
+
+def test_unpartial():
+    def func1(a, b, c):
+        pass
+
+    func2 = functools.partial(func1, 1)
+    func2.__doc__ = "func2"
+    func3 = functools.partial(func2, 2)  # nested partial object
+
+    assert inspect.unpartial(func2) is func1
+    assert inspect.unpartial(func3) is func1