Merge branch '2.0' into 6785_attr_can_refer_props

CHANGES

@@ -22,6 +22,7 @@ Deprecated
 * ``sphinx.util.detect_encoding()``
 * ``sphinx.util.get_module_source()``
 * ``sphinx.util.inspect.Signature``
+* ``sphinx.util.inspect.safe_getmembers()``
 
 Features added
 --------------
@@ -39,8 +40,14 @@ Features added
 * #2755: autodoc: Support type_comment style (ex. ``# type: (str) -> str``)
   annotation (python3.8+ or `typed_ast <https://github.com/python/typed_ast>`_
   is required)
+* #7051: autodoc: Support instance variables without defaults (PEP-526)
+* #6418: autodoc: Add a new extension ``sphinx.ext.autodoc.typehints``. It shows
+  typehints as object description if ``autodoc_typehints = "description"`` set.
+  This is an experimental extension and it will be integrated into autodoc core
+  in Sphinx-3.0
 * SphinxTranslator now calls visitor/departure method for super node class if
   visitor/departure method for original node class not found
+* #6418: Add new event: :event:`object-description-transform`
 * #6785: py domain: ``:py:attr:`` is able to refer properties again
 
 Bugs fixed

doc/extdev/appapi.rst

@@ -218,6 +218,14 @@ connect handlers to the events.  Example:
 
    .. versionadded:: 0.5
 
+.. event:: object-description-transform (app, domain, objtype, contentnode)
+
+   Emitted when an object description directive has run.  The *domain* and
+   *objtype* arguments are strings indicating object description of the object.
+   And *contentnode* is a content for the object.  It can be modified in-place.
+
+   .. versionadded:: 2.4
+
 .. event:: doctree-read (app, doctree)
 
    Emitted when a doctree has been parsed and read by the environment, and is

doc/extdev/deprecated.rst

@@ -87,6 +87,11 @@ The following is a list of deprecated interfaces.
      - ``sphinx.util.inspect.signature`` and
        ``sphinx.util.inspect.stringify_signature()``
 
+   * - ``sphinx.util.inspect.safe_getmembers()``
+     - 2.4
+     - 4.0
+     - ``inspect.getmembers()``
+
    * - ``sphinx.builders.gettext.POHEADER``
      - 2.3
      - 4.0

doc/usage/extensions/autodoc.rst

@@ -567,3 +567,24 @@ member should be included in the documentation by using the following event:
       ``inherited_members``, ``undoc_members``, ``show_inheritance`` and
       ``noindex`` that are true if the flag option of same name was given to the
       auto directive
+
+Generating documents from type annotations
+------------------------------------------
+
+As an experimental feature, autodoc provides ``sphinx.ext.autodoc.typehints`` as
+an additional extension.  It extends autodoc itself to generate function document
+from its type annotations.
+
+To enable the feature, please add ``sphinx.ext.autodoc.typehints`` to list of
+extensions and set `'description'` to :confval:`autodoc_typehints`:
+
+.. code-block:: python
+
+   extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autodoc.typehints']
+
+   autodoc_typehints = 'description'
+
+.. versionadded:: 2.4
+
+   Added as an experimental feature.  This will be integrated into autodoc core
+   in Sphinx-3.0.

sphinx/directives/__init__.py

@@ -193,6 +193,8 @@ class ObjectDescription(SphinxDirective):
             self.env.temp_data['object'] = self.names[0]
         self.before_content()
         self.state.nested_parse(self.content, self.content_offset, contentnode)
+        self.env.app.emit('object-description-transform',
+                          self.domain, self.objtype, contentnode)
         DocFieldTransformer(self).transform_all(contentnode)
         self.env.temp_data['object'] = None
         self.after_content()
@@ -295,6 +297,8 @@ def setup(app: "Sphinx") -> Dict[str, Any]:
     # new, more consistent, name
     directives.register_directive('object', ObjectDescription)
 
+    app.add_event('object-description-transform')
+
     return {
         'version': 'builtin',
         'parallel_read_safe': True,

sphinx/ext/autodoc/__init__.py

@@ -24,7 +24,7 @@ from sphinx.deprecation import (
     RemovedInSphinx30Warning, RemovedInSphinx40Warning, deprecated_alias
 )
 from sphinx.environment import BuildEnvironment
-from sphinx.ext.autodoc.importer import import_object, get_object_members
+from sphinx.ext.autodoc.importer import import_object, get_module_members, get_object_members
 from sphinx.ext.autodoc.mock import mock
 from sphinx.locale import _, __
 from sphinx.pycode import ModuleAnalyzer, PycodeError
@@ -32,9 +32,7 @@ from sphinx.util import inspect
 from sphinx.util import logging
 from sphinx.util import rpartition
 from sphinx.util.docstrings import prepare_docstring
-from sphinx.util.inspect import (
+from sphinx.util.inspect import getdoc, object_description, safe_getattr, stringify_signature
-    getdoc, object_description, safe_getattr, safe_getmembers, stringify_signature
-)
 
 if False:
     # For type annotation
@@ -529,7 +527,10 @@ class Documenter:
         # process members and determine which to skip
         for (membername, member) in members:
             # if isattr is True, the member is documented as an attribute
-            isattr = False
+            if member is INSTANCEATTR:
+                isattr = True
+            else:
+                isattr = False
 
             doc = getdoc(member, self.get_attr, self.env.config.autodoc_inherit_docstrings)
 
@@ -793,7 +794,7 @@ class ModuleDocumenter(Documenter):
                     hasattr(self.object, '__all__')):
                 # for implicit module members, check __module__ to avoid
                 # documenting imported objects
-                return True, safe_getmembers(self.object)
+                return True, get_module_members(self.object)
             else:
                 memberlist = self.object.__all__
                 # Sometimes __all__ is broken...
@@ -806,7 +807,7 @@ class ModuleDocumenter(Documenter):
                         type='autodoc'
                     )
                     # fall back to all members
-                    return True, safe_getmembers(self.object)
+                    return True, get_module_members(self.object)
         else:
             memberlist = self.options.members or []
         ret = []
@@ -1251,6 +1252,37 @@ class DataDocumenter(ModuleLevelDocumenter):
             or self.modname
 
 
+class DataDeclarationDocumenter(DataDocumenter):
+    """
+    Specialized Documenter subclass for data that cannot be imported
+    because they are declared without initial value (refs: PEP-526).
+    """
+    objtype = 'datadecl'
+    directivetype = 'data'
+    member_order = 60
+
+    # must be higher than AttributeDocumenter
+    priority = 11
+
+    @classmethod
+    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any
+                            ) -> bool:
+        """This documents only INSTANCEATTR members."""
+        return (isinstance(parent, ModuleDocumenter) and
+                isattr and
+                member is INSTANCEATTR)
+
+    def import_object(self) -> bool:
+        """Never import anything."""
+        # disguise as a data
+        self.objtype = 'data'
+        return True
+
+    def add_content(self, more_content: Any, no_docstring: bool = False) -> None:
+        """Never try to get a docstring from the object."""
+        super().add_content(more_content, no_docstring=True)
+
+
 class MethodDocumenter(DocstringSignatureMixin, ClassLevelDocumenter):  # type: ignore
     """
     Specialized Documenter subclass for methods (normal, static and class).
@@ -1438,7 +1470,9 @@ class InstanceAttributeDocumenter(AttributeDocumenter):
     def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any
                             ) -> bool:
         """This documents only INSTANCEATTR members."""
-        return isattr and (member is INSTANCEATTR)
+        return (not isinstance(parent, ModuleDocumenter) and
+                isattr and
+                member is INSTANCEATTR)
 
     def import_object(self) -> bool:
         """Never import anything."""
@@ -1550,6 +1584,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_autodocumenter(ClassDocumenter)
     app.add_autodocumenter(ExceptionDocumenter)
     app.add_autodocumenter(DataDocumenter)
+    app.add_autodocumenter(DataDeclarationDocumenter)
     app.add_autodocumenter(FunctionDocumenter)
     app.add_autodocumenter(DecoratorDocumenter)
     app.add_autodocumenter(MethodDocumenter)

sphinx/ext/autodoc/importer.py

@@ -12,7 +12,7 @@ import importlib
 import traceback
 import warnings
 from collections import namedtuple
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Tuple
 
 from sphinx.deprecation import RemovedInSphinx40Warning, deprecated_alias
 from sphinx.util import logging
@@ -101,12 +101,35 @@ def import_object(modname: str, objpath: List[str], objtype: str = '',
         raise ImportError(errmsg)
 
 
+def get_module_members(module: Any) -> List[Tuple[str, Any]]:
+    """Get members of target module."""
+    from sphinx.ext.autodoc import INSTANCEATTR
+
+    members = {}  # type: Dict[str, Tuple[str, Any]]
+    for name in dir(module):
+        try:
+            value = safe_getattr(module, name, None)
+            members[name] = (name, value)
+        except AttributeError:
+            continue
+
+    # annotation only member (ex. attr: int)
+    if hasattr(module, '__annotations__'):
+        for name in module.__annotations__:
+            if name not in members:
+                members[name] = (name, INSTANCEATTR)
+
+    return sorted(list(members.values()))
+
+
 Attribute = namedtuple('Attribute', ['name', 'directly_defined', 'value'])
 
 
 def get_object_members(subject: Any, objpath: List[str], attrgetter: Callable,
                        analyzer: Any = None) -> Dict[str, Attribute]:
     """Get members and attributes of target object."""
+    from sphinx.ext.autodoc import INSTANCEATTR
+
     # the members directly defined in the class
     obj_dict = attrgetter(subject, '__dict__', {})
 
@@ -140,10 +163,14 @@ def get_object_members(subject: Any, objpath: List[str], attrgetter: Callable,
         except AttributeError:
             continue
 
+    # annotation only member (ex. attr: int)
+    if hasattr(subject, '__annotations__'):
+        for name in subject.__annotations__:
+            if name not in members:
+                members[name] = Attribute(name, True, INSTANCEATTR)
+
     if analyzer:
         # append instance attributes (cf. self.attr1) if analyzer knows
-        from sphinx.ext.autodoc import INSTANCEATTR
-
         namespace = '.'.join(objpath)
         for (ns, name) in analyzer.find_attr_docs():
             if namespace == ns and name not in members:

sphinx/ext/autodoc/typehints.py

@@ -0,0 +1,144 @@
+"""
+    sphinx.ext.autodoc.typehints
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Generating content for autodoc using typehints
+
+    :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+from typing import Any, Dict, Iterable
+from typing import cast
+
+from docutils import nodes
+from docutils.nodes import Element
+
+from sphinx import addnodes
+from sphinx.application import Sphinx
+from sphinx.config import ENUM
+from sphinx.util import inspect, typing
+
+
+def config_inited(app, config):
+    if config.autodoc_typehints == 'description':
+        # HACK: override this to make autodoc suppressing typehints in signatures
+        config.autodoc_typehints = 'none'
+
+        # preserve user settings
+        app._autodoc_typehints_description = True
+    else:
+        app._autodoc_typehints_description = False
+
+
+def record_typehints(app: Sphinx, objtype: str, name: str, obj: Any,
+                     options: Dict, args: str, retann: str) -> None:
+    """Record type hints to env object."""
+    try:
+        if callable(obj):
+            annotations = app.env.temp_data.setdefault('annotations', {}).setdefault(name, {})
+            sig = inspect.signature(obj)
+            for param in sig.parameters.values():
+                if param.annotation is not param.empty:
+                    annotations[param.name] = typing.stringify(param.annotation)
+            if sig.return_annotation is not sig.empty:
+                annotations['return'] = typing.stringify(sig.return_annotation)
+    except TypeError:
+        pass
+
+
+def merge_typehints(app: Sphinx, domain: str, objtype: str, contentnode: Element) -> None:
+    if domain != 'py':
+        return
+    if app._autodoc_typehints_description is False:  # type: ignore
+        return
+
+    signature = cast(addnodes.desc_signature, contentnode.parent[0])
+    fullname = '.'.join([signature['module'], signature['fullname']])
+    annotations = app.env.temp_data.get('annotations', {})
+    if annotations.get(fullname, {}):
+        field_lists = [n for n in contentnode if isinstance(n, nodes.field_list)]
+        if field_lists == []:
+            field_list = insert_field_list(contentnode)
+            field_lists.append(field_list)
+
+        for field_list in field_lists:
+            modify_field_list(field_list, annotations[fullname])
+
+
+def insert_field_list(node: Element) -> nodes.field_list:
+    field_list = nodes.field_list()
+    desc = [n for n in node if isinstance(n, addnodes.desc)]
+    if desc:
+        # insert just before sub object descriptions (ex. methods, nested classes, etc.)
+        index = node.index(desc[0])
+        node.insert(index - 1, [field_list])
+    else:
+        node += field_list
+
+    return field_list
+
+
+def modify_field_list(node: nodes.field_list, annotations: Dict[str, str]) -> None:
+    arguments = {}  # type: Dict[str, Dict[str, bool]]
+    fields = cast(Iterable[nodes.field], node)
+    for field in fields:
+        field_name = field[0].astext()
+        parts = re.split(' +', field_name)
+        if parts[0] == 'param':
+            if len(parts) == 2:
+                # :param xxx:
+                arg = arguments.setdefault(parts[1], {})
+                arg['param'] = True
+            elif len(parts) > 2:
+                # :param xxx yyy:
+                name = ' '.join(parts[2:])
+                arg = arguments.setdefault(name, {})
+                arg['param'] = True
+                arg['type'] = True
+        elif parts[0] == 'type':
+            name = ' '.join(parts[1:])
+            arg = arguments.setdefault(name, {})
+            arg['type'] = True
+        elif parts[0] == 'rtype':
+            arguments['return'] = {'type': True}
+
+    for name, annotation in annotations.items():
+        if name == 'return':
+            continue
+
+        arg = arguments.get(name, {})
+        field = nodes.field()
+        if arg.get('param') and arg.get('type'):
+            # both param and type are already filled manually
+            continue
+        elif arg.get('param'):
+            # only param: fill type field
+            field += nodes.field_name('', 'type ' + name)
+            field += nodes.field_body('', nodes.paragraph('', annotation))
+        elif arg.get('type'):
+            # only type: It's odd...
+            field += nodes.field_name('', 'param ' + name)
+            field += nodes.field_body('', nodes.paragraph('', ''))
+        else:
+            # both param and type are not found
+            field += nodes.field_name('', 'param ' + annotation + ' ' + name)
+            field += nodes.field_body('', nodes.paragraph('', ''))
+
+        node += field
+
+    if 'return' in annotations and 'return' not in arguments:
+        field = nodes.field()
+        field += nodes.field_name('', 'rtype')
+        field += nodes.field_body('', nodes.paragraph('', annotation))
+        node += field
+
+
+def setup(app):
+    app.setup_extension('sphinx.ext.autodoc')
+    app.config.values['autodoc_typehints'] = ('signature', True,
+                                              ENUM("signature", "description", "none"))
+    app.connect('config-inited', config_inited)
+    app.connect('autodoc-process-signature', record_typehints)
+    app.connect('object-description-transform', merge_typehints)

sphinx/ext/autosummary/generate.py

@@ -71,13 +71,13 @@ def setup_documenters(app: Any) -> None:
         ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter,
         FunctionDocumenter, MethodDocumenter, AttributeDocumenter,
         InstanceAttributeDocumenter, DecoratorDocumenter, PropertyDocumenter,
-        SlotsAttributeDocumenter,
+        SlotsAttributeDocumenter, DataDeclarationDocumenter,
     )
     documenters = [
         ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter,
         FunctionDocumenter, MethodDocumenter, AttributeDocumenter,
         InstanceAttributeDocumenter, DecoratorDocumenter, PropertyDocumenter,
-        SlotsAttributeDocumenter,
+        SlotsAttributeDocumenter, DataDeclarationDocumenter,
     ]  # type: List[Type[Documenter]]
     for documenter in documenters:
         app.registry.add_documenter(documenter.objtype, documenter)

sphinx/util/inspect.py

@@ -257,6 +257,8 @@ def safe_getattr(obj: Any, name: str, *defargs: Any) -> Any:
 def safe_getmembers(object: Any, predicate: Callable[[str], bool] = None,
                     attr_getter: Callable = safe_getattr) -> List[Tuple[str, Any]]:
     """A version of inspect.getmembers() that uses safe_getattr()."""
+    warnings.warn('safe_getmembers() is deprecated', RemovedInSphinx40Warning)
+
     results = []  # type: List[Tuple[str, Any]]
     for key in dir(object):
         try:
@@ -450,6 +452,8 @@ class Signature:
     its return annotation.
     """
 
+    empty = inspect.Signature.empty
+
     def __init__(self, subject: Callable, bound_method: bool = False,
                  has_retval: bool = True) -> None:
         warnings.warn('sphinx.util.inspect.Signature() is deprecated',

tests/roots/test-ext-autodoc/index.rst

@@ -7,3 +7,5 @@
 
 .. automodule:: autodoc_dummy_bar
    :members:
+
+.. autofunction:: target.typehints.incr

tests/roots/test-ext-autodoc/target/typed_vars.py

@@ -0,0 +1,13 @@
+#: attr1
+attr1: str = ''
+#: attr2
+attr2: str
+
+
+class Class:
+    attr1: int = 0
+    attr2: int
+
+    def __init__(self):
+        self.attr3: int = 0     #: attr3
+        self.attr4: int         #: attr4

tests/test_autodoc.py

@@ -1388,6 +1388,61 @@ def test_partialmethod_undoc_members(app):
     assert list(actual) == expected
 
 
+@pytest.mark.skipif(sys.version_info < (3, 6), reason='py36+ is available since python3.6.')
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autodoc_typed_instance_variables(app):
+    options = {"members": None,
+               "undoc-members": True}
+    actual = do_autodoc(app, 'module', 'target.typed_vars', options)
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.typed_vars',
+        '',
+        '',
+        '.. py:class:: Class()',
+        '   :module: target.typed_vars',
+        '',
+        '   ',
+        '   .. py:attribute:: Class.attr1',
+        '      :module: target.typed_vars',
+        '      :annotation: = 0',
+        '   ',
+        '   ',
+        '   .. py:attribute:: Class.attr2',
+        '      :module: target.typed_vars',
+        '      :annotation: = None',
+        '   ',
+        '   ',
+        '   .. py:attribute:: Class.attr3',
+        '      :module: target.typed_vars',
+        '      :annotation: = None',
+        '   ',
+        '      attr3',
+        '      ',
+        '   ',
+        '   .. py:attribute:: Class.attr4',
+        '      :module: target.typed_vars',
+        '      :annotation: = None',
+        '   ',
+        '      attr4',
+        '      ',
+        '',
+        '.. py:data:: attr1',
+        '   :module: target.typed_vars',
+        "   :annotation: = ''",
+        '',
+        '   attr1',
+        '   ',
+        '',
+        '.. py:data:: attr2',
+        '   :module: target.typed_vars',
+        "   :annotation: = None",
+        '',
+        '   attr2',
+        '   '
+    ]
+
+
 @pytest.mark.sphinx('html', testroot='pycode-egg')
 def test_autodoc_for_egged_code(app):
     options = {"members": None,

tests/test_ext_autodoc_configs.py

@@ -540,6 +540,24 @@ def test_autodoc_typehints_none(app):
     ]
 
 
+@pytest.mark.sphinx('text', testroot='ext-autodoc',
+                    confoverrides={'extensions': ['sphinx.ext.autodoc.typehints'],
+                                   'autodoc_typehints': 'description'})
+def test_autodoc_typehints_description(app):
+    app.build()
+    context = (app.outdir / 'index.txt').text()
+    assert ('target.typehints.incr(a, b=1)\n'
+            '\n'
+            '   Parameters:\n'
+            '      * **a** (*int*) --\n'
+            '\n'
+            '      * **b** (*int*) --\n'
+            '\n'
+            '   Return type:\n'
+            '      int\n'
+            in context)
+
+
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
 @pytest.mark.filterwarnings('ignore:autodoc_default_flags is now deprecated.')
 def test_merge_autodoc_default_flags1(app):