Merge branch '3.x' into 8132_project_copyright

CHANGES

@@ -10,9 +10,14 @@ Incompatible changes
 Deprecated
 ----------
 
+* ``sphinx.ext.autodoc.importer.get_module_members()``
+
 Features added
 --------------
 
+* #8022: autodoc: autodata and autoattribute directives does not show right-hand
+  value of the variable if docstring contains ``:meta hide-value:`` in
+  info-field-list
 * #8132: Add :confval:`project_copyright` as an alias of :confval:`copyright`
 
 Bugs fixed
@@ -20,6 +25,7 @@ Bugs fixed
 
 * #741: autodoc: inherited-members doesn't work for instance attributes on super
   class
+* #8592: autodoc: ``:meta public:`` does not effect to variables
 
 Testing
 --------

doc/extdev/deprecated.rst

@@ -26,6 +26,11 @@ The following is a list of deprecated interfaces.
      - (will be) Removed
      - Alternatives
 
+   * - ``sphinx.ext.autodoc.importer.get_module_members()``
+     - 3.5
+     - 5.0
+     - ``sphinx.ext.autodoc.ModuleDocumenter.get_module_members()``
+
    * - The ``follow_wrapped`` argument of ``sphinx.util.inspect.signature()``
      - 3.4
      - 5.0

doc/usage/extensions/autodoc.rst

@@ -182,6 +182,16 @@ inserting them into the page source under a suitable :rst:dir:`py:module`,
 
      .. versionadded:: 3.1
 
+   * autodoc considers a variable member does not have any default value if its
+     docstring contains ``:meta hide-value:`` in its :ref:`info-field-lists`.
+     Example:
+
+     .. code-block:: rst
+
+        var1 = None  #: :meta hide-value:
+
+     .. versionadded:: 3.5
+
    * Python "special" members (that is, those named like ``__special__``) will
      be included if the ``special-members`` flag option is given::
 

sphinx/ext/autodoc/__init__.py

@@ -25,8 +25,8 @@ from sphinx.config import ENUM, Config
 from sphinx.deprecation import (RemovedInSphinx40Warning, RemovedInSphinx50Warning,
                                 RemovedInSphinx60Warning)
 from sphinx.environment import BuildEnvironment
-from sphinx.ext.autodoc.importer import (ClassAttribute, get_class_members, get_module_members,
+from sphinx.ext.autodoc.importer import (ClassAttribute, get_class_members, get_object_members,
-                                         get_object_members, import_module, import_object)
+                                         import_module, import_object)
 from sphinx.ext.autodoc.mock import mock
 from sphinx.locale import _, __
 from sphinx.pycode import ModuleAnalyzer, PycodeError
@@ -538,8 +538,12 @@ class Documenter:
             # etc. don't support a prepended module name
             self.add_line('   :module: %s' % self.modname, sourcename)
 
-    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+    def get_doc(self, encoding: str = None, ignore: int = None) -> Optional[List[List[str]]]:
-        """Decode and return lines of the docstring(s) for the object."""
+        """Decode and return lines of the docstring(s) for the object.
+
+        When it returns None value, autodoc-process-docstring will not be called for this
+        object.
+        """
         if encoding is not None:
             warnings.warn("The 'encoding' argument to autodoc.%s.get_doc() is deprecated."
                           % self.__class__.__name__,
@@ -587,12 +591,10 @@ class Documenter:
     def add_content(self, more_content: Optional[StringList], no_docstring: bool = False
                     ) -> None:
         """Add content from docstrings, attribute documentation and user."""
-        # Suspended temporarily (see https://github.com/sphinx-doc/sphinx/pull/8533)
+        if no_docstring:
-        #
+            warnings.warn("The 'no_docstring' argument to %s.add_content() is deprecated."
-        # if no_docstring:
+                          % self.__class__.__name__,
-        #     warnings.warn("The 'no_docstring' argument to %s.add_content() is deprecated."
+                          RemovedInSphinx50Warning, stacklevel=2)
-        #                   % self.__class__.__name__,
-        #                   RemovedInSphinx50Warning, stacklevel=2)
 
         # set sourcename and add content from attribute documentation
         sourcename = self.get_sourcename()
@@ -612,13 +614,17 @@ class Documenter:
         # add content from docstrings
         if not no_docstring:
             docstrings = self.get_doc()
-            if not docstrings:
+            if docstrings is None:
-                # append at least a dummy docstring, so that the event
+                # Do not call autodoc-process-docstring on get_doc() returns None.
-                # autodoc-process-docstring is fired and can add some
+                pass
-                # content if desired
+            else:
-                docstrings.append([])
+                if not docstrings:
-            for i, line in enumerate(self.process_doc(docstrings)):
+                    # append at least a dummy docstring, so that the event
-                self.add_line(line, sourcename, i)
+                    # autodoc-process-docstring is fired and can add some
+                    # content if desired
+                    docstrings.append([])
+                for i, line in enumerate(self.process_doc(docstrings)):
+                    self.add_line(line, sourcename, i)
 
         # add additional content (e.g. from document), if present
         if more_content:
@@ -1037,30 +1043,54 @@ class ModuleDocumenter(Documenter):
         if self.options.deprecated:
             self.add_line('   :deprecated:', sourcename)
 
+    def get_module_members(self) -> Dict[str, ObjectMember]:
+        """Get members of target module."""
+        if self.analyzer:
+            attr_docs = self.analyzer.attr_docs
+        else:
+            attr_docs = {}
+
+        members = {}  # type: Dict[str, ObjectMember]
+        for name in dir(self.object):
+            try:
+                value = safe_getattr(self.object, name, None)
+                docstring = attr_docs.get(('', name), [])
+                members[name] = ObjectMember(name, value, docstring="\n".join(docstring))
+            except AttributeError:
+                continue
+
+        # annotation only member (ex. attr: int)
+        try:
+            for name in inspect.getannotations(self.object):
+                if name not in members:
+                    docstring = attr_docs.get(('', name), [])
+                    members[name] = ObjectMember(name, INSTANCEATTR,
+                                                 docstring="\n".join(docstring))
+        except AttributeError:
+            pass
+
+        return members
+
     def get_object_members(self, want_all: bool) -> Tuple[bool, ObjectMembers]:
+        members = self.get_module_members()
         if want_all:
-            members = get_module_members(self.object)
             if not self.__all__:
                 # for implicit module members, check __module__ to avoid
                 # documenting imported objects
-                return True, members
+                return True, list(members.values())
             else:
-                ret = []
+                for member in members.values():
-                for name, value in members:
+                    if member.__name__ not in self.__all__:
-                    if name in self.__all__:
+                        member.skipped = True
-                        ret.append(ObjectMember(name, value))
-                    else:
-                        ret.append(ObjectMember(name, value, skipped=True))
 
-                return False, ret
+                return False, list(members.values())
         else:
             memberlist = self.options.members or []
             ret = []
             for name in memberlist:
-                try:
+                if name in members:
-                    value = safe_getattr(self.object, name)
+                    ret.append(members[name])
-                    ret.append(ObjectMember(name, value))
+                else:
-                except AttributeError:
                     logger.warning(__('missing attribute mentioned in :members: option: '
                                       'module %s, attribute %s') %
                                    (safe_getattr(self.object, '__name__', '???'), name),
@@ -1213,7 +1243,7 @@ class DocstringSignatureMixin:
 
         return result
 
-    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+    def get_doc(self, encoding: str = None, ignore: int = None) -> Optional[List[List[str]]]:
         if encoding is not None:
             warnings.warn("The 'encoding' argument to autodoc.%s.get_doc() is deprecated."
                           % self.__class__.__name__,
@@ -1611,14 +1641,14 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
         else:
             return False, [convert(m) for m in members.values() if m.class_ == self.object]
 
-    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+    def get_doc(self, encoding: str = None, ignore: int = None) -> Optional[List[List[str]]]:
         if encoding is not None:
             warnings.warn("The 'encoding' argument to autodoc.%s.get_doc() is deprecated."
                           % self.__class__.__name__,
                           RemovedInSphinx40Warning, stacklevel=2)
         if self.doc_as_attr:
             # Don't show the docstring of the class when it is an alias.
-            return []
+            return None
 
         lines = getattr(self, '_new_docstrings', None)
         if lines is not None:
@@ -1667,9 +1697,8 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
                     ) -> None:
         if self.doc_as_attr:
             more_content = StringList([_('alias of %s') % restify(self.object)], source='')
-            super().add_content(more_content, no_docstring=True)
+
-        else:
+        super().add_content(more_content)
-            super().add_content(more_content)
 
     def document_members(self, all_members: bool = False) -> None:
         if self.doc_as_attr:
@@ -1774,7 +1803,7 @@ class TypeVarMixin(DataDocumenterMixinBase):
         return (isinstance(self.object, TypeVar) or
                 super().should_suppress_directive_header())
 
-    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+    def get_doc(self, encoding: str = None, ignore: int = None) -> Optional[List[List[str]]]:
         if ignore is not None:
             warnings.warn("The 'ignore' argument to autodoc.%s.get_doc() is deprecated."
                           % self.__class__.__name__,
@@ -1838,7 +1867,7 @@ class UninitializedGlobalVariableMixin(DataDocumenterMixinBase):
         return (self.object is UNINITIALIZED_ATTR or
                 super().should_suppress_value_header())
 
-    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+    def get_doc(self, encoding: str = None, ignore: int = None) -> Optional[List[List[str]]]:
         if self.object is UNINITIALIZED_ATTR:
             return []
         else:
@@ -1883,6 +1912,17 @@ class DataDocumenter(GenericAliasMixin, NewTypeMixin, TypeVarMixin,
 
         return ret
 
+    def should_suppress_value_header(self) -> bool:
+        if super().should_suppress_value_header():
+            return True
+        else:
+            doc = self.get_doc()
+            metadata = extract_metadata('\n'.join(sum(doc, [])))
+            if 'hide-value' in metadata:
+                return True
+
+        return False
+
     def add_directive_header(self, sig: str) -> None:
         super().add_directive_header(sig)
         sourcename = self.get_sourcename()
@@ -1914,8 +1954,32 @@ class DataDocumenter(GenericAliasMixin, NewTypeMixin, TypeVarMixin,
         return self.get_attr(self.parent or self.object, '__module__', None) \
             or self.modname
 
+    def get_module_comment(self, attrname: str) -> Optional[List[str]]:
+        try:
+            analyzer = ModuleAnalyzer.for_module(self.modname)
+            analyzer.analyze()
+            key = ('', attrname)
+            if key in analyzer.attr_docs:
+                return list(analyzer.attr_docs[key])
+        except PycodeError:
+            pass
+
+        return None
+
+    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+        # Check the variable has a docstring-comment
+        comment = self.get_module_comment(self.objpath[-1])
+        if comment:
+            return [comment]
+        else:
+            return super().get_doc(encoding, ignore)
+
     def add_content(self, more_content: Optional[StringList], no_docstring: bool = False
                     ) -> None:
+        # Disable analyzing variable comment on Documenter.add_content() to control it on
+        # DataDocumenter.add_content()
+        self.analyzer = None
+
         if not more_content:
             more_content = StringList()
 
@@ -2102,7 +2166,7 @@ class NonDataDescriptorMixin(DataDocumenterMixinBase):
         return (inspect.isattributedescriptor(self.object) or
                 super().should_suppress_directive_header())
 
-    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+    def get_doc(self, encoding: str = None, ignore: int = None) -> Optional[List[List[str]]]:
         if not inspect.isattributedescriptor(self.object):
             # the docstring of non datadescriptor is very probably the wrong thing
             # to display
@@ -2141,7 +2205,7 @@ class SlotsMixin(DataDocumenterMixinBase):
         else:
             return super().should_suppress_directive_header()
 
-    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+    def get_doc(self, encoding: str = None, ignore: int = None) -> Optional[List[List[str]]]:
         if self.object is SLOTSATTR:
             try:
                 __slots__ = inspect.getslots(self.parent)
@@ -2352,6 +2416,17 @@ class AttributeDocumenter(GenericAliasMixin, NewTypeMixin, SlotsMixin,  # type:
         return self.get_attr(self.parent or self.object, '__module__', None) \
             or self.modname
 
+    def should_suppress_value_header(self) -> bool:
+        if super().should_suppress_value_header():
+            return True
+        else:
+            doc = self.get_doc()
+            metadata = extract_metadata('\n'.join(sum(doc, [])))
+            if 'hide-value' in metadata:
+                return True
+
+        return False
+
     def add_directive_header(self, sig: str) -> None:
         super().add_directive_header(sig)
         sourcename = self.get_sourcename()
@@ -2395,7 +2470,7 @@ class AttributeDocumenter(GenericAliasMixin, NewTypeMixin, SlotsMixin,  # type:
 
         return None
 
-    def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
+    def get_doc(self, encoding: str = None, ignore: int = None) -> Optional[List[List[str]]]:
         # Check the attribute has a docstring-comment
         comment = self.get_attribute_comment(self.parent, self.objpath[-1])
         if comment:

sphinx/ext/autodoc/importer.py

@@ -13,7 +13,8 @@ import traceback
 import warnings
 from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Optional, Tuple
 
-from sphinx.deprecation import RemovedInSphinx40Warning, deprecated_alias
+from sphinx.deprecation import (RemovedInSphinx40Warning, RemovedInSphinx50Warning,
+                                deprecated_alias)
 from sphinx.pycode import ModuleAnalyzer, PycodeError
 from sphinx.util import logging
 from sphinx.util.inspect import (getannotations, getmro, getslots, isclass, isenumclass,
@@ -141,6 +142,9 @@ def get_module_members(module: Any) -> List[Tuple[str, Any]]:
     """Get members of target module."""
     from sphinx.ext.autodoc import INSTANCEATTR
 
+    warnings.warn('sphinx.ext.autodoc.importer.get_module_members() is deprecated.',
+                  RemovedInSphinx50Warning)
+
     members = {}  # type: Dict[str, Tuple[str, Any]]
     for name in dir(module):
         try:

tests/roots/test-ext-autodoc/target/classes.py

@@ -27,3 +27,6 @@ class Qux:
 class Quux(List[Union[int, float]]):
     """A subclass of List[Union[int, float]]"""
     pass
+
+
+Alias = Foo

tests/roots/test-ext-autodoc/target/hide_value.py

@@ -0,0 +1,19 @@
+#: docstring
+#:
+#: :meta hide-value:
+SENTINEL1 = object()
+
+#: :meta hide-value:
+SENTINEL2 = object()
+
+
+class Foo:
+    """docstring"""
+
+    #: docstring
+    #:
+    #: :meta hide-value:
+    SENTINEL1 = object()
+
+    #: :meta hide-value:
+    SENTINEL2 = object()

tests/roots/test-ext-autodoc/target/private.py

@@ -9,3 +9,7 @@ def _public_function(name):
 
     :meta public:
     """
+
+
+PRIVATE_CONSTANT = None  #: :meta private:
+_PUBLIC_CONSTANT = None  #: :meta public:

tests/test_ext_autodoc.py

@@ -2235,3 +2235,49 @@ def test_name_mangling(app):
         '      name of Foo',
         '',
     ]
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason='python 3.6+ is required.')
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_hide_value(app):
+    options = {'members': True}
+    actual = do_autodoc(app, 'module', 'target.hide_value', options)
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.hide_value',
+        '',
+        '',
+        '.. py:class:: Foo()',
+        '   :module: target.hide_value',
+        '',
+        '   docstring',
+        '',
+        '',
+        '   .. py:attribute:: Foo.SENTINEL1',
+        '      :module: target.hide_value',
+        '',
+        '      docstring',
+        '',
+        '      :meta hide-value:',
+        '',
+        '',
+        '   .. py:attribute:: Foo.SENTINEL2',
+        '      :module: target.hide_value',
+        '',
+        '      :meta hide-value:',
+        '',
+        '',
+        '.. py:data:: SENTINEL1',
+        '   :module: target.hide_value',
+        '',
+        '   docstring',
+        '',
+        '   :meta hide-value:',
+        '',
+        '',
+        '.. py:data:: SENTINEL2',
+        '   :module: target.hide_value',
+        '',
+        '   :meta hide-value:',
+        '',
+    ]

tests/test_ext_autodoc_autoattribute.py

@@ -189,3 +189,29 @@ def test_autoattribute_TypeVar(app):
         "   alias of TypeVar('T1')",
         '',
     ]
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason='python 3.6+ is required.')
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autoattribute_hide_value(app):
+    actual = do_autodoc(app, 'attribute', 'target.hide_value.Foo.SENTINEL1')
+    assert list(actual) == [
+        '',
+        '.. py:attribute:: Foo.SENTINEL1',
+        '   :module: target.hide_value',
+        '',
+        '   docstring',
+        '',
+        '   :meta hide-value:',
+        '',
+    ]
+
+    actual = do_autodoc(app, 'attribute', 'target.hide_value.Foo.SENTINEL2')
+    assert list(actual) == [
+        '',
+        '.. py:attribute:: Foo.SENTINEL2',
+        '   :module: target.hide_value',
+        '',
+        '   :meta hide-value:',
+        '',
+    ]

tests/test_ext_autodoc_autoclass.py

@@ -173,3 +173,21 @@ def test_show_inheritance_for_subclass_of_generic_type(app):
         '   A subclass of List[Union[int, float]]',
         '',
     ]
+
+
+def test_class_alias(app):
+    def autodoc_process_docstring(*args):
+        """A handler always raises an error.
+        This confirms this handler is never called for class aliases.
+        """
+        raise
+
+    app.connect('autodoc-process-docstring', autodoc_process_docstring)
+    actual = do_autodoc(app, 'class', 'target.classes.Alias')
+    assert list(actual) == [
+        '',
+        '.. py:attribute:: Alias',
+        '   :module: target.classes',
+        '',
+        '   alias of :class:`target.classes.Foo`',
+    ]

tests/test_ext_autodoc_autodata.py

@@ -129,3 +129,29 @@ def test_autodata_TypeVar(app):
         "   alias of TypeVar('T1')",
         '',
     ]
+
+
+@pytest.mark.skipif(sys.version_info < (3, 6), reason='python 3.6+ is required.')
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autodata_hide_value(app):
+    actual = do_autodoc(app, 'data', 'target.hide_value.SENTINEL1')
+    assert list(actual) == [
+        '',
+        '.. py:data:: SENTINEL1',
+        '   :module: target.hide_value',
+        '',
+        '   docstring',
+        '',
+        '   :meta hide-value:',
+        '',
+    ]
+
+    actual = do_autodoc(app, 'data', 'target.hide_value.SENTINEL2')
+    assert list(actual) == [
+        '',
+        '.. py:data:: SENTINEL2',
+        '   :module: target.hide_value',
+        '',
+        '   :meta hide-value:',
+        '',
+    ]

tests/test_ext_autodoc_private_members.py

@@ -23,6 +23,13 @@ def test_private_field(app):
         '.. py:module:: target.private',
         '',
         '',
+        '.. py:data:: _PUBLIC_CONSTANT',
+        '   :module: target.private',
+        '   :value: None',
+        '',
+        '   :meta public:',
+        '',
+        '',
         '.. py:function:: _public_function(name)',
         '   :module: target.private',
         '',
@@ -44,6 +51,20 @@ def test_private_field_and_private_members(app):
         '.. py:module:: target.private',
         '',
         '',
+        '.. py:data:: PRIVATE_CONSTANT',
+        '   :module: target.private',
+        '   :value: None',
+        '',
+        '   :meta private:',
+        '',
+        '',
+        '.. py:data:: _PUBLIC_CONSTANT',
+        '   :module: target.private',
+        '   :value: None',
+        '',
+        '   :meta public:',
+        '',
+        '',
         '.. py:function:: _public_function(name)',
         '   :module: target.private',
         '',
@@ -66,13 +87,20 @@ def test_private_field_and_private_members(app):
 def test_private_members(app):
     app.config.autoclass_content = 'class'
     options = {"members": None,
-               "private-members": "_public_function"}
+               "private-members": "_PUBLIC_CONSTANT,_public_function"}
     actual = do_autodoc(app, 'module', 'target.private', options)
     assert list(actual) == [
         '',
         '.. py:module:: target.private',
         '',
         '',
+        '.. py:data:: _PUBLIC_CONSTANT',
+        '   :module: target.private',
+        '   :value: None',
+        '',
+        '   :meta public:',
+        '',
+        '',
         '.. py:function:: _public_function(name)',
         '   :module: target.private',
         '',