Close #6830: autodoc: consider a member private if docstring has "private" metadata

CHANGES

@@ -16,6 +16,8 @@ Incompatible changes
   :confval:`autosummary_generate_overwrite` to change the behavior
 * #5923: autodoc: the members of ``object`` class are not documented by default
   when ``:inherited-members:`` and ``:special-members:`` are given.
+* #6830: py domain: ``meta`` fields in info-field-list becomes reserved.  They
+  are not displayed on output document now
 
 Deprecated
 ----------
@@ -29,8 +31,11 @@ Features added
   old stub file
 * #5923: autodoc: ``:inherited-members:`` option takes a name of anchestor class
   not to document inherited members of the class and uppers
+* #6830: autodoc: consider a member private if docstring contains
+  ``:meta private:`` in info-field-list
 * #6558: glossary: emit a warning for duplicated glossary entry
 * #6558: std domain: emit a warning for duplicated generic objects
+* #6830: py domain: Add new event: :event:`object-description-transform`
 
 Bugs fixed
 ----------

doc/extdev/appapi.rst

@@ -216,6 +216,14 @@ connect handlers to the events.  Example:
 
    .. versionadded:: 0.5
 
+.. event:: object-description-transform (app, domain, objtype, contentnode)
+
+   Emitted when an object description directive has run.  The *domain* and
+   *objtype* arguments are strings indicating object description of the object.
+   And *contentnode* is a content for the object.  It can be modified in-place.
+
+   .. versionadded:: 3.0
+
 .. event:: doctree-read (app, doctree)
 
    Emitted when a doctree has been parsed and read by the environment, and is

doc/usage/extensions/autodoc.rst

@@ -140,6 +140,20 @@ inserting them into the page source under a suitable :rst:dir:`py:module`,
 
      .. versionadded:: 1.1
 
+   * autodoc considers a member private if its docstring contains
+     ``:meta private:`` in its :ref:`info-field-lists`.
+     For example:
+
+     .. code-block:: rst
+
+        def my_function(my_arg, my_other_arg):
+            """blah blah blah
+
+            :meta private:
+            """
+
+     .. versionadded:: 3.0
+
    * Python "special" members (that is, those named like ``__special__``) will
      be included if the ``special-members`` flag option is given::
 

doc/usage/restructuredtext/domains.rst

@@ -354,6 +354,9 @@ Info field lists
 ~~~~~~~~~~~~~~~~
 
 .. versionadded:: 0.4
+.. versionchanged:: 3.0
+
+   meta fields are added.
 
 Inside Python object description directives, reST field lists with these fields
 are recognized and formatted nicely:
@@ -367,6 +370,10 @@ are recognized and formatted nicely:
 * ``vartype``: Type of a variable.  Creates a link if possible.
 * ``returns``, ``return``: Description of the return value.
 * ``rtype``: Return type.  Creates a link if possible.
+* ``meta``: Add metadata to description of the python object.  The metadata will
+  not be shown on output document.  For example, ``:meta private:`` indicates
+  the python object is private member.  It is used in
+  :py:mod:`sphinx.ext.autodoc` for filtering members.
 
 .. note::
 

sphinx/directives/__init__.py

@@ -193,6 +193,8 @@ class ObjectDescription(SphinxDirective):
             self.env.temp_data['object'] = self.names[0]
         self.before_content()
         self.state.nested_parse(self.content, self.content_offset, contentnode)
+        self.env.app.emit('object-description-transform',
+                          self.domain, self.objtype, contentnode)
         DocFieldTransformer(self).transform_all(contentnode)
         self.env.temp_data['object'] = None
         self.after_content()
@@ -295,6 +297,8 @@ def setup(app: "Sphinx") -> Dict[str, Any]:
     # new, more consistent, name
     directives.register_directive('object', ObjectDescription)
 
+    app.add_event('object-description-transform')
+
     return {
         'version': 'builtin',
         'parallel_read_safe': True,

sphinx/domains/python.py

@@ -764,6 +764,21 @@ class PyXRefRole(XRefRole):
         return title, target
 
 
+def filter_meta_fields(app: Sphinx, domain: str, objtype: str, content: Element) -> None:
+    """Filter ``:meta:`` field from its docstring."""
+    if domain != 'py':
+        return
+
+    for node in content:
+        if isinstance(node, nodes.field_list):
+            fields = cast(List[nodes.field], node)
+            for field in fields:
+                field_name = cast(nodes.field_body, field[0]).astext().strip()
+                if field_name == 'meta' or field_name.startswith('meta '):
+                    node.remove(field)
+                    break
+
+
 class PythonModuleIndex(Index):
     """
     Index subclass to provide the Python module index.
@@ -1067,7 +1082,10 @@ class PythonDomain(Domain):
 
 
 def setup(app: Sphinx) -> Dict[str, Any]:
+    app.setup_extension('sphinx.directives')
+
     app.add_domain(PythonDomain)
+    app.connect('object-description-transform', filter_meta_fields)
 
     return {
         'version': 'builtin',

sphinx/ext/autodoc/__init__.py

@@ -29,7 +29,7 @@ from sphinx.pycode import ModuleAnalyzer, PycodeError
 from sphinx.util import inspect
 from sphinx.util import logging
 from sphinx.util import rpartition
-from sphinx.util.docstrings import prepare_docstring
+from sphinx.util.docstrings import extract_metadata, prepare_docstring
 from sphinx.util.inspect import (
     Signature, getdoc, object_description, safe_getattr, safe_getmembers
 )
@@ -560,6 +560,13 @@ class Documenter:
                     doc = None
             has_doc = bool(doc)
 
+            metadata = extract_metadata(doc)
+            if 'private' in metadata:
+                # consider a member private if docstring has "private" metadata
+                isprivate = True
+            else:
+                isprivate = membername.startswith('_')
+
             keep = False
             if want_all and membername.startswith('__') and \
                     membername.endswith('__') and len(membername) > 4:
@@ -575,14 +582,14 @@ class Documenter:
                     if membername in self.options.special_members:
                         keep = has_doc or self.options.undoc_members
             elif (namespace, membername) in attr_docs:
-                if want_all and membername.startswith('_'):
+                if want_all and isprivate:
                     # ignore members whose name starts with _ by default
                     keep = self.options.private_members
                 else:
                     # keep documented attributes
                     keep = True
                 isattr = True
-            elif want_all and membername.startswith('_'):
+            elif want_all and isprivate:
                 # ignore members whose name starts with _ by default
                 keep = self.options.private_members and \
                     (has_doc or self.options.undoc_members)

sphinx/util/docstrings.py

@@ -8,8 +8,38 @@
     :license: BSD, see LICENSE for details.
 """
 
+import re
 import sys
-from typing import List
+from typing import Dict, List
+
+from docutils.parsers.rst.states import Body
+
+
+field_list_item_re = re.compile(Body.patterns['field_marker'])
+
+
+def extract_metadata(s: str) -> Dict[str, str]:
+    """Extract metadata from docstring."""
+    in_other_element = False
+    metadata = {}  # type: Dict[str, str]
+
+    if not s:
+        return metadata
+
+    for line in prepare_docstring(s):
+        if line.strip() == '':
+            in_other_element = False
+        else:
+            matched = field_list_item_re.match(line)
+            if matched and not in_other_element:
+                field_name = matched.group()[1:].split(':', 1)[0]
+                if field_name.startswith('meta '):
+                    name = field_name[5:].strip()
+                    metadata[name] = line[matched.end():].strip()
+            else:
+                in_other_element = True
+
+    return metadata
 
 
 def prepare_docstring(s: str, ignore: int = 1, tabsize: int = 8) -> List[str]:

tests/roots/test-ext-autodoc/target/private.py

@@ -0,0 +1,5 @@
+def private_function(name):
+    """private_function is a docstring().
+
+    :meta private:
+    """

tests/test_ext_autodoc_private_members.py

@@ -0,0 +1,46 @@
+"""
+    test_ext_autodoc_private_members
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Test the autodoc extension.  This tests mainly for private-members option.
+
+    :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import pytest
+
+from test_autodoc import do_autodoc
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_private_field(app):
+    app.config.autoclass_content = 'class'
+    options = {"members": None}
+    actual = do_autodoc(app, 'module', 'target.private', options)
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.private',
+        '',
+    ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_private_field_and_private_members(app):
+    app.config.autoclass_content = 'class'
+    options = {"members": None,
+               "private-members": None}
+    actual = do_autodoc(app, 'module', 'target.private', options)
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.private',
+        '',
+        '',
+        '.. py:function:: private_function(name)',
+        '   :module: target.private',
+        '',
+        '   private_function is a docstring().',
+        '   ',
+        '   :meta private:',
+        '   '
+    ]

tests/test_util_docstrings.py

@@ -8,7 +8,34 @@
     :license: BSD, see LICENSE for details.
 """
 
-from sphinx.util.docstrings import prepare_docstring, prepare_commentdoc
+from sphinx.util.docstrings import (
+    extract_metadata, prepare_docstring, prepare_commentdoc
+)
+
+
+def test_extract_metadata():
+    metadata = extract_metadata(":meta foo: bar\n"
+                                ":meta baz:\n")
+    assert metadata == {'foo': 'bar', 'baz': ''}
+
+    # field_list like text following just after paragaph is not a field_list
+    metadata = extract_metadata("blah blah blah\n"
+                                ":meta foo: bar\n"
+                                ":meta baz:\n")
+    assert metadata == {}
+
+    # field_list like text following after blank line is a field_list
+    metadata = extract_metadata("blah blah blah\n"
+                                "\n"
+                                ":meta foo: bar\n"
+                                ":meta baz:\n")
+    assert metadata == {'foo': 'bar', 'baz': ''}
+
+    # non field_list item breaks field_list
+    metadata = extract_metadata(":meta foo: bar\n"
+                                "blah blah blah\n"
+                                ":meta baz:\n")
+    assert metadata == {'foo': 'bar'}
 
 
 def test_prepare_docstring():