Calculate and use the `docstring_lines` attribute (#14015)

sphinx/ext/autodoc/_docstrings.py

@@ -16,7 +16,7 @@ from sphinx.util.docstrings import prepare_docstring
 from sphinx.util.inspect import getdoc
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator, Mapping, Sequence
+    from collections.abc import Iterator, Mapping
     from typing import Any, Literal
 
     from sphinx.events import EventManager
@@ -26,26 +26,64 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger('sphinx.ext.autodoc')
 
-_OBJECT_INIT_DOCSTRING = (tuple(prepare_docstring(object.__init__.__doc__ or '')),)
-_OBJECT_NEW_DOCSTRING = (tuple(prepare_docstring(object.__new__.__doc__ or '')),)
+_OBJECT_INIT_DOCSTRING = [prepare_docstring(object.__init__.__doc__ or '')]
+_OBJECT_NEW_DOCSTRING = [prepare_docstring(object.__new__.__doc__ or '')]
+
+
+def _docstring_lines_for_props(
+    docstrings: list[list[str]] | None,
+    /,
+    *,
+    props: _ItemProperties,
+    real_modname: str | None,
+    events: EventManager,
+    options: _AutoDocumenterOptions,
+) -> tuple[str, ...]:
+    guess_modname = props._obj___module__ or props.module_name
+    if props.obj_type in {'class', 'exception'}:
+        # Do not pass real_modname and use the name from the __module__
+        # attribute of the class.
+        # If a class gets imported into the module real_modname
+        # the analyzer won't find the source of the class, if
+        # it looks in real_modname.
+        real_modname = guess_modname
+    else:
+        real_modname = real_modname or guess_modname
+    try:
+        analyzer = ModuleAnalyzer.for_module(real_modname)
+        # parse right now, to get PycodeErrors on parsing (results will
+        # be cached anyway)
+        analyzer.analyze()
+    except PycodeError as exc:
+        logger.debug('[autodoc] module analyzer failed: %s', exc)
+        # no source file -- e.g. for builtin and C modules
+        analyzer = None
+
+    attr_docs = {} if analyzer is None else analyzer.attr_docs
+    prepared_docstrings = _prepare_docstrings(
+        docstrings, props=props, attr_docs=attr_docs
+    )
+    docstring_lines = _process_docstrings(
+        prepared_docstrings,
+        events=events,
+        props=props,
+        options=options,
+    )
+    return tuple(docstring_lines)
 
 
 def _prepare_docstrings(
+    docstrings: list[list[str]] | None,
     *,
     props: _ItemProperties,
     attr_docs: Mapping[tuple[str, str], list[str]],
 ) -> list[list[str]] | None:
     """Add content from docstrings, attribute documentation and user."""
-    if props._docstrings is not None:
-        if props._docstrings:
-            docstrings = [list(doc) for doc in props._docstrings]
-        else:
-            # append at least a dummy docstring, so that the event
-            # autodoc-process-docstring is fired and can add some
-            # content if desired
-            docstrings = [[]]
-    else:
-        docstrings = None
+    if docstrings is not None and not docstrings:
+        # append at least a dummy docstring, so that the event
+        # autodoc-process-docstring is fired and can add some
+        # content if desired
+        docstrings = [[]]
 
     if props.obj_type in {'data', 'attribute'}:
         return docstrings
@@ -107,8 +145,7 @@ def _get_docstring_lines(
     inherit_docstrings: bool,
     parent: Any,
     tab_width: int,
-    _new_docstrings: Sequence[Sequence[str]] | None = None,
-) -> Sequence[Sequence[str]] | None:
+) -> list[list[str]] | None:
     obj = props._obj
 
     if props.obj_type in {'class', 'exception'}:
@@ -116,20 +153,16 @@ def _get_docstring_lines(
 
         if isinstance(obj, TypeVar):
             if obj.__doc__ == TypeVar.__doc__:
-                return ()
+                return []
         if props.doc_as_attr:
             # Don't show the docstring of the class when it is an alias.
             if _class_variable_comment(props):
-                return ()
+                return []
             return None
 
-        if _new_docstrings is not None:
-            return tuple(tuple(doc) for doc in _new_docstrings)
-
         docstrings = []
-        attrdocstring = getdoc(obj)
-        if attrdocstring:
-            docstrings.append(attrdocstring)
+        if attr_docstring := getdoc(obj):
+            docstrings.append(attr_docstring)
 
         # for classes, what the "docstring" is can be controlled via a
         # config value; the default is only the class docstring
@@ -162,25 +195,20 @@ def _get_docstring_lines(
                 else:
                     docstrings.append(init_docstring)
 
-        return tuple(
-            tuple(prepare_docstring(docstring, tab_width)) for docstring in docstrings
-        )
+        return [prepare_docstring(docstring, tab_width) for docstring in docstrings]
 
     if props.obj_type == 'method':
         docstring = _get_doc(
             obj,
             props=props,
             inherit_docstrings=inherit_docstrings,
-            _new_docstrings=_new_docstrings,
             parent=parent,
             tab_width=tab_width,
         )
-        if props.name == '__init__':
-            if docstring == _OBJECT_INIT_DOCSTRING:
-                return ()
-        if props.name == '__new__':
-            if docstring == _OBJECT_NEW_DOCSTRING:
-                return ()
+        if props.name == '__init__' and docstring == _OBJECT_INIT_DOCSTRING:
+            return []
+        if props.name == '__new__' and docstring == _OBJECT_NEW_DOCSTRING:
+            return []
         return docstring
 
     if props.obj_type == 'data':
@@ -193,17 +221,16 @@ def _get_docstring_lines(
             analyzer.analyze()
             key = ('', props.name)
             if key in analyzer.attr_docs:
-                comment = tuple(analyzer.attr_docs[key])
+                comment = list(analyzer.attr_docs[key])
         except PycodeError:
             pass
 
         if comment:
-            return (comment,)
+            return [comment]
         return _get_doc(
             obj,
             props=props,
             inherit_docstrings=inherit_docstrings,
-            _new_docstrings=_new_docstrings,
             parent=parent,
             tab_width=tab_width,
         )
@@ -219,7 +246,7 @@ def _get_docstring_lines(
             parent=parent, obj_path=props.parts, attrname=props.parts[-1]
         )
         if comment:
-            return (comment,)
+            return [comment]
 
         # Disable `autodoc_inherit_docstring` to avoid to obtain
         # a docstring from the value which descriptor returns unexpectedly.
@@ -231,16 +258,15 @@ def _get_docstring_lines(
             try:
                 parent___slots__ = inspect.getslots(parent)
                 if parent___slots__ and (docstring := parent___slots__.get(props.name)):
-                    docstring = tuple(prepare_docstring(docstring))
-                    return (docstring,)
-                return ()
+                    return [prepare_docstring(docstring)]
+                return []
             except ValueError as exc:
                 logger.warning(
                     __('Invalid __slots__ found on %s. Ignored.'),
                     (parent.__qualname__, exc),
                     type='autodoc',
                 )
-                return ()
+                return []
 
         if (
             obj is RUNTIME_INSTANCE_ATTRIBUTE
@@ -262,7 +288,6 @@ def _get_docstring_lines(
             obj,
             props=props,
             inherit_docstrings=inherit_docstrings,
-            _new_docstrings=_new_docstrings,
             parent=parent,
             tab_width=tab_width,
         )
@@ -271,7 +296,6 @@ def _get_docstring_lines(
         obj,
         props=props,
         inherit_docstrings=inherit_docstrings,
-        _new_docstrings=_new_docstrings,
         parent=parent,
         tab_width=tab_width,
     )
@@ -283,23 +307,16 @@ def _get_doc(
     *,
     props: _ItemProperties,
     inherit_docstrings: bool,
-    _new_docstrings: Sequence[Sequence[str]] | None,
     parent: Any,
     tab_width: int,
-) -> Sequence[Sequence[str]] | None:
+) -> list[list[str]] | None:
     """Decode and return lines of the docstring(s) for the object.
 
     When it returns None, autodoc-process-docstring will not be called for this
     object.
     """
     if obj is UNINITIALIZED_ATTR:
-        return ()
-
-    if props.obj_type not in {'module', 'data'} and _new_docstrings is not None:
-        # docstring already returned previously, then modified due to
-        # ``_extract_signatures_from_docstring()``. Just return the
-        # previously-computed result, so that we don't lose the processing.
-        return _new_docstrings
+        return []
 
     docstring = getdoc(
         obj,
@@ -308,8 +325,8 @@ def _get_doc(
         name=props.object_name,
     )
     if docstring:
-        return (tuple(prepare_docstring(docstring, tab_width)),)
-    return ()
+        return [prepare_docstring(docstring, tab_width)]
+    return []
 
 
 def _class_variable_comment(props: _ItemProperties) -> bool:

sphinx/ext/autodoc/_generate.py

@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
 from docutils.statemachine import StringList
 
 from sphinx.errors import PycodeError
-from sphinx.ext.autodoc._docstrings import _prepare_docstrings, _process_docstrings
 from sphinx.ext.autodoc._member_finder import _gather_members
 from sphinx.ext.autodoc._renderer import _add_content, _directive_header_lines
 from sphinx.ext.autodoc._sentinels import ALL
@@ -104,7 +103,7 @@ def _generate_directives(
         except PycodeError:
             pass
 
-    has_docstring = any(props._docstrings or ())
+    has_docstring = bool(props.docstring_lines)
     if ismock(props._obj) and not has_docstring:
         logger.warning(
             __('A mocked object is detected: %r'),
@@ -182,14 +181,7 @@ def _add_directive_lines(
     header_lines = StringList(list(lines), source='')
 
     # add content from docstrings or attribute documentation
-    docstrings = _prepare_docstrings(props=props, attr_docs=attr_docs)
-    lines = _process_docstrings(
-        docstrings,
-        events=events,
-        props=props,
-        options=options,
-    )
-    docstring_lines = StringList(list(lines), source=source_name)
+    docstring_lines = StringList(props.docstring_lines, source=source_name)
 
     # add alias information, if applicable
     lines = _body_alias_lines(
@@ -252,6 +244,7 @@ def _document_members(
         events=events,
         get_attr=get_attr,
         options=options,
+        real_modname=real_modname,
         props=props,
     )
 

sphinx/ext/autodoc/_member_finder.py

@@ -100,6 +100,7 @@ def _gather_members(
     events: EventManager,
     get_attr: _AttrGetter,
     options: _AutoDocumenterOptions,
+    real_modname: str,
     props: _ItemProperties,
 ) -> list[tuple[_ItemProperties, bool, str]]:
     """Generate reST for member documentation.
@@ -182,6 +183,7 @@ def _gather_members(
             events=events,
             get_attr=get_attr,
             options=options,
+            real_modname=real_modname,
         )
         if member_props is None:
             continue

sphinx/ext/autodoc/_property_types.py

@@ -51,7 +51,6 @@ class _ItemProperties:
     #: The item's signature lines, for use in the directive
     signatures: tuple[str, ...] = ()
 
-    _docstrings: Sequence[Sequence[str]] | None = None
     _docstrings_has_hide_value: bool = False
     _obj: Any
     _obj___module__: str | None

sphinx/ext/autodoc/importer.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 import contextlib
 import importlib
-import itertools
 import os
 import re
 import sys
@@ -20,7 +19,10 @@ from typing import TYPE_CHECKING, NewType, TypeVar
 from weakref import WeakSet
 
 from sphinx.errors import PycodeError
-from sphinx.ext.autodoc._docstrings import _get_docstring_lines
+from sphinx.ext.autodoc._docstrings import (
+    _docstring_lines_for_props,
+    _get_docstring_lines,
+)
 from sphinx.ext.autodoc._property_types import (
     _AssignStatementProperties,
     _ClassDefProperties,
@@ -446,7 +448,7 @@ def _is_runtime_instance_attribute_not_commented(
 
 def _get_attribute_comment(
     parent: Any, obj_path: Sequence[str], attrname: str
-) -> tuple[str, ...] | None:
+) -> list[str] | None:
     for cls in inspect.getmro(parent):
         try:
             module = safe_getattr(cls, '__module__')
@@ -457,7 +459,7 @@ def _get_attribute_comment(
             if qualname and obj_path:
                 key = (qualname, attrname)
                 if key in analyzer.attr_docs:
-                    return tuple(analyzer.attr_docs[key])
+                    return list(analyzer.attr_docs[key])
         except (AttributeError, PycodeError):
             pass
 
@@ -495,6 +497,7 @@ def _load_object_by_name(
     events: EventManager,
     get_attr: _AttrGetter,
     options: _AutoDocumenterOptions,
+    real_modname: str | None = None,
 ) -> _ItemProperties | None:
     """Import and load the object given by *name*."""
     parsed = _parse_name(
@@ -883,7 +886,8 @@ def _load_object_by_name(
         class_doc_from = options.class_doc_from
     else:
         class_doc_from = config.autoclass_content
-    props._docstrings = _get_docstring_lines(
+
+    docstrings = _get_docstring_lines(
         props,
         class_doc_from=class_doc_from,
         get_attr=get_attr,
@@ -891,10 +895,12 @@ def _load_object_by_name(
         parent=parent,
         tab_width=options._tab_width,
     )
-    for line in itertools.chain.from_iterable(props._docstrings or ()):
-        if _hide_value_re.match(line):
-            props._docstrings_has_hide_value = True
-            break
+    if docstrings:
+        for docstring_lines in docstrings:
+            for line in docstring_lines:
+                if _hide_value_re.match(line):
+                    props._docstrings_has_hide_value = True
+                    break
 
     # format the object's signature, if any
     try:
@@ -902,6 +908,7 @@ def _load_object_by_name(
             args=args,
             retann=retann,
             config=config,
+            docstrings=docstrings,
             events=events,
             get_attr=get_attr,
             parent=parent,
@@ -916,6 +923,14 @@ def _load_object_by_name(
         f'{args} -> {retann}' if retann else str(args) for args, retann in signatures
     )
 
+    props.docstring_lines = _docstring_lines_for_props(
+        docstrings,
+        props=props,
+        real_modname=real_modname,
+        events=events,
+        options=options,
+    )
+
     return props
 
 
@@ -1154,6 +1169,7 @@ def _update_module_annotations_from_type_comments(mod: ModuleType) -> None:
 def _format_signatures(
     *,
     config: Config,
+    docstrings: list[list[str]] | None,
     events: EventManager,
     get_attr: _AttrGetter,
     parent: Any,
@@ -1191,15 +1207,12 @@ def _format_signatures(
         not signatures
         and config.autodoc_docstring_signature
         and props.obj_type not in {'module', 'data', 'type'}
+        and docstrings is not None
     ):
         # only act if a signature is not explicitly given already,
         # and if the feature is enabled
-        signatures[:] = _extract_signatures_from_docstring(
-            config=config,
-            get_attr=get_attr,
-            options=options,
-            parent=parent,
-            props=props,
+        signatures[:] = _extract_signatures_from_docstrings(
+            docstrings, props=props, tab_width=options._tab_width
         )
 
     if not signatures:
@@ -1291,6 +1304,7 @@ def _format_signatures(
                 )
                 signatures += _format_signatures(
                     config=config,
+                    docstrings=None,
                     events=events,
                     get_attr=get_attr,
                     parent=None,
@@ -1402,6 +1416,7 @@ def _format_signatures(
                 )
                 signatures += _format_signatures(
                     config=config,
+                    docstrings=None,
                     events=events,
                     get_attr=get_attr,
                     parent=parent,
@@ -1437,16 +1452,12 @@ def _format_signatures(
     return signatures
 
 
-def _extract_signatures_from_docstring(
-    config: Config,
-    get_attr: _AttrGetter,
-    options: _AutoDocumenterOptions,
-    parent: Any,
+def _extract_signatures_from_docstrings(
+    docstrings: list[list[str]],
+    /,
     props: _ItemProperties,
+    tab_width: int,
 ) -> list[_FormattedSignature]:
-    if props._docstrings is None:
-        return []
-
     signatures: list[_FormattedSignature] = []
 
     # candidates of the object name
@@ -1456,11 +1467,10 @@ def _extract_signatures_from_docstring(
         if hasattr(props._obj, '__mro__'):
             valid_names |= {cls.__name__ for cls in props._obj.__mro__}
 
-    docstrings = props._docstrings or ()
-    _new_docstrings = [list(l) for l in docstrings]
-    tab_width = options._tab_width
+    stripped_docstrings = [list(l) for l in (docstrings or ())]
     for i, doclines in enumerate(docstrings):
-        for j, line in enumerate(doclines):
+        j = 0
+        for j, line in enumerate(doclines):  # NoQA: B007
             if not line:
                 # no lines in docstring, no match
                 break
@@ -1480,11 +1490,6 @@ def _extract_signatures_from_docstring(
             if base not in valid_names:
                 break
 
-            # re-prepare docstring to ignore more leading indentation
-            _new_docstrings[i] = prepare_docstring(
-                '\n'.join(doclines[j + 1 :]), tab_width
-            )
-
             if props.obj_type in {'class', 'exception'} and retann == 'None':
                 # Strip a return value from signatures of constructor in docstring
                 signatures.append((args, ''))
@@ -1492,23 +1497,31 @@ def _extract_signatures_from_docstring(
                 signatures.append((args, retann or ''))
 
         if signatures:
-            # finish the loop when signature found
+            # re-prepare docstring to ignore more leading indentation
+            stripped_docstrings[i] = prepare_docstring(
+                '\n'.join(doclines[j:]), tab_width
+            )
+
+            # finish the loop after finding at least one signature
             break
 
     if not signatures:
         return []
 
-    props._docstrings = _get_docstring_lines(
-        props,
-        class_doc_from=options.class_doc_from
-        if options.class_doc_from is not None
-        else config.autoclass_content,
-        get_attr=get_attr,
-        inherit_docstrings=config.autodoc_inherit_docstrings,
-        parent=parent,
-        tab_width=tab_width,
-        _new_docstrings=tuple(tuple(doc) for doc in _new_docstrings),
-    )
+    # Update docstrings from stripped_docstrings if needed
+    if props.obj_type in {
+        'class',
+        'exception',
+        'function',
+        'method',
+        'property',
+        'decorator',
+    } or (
+        props.obj_type == 'attribute'
+        and isinstance(props, _AssignStatementProperties)
+        and props._obj_is_attribute_descriptor
+    ):
+        docstrings[:] = stripped_docstrings
 
     return signatures
 

sphinx/ext/autosummary/__init__.py

@@ -67,7 +67,6 @@ import sphinx
 from sphinx import addnodes
 from sphinx.errors import PycodeError
 from sphinx.ext.autodoc._directive_options import _AutoDocumenterOptions
-from sphinx.ext.autodoc._docstrings import _prepare_docstrings, _process_docstrings
 from sphinx.ext.autodoc._member_finder import _best_object_type_for_member
 from sphinx.ext.autodoc._sentinels import INSTANCE_ATTR
 from sphinx.ext.autodoc.directive import _AutodocAttrGetter
@@ -388,15 +387,7 @@ class Autosummary(SphinxDirective):
             # -- Grab the summary
 
             # get content from docstrings or attribute documentation
-            attr_docs = {} if analyzer is None else analyzer.attr_docs
-            docstrings = _prepare_docstrings(props=props, attr_docs=attr_docs)
-            docstring_lines = _process_docstrings(
-                docstrings,
-                events=events,
-                props=props,
-                options=opts,
-            )
-            summary = extract_summary(list(docstring_lines), document_settings)
+            summary = extract_summary(props.docstring_lines, document_settings)
 
             items.append((display_name, sig, summary, real_name))
 

tests/test_ext_autodoc/test_ext_autodoc.py

@@ -124,7 +124,7 @@ def format_sig(obj_type, name, obj, *, app, args=None, retann=None):
         _obj_is_new_type=False,
         _obj_is_typevar=False,
     )
-    props._docstrings = _get_docstring_lines(
+    docstrings = _get_docstring_lines(
         props,
         class_doc_from=app.config.autoclass_content,
         get_attr=get_attr,
@@ -134,6 +134,7 @@ def format_sig(obj_type, name, obj, *, app, args=None, retann=None):
     )
     signatures = _format_signatures(
         config=app.config,
+        docstrings=docstrings,
         events=app.events,
         get_attr=get_attr,
         options=options,
@@ -393,6 +394,7 @@ def test_autodoc_process_signature_typehints(app):
     options = _AutoDocumenterOptions()
     _format_signatures(
         config=app.config,
+        docstrings=None,
         events=app.events,
         get_attr=get_attr,
         options=options,