Document `typing.NewType` as a class (#10700)

sphinx/ext/autodoc/__init__.py

@@ -8,6 +8,7 @@ for those who like elaborate docstrings.
 from __future__ import annotations
 
 import re
+import sys
 from inspect import Parameter, Signature
 from types import ModuleType
 from typing import (TYPE_CHECKING, Any, Callable, Iterator, List, Sequence, Tuple, TypeVar,
@@ -1420,6 +1421,11 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
         'class-doc-from': class_doc_from_option,
     }
 
+    # Must be higher than FunctionDocumenter, ClassDocumenter, and
+    # AttributeDocumenter as NewType can be an attribute and is a class
+    # after Python 3.10. Before 3.10 it is a kind of function object
+    priority = 15
+
     _signature_class: Any = None
     _signature_method_name: str = None
 
@@ -1441,7 +1447,8 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
     @classmethod
     def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any
                             ) -> bool:
-        return isinstance(member, type)
+        return isinstance(member, type) or (
+            isattr and (inspect.isNewType(member) or isinstance(member, TypeVar)))
 
     def import_object(self, raiseerror: bool = False) -> bool:
         ret = super().import_object(raiseerror)
@@ -1452,9 +1459,19 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
                 self.doc_as_attr = (self.objpath[-1] != self.object.__name__)
             else:
                 self.doc_as_attr = True
+            if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
+                modname = getattr(self.object, '__module__', self.modname)
+                if modname != self.modname and self.modname.startswith(modname):
+                    bases = self.modname[len(modname):].strip('.').split('.')
+                    self.objpath = bases + self.objpath
+                    self.modname = modname
         return ret
 
     def _get_signature(self) -> tuple[Any | None, str | None, Signature | None]:
+        if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
+            # Supress signature
+            return None, None, None
+
         def get_user_defined_function_or_method(obj: Any, attr: str) -> Any:
             """ Get the `attr` function or method from `obj`, if it is user-defined. """
             if inspect.is_builtin_class_method(obj, attr):
@@ -1635,11 +1652,15 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
             self.directivetype = 'attribute'
         super().add_directive_header(sig)
 
+        if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
+            return
+
         if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals:
             self.add_line('   :final:', sourcename)
 
         canonical_fullname = self.get_canonical_fullname()
-        if not self.doc_as_attr and canonical_fullname and self.fullname != canonical_fullname:
+        if (not self.doc_as_attr and not inspect.isNewType(self.object)
+                and canonical_fullname and self.fullname != canonical_fullname):
             self.add_line('   :canonical: %s' % canonical_fullname, sourcename)
 
         # add inheritance info, if wanted
@@ -1687,6 +1708,27 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
             return False, [m for m in members.values() if m.class_ == self.object]
 
     def get_doc(self) -> list[list[str]] | None:
+        if isinstance(self.object, TypeVar):
+            if self.object.__doc__ == TypeVar.__doc__:
+                return []
+        if sys.version_info[:2] < (3, 10):
+            if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
+                parts = self.modname.strip('.').split('.')
+                orig_objpath = self.objpath
+                for i in range(len(parts)):
+                    new_modname = '.'.join(parts[:len(parts) - i])
+                    new_objpath = parts[len(parts) - i:] + orig_objpath
+                    try:
+                        analyzer = ModuleAnalyzer.for_module(new_modname)
+                        analyzer.analyze()
+                        key = ('', new_objpath[-1])
+                        comment = list(analyzer.attr_docs.get(key, []))
+                        if comment:
+                            self.objpath = new_objpath
+                            self.modname = new_modname
+                            return [comment]
+                    except PycodeError:
+                        pass
         if self.doc_as_attr:
             # Don't show the docstring of the class when it is an alias.
             comment = self.get_variable_comment()
@@ -1751,6 +1793,35 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
             return None
 
     def add_content(self, more_content: StringList | None) -> None:
+        if inspect.isNewType(self.object):
+            if self.config.autodoc_typehints_format == "short":
+                supertype = restify(self.object.__supertype__, "smart")
+            else:
+                supertype = restify(self.object.__supertype__)
+
+            more_content = StringList([_('alias of %s') % supertype, ''], source='')
+        if isinstance(self.object, TypeVar):
+            attrs = [repr(self.object.__name__)]
+            for constraint in self.object.__constraints__:
+                if self.config.autodoc_typehints_format == "short":
+                    attrs.append(stringify_annotation(constraint, "smart"))
+                else:
+                    attrs.append(stringify_annotation(constraint))
+            if self.object.__bound__:
+                if self.config.autodoc_typehints_format == "short":
+                    bound = restify(self.object.__bound__, "smart")
+                else:
+                    bound = restify(self.object.__bound__)
+                attrs.append(r"bound=\ " + bound)
+            if self.object.__covariant__:
+                attrs.append("covariant=True")
+            if self.object.__contravariant__:
+                attrs.append("contravariant=True")
+
+            more_content = StringList(
+                [_('alias of TypeVar(%s)') % ", ".join(attrs), ''],
+                source=''
+            )
         if self.doc_as_attr and self.modname != self.get_real_modname():
             try:
                 # override analyzer to obtain doccomment around its definition.
@@ -1801,7 +1872,7 @@ class ExceptionDocumenter(ClassDocumenter):
     member_order = 10
 
     # needs a higher priority than ClassDocumenter
-    priority = 10
+    priority = ClassDocumenter.priority + 5
 
     @classmethod
     def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any
@@ -1827,7 +1898,7 @@ class DataDocumenterMixinBase:
         return False
 
     def update_content(self, more_content: StringList) -> None:
-        """Update docstring for the NewType object."""
+        """Update docstring, for example with TypeVar variance."""
         pass
 
 
@@ -1854,74 +1925,6 @@ class GenericAliasMixin(DataDocumenterMixinBase):
         super().update_content(more_content)
 
 
-class NewTypeMixin(DataDocumenterMixinBase):
-    """
-    Mixin for DataDocumenter and AttributeDocumenter to provide the feature for
-    supporting NewTypes.
-    """
-
-    def should_suppress_directive_header(self) -> bool:
-        return (inspect.isNewType(self.object) or
-                super().should_suppress_directive_header())
-
-    def update_content(self, more_content: StringList) -> None:
-        if inspect.isNewType(self.object):
-            if self.config.autodoc_typehints_format == "short":
-                supertype = restify(self.object.__supertype__, "smart")
-            else:
-                supertype = restify(self.object.__supertype__)
-
-            more_content.append(_('alias of %s') % supertype, '')
-            more_content.append('', '')
-
-        super().update_content(more_content)
-
-
-class TypeVarMixin(DataDocumenterMixinBase):
-    """
-    Mixin for DataDocumenter and AttributeDocumenter to provide the feature for
-    supporting TypeVars.
-    """
-
-    def should_suppress_directive_header(self) -> bool:
-        return (isinstance(self.object, TypeVar) or
-                super().should_suppress_directive_header())
-
-    def get_doc(self) -> list[list[str]] | None:
-        if isinstance(self.object, TypeVar):
-            if self.object.__doc__ != TypeVar.__doc__:
-                return super().get_doc()  # type: ignore
-            else:
-                return []
-        else:
-            return super().get_doc()  # type: ignore
-
-    def update_content(self, more_content: StringList) -> None:
-        if isinstance(self.object, TypeVar):
-            attrs = [repr(self.object.__name__)]
-            for constraint in self.object.__constraints__:
-                if self.config.autodoc_typehints_format == "short":
-                    attrs.append(stringify_annotation(constraint, "smart"))
-                else:
-                    attrs.append(stringify_annotation(constraint,
-                                                      "fully-qualified-except-typing"))
-            if self.object.__bound__:
-                if self.config.autodoc_typehints_format == "short":
-                    bound = restify(self.object.__bound__, "smart")
-                else:
-                    bound = restify(self.object.__bound__)
-                attrs.append(r"bound=\ " + bound)
-            if self.object.__covariant__:
-                attrs.append("covariant=True")
-            if self.object.__contravariant__:
-                attrs.append("contravariant=True")
-
-            more_content.append(_('alias of TypeVar(%s)') % ", ".join(attrs), '')
-            more_content.append('', '')
-
-        super().update_content(more_content)
-
-
 class UninitializedGlobalVariableMixin(DataDocumenterMixinBase):
     """
     Mixin for DataDocumenter to provide the feature for supporting uninitialized
@@ -1963,7 +1966,7 @@ class UninitializedGlobalVariableMixin(DataDocumenterMixinBase):
             return super().get_doc()  # type: ignore
 
 
-class DataDocumenter(GenericAliasMixin, NewTypeMixin, TypeVarMixin,
+class DataDocumenter(GenericAliasMixin,
                      UninitializedGlobalVariableMixin, ModuleLevelDocumenter):
     """
     Specialized Documenter subclass for data items.
@@ -2083,24 +2086,6 @@ class DataDocumenter(GenericAliasMixin, NewTypeMixin, TypeVarMixin,
         super().add_content(more_content)
 
 
-class NewTypeDataDocumenter(DataDocumenter):
-    """
-    Specialized Documenter subclass for NewTypes.
-
-    Note: This must be invoked before FunctionDocumenter because NewType is a kind of
-    function object.
-    """
-
-    objtype = 'newtypedata'
-    directivetype = 'data'
-    priority = FunctionDocumenter.priority + 1
-
-    @classmethod
-    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any
-                            ) -> bool:
-        return inspect.isNewType(member) and isattr
-
-
 class MethodDocumenter(DocstringSignatureMixin, ClassLevelDocumenter):  # type: ignore
     """
     Specialized Documenter subclass for methods (normal, static and class).
@@ -2520,8 +2505,8 @@ class UninitializedInstanceAttributeMixin(DataDocumenterMixinBase):
             return super().get_doc()  # type: ignore
 
 
-class AttributeDocumenter(GenericAliasMixin, NewTypeMixin, SlotsMixin,  # type: ignore
+class AttributeDocumenter(GenericAliasMixin, SlotsMixin,  # type: ignore
-                          TypeVarMixin, RuntimeInstanceAttributeMixin,
+                          RuntimeInstanceAttributeMixin,
                           UninitializedInstanceAttributeMixin, NonDataDescriptorMixin,
                           DocstringStripSignatureMixin, ClassLevelDocumenter):
     """
@@ -2759,24 +2744,6 @@ class PropertyDocumenter(DocstringStripSignatureMixin, ClassLevelDocumenter):  #
                 return None
 
 
-class NewTypeAttributeDocumenter(AttributeDocumenter):
-    """
-    Specialized Documenter subclass for NewTypes.
-
-    Note: This must be invoked before MethodDocumenter because NewType is a kind of
-    function object.
-    """
-
-    objtype = 'newvarattribute'
-    directivetype = 'attribute'
-    priority = MethodDocumenter.priority + 1
-
-    @classmethod
-    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any
-                            ) -> bool:
-        return not isinstance(parent, ModuleDocumenter) and inspect.isNewType(member)
-
-
 def autodoc_attrgetter(app: Sphinx, obj: Any, name: str, *defargs: Any) -> Any:
     """Alternative getattr() for types"""
     for typ, func in app.registry.autodoc_attrgettrs.items():
@@ -2791,13 +2758,11 @@ def setup(app: Sphinx) -> dict[str, Any]:
     app.add_autodocumenter(ClassDocumenter)
     app.add_autodocumenter(ExceptionDocumenter)
     app.add_autodocumenter(DataDocumenter)
-    app.add_autodocumenter(NewTypeDataDocumenter)
     app.add_autodocumenter(FunctionDocumenter)
     app.add_autodocumenter(DecoratorDocumenter)
     app.add_autodocumenter(MethodDocumenter)
     app.add_autodocumenter(AttributeDocumenter)
     app.add_autodocumenter(PropertyDocumenter)
-    app.add_autodocumenter(NewTypeAttributeDocumenter)
 
     app.add_config_value('autoclass_content', 'class', True, ENUM('both', 'class', 'init'))
     app.add_config_value('autodoc_member_order', 'alphabetical', True,

sphinx/ext/autosummary/generate.py

@@ -82,12 +82,11 @@ def setup_documenters(app: Any) -> None:
     from sphinx.ext.autodoc import (AttributeDocumenter, ClassDocumenter, DataDocumenter,
                                     DecoratorDocumenter, ExceptionDocumenter,
                                     FunctionDocumenter, MethodDocumenter, ModuleDocumenter,
-                                    NewTypeAttributeDocumenter, NewTypeDataDocumenter,
                                     PropertyDocumenter)
     documenters: list[type[Documenter]] = [
         ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter,
-        FunctionDocumenter, MethodDocumenter, NewTypeAttributeDocumenter,
+        FunctionDocumenter, MethodDocumenter,
-        NewTypeDataDocumenter, AttributeDocumenter, DecoratorDocumenter, PropertyDocumenter,
+        AttributeDocumenter, DecoratorDocumenter, PropertyDocumenter,
     ]
     for documenter in documenters:
         app.registry.add_documenter(documenter.objtype, documenter)

tests/test_ext_autodoc.py

@@ -1911,7 +1911,7 @@ def test_autodoc_TypeVar(app):
         '   :module: target.typevar',
         '',
         '',
-        '   .. py:attribute:: Class.T1',
+        '   .. py:class:: Class.T1',
         '      :module: target.typevar',
         '',
         '      T1',
@@ -1919,7 +1919,7 @@ def test_autodoc_TypeVar(app):
         "      alias of TypeVar('T1')",
         '',
         '',
-        '   .. py:attribute:: Class.T6',
+        '   .. py:class:: Class.T6',
         '      :module: target.typevar',
         '',
         '      T6',
@@ -1927,7 +1927,7 @@ def test_autodoc_TypeVar(app):
         '      alias of :py:class:`~datetime.date`',
         '',
         '',
-        '.. py:data:: T1',
+        '.. py:class:: T1',
         '   :module: target.typevar',
         '',
         '   T1',
@@ -1935,7 +1935,7 @@ def test_autodoc_TypeVar(app):
         "   alias of TypeVar('T1')",
         '',
         '',
-        '.. py:data:: T3',
+        '.. py:class:: T3',
         '   :module: target.typevar',
         '',
         '   T3',
@@ -1943,7 +1943,7 @@ def test_autodoc_TypeVar(app):
         "   alias of TypeVar('T3', int, str)",
         '',
         '',
-        '.. py:data:: T4',
+        '.. py:class:: T4',
         '   :module: target.typevar',
         '',
         '   T4',
@@ -1951,7 +1951,7 @@ def test_autodoc_TypeVar(app):
         "   alias of TypeVar('T4', covariant=True)",
         '',
         '',
-        '.. py:data:: T5',
+        '.. py:class:: T5',
         '   :module: target.typevar',
         '',
         '   T5',
@@ -1959,7 +1959,7 @@ def test_autodoc_TypeVar(app):
         "   alias of TypeVar('T5', contravariant=True)",
         '',
         '',
-        '.. py:data:: T6',
+        '.. py:class:: T6',
         '   :module: target.typevar',
         '',
         '   T6',
@@ -1967,7 +1967,7 @@ def test_autodoc_TypeVar(app):
         '   alias of :py:class:`~datetime.date`',
         '',
         '',
-        '.. py:data:: T7',
+        '.. py:class:: T7',
         '   :module: target.typevar',
         '',
         '   T7',

tests/test_ext_autodoc_autoattribute.py

@@ -151,36 +151,6 @@ def test_autoattribute_GenericAlias(app):
     ]
 
 
-@pytest.mark.sphinx('html', testroot='ext-autodoc')
-def test_autoattribute_NewType(app):
-    actual = do_autodoc(app, 'attribute', 'target.typevar.Class.T6')
-    assert list(actual) == [
-        '',
-        '.. py:attribute:: Class.T6',
-        '   :module: target.typevar',
-        '',
-        '   T6',
-        '',
-        '   alias of :py:class:`~datetime.date`',
-        '',
-    ]
-
-
-@pytest.mark.sphinx('html', testroot='ext-autodoc')
-def test_autoattribute_TypeVar(app):
-    actual = do_autodoc(app, 'attribute', 'target.typevar.Class.T1')
-    assert list(actual) == [
-        '',
-        '.. py:attribute:: Class.T1',
-        '   :module: target.typevar',
-        '',
-        '   T1',
-        '',
-        "   alias of TypeVar('T1')",
-        '',
-    ]
-
-
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
 def test_autoattribute_hide_value(app):
     actual = do_autodoc(app, 'attribute', 'target.hide_value.Foo.SENTINEL1')

tests/test_ext_autodoc_autoclass.py

@@ -439,3 +439,63 @@ def test_coroutine(app):
         '      A documented coroutine staticmethod',
         '',
     ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autodata_NewType_module_level(app):
+    actual = do_autodoc(app, 'class', 'target.typevar.T6')
+    assert list(actual) == [
+        '',
+        '.. py:class:: T6',
+        '   :module: target.typevar',
+        '',
+        '   T6',
+        '',
+        '   alias of :py:class:`~datetime.date`',
+        '',
+    ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autoattribute_NewType_class_level(app):
+    actual = do_autodoc(app, 'class', 'target.typevar.Class.T6')
+    assert list(actual) == [
+        '',
+        '.. py:class:: Class.T6',
+        '   :module: target.typevar',
+        '',
+        '   T6',
+        '',
+        '   alias of :py:class:`~datetime.date`',
+        '',
+    ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autodata_TypeVar_class_level(app):
+    actual = do_autodoc(app, 'class', 'target.typevar.T1')
+    assert list(actual) == [
+        '',
+        '.. py:class:: T1',
+        '   :module: target.typevar',
+        '',
+        '   T1',
+        '',
+        "   alias of TypeVar('T1')",
+        '',
+    ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autoattribute_TypeVar_module_level(app):
+    actual = do_autodoc(app, 'class', 'target.typevar.Class.T1')
+    assert list(actual) == [
+        '',
+        '.. py:class:: Class.T1',
+        '   :module: target.typevar',
+        '',
+        '   T1',
+        '',
+        "   alias of TypeVar('T1')",
+        '',
+    ]

tests/test_ext_autodoc_autodata.py

@@ -81,36 +81,6 @@ def test_autodata_GenericAlias(app):
     ]
 
 
-@pytest.mark.sphinx('html', testroot='ext-autodoc')
-def test_autodata_NewType(app):
-    actual = do_autodoc(app, 'data', 'target.typevar.T6')
-    assert list(actual) == [
-        '',
-        '.. py:data:: T6',
-        '   :module: target.typevar',
-        '',
-        '   T6',
-        '',
-        '   alias of :py:class:`~datetime.date`',
-        '',
-    ]
-
-
-@pytest.mark.sphinx('html', testroot='ext-autodoc')
-def test_autodata_TypeVar(app):
-    actual = do_autodoc(app, 'data', 'target.typevar.T1')
-    assert list(actual) == [
-        '',
-        '.. py:data:: T1',
-        '   :module: target.typevar',
-        '',
-        '   T1',
-        '',
-        "   alias of TypeVar('T1')",
-        '',
-    ]
-
-
 @pytest.mark.sphinx('html', testroot='ext-autodoc')
 def test_autodata_hide_value(app):
     actual = do_autodoc(app, 'data', 'target.hide_value.SENTINEL1')

tests/test_ext_autodoc_configs.py

@@ -754,7 +754,7 @@ def test_autodoc_typehints_signature(app):
         '   :module: target.typehints',
         '',
         '',
-        '.. py:data:: T',
+        '.. py:class:: T',
         '   :module: target.typehints',
         '',
         '   docstring',
@@ -868,7 +868,7 @@ def test_autodoc_typehints_none(app):
         '   :module: target.typehints',
         '',
         '',
-        '.. py:data:: T',
+        '.. py:class:: T',
         '   :module: target.typehints',
         '',
         '   docstring',
@@ -1501,7 +1501,7 @@ def test_autodoc_typehints_format_fully_qualified(app):
         '   :module: target.typehints',
         '',
         '',
-        '.. py:data:: T',
+        '.. py:class:: T',
         '   :module: target.typehints',
         '',
         '   docstring',
@@ -1564,10 +1564,10 @@ def test_autodoc_typehints_format_fully_qualified_for_generic_alias(app):
 @pytest.mark.sphinx('html', testroot='ext-autodoc',
                     confoverrides={'autodoc_typehints_format': "fully-qualified"})
 def test_autodoc_typehints_format_fully_qualified_for_newtype_alias(app):
-    actual = do_autodoc(app, 'data', 'target.typevar.T6')
+    actual = do_autodoc(app, 'class', 'target.typevar.T6')
     assert list(actual) == [
         '',
-        '.. py:data:: T6',
+        '.. py:class:: T6',
         '   :module: target.typevar',
         '',
         '   T6',