Merge branch '2.0'

This commit is contained in:
Takeshi KOMIYA
2020-01-19 22:47:02 +09:00
17 changed files with 459 additions and 50 deletions

View File

@@ -79,6 +79,13 @@ Features added
* #6696: html: ``:scale:`` option of image/figure directive not working for SVG
images (imagesize-1.2.0 or above is required)
* #6994: imgconverter: Support illustrator file (.ai) to .png conversion
* autodoc: Support Positional-Only Argument separator (PEP-570 compliant)
* #2755: autodoc: Add new event: :event:`autodoc-before-process-signature`
* #2755: autodoc: Support type_comment style (ex. ``# type: (str) -> str``)
annotation (python3.8+ or `typed_ast <https://github.com/python/typed_ast>`_
is required)
* SphinxTranslator now calls visitor/departure method for super node class if
visitor/departure method for original node class not found
Bugs fixed
----------
@@ -89,6 +96,7 @@ Bugs fixed
* #6559: Wrong node-ids are generated in glossary directive
* #6986: apidoc: misdetects module name for .so file inside module
* #6999: napoleon: fails to parse tilde in :exc: role
* #7023: autodoc: nested partial functions are not listed
Testing
--------

View File

@@ -526,6 +526,17 @@ autodoc provides the following additional events:
auto directive
:param lines: the lines of the docstring, see above
.. event:: autodoc-before-process-signature (app, obj, bound_method)
.. versionadded:: 2.4
Emitted before autodoc formats a signature for an object. The event handler
can modify an object to change its signature.
:param app: the Sphinx application object
:param obj: the object itself
:param bound_method: a boolean indicates an object is bound method or not
.. event:: autodoc-process-signature (app, what, name, obj, options, signature, return_annotation)
.. versionadded:: 0.5

View File

@@ -42,7 +42,7 @@ extras_require = {
'sphinxcontrib-websupport',
],
'test': [
'pytest',
'pytest < 5.3.3',
'pytest-cov',
'html5lib',
'flake8>=3.5.0',

View File

@@ -1013,8 +1013,11 @@ class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter): # typ
not inspect.isbuiltin(self.object) and
not inspect.isclass(self.object) and
hasattr(self.object, '__call__')):
self.env.app.emit('autodoc-before-process-signature',
self.object.__call__, False)
sig = inspect.signature(self.object.__call__)
else:
self.env.app.emit('autodoc-before-process-signature', self.object, False)
sig = inspect.signature(self.object)
args = stringify_signature(sig, **kwargs)
except TypeError:
@@ -1026,9 +1029,13 @@ class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter): # typ
# typing) we try to use the constructor signature as function
# signature without the first argument.
try:
self.env.app.emit('autodoc-before-process-signature',
self.object.__new__, True)
sig = inspect.signature(self.object.__new__, bound_method=True)
args = stringify_signature(sig, show_return_annotation=False, **kwargs)
except TypeError:
self.env.app.emit('autodoc-before-process-signature',
self.object.__init__, True)
sig = inspect.signature(self.object.__init__, bound_method=True)
args = stringify_signature(sig, show_return_annotation=False, **kwargs)
@@ -1111,6 +1118,7 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter): # type:
not(inspect.ismethod(initmeth) or inspect.isfunction(initmeth)):
return None
try:
self.env.app.emit('autodoc-before-process-signature', initmeth, True)
sig = inspect.signature(initmeth, bound_method=True)
return stringify_signature(sig, show_return_annotation=False, **kwargs)
except TypeError:
@@ -1314,8 +1322,10 @@ class MethodDocumenter(DocstringSignatureMixin, ClassLevelDocumenter): # type:
# can never get arguments of a C function or method
return None
if inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name):
self.env.app.emit('autodoc-before-process-signature', self.object, False)
sig = inspect.signature(self.object, bound_method=False)
else:
self.env.app.emit('autodoc-before-process-signature', self.object, True)
sig = inspect.signature(self.object, bound_method=True)
args = stringify_signature(sig, **kwargs)
@@ -1556,8 +1566,11 @@ def setup(app: Sphinx) -> Dict[str, Any]:
app.add_config_value('autodoc_typehints', "signature", True, ENUM("signature", "none"))
app.add_config_value('autodoc_warningiserror', True, True)
app.add_config_value('autodoc_inherit_docstrings', True, True)
app.add_event('autodoc-before-process-signature')
app.add_event('autodoc-process-docstring')
app.add_event('autodoc-process-signature')
app.add_event('autodoc-skip-member')
app.setup_extension('sphinx.ext.autodoc.type_comment')
return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

View File

@@ -0,0 +1,74 @@
"""
sphinx.ext.autodoc.type_comment
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Update annotations info of living objects using type_comments.
:copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import ast
from inspect import getsource
from typing import Any, Dict
from typing import cast
import sphinx
from sphinx.application import Sphinx
from sphinx.pycode.ast import parse as ast_parse
from sphinx.pycode.ast import unparse as ast_unparse
from sphinx.util import inspect
from sphinx.util import logging
logger = logging.getLogger(__name__)
def get_type_comment(obj: Any) -> ast.FunctionDef:
"""Get type_comment'ed FunctionDef object from living object.
This tries to parse original code for living object and returns
AST node for given *obj*. It requires py38+ or typed_ast module.
"""
try:
source = getsource(obj)
if source.startswith((' ', r'\t')):
# subject is placed inside class or block. To read its docstring,
# this adds if-block before the declaration.
module = ast_parse('if True:\n' + source)
subject = cast(ast.FunctionDef, module.body[0].body[0]) # type: ignore
else:
module = ast_parse(source)
subject = cast(ast.FunctionDef, module.body[0]) # type: ignore
if getattr(subject, "type_comment", None):
return ast_parse(subject.type_comment, mode='func_type') # type: ignore
else:
return None
except (OSError, TypeError): # failed to load source code
return None
except SyntaxError: # failed to parse type_comments
return None
def update_annotations_using_type_comments(app: Sphinx, obj: Any, bound_method: bool) -> None:
"""Update annotations info of *obj* using type_comments."""
try:
function = get_type_comment(obj)
if function and hasattr(function, 'argtypes'):
if function.argtypes != [ast.Ellipsis]: # type: ignore
sig = inspect.signature(obj, bound_method)
for i, param in enumerate(sig.parameters.values()):
if param.name not in obj.__annotations__:
annotation = ast_unparse(function.argtypes[i]) # type: ignore
obj.__annotations__[param.name] = annotation
if 'return' not in obj.__annotations__:
obj.__annotations__['return'] = ast_unparse(function.returns) # type: ignore
except NotImplementedError as exc: # failed to ast.unparse()
logger.warning("Failed to parse type_comment for %r: %s", obj, exc)
def setup(app: Sphinx) -> Dict[str, Any]:
app.connect('autodoc-before-process-signature', update_annotations_using_type_comments)
return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

80
sphinx/pycode/ast.py Normal file
View File

@@ -0,0 +1,80 @@
"""
sphinx.pycode.ast
~~~~~~~~~~~~~~~~~
Helpers for AST (Abstract Syntax Tree).
:copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import sys
if sys.version_info > (3, 8):
import ast
else:
try:
# use typed_ast module if installed
from typed_ast import ast3 as ast
except ImportError:
import ast # type: ignore
def parse(code: str, mode: str = 'exec') -> "ast.AST":
"""Parse the *code* using built-in ast or typed_ast.
This enables "type_comments" feature if possible.
"""
try:
# type_comments parameter is available on py38+
return ast.parse(code, mode=mode, type_comments=True) # type: ignore
except TypeError:
# fallback to ast module.
# typed_ast is used to parse type_comments if installed.
return ast.parse(code, mode=mode)
def unparse(node: ast.AST) -> str:
"""Unparse an AST to string."""
if node is None:
return None
elif isinstance(node, ast.Attribute):
return "%s.%s" % (unparse(node.value), node.attr)
elif isinstance(node, ast.Bytes):
return repr(node.s)
elif isinstance(node, ast.Call):
args = ([unparse(e) for e in node.args] +
["%s=%s" % (k.arg, unparse(k.value)) for k in node.keywords])
return "%s(%s)" % (unparse(node.func), ", ".join(args))
elif isinstance(node, ast.Dict):
keys = (unparse(k) for k in node.keys)
values = (unparse(v) for v in node.values)
items = (k + ": " + v for k, v in zip(keys, values))
return "{" + ", ".join(items) + "}"
elif isinstance(node, ast.Ellipsis):
return "..."
elif isinstance(node, ast.Index):
return unparse(node.value)
elif isinstance(node, ast.Lambda):
return "<function <lambda>>" # TODO
elif isinstance(node, ast.List):
return "[" + ", ".join(unparse(e) for e in node.elts) + "]"
elif isinstance(node, ast.Name):
return node.id
elif isinstance(node, ast.NameConstant):
return repr(node.value)
elif isinstance(node, ast.Num):
return repr(node.n)
elif isinstance(node, ast.Set):
return "{" + ", ".join(unparse(e) for e in node.elts) + "}"
elif isinstance(node, ast.Str):
return repr(node.s)
elif isinstance(node, ast.Subscript):
return "%s[%s]" % (unparse(node.value), unparse(node.slice))
elif isinstance(node, ast.Tuple):
return ", ".join(unparse(e) for e in node.elts)
elif sys.version_info > (3, 6) and isinstance(node, ast.Constant):
# this branch should be placed at last
return repr(node.value)
else:
raise NotImplementedError('Unable to parse %s object' % type(node).__name__)

View File

@@ -429,7 +429,10 @@ class ReferenceRole(SphinxRole):
class SphinxTranslator(nodes.NodeVisitor):
"""A base class for Sphinx translators.
This class provides helper methods for Sphinx translators.
This class adds a support for visitor/departure method for super node class
if visitor/departure method for node class is not found.
It also provides helper methods for Sphinx translators.
.. note:: The subclasses of this class might not work with docutils.
This class is strongly coupled with Sphinx.
@@ -441,6 +444,42 @@ class SphinxTranslator(nodes.NodeVisitor):
self.config = builder.config
self.settings = document.settings
def dispatch_visit(self, node):
"""
Dispatch node to appropriate visitor method.
The priority of visitor method is:
1. ``self.visit_{node_class}()``
2. ``self.visit_{supre_node_class}()``
3. ``self.unknown_visit()``
"""
for node_class in node.__class__.__mro__:
method = getattr(self, 'visit_%s' % (node_class.__name__), None)
if method:
logger.debug('SphinxTranslator.dispatch_visit calling %s for %s' %
(method.__name__, node))
return method(node)
else:
super().dispatch_visit(node)
def dispatch_departure(self, node):
"""
Dispatch node to appropriate departure method.
The priority of departure method is:
1. ``self.depart_{node_class}()``
2. ``self.depart_{super_node_class}()``
3. ``self.unknown_departure()``
"""
for node_class in node.__class__.__mro__:
method = getattr(self, 'depart_%s' % (node_class.__name__), None)
if method:
logger.debug('SphinxTranslator.dispatch_departure calling %s for %s' %
(method.__name__, node))
return method(node)
else:
super().dispatch_departure(node)
# cache a vanilla instance of nodes.document
# Used in new_document() function

View File

@@ -121,6 +121,17 @@ def isenumattribute(x: Any) -> bool:
return isinstance(x, enum.Enum)
def unpartial(obj: Any) -> Any:
"""Get an original object from partial object.
This returns given object itself if not partial.
"""
while ispartial(obj):
obj = obj.func
return obj
def ispartial(obj: Any) -> bool:
"""Check if the object is partial."""
return isinstance(obj, (partial, partialmethod))
@@ -197,24 +208,21 @@ def isattributedescriptor(obj: Any) -> bool:
def isfunction(obj: Any) -> bool:
"""Check if the object is function."""
return inspect.isfunction(obj) or ispartial(obj) and inspect.isfunction(obj.func)
return inspect.isfunction(unpartial(obj))
def isbuiltin(obj: Any) -> bool:
"""Check if the object is builtin."""
return inspect.isbuiltin(obj) or ispartial(obj) and inspect.isbuiltin(obj.func)
return inspect.isbuiltin(unpartial(obj))
def iscoroutinefunction(obj: Any) -> bool:
"""Check if the object is coroutine-function."""
obj = unpartial(obj)
if hasattr(obj, '__code__') and inspect.iscoroutinefunction(obj):
# check obj.__code__ because iscoroutinefunction() crashes for custom method-like
# objects (see https://github.com/sphinx-doc/sphinx/issues/6605)
return True
elif (ispartial(obj) and hasattr(obj.func, '__code__') and
inspect.iscoroutinefunction(obj.func)):
# partialed
return True
else:
return False
@@ -374,44 +382,40 @@ def stringify_signature(sig: inspect.Signature, show_annotation: bool = True,
args = []
last_kind = None
for param in sig.parameters.values():
# insert '*' between POSITIONAL args and KEYWORD_ONLY args::
# func(a, b, *, c, d):
if param.kind == param.KEYWORD_ONLY and last_kind in (param.POSITIONAL_OR_KEYWORD,
param.POSITIONAL_ONLY,
None):
if param.kind != param.POSITIONAL_ONLY and last_kind == param.POSITIONAL_ONLY:
# PEP-570: Separator for Positional Only Parameter: /
args.append('/')
elif param.kind == param.KEYWORD_ONLY and last_kind in (param.POSITIONAL_OR_KEYWORD,
param.POSITIONAL_ONLY,
None):
# PEP-3102: Separator for Keyword Only Parameter: *
args.append('*')
arg = StringIO()
if param.kind in (param.POSITIONAL_ONLY,
param.POSITIONAL_OR_KEYWORD,
param.KEYWORD_ONLY):
arg.write(param.name)
if show_annotation and param.annotation is not param.empty:
arg.write(': ')
arg.write(stringify_annotation(param.annotation))
if param.default is not param.empty:
if show_annotation and param.annotation is not param.empty:
arg.write(' = ')
arg.write(object_description(param.default))
else:
arg.write('=')
arg.write(object_description(param.default))
elif param.kind == param.VAR_POSITIONAL:
arg.write('*')
arg.write(param.name)
if show_annotation and param.annotation is not param.empty:
arg.write(': ')
arg.write(stringify_annotation(param.annotation))
if param.kind == param.VAR_POSITIONAL:
arg.write('*' + param.name)
elif param.kind == param.VAR_KEYWORD:
arg.write('**')
arg.write('**' + param.name)
else:
arg.write(param.name)
if show_annotation and param.annotation is not param.empty:
arg.write(': ')
arg.write(stringify_annotation(param.annotation))
if param.default is not param.empty:
if show_annotation and param.annotation is not param.empty:
arg.write(': ')
arg.write(stringify_annotation(param.annotation))
arg.write(' = ')
else:
arg.write('=')
arg.write(object_description(param.default))
args.append(arg.getvalue())
last_kind = param.kind
if last_kind == Parameter.POSITIONAL_ONLY:
# PEP-570: Separator for Positional Only Parameter: /
args.append('/')
if (sig.return_annotation is Parameter.empty or
show_annotation is False or
show_return_annotation is False):

View File

@@ -1,11 +1,12 @@
from functools import partial
def func1():
def func1(a, b, c):
"""docstring of func1"""
pass
func2 = partial(func1)
func3 = partial(func1)
func2 = partial(func1, 1)
func3 = partial(func2, 2)
func3.__doc__ = "docstring of func3"
func4 = partial(func3, 3)

View File

@@ -14,5 +14,5 @@ class Cell(object):
#: Make a cell alive.
set_alive = partialmethod(set_state, True)
# a partialmethod with no docstring
set_dead = partialmethod(set_state, False)
"""Make a cell dead."""

View File

@@ -0,0 +1,5 @@
def foo(a, b, /, c, d):
pass
def bar(a, b, /):
pass

View File

@@ -2,9 +2,23 @@ def incr(a: int, b: int = 1) -> int:
return a + b
def decr(a, b = 1):
# type: (int, int) -> int
return a - b
class Math:
def __init__(self, s: str, o: object = None) -> None:
pass
def incr(self, a: int, b: int = 1) -> int:
return a + b
def decr(self, a, b = 1):
# type: (int, int) -> int
return a - b
def complex_func(arg1, arg2, arg3=None, *args, **kwargs):
# type: (str, List[int], Tuple[int, Union[str, Unknown]], *str, **str) -> None
pass

View File

@@ -1265,19 +1265,25 @@ def test_partialfunction():
'.. py:module:: target.partialfunction',
'',
'',
'.. py:function:: func1()',
'.. py:function:: func1(a, b, c)',
' :module: target.partialfunction',
'',
' docstring of func1',
' ',
'',
'.. py:function:: func2()',
'.. py:function:: func2(b, c)',
' :module: target.partialfunction',
'',
' docstring of func1',
' ',
'',
'.. py:function:: func3()',
'.. py:function:: func3(c)',
' :module: target.partialfunction',
'',
' docstring of func3',
' ',
'',
'.. py:function:: func4()',
' :module: target.partialfunction',
'',
' docstring of func3',
@@ -1348,12 +1354,6 @@ def test_partialmethod(app):
' Make a cell alive.',
' ',
' ',
' .. py:method:: Cell.set_dead()',
' :module: target.partialmethod',
' ',
' Make a cell dead.',
' ',
' ',
' .. py:method:: Cell.set_state(state)',
' :module: target.partialmethod',
' ',
@@ -1366,6 +1366,41 @@ def test_partialmethod(app):
assert list(actual) == expected
@pytest.mark.sphinx('html', testroot='ext-autodoc')
def test_partialmethod_undoc_members(app):
expected = [
'',
'.. py:class:: Cell',
' :module: target.partialmethod',
'',
' An example for partialmethod.',
' ',
' refs: https://docs.python.jp/3/library/functools.html#functools.partialmethod',
' ',
' ',
' .. py:method:: Cell.set_alive()',
' :module: target.partialmethod',
' ',
' Make a cell alive.',
' ',
' ',
' .. py:method:: Cell.set_dead()',
' :module: target.partialmethod',
' ',
' ',
' .. py:method:: Cell.set_state(state)',
' :module: target.partialmethod',
' ',
' Update state of cell to *state*.',
' ',
]
options = {"members": None,
"undoc-members": None}
actual = do_autodoc(app, 'class', 'target.partialmethod.Cell', options)
assert list(actual) == expected
@pytest.mark.sphinx('html', testroot='pycode-egg')
def test_autodoc_for_egged_code(app):
options = {"members": None,

View File

@@ -478,10 +478,23 @@ def test_autodoc_typehints_signature(app):
' :module: target.typehints',
'',
' ',
' .. py:method:: Math.decr(a: int, b: int = 1) -> int',
' :module: target.typehints',
' ',
' ',
' .. py:method:: Math.incr(a: int, b: int = 1) -> int',
' :module: target.typehints',
' ',
'',
'.. py:function:: complex_func(arg1: str, arg2: List[int], arg3: Tuple[int, '
'Union[str, Unknown]] = None, *args: str, **kwargs: str) -> None',
' :module: target.typehints',
'',
'',
'.. py:function:: decr(a: int, b: int = 1) -> int',
' :module: target.typehints',
'',
'',
'.. py:function:: incr(a: int, b: int = 1) -> int',
' :module: target.typehints',
''
@@ -504,10 +517,22 @@ def test_autodoc_typehints_none(app):
' :module: target.typehints',
'',
' ',
' .. py:method:: Math.decr(a, b=1)',
' :module: target.typehints',
' ',
' ',
' .. py:method:: Math.incr(a, b=1)',
' :module: target.typehints',
' ',
'',
'.. py:function:: complex_func(arg1, arg2, arg3=None, *args, **kwargs)',
' :module: target.typehints',
'',
'',
'.. py:function:: decr(a, b=1)',
' :module: target.typehints',
'',
'',
'.. py:function:: incr(a, b=1)',
' :module: target.typehints',
''

40
tests/test_pycode_ast.py Normal file
View File

@@ -0,0 +1,40 @@
"""
test_pycode_ast
~~~~~~~~~~~~~~~
Test pycode.ast
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import pytest
from sphinx.pycode import ast
@pytest.mark.parametrize('source,expected', [
("os.path", "os.path"), # Attribute
("b'bytes'", "b'bytes'"), # Bytes
("object()", "object()"), # Call
("1234", "1234"), # Constant
("{'key1': 'value1', 'key2': 'value2'}",
"{'key1': 'value1', 'key2': 'value2'}"), # Dict
("...", "..."), # Ellipsis
("Tuple[int, int]", "Tuple[int, int]"), # Index, Subscript
("lambda x, y: x + y",
"<function <lambda>>"), # Lambda
("[1, 2, 3]", "[1, 2, 3]"), # List
("sys", "sys"), # Name, NameConstant
("1234", "1234"), # Num
("{1, 2, 3}", "{1, 2, 3}"), # Set
("'str'", "'str'"), # Str
("(1, 2, 3)", "1, 2, 3"), # Tuple
])
def test_unparse(source, expected):
module = ast.parse(source)
assert ast.unparse(module.body[0].value) == expected
def test_unparse_None():
assert ast.unparse(None) is None

View File

@@ -12,7 +12,9 @@ import os
from docutils import nodes
from sphinx.util.docutils import SphinxFileOutput, docutils_namespace, register_node
from sphinx.util.docutils import (
SphinxFileOutput, SphinxTranslator, docutils_namespace, new_document, register_node
)
def test_register_node():
@@ -61,3 +63,34 @@ def test_SphinxFileOutput(tmpdir):
# overrite it again (content changed)
output.write(content + "; content change")
assert os.stat(filename).st_mtime != 0 # updated
def test_SphinxTranslator(app):
class CustomNode(nodes.inline):
pass
class MyTranslator(SphinxTranslator):
def __init__(self, *args):
self.called = []
super().__init__(*args)
def visit_document(self, node):
pass
def depart_document(self, node):
pass
def visit_inline(self, node):
self.called.append('visit_inline')
def depart_inline(self, node):
self.called.append('depart_inline')
document = new_document('')
document += CustomNode()
translator = MyTranslator(document, app.builder)
document.walkabout(translator)
# MyTranslator does not have visit_CustomNode. But it calls visit_inline instead.
assert translator.called == ['visit_inline', 'depart_inline']

View File

@@ -294,6 +294,21 @@ def test_signature_annotations():
sig = inspect.signature(f7)
assert stringify_signature(sig, show_return_annotation=False) == '(x: int = None, y: dict = {})'
@pytest.mark.skipif(sys.version_info < (3, 8), reason='python 3.8+ is required.')
@pytest.mark.sphinx(testroot='ext-autodoc')
def test_signature_annotations_py38(app):
from target.pep570 import foo, bar
# case: separator in the middle
sig = inspect.signature(foo)
assert stringify_signature(sig) == '(a, b, /, c, d)'
# case: separator at tail
sig = inspect.signature(bar)
assert stringify_signature(sig) == '(a, b, /)'
def test_safe_getattr_with_default():
class Foo:
def __getattr__(self, item):
@@ -496,3 +511,15 @@ def test_isproperty(app):
assert inspect.isproperty(Base.meth) is False # method of class
assert inspect.isproperty(Base().meth) is False # method of instance
assert inspect.isproperty(func) is False # function
def test_unpartial():
def func1(a, b, c):
pass
func2 = functools.partial(func1, 1)
func2.__doc__ = "func2"
func3 = functools.partial(func2, 2) # nested partial object
assert inspect.unpartial(func2) is func1
assert inspect.unpartial(func3) is func1