From 0b32e72635f6e117824b5cba3b8b38254a6a2644 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 4 Oct 2020 01:45:47 +0900 Subject: [PATCH 1/2] pycode: ast.unparse() construct number literals using source code Developers can write number literals in several ways. For example, decimal (1234), hexadecimal (0x1234), octal decimal (0o1234) and so on. But, AST module don't mind how the numbers written in the code. As a result, ast.unparse() could not reproduce the original form of number literals. This allows to construct number literals as possible using original source code. Note: This is only available in Python 3.8+. --- sphinx/pycode/ast.py | 11 +++++++++-- tests/test_pycode_ast.py | 14 +++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/sphinx/pycode/ast.py b/sphinx/pycode/ast.py index 2583448d5..17d78f4eb 100644 --- a/sphinx/pycode/ast.py +++ b/sphinx/pycode/ast.py @@ -58,17 +58,19 @@ def parse(code: str, mode: str = 'exec') -> "ast.AST": return ast.parse(code, mode=mode) -def unparse(node: Optional[ast.AST]) -> Optional[str]: +def unparse(node: Optional[ast.AST], code: str = '') -> Optional[str]: """Unparse an AST to string.""" if node is None: return None elif isinstance(node, str): return node - return _UnparseVisitor().visit(node) + return _UnparseVisitor(code).visit(node) # a greatly cut-down version of `ast._Unparser` class _UnparseVisitor(ast.NodeVisitor): + def __init__(self, code: str = '') -> None: + self.code = code def _visit_op(self, node: ast.AST) -> str: return OPERATORS[node.__class__] @@ -195,6 +197,11 @@ class _UnparseVisitor(ast.NodeVisitor): def visit_Constant(self, node: ast.Constant) -> str: if node.value is Ellipsis: return "..." + elif isinstance(node.value, (int, float, complex)): + if self.code and sys.version_info > (3, 8): + return ast.get_source_segment(self.code, node) + else: + return repr(node.value) else: return repr(node.value) diff --git a/tests/test_pycode_ast.py b/tests/test_pycode_ast.py index 32a784b74..bbff64dd0 100644 --- a/tests/test_pycode_ast.py +++ b/tests/test_pycode_ast.py @@ -58,7 +58,7 @@ from sphinx.pycode import ast ]) def test_unparse(source, expected): module = ast.parse(source) - assert ast.unparse(module.body[0].value) == expected + assert ast.unparse(module.body[0].value, source) == expected def test_unparse_None(): @@ -66,8 +66,12 @@ def test_unparse_None(): @pytest.mark.skipif(sys.version_info < (3, 8), reason='python 3.8+ is required.') -def test_unparse_py38(): - source = "lambda x=0, /, y=1, *args, z, **kwargs: x + y + z" - expected = "lambda x=0, /, y=1, *args, z, **kwargs: ..." +@pytest.mark.parametrize('source,expected', [ + ("lambda x=0, /, y=1, *args, z, **kwargs: x + y + z", + "lambda x=0, /, y=1, *args, z, **kwargs: ..."), # posonlyargs + ("0x1234", "0x1234"), # Constant + ("1_000_000", "1_000_000"), # Constant +]) +def test_unparse_py38(source, expected): module = ast.parse(source) - assert ast.unparse(module.body[0].value) == expected + assert ast.unparse(module.body[0].value, source) == expected From cc941db40b946534da8897a29631325d96313a6e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 4 Oct 2020 10:31:02 +0900 Subject: [PATCH 2/2] Fix #8255: py domain: number in defarg is changed to decimal Number literals in default argument value is converted to decimal form unexpectedly by AST module. This fixes the signature parsing code to recosntruct it correctly. Note: This is only available in Python 3.8+. --- CHANGES | 2 ++ sphinx/util/inspect.py | 25 +++++++++++++------------ tests/test_domain_py.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index 600efc466..c907992dc 100644 --- a/CHANGES +++ b/CHANGES @@ -49,6 +49,8 @@ Bugs fixed * #8277: sphinx-build: missing and redundant spacing (and etc) for console output on building * #7973: imgconverter: Check availability of imagemagick many times +* #8255: py domain: number in default argument value is changed from hexadecimal + to decimal * #8093: The highlight warning has wrong location in some builders (LaTeX, singlehtml and so on) * #8239: Failed to refer a token in productionlist if it is indented diff --git a/sphinx/util/inspect.py b/sphinx/util/inspect.py index 378174993..f2cd8070b 100644 --- a/sphinx/util/inspect.py +++ b/sphinx/util/inspect.py @@ -600,13 +600,14 @@ def stringify_signature(sig: inspect.Signature, show_annotation: bool = True, def signature_from_str(signature: str) -> inspect.Signature: """Create a Signature object from string.""" - module = ast.parse('def func' + signature + ': pass') + code = 'def func' + signature + ': pass' + module = ast.parse(code) function = cast(ast.FunctionDef, module.body[0]) # type: ignore - return signature_from_ast(function) + return signature_from_ast(function, code) -def signature_from_ast(node: ast.FunctionDef) -> inspect.Signature: +def signature_from_ast(node: ast.FunctionDef, code: str = '') -> inspect.Signature: """Create a Signature object from AST *node*.""" args = node.args defaults = list(args.defaults) @@ -626,9 +627,9 @@ def signature_from_ast(node: ast.FunctionDef) -> inspect.Signature: if defaults[i] is Parameter.empty: default = Parameter.empty else: - default = ast_unparse(defaults[i]) + default = ast_unparse(defaults[i], code) - annotation = ast_unparse(arg.annotation) or Parameter.empty + annotation = ast_unparse(arg.annotation, code) or Parameter.empty params.append(Parameter(arg.arg, Parameter.POSITIONAL_ONLY, default=default, annotation=annotation)) @@ -636,29 +637,29 @@ def signature_from_ast(node: ast.FunctionDef) -> inspect.Signature: if defaults[i + posonlyargs] is Parameter.empty: default = Parameter.empty else: - default = ast_unparse(defaults[i + posonlyargs]) + default = ast_unparse(defaults[i + posonlyargs], code) - annotation = ast_unparse(arg.annotation) or Parameter.empty + annotation = ast_unparse(arg.annotation, code) or Parameter.empty params.append(Parameter(arg.arg, Parameter.POSITIONAL_OR_KEYWORD, default=default, annotation=annotation)) if args.vararg: - annotation = ast_unparse(args.vararg.annotation) or Parameter.empty + annotation = ast_unparse(args.vararg.annotation, code) or Parameter.empty params.append(Parameter(args.vararg.arg, Parameter.VAR_POSITIONAL, annotation=annotation)) for i, arg in enumerate(args.kwonlyargs): - default = ast_unparse(args.kw_defaults[i]) or Parameter.empty - annotation = ast_unparse(arg.annotation) or Parameter.empty + default = ast_unparse(args.kw_defaults[i], code) or Parameter.empty + annotation = ast_unparse(arg.annotation, code) or Parameter.empty params.append(Parameter(arg.arg, Parameter.KEYWORD_ONLY, default=default, annotation=annotation)) if args.kwarg: - annotation = ast_unparse(args.kwarg.annotation) or Parameter.empty + annotation = ast_unparse(args.kwarg.annotation, code) or Parameter.empty params.append(Parameter(args.kwarg.arg, Parameter.VAR_KEYWORD, annotation=annotation)) - return_annotation = ast_unparse(node.returns) or Parameter.empty + return_annotation = ast_unparse(node.returns, code) or Parameter.empty return inspect.Signature(params, return_annotation=return_annotation) diff --git a/tests/test_domain_py.py b/tests/test_domain_py.py index b98f37912..8040af9cc 100644 --- a/tests/test_domain_py.py +++ b/tests/test_domain_py.py @@ -386,6 +386,19 @@ def test_pyfunction_signature_full_py38(app): [desc_parameter, desc_sig_operator, "/"])]) +@pytest.mark.skipif(sys.version_info < (3, 8), reason='python 3.8+ is required.') +def test_pyfunction_with_number_literals(app): + text = ".. py:function:: hello(age=0x10, height=1_6_0)" + doctree = restructuredtext.parse(app, text) + assert_node(doctree[1][0][1], + [desc_parameterlist, ([desc_parameter, ([desc_sig_name, "age"], + [desc_sig_operator, "="], + [nodes.inline, "0x10"])], + [desc_parameter, ([desc_sig_name, "height"], + [desc_sig_operator, "="], + [nodes.inline, "1_6_0"])])]) + + def test_optional_pyfunction_signature(app): text = ".. py:function:: compile(source [, filename [, symbol]]) -> ast object" doctree = restructuredtext.parse(app, text)