mirror of
https://github.com/sphinx-doc/sphinx.git
synced 2025-02-25 18:55:22 -06:00
Merge pull request #6554 from tk0miya/refactor_type_annotation_pycode
Migrate to py3 style type annotation: sphinx.pycode
This commit is contained in:
Takeshi KOMIYA
GitHub
commit
ac158d0786
@ -11,29 +11,25 @@
|
||||
import re
|
||||
from io import StringIO
|
||||
from os import path
|
||||
from typing import Any, Dict, IO, List, Tuple
|
||||
from zipfile import ZipFile
|
||||
|
||||
from sphinx.errors import PycodeError
|
||||
from sphinx.pycode.parser import Parser
|
||||
from sphinx.util import get_module_source, detect_encoding
|
||||
|
||||
if False:
|
||||
# For type annotation
|
||||
from typing import Any, Dict, IO, List, Tuple # NOQA
|
||||
|
||||
|
||||
class ModuleAnalyzer:
|
||||
# cache for analyzer objects -- caches both by module and file name
|
||||
cache = {} # type: Dict[Tuple[str, str], Any]
|
||||
|
||||
@classmethod
|
||||
def for_string(cls, string, modname, srcname='<string>'):
|
||||
# type: (str, str, str) -> ModuleAnalyzer
|
||||
def for_string(cls, string: str, modname: str, srcname: str = '<string>'
|
||||
) -> "ModuleAnalyzer":
|
||||
return cls(StringIO(string), modname, srcname, decoded=True)
|
||||
|
||||
@classmethod
|
||||
def for_file(cls, filename, modname):
|
||||
# type: (str, str) -> ModuleAnalyzer
|
||||
def for_file(cls, filename: str, modname: str) -> "ModuleAnalyzer":
|
||||
if ('file', filename) in cls.cache:
|
||||
return cls.cache['file', filename]
|
||||
try:
|
||||
@ -48,8 +44,7 @@ class ModuleAnalyzer:
|
||||
return obj
|
||||
|
||||
@classmethod
|
||||
def for_egg(cls, filename, modname):
|
||||
# type: (str, str) -> ModuleAnalyzer
|
||||
def for_egg(cls, filename: str, modname: str) -> "ModuleAnalyzer":
|
||||
SEP = re.escape(path.sep)
|
||||
eggpath, relpath = re.split('(?<=\\.egg)' + SEP, filename)
|
||||
try:
|
||||
@ -60,8 +55,7 @@ class ModuleAnalyzer:
|
||||
raise PycodeError('error opening %r' % filename, exc)
|
||||
|
||||
@classmethod
|
||||
def for_module(cls, modname):
|
||||
# type: (str) -> ModuleAnalyzer
|
||||
def for_module(cls, modname: str) -> "ModuleAnalyzer":
|
||||
if ('module', modname) in cls.cache:
|
||||
entry = cls.cache['module', modname]
|
||||
if isinstance(entry, PycodeError):
|
||||
@ -80,8 +74,7 @@ class ModuleAnalyzer:
|
||||
cls.cache['module', modname] = obj
|
||||
return obj
|
||||
|
||||
def __init__(self, source, modname, srcname, decoded=False):
|
||||
# type: (IO, str, str, bool) -> None
|
||||
def __init__(self, source: IO, modname: str, srcname: str, decoded: bool = False) -> None:
|
||||
self.modname = modname # name of the module
|
||||
self.srcname = srcname # name of the source file
|
||||
|
||||
@ -100,8 +93,7 @@ class ModuleAnalyzer:
|
||||
self.tagorder = None # type: Dict[str, int]
|
||||
self.tags = None # type: Dict[str, Tuple[str, int, int]]
|
||||
|
||||
def parse(self):
|
||||
# type: () -> None
|
||||
def parse(self) -> None:
|
||||
"""Parse the source code."""
|
||||
try:
|
||||
parser = Parser(self.code, self.encoding)
|
||||
@ -119,16 +111,14 @@ class ModuleAnalyzer:
|
||||
except Exception as exc:
|
||||
raise PycodeError('parsing %r failed: %r' % (self.srcname, exc))
|
||||
|
||||
def find_attr_docs(self):
|
||||
# type: () -> Dict[Tuple[str, str], List[str]]
|
||||
def find_attr_docs(self) -> Dict[Tuple[str, str], List[str]]:
|
||||
"""Find class and module-level attributes and their documentation."""
|
||||
if self.attr_docs is None:
|
||||
self.parse()
|
||||
|
||||
return self.attr_docs
|
||||
|
||||
def find_tags(self):
|
||||
# type: () -> Dict[str, Tuple[str, int, int]]
|
||||
def find_tags(self) -> Dict[str, Tuple[str, int, int]]:
|
||||
"""Find class, function and method definitions and their location."""
|
||||
if self.tags is None:
|
||||
self.parse()
|
||||
|
||||
@ -15,10 +15,8 @@ import sys
|
||||
import tokenize
|
||||
from token import NAME, NEWLINE, INDENT, DEDENT, NUMBER, OP, STRING
|
||||
from tokenize import COMMENT, NL
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
if False:
|
||||
# For type annotation
|
||||
from typing import Any, Dict, List, Tuple # NOQA
|
||||
|
||||
comment_re = re.compile('^\\s*#: ?(.*)\r?\n?$')
|
||||
indent_re = re.compile('^\\s*$')
|
||||
@ -31,13 +29,11 @@ else:
|
||||
ASSIGN_NODES = (ast.Assign)
|
||||
|
||||
|
||||
def filter_whitespace(code):
|
||||
# type: (str) -> str
|
||||
def filter_whitespace(code: str) -> str:
|
||||
return code.replace('\f', ' ') # replace FF (form feed) with whitespace
|
||||
|
||||
|
||||
def get_assign_targets(node):
|
||||
# type: (ast.AST) -> List[ast.expr]
|
||||
def get_assign_targets(node: ast.AST) -> List[ast.expr]:
|
||||
"""Get list of targets from Assign and AnnAssign node."""
|
||||
if isinstance(node, ast.Assign):
|
||||
return node.targets
|
||||
@ -45,8 +41,7 @@ def get_assign_targets(node):
|
||||
return [node.target] # type: ignore
|
||||
|
||||
|
||||
def get_lvar_names(node, self=None):
|
||||
# type: (ast.AST, ast.arg) -> List[str]
|
||||
def get_lvar_names(node: ast.AST, self: ast.arg = None) -> List[str]:
|
||||
"""Convert assignment-AST to variable names.
|
||||
|
||||
This raises `TypeError` if the assignment does not create new variable::
|
||||
@ -88,11 +83,9 @@ def get_lvar_names(node, self=None):
|
||||
raise NotImplementedError('Unexpected node name %r' % node_name)
|
||||
|
||||
|
||||
def dedent_docstring(s):
|
||||
# type: (str) -> str
|
||||
def dedent_docstring(s: str) -> str:
|
||||
"""Remove common leading indentation from docstring."""
|
||||
def dummy():
|
||||
# type: () -> None
|
||||
def dummy() -> None:
|
||||
# dummy function to mock `inspect.getdoc`.
|
||||
pass
|
||||
|
||||
@ -104,16 +97,15 @@ def dedent_docstring(s):
|
||||
class Token:
|
||||
"""Better token wrapper for tokenize module."""
|
||||
|
||||
def __init__(self, kind, value, start, end, source):
|
||||
# type: (int, Any, Tuple[int, int], Tuple[int, int], str) -> None
|
||||
def __init__(self, kind: int, value: Any, start: Tuple[int, int], end: Tuple[int, int],
|
||||
source: str) -> None:
|
||||
self.kind = kind
|
||||
self.value = value
|
||||
self.start = start
|
||||
self.end = end
|
||||
self.source = source
|
||||
|
||||
def __eq__(self, other):
|
||||
# type: (Any) -> bool
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
if isinstance(other, int):
|
||||
return self.kind == other
|
||||
elif isinstance(other, str):
|
||||
@ -125,32 +117,27 @@ class Token:
|
||||
else:
|
||||
raise ValueError('Unknown value: %r' % other)
|
||||
|
||||
def match(self, *conditions):
|
||||
# type: (Any) -> bool
|
||||
def match(self, *conditions) -> bool:
|
||||
return any(self == candidate for candidate in conditions)
|
||||
|
||||
def __repr__(self):
|
||||
# type: () -> str
|
||||
def __repr__(self) -> str:
|
||||
return '<Token kind=%r value=%r>' % (tokenize.tok_name[self.kind],
|
||||
self.value.strip())
|
||||
|
||||
|
||||
class TokenProcessor:
|
||||
def __init__(self, buffers):
|
||||
# type: (List[str]) -> None
|
||||
def __init__(self, buffers: List[str]) -> None:
|
||||
lines = iter(buffers)
|
||||
self.buffers = buffers
|
||||
self.tokens = tokenize.generate_tokens(lambda: next(lines))
|
||||
self.current = None # type: Token
|
||||
self.previous = None # type: Token
|
||||
|
||||
def get_line(self, lineno):
|
||||
# type: (int) -> str
|
||||
def get_line(self, lineno: int) -> str:
|
||||
"""Returns specified line."""
|
||||
return self.buffers[lineno - 1]
|
||||
|
||||
def fetch_token(self):
|
||||
# type: () -> Token
|
||||
def fetch_token(self) -> Token:
|
||||
"""Fetch a next token from source code.
|
||||
|
||||
Returns ``False`` if sequence finished.
|
||||
@ -163,8 +150,7 @@ class TokenProcessor:
|
||||
|
||||
return self.current
|
||||
|
||||
def fetch_until(self, condition):
|
||||
# type: (Any) -> List[Token]
|
||||
def fetch_until(self, condition: Any) -> List[Token]:
|
||||
"""Fetch tokens until specified token appeared.
|
||||
|
||||
.. note:: This also handles parenthesis well.
|
||||
@ -191,13 +177,11 @@ class AfterCommentParser(TokenProcessor):
|
||||
and returns the comments for variable if exists.
|
||||
"""
|
||||
|
||||
def __init__(self, lines):
|
||||
# type: (List[str]) -> None
|
||||
def __init__(self, lines: List[str]) -> None:
|
||||
super().__init__(lines)
|
||||
self.comment = None # type: str
|
||||
|
||||
def fetch_rvalue(self):
|
||||
# type: () -> List[Token]
|
||||
def fetch_rvalue(self) -> List[Token]:
|
||||
"""Fetch right-hand value of assignment."""
|
||||
tokens = []
|
||||
while self.fetch_token():
|
||||
@ -217,8 +201,7 @@ class AfterCommentParser(TokenProcessor):
|
||||
|
||||
return tokens
|
||||
|
||||
def parse(self):
|
||||
# type: () -> None
|
||||
def parse(self) -> None:
|
||||
"""Parse the code and obtain comment after assignment."""
|
||||
# skip lvalue (or whole of AnnAssign)
|
||||
while not self.fetch_token().match([OP, '='], NEWLINE, COMMENT):
|
||||
@ -235,8 +218,7 @@ class AfterCommentParser(TokenProcessor):
|
||||
class VariableCommentPicker(ast.NodeVisitor):
|
||||
"""Python source code parser to pick up variable comments."""
|
||||
|
||||
def __init__(self, buffers, encoding):
|
||||
# type: (List[str], str) -> None
|
||||
def __init__(self, buffers: List[str], encoding: str) -> None:
|
||||
self.counter = itertools.count()
|
||||
self.buffers = buffers
|
||||
self.encoding = encoding
|
||||
@ -248,8 +230,7 @@ class VariableCommentPicker(ast.NodeVisitor):
|
||||
self.deforders = {} # type: Dict[str, int]
|
||||
super().__init__()
|
||||
|
||||
def add_entry(self, name):
|
||||
# type: (str) -> None
|
||||
def add_entry(self, name: str) -> None:
|
||||
if self.current_function:
|
||||
if self.current_classes and self.context[-1] == "__init__":
|
||||
# store variable comments inside __init__ method of classes
|
||||
@ -261,8 +242,7 @@ class VariableCommentPicker(ast.NodeVisitor):
|
||||
|
||||
self.deforders[".".join(definition)] = next(self.counter)
|
||||
|
||||
def add_variable_comment(self, name, comment):
|
||||
# type: (str, str) -> None
|
||||
def add_variable_comment(self, name: str, comment: str) -> None:
|
||||
if self.current_function:
|
||||
if self.current_classes and self.context[-1] == "__init__":
|
||||
# store variable comments inside __init__ method of classes
|
||||
@ -274,27 +254,23 @@ class VariableCommentPicker(ast.NodeVisitor):
|
||||
|
||||
self.comments[(context, name)] = comment
|
||||
|
||||
def get_self(self):
|
||||
# type: () -> ast.arg
|
||||
def get_self(self) -> ast.arg:
|
||||
"""Returns the name of first argument if in function."""
|
||||
if self.current_function and self.current_function.args.args:
|
||||
return self.current_function.args.args[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_line(self, lineno):
|
||||
# type: (int) -> str
|
||||
def get_line(self, lineno: int) -> str:
|
||||
"""Returns specified line."""
|
||||
return self.buffers[lineno - 1]
|
||||
|
||||
def visit(self, node):
|
||||
# type: (ast.AST) -> None
|
||||
def visit(self, node: ast.AST) -> None:
|
||||
"""Updates self.previous to ."""
|
||||
super().visit(node)
|
||||
self.previous = node
|
||||
|
||||
def visit_Assign(self, node):
|
||||
# type: (ast.Assign) -> None
|
||||
def visit_Assign(self, node: ast.Assign) -> None:
|
||||
"""Handles Assign node and pick up a variable comment."""
|
||||
try:
|
||||
targets = get_assign_targets(node)
|
||||
@ -334,13 +310,11 @@ class VariableCommentPicker(ast.NodeVisitor):
|
||||
for varname in varnames:
|
||||
self.add_entry(varname)
|
||||
|
||||
def visit_AnnAssign(self, node):
|
||||
# type: (ast.AST) -> None
|
||||
def visit_AnnAssign(self, node: ast.AST) -> None: # Note: ast.AnnAssign not found in py35
|
||||
"""Handles AnnAssign node and pick up a variable comment."""
|
||||
self.visit_Assign(node) # type: ignore
|
||||
|
||||
def visit_Expr(self, node):
|
||||
# type: (ast.Expr) -> None
|
||||
def visit_Expr(self, node: ast.Expr) -> None:
|
||||
"""Handles Expr node and pick up a comment if string."""
|
||||
if (isinstance(self.previous, ASSIGN_NODES) and isinstance(node.value, ast.Str)):
|
||||
try:
|
||||
@ -357,8 +331,7 @@ class VariableCommentPicker(ast.NodeVisitor):
|
||||
except TypeError:
|
||||
pass # this assignment is not new definition!
|
||||
|
||||
def visit_Try(self, node):
|
||||
# type: (ast.Try) -> None
|
||||
def visit_Try(self, node: ast.Try) -> None:
|
||||
"""Handles Try node and processes body and else-clause.
|
||||
|
||||
.. note:: pycode parser ignores objects definition in except-clause.
|
||||
@ -368,8 +341,7 @@ class VariableCommentPicker(ast.NodeVisitor):
|
||||
for subnode in node.orelse:
|
||||
self.visit(subnode)
|
||||
|
||||
def visit_ClassDef(self, node):
|
||||
# type: (ast.ClassDef) -> None
|
||||
def visit_ClassDef(self, node: ast.ClassDef) -> None:
|
||||
"""Handles ClassDef node and set context."""
|
||||
self.current_classes.append(node.name)
|
||||
self.add_entry(node.name)
|
||||
@ -380,8 +352,7 @@ class VariableCommentPicker(ast.NodeVisitor):
|
||||
self.context.pop()
|
||||
self.current_classes.pop()
|
||||
|
||||
def visit_FunctionDef(self, node):
|
||||
# type: (ast.FunctionDef) -> None
|
||||
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
|
||||
"""Handles FunctionDef node and set context."""
|
||||
if self.current_function is None:
|
||||
self.add_entry(node.name) # should be called before setting self.current_function
|
||||
@ -392,8 +363,7 @@ class VariableCommentPicker(ast.NodeVisitor):
|
||||
self.context.pop()
|
||||
self.current_function = None
|
||||
|
||||
def visit_AsyncFunctionDef(self, node):
|
||||
# type: (ast.AsyncFunctionDef) -> None
|
||||
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
|
||||
"""Handles AsyncFunctionDef node and set context."""
|
||||
self.visit_FunctionDef(node) # type: ignore
|
||||
|
||||
@ -403,16 +373,14 @@ class DefinitionFinder(TokenProcessor):
|
||||
classes and methods.
|
||||
"""
|
||||
|
||||
def __init__(self, lines):
|
||||
# type: (List[str]) -> None
|
||||
def __init__(self, lines: List[str]) -> None:
|
||||
super().__init__(lines)
|
||||
self.decorator = None # type: Token
|
||||
self.context = [] # type: List[str]
|
||||
self.indents = [] # type: List
|
||||
self.definitions = {} # type: Dict[str, Tuple[str, int, int]]
|
||||
|
||||
def add_definition(self, name, entry):
|
||||
# type: (str, Tuple[str, int, int]) -> None
|
||||
def add_definition(self, name: str, entry: Tuple[str, int, int]) -> None:
|
||||
"""Add a location of definition."""
|
||||
if self.indents and self.indents[-1][0] == 'def' and entry[0] == 'def':
|
||||
# ignore definition of inner function
|
||||
@ -420,8 +388,7 @@ class DefinitionFinder(TokenProcessor):
|
||||
else:
|
||||
self.definitions[name] = entry
|
||||
|
||||
def parse(self):
|
||||
# type: () -> None
|
||||
def parse(self) -> None:
|
||||
"""Parse the code to obtain location of definitions."""
|
||||
while True:
|
||||
token = self.fetch_token()
|
||||
@ -442,8 +409,7 @@ class DefinitionFinder(TokenProcessor):
|
||||
elif token == DEDENT:
|
||||
self.finalize_block()
|
||||
|
||||
def parse_definition(self, typ):
|
||||
# type: (str) -> None
|
||||
def parse_definition(self, typ: str) -> None:
|
||||
"""Parse AST of definition."""
|
||||
name = self.fetch_token()
|
||||
self.context.append(name.value)
|
||||
@ -464,8 +430,7 @@ class DefinitionFinder(TokenProcessor):
|
||||
self.add_definition(funcname, (typ, start_pos, name.end[0]))
|
||||
self.context.pop()
|
||||
|
||||
def finalize_block(self):
|
||||
# type: () -> None
|
||||
def finalize_block(self) -> None:
|
||||
"""Finalize definition block."""
|
||||
definition = self.indents.pop()
|
||||
if definition[0] != 'other':
|
||||
@ -484,22 +449,19 @@ class Parser:
|
||||
This is a better wrapper for ``VariableCommentPicker``.
|
||||
"""
|
||||
|
||||
def __init__(self, code, encoding='utf-8'):
|
||||
# type: (str, str) -> None
|
||||
def __init__(self, code: str, encoding: str = 'utf-8') -> None:
|
||||
self.code = filter_whitespace(code)
|
||||
self.encoding = encoding
|
||||
self.comments = {} # type: Dict[Tuple[str, str], str]
|
||||
self.deforders = {} # type: Dict[str, int]
|
||||
self.definitions = {} # type: Dict[str, Tuple[str, int, int]]
|
||||
|
||||
def parse(self):
|
||||
# type: () -> None
|
||||
def parse(self) -> None:
|
||||
"""Parse the source code."""
|
||||
self.parse_comments()
|
||||
self.parse_definition()
|
||||
|
||||
def parse_comments(self):
|
||||
# type: () -> None
|
||||
def parse_comments(self) -> None:
|
||||
"""Parse the code and pick up comments."""
|
||||
tree = ast.parse(self.code.encode())
|
||||
picker = VariableCommentPicker(self.code.splitlines(True), self.encoding)
|
||||
@ -507,8 +469,7 @@ class Parser:
|
||||
self.comments = picker.comments
|
||||
self.deforders = picker.deforders
|
||||
|
||||
def parse_definition(self):
|
||||
# type: () -> None
|
||||
def parse_definition(self) -> None:
|
||||
"""Parse the location of definitions from the code."""
|
||||
parser = DefinitionFinder(self.code.splitlines(True))
|
||||
parser.parse()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user