diff --git a/sphinx/pycode/parser.py b/sphinx/pycode/parser.py index 1746537bb..3603e3cc3 100644 --- a/sphinx/pycode/parser.py +++ b/sphinx/pycode/parser.py @@ -15,10 +15,8 @@ import sys import tokenize from token import NAME, NEWLINE, INDENT, DEDENT, NUMBER, OP, STRING from tokenize import COMMENT, NL +from typing import Any, Dict, List, Tuple -if False: - # For type annotation - from typing import Any, Dict, List, Tuple # NOQA comment_re = re.compile('^\\s*#: ?(.*)\r?\n?$') indent_re = re.compile('^\\s*$') @@ -31,13 +29,11 @@ else: ASSIGN_NODES = (ast.Assign) -def filter_whitespace(code): - # type: (str) -> str +def filter_whitespace(code: str) -> str: return code.replace('\f', ' ') # replace FF (form feed) with whitespace -def get_assign_targets(node): - # type: (ast.AST) -> List[ast.expr] +def get_assign_targets(node: ast.AST) -> List[ast.expr]: """Get list of targets from Assign and AnnAssign node.""" if isinstance(node, ast.Assign): return node.targets @@ -45,8 +41,7 @@ def get_assign_targets(node): return [node.target] # type: ignore -def get_lvar_names(node, self=None): - # type: (ast.AST, ast.arg) -> List[str] +def get_lvar_names(node: ast.AST, self: ast.arg = None) -> List[str]: """Convert assignment-AST to variable names. This raises `TypeError` if the assignment does not create new variable:: @@ -88,11 +83,9 @@ def get_lvar_names(node, self=None): raise NotImplementedError('Unexpected node name %r' % node_name) -def dedent_docstring(s): - # type: (str) -> str +def dedent_docstring(s: str) -> str: """Remove common leading indentation from docstring.""" - def dummy(): - # type: () -> None + def dummy() -> None: # dummy function to mock `inspect.getdoc`. pass @@ -104,16 +97,15 @@ def dedent_docstring(s): class Token: """Better token wrapper for tokenize module.""" - def __init__(self, kind, value, start, end, source): - # type: (int, Any, Tuple[int, int], Tuple[int, int], str) -> None + def __init__(self, kind: int, value: Any, start: Tuple[int, int], end: Tuple[int, int], + source: str) -> None: self.kind = kind self.value = value self.start = start self.end = end self.source = source - def __eq__(self, other): - # type: (Any) -> bool + def __eq__(self, other: Any) -> bool: if isinstance(other, int): return self.kind == other elif isinstance(other, str): @@ -125,32 +117,27 @@ class Token: else: raise ValueError('Unknown value: %r' % other) - def match(self, *conditions): - # type: (Any) -> bool + def match(self, *conditions) -> bool: return any(self == candidate for candidate in conditions) - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: return '' % (tokenize.tok_name[self.kind], self.value.strip()) class TokenProcessor: - def __init__(self, buffers): - # type: (List[str]) -> None + def __init__(self, buffers: List[str]) -> None: lines = iter(buffers) self.buffers = buffers self.tokens = tokenize.generate_tokens(lambda: next(lines)) self.current = None # type: Token self.previous = None # type: Token - def get_line(self, lineno): - # type: (int) -> str + def get_line(self, lineno: int) -> str: """Returns specified line.""" return self.buffers[lineno - 1] - def fetch_token(self): - # type: () -> Token + def fetch_token(self) -> Token: """Fetch a next token from source code. Returns ``False`` if sequence finished. @@ -163,8 +150,7 @@ class TokenProcessor: return self.current - def fetch_until(self, condition): - # type: (Any) -> List[Token] + def fetch_until(self, condition: Any) -> List[Token]: """Fetch tokens until specified token appeared. .. note:: This also handles parenthesis well. @@ -191,13 +177,11 @@ class AfterCommentParser(TokenProcessor): and returns the comments for variable if exists. """ - def __init__(self, lines): - # type: (List[str]) -> None + def __init__(self, lines: List[str]) -> None: super().__init__(lines) self.comment = None # type: str - def fetch_rvalue(self): - # type: () -> List[Token] + def fetch_rvalue(self) -> List[Token]: """Fetch right-hand value of assignment.""" tokens = [] while self.fetch_token(): @@ -217,8 +201,7 @@ class AfterCommentParser(TokenProcessor): return tokens - def parse(self): - # type: () -> None + def parse(self) -> None: """Parse the code and obtain comment after assignment.""" # skip lvalue (or whole of AnnAssign) while not self.fetch_token().match([OP, '='], NEWLINE, COMMENT): @@ -235,8 +218,7 @@ class AfterCommentParser(TokenProcessor): class VariableCommentPicker(ast.NodeVisitor): """Python source code parser to pick up variable comments.""" - def __init__(self, buffers, encoding): - # type: (List[str], str) -> None + def __init__(self, buffers: List[str], encoding: str) -> None: self.counter = itertools.count() self.buffers = buffers self.encoding = encoding @@ -248,8 +230,7 @@ class VariableCommentPicker(ast.NodeVisitor): self.deforders = {} # type: Dict[str, int] super().__init__() - def add_entry(self, name): - # type: (str) -> None + def add_entry(self, name: str) -> None: if self.current_function: if self.current_classes and self.context[-1] == "__init__": # store variable comments inside __init__ method of classes @@ -261,8 +242,7 @@ class VariableCommentPicker(ast.NodeVisitor): self.deforders[".".join(definition)] = next(self.counter) - def add_variable_comment(self, name, comment): - # type: (str, str) -> None + def add_variable_comment(self, name: str, comment: str) -> None: if self.current_function: if self.current_classes and self.context[-1] == "__init__": # store variable comments inside __init__ method of classes @@ -274,27 +254,23 @@ class VariableCommentPicker(ast.NodeVisitor): self.comments[(context, name)] = comment - def get_self(self): - # type: () -> ast.arg + def get_self(self) -> ast.arg: """Returns the name of first argument if in function.""" if self.current_function and self.current_function.args.args: return self.current_function.args.args[0] else: return None - def get_line(self, lineno): - # type: (int) -> str + def get_line(self, lineno: int) -> str: """Returns specified line.""" return self.buffers[lineno - 1] - def visit(self, node): - # type: (ast.AST) -> None + def visit(self, node: ast.AST) -> None: """Updates self.previous to .""" super().visit(node) self.previous = node - def visit_Assign(self, node): - # type: (ast.Assign) -> None + def visit_Assign(self, node: ast.Assign) -> None: """Handles Assign node and pick up a variable comment.""" try: targets = get_assign_targets(node) @@ -334,13 +310,11 @@ class VariableCommentPicker(ast.NodeVisitor): for varname in varnames: self.add_entry(varname) - def visit_AnnAssign(self, node): - # type: (ast.AST) -> None + def visit_AnnAssign(self, node: ast.AST) -> None: # Note: ast.AnnAssign not found in py35 """Handles AnnAssign node and pick up a variable comment.""" self.visit_Assign(node) # type: ignore - def visit_Expr(self, node): - # type: (ast.Expr) -> None + def visit_Expr(self, node: ast.Expr) -> None: """Handles Expr node and pick up a comment if string.""" if (isinstance(self.previous, ASSIGN_NODES) and isinstance(node.value, ast.Str)): try: @@ -357,8 +331,7 @@ class VariableCommentPicker(ast.NodeVisitor): except TypeError: pass # this assignment is not new definition! - def visit_Try(self, node): - # type: (ast.Try) -> None + def visit_Try(self, node: ast.Try) -> None: """Handles Try node and processes body and else-clause. .. note:: pycode parser ignores objects definition in except-clause. @@ -368,8 +341,7 @@ class VariableCommentPicker(ast.NodeVisitor): for subnode in node.orelse: self.visit(subnode) - def visit_ClassDef(self, node): - # type: (ast.ClassDef) -> None + def visit_ClassDef(self, node: ast.ClassDef) -> None: """Handles ClassDef node and set context.""" self.current_classes.append(node.name) self.add_entry(node.name) @@ -380,8 +352,7 @@ class VariableCommentPicker(ast.NodeVisitor): self.context.pop() self.current_classes.pop() - def visit_FunctionDef(self, node): - # type: (ast.FunctionDef) -> None + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: """Handles FunctionDef node and set context.""" if self.current_function is None: self.add_entry(node.name) # should be called before setting self.current_function @@ -392,8 +363,7 @@ class VariableCommentPicker(ast.NodeVisitor): self.context.pop() self.current_function = None - def visit_AsyncFunctionDef(self, node): - # type: (ast.AsyncFunctionDef) -> None + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: """Handles AsyncFunctionDef node and set context.""" self.visit_FunctionDef(node) # type: ignore @@ -403,16 +373,14 @@ class DefinitionFinder(TokenProcessor): classes and methods. """ - def __init__(self, lines): - # type: (List[str]) -> None + def __init__(self, lines: List[str]) -> None: super().__init__(lines) self.decorator = None # type: Token self.context = [] # type: List[str] self.indents = [] # type: List self.definitions = {} # type: Dict[str, Tuple[str, int, int]] - def add_definition(self, name, entry): - # type: (str, Tuple[str, int, int]) -> None + def add_definition(self, name: str, entry: Tuple[str, int, int]) -> None: """Add a location of definition.""" if self.indents and self.indents[-1][0] == 'def' and entry[0] == 'def': # ignore definition of inner function @@ -420,8 +388,7 @@ class DefinitionFinder(TokenProcessor): else: self.definitions[name] = entry - def parse(self): - # type: () -> None + def parse(self) -> None: """Parse the code to obtain location of definitions.""" while True: token = self.fetch_token() @@ -442,8 +409,7 @@ class DefinitionFinder(TokenProcessor): elif token == DEDENT: self.finalize_block() - def parse_definition(self, typ): - # type: (str) -> None + def parse_definition(self, typ: str) -> None: """Parse AST of definition.""" name = self.fetch_token() self.context.append(name.value) @@ -464,8 +430,7 @@ class DefinitionFinder(TokenProcessor): self.add_definition(funcname, (typ, start_pos, name.end[0])) self.context.pop() - def finalize_block(self): - # type: () -> None + def finalize_block(self) -> None: """Finalize definition block.""" definition = self.indents.pop() if definition[0] != 'other': @@ -484,22 +449,19 @@ class Parser: This is a better wrapper for ``VariableCommentPicker``. """ - def __init__(self, code, encoding='utf-8'): - # type: (str, str) -> None + def __init__(self, code: str, encoding: str = 'utf-8') -> None: self.code = filter_whitespace(code) self.encoding = encoding self.comments = {} # type: Dict[Tuple[str, str], str] self.deforders = {} # type: Dict[str, int] self.definitions = {} # type: Dict[str, Tuple[str, int, int]] - def parse(self): - # type: () -> None + def parse(self) -> None: """Parse the source code.""" self.parse_comments() self.parse_definition() - def parse_comments(self): - # type: () -> None + def parse_comments(self) -> None: """Parse the code and pick up comments.""" tree = ast.parse(self.code.encode()) picker = VariableCommentPicker(self.code.splitlines(True), self.encoding) @@ -507,8 +469,7 @@ class Parser: self.comments = picker.comments self.deforders = picker.deforders - def parse_definition(self): - # type: () -> None + def parse_definition(self) -> None: """Parse the location of definitions from the code.""" parser = DefinitionFinder(self.code.splitlines(True)) parser.parse()