IntenseWebs/sphinx
3
0
Fork 0
mirror of https://github.com/sphinx-doc/sphinx.git synced 2025-02-25 18:55:22 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6554 from tk0miya/refactor_type_annotation_pycode

Browse Source 
Migrate to py3 style type annotation: sphinx.pycode
This commit is contained in:
Takeshi KOMIYA 2019-07-06 14:54:13 +09:00 committed by GitHub
commit ac158d0786
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 50 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
sphinx/pycode/__init__.py
View File

@ -11,29 +11,25 @@
import re import re
from io import StringIO from io import StringIO
from os import path from os import path
from typing import Any, Dict, IO, List, Tuple
from zipfile import ZipFile from zipfile import ZipFile
from sphinx.errors import PycodeError from sphinx.errors import PycodeError
from sphinx.pycode.parser import Parser from sphinx.pycode.parser import Parser
from sphinx.util import get_module_source, detect_encoding from sphinx.util import get_module_source, detect_encoding
if False:
# For type annotation
from typing import Any, Dict, IO, List, Tuple # NOQA
class ModuleAnalyzer: class ModuleAnalyzer:
# cache for analyzer objects -- caches both by module and file name # cache for analyzer objects -- caches both by module and file name
cache = {} # type: Dict[Tuple[str, str], Any] cache = {} # type: Dict[Tuple[str, str], Any]
@classmethod @classmethod
def for_string(cls, string, modname, srcname='<string>'): def for_string(cls, string: str, modname: str, srcname: str = '<string>'
# type: (str, str, str) -> ModuleAnalyzer ) -> "ModuleAnalyzer":
return cls(StringIO(string), modname, srcname, decoded=True) return cls(StringIO(string), modname, srcname, decoded=True)
@classmethod @classmethod
def for_file(cls, filename, modname): def for_file(cls, filename: str, modname: str) -> "ModuleAnalyzer":
# type: (str, str) -> ModuleAnalyzer
if ('file', filename) in cls.cache: if ('file', filename) in cls.cache:
return cls.cache['file', filename] return cls.cache['file', filename]
try: try:
@ -48,8 +44,7 @@ class ModuleAnalyzer:
return obj return obj
@classmethod @classmethod
def for_egg(cls, filename, modname): def for_egg(cls, filename: str, modname: str) -> "ModuleAnalyzer":
# type: (str, str) -> ModuleAnalyzer
SEP = re.escape(path.sep) SEP = re.escape(path.sep)
eggpath, relpath = re.split('(?<=\\.egg)' + SEP, filename) eggpath, relpath = re.split('(?<=\\.egg)' + SEP, filename)
try: try:
@ -60,8 +55,7 @@ class ModuleAnalyzer:
raise PycodeError('error opening %r' % filename, exc) raise PycodeError('error opening %r' % filename, exc)
@classmethod @classmethod
def for_module(cls, modname): def for_module(cls, modname: str) -> "ModuleAnalyzer":
# type: (str) -> ModuleAnalyzer
if ('module', modname) in cls.cache: if ('module', modname) in cls.cache:
entry = cls.cache['module', modname] entry = cls.cache['module', modname]
if isinstance(entry, PycodeError): if isinstance(entry, PycodeError):
@ -80,8 +74,7 @@ class ModuleAnalyzer:
cls.cache['module', modname] = obj cls.cache['module', modname] = obj
return obj return obj
def __init__(self, source, modname, srcname, decoded=False): def __init__(self, source: IO, modname: str, srcname: str, decoded: bool = False) -> None:
# type: (IO, str, str, bool) -> None
self.modname = modname # name of the module self.modname = modname # name of the module
self.srcname = srcname # name of the source file self.srcname = srcname # name of the source file
@ -100,8 +93,7 @@ class ModuleAnalyzer:
self.tagorder = None # type: Dict[str, int] self.tagorder = None # type: Dict[str, int]
self.tags = None # type: Dict[str, Tuple[str, int, int]] self.tags = None # type: Dict[str, Tuple[str, int, int]]
def parse(self): def parse(self) -> None:
# type: () -> None
"""Parse the source code.""" """Parse the source code."""
try: try:
parser = Parser(self.code, self.encoding) parser = Parser(self.code, self.encoding)
@ -119,16 +111,14 @@ class ModuleAnalyzer:
except Exception as exc: except Exception as exc:
raise PycodeError('parsing %r failed: %r' % (self.srcname, exc)) raise PycodeError('parsing %r failed: %r' % (self.srcname, exc))
def find_attr_docs(self): def find_attr_docs(self) -> Dict[Tuple[str, str], List[str]]:
# type: () -> Dict[Tuple[str, str], List[str]]
"""Find class and module-level attributes and their documentation.""" """Find class and module-level attributes and their documentation."""
if self.attr_docs is None: if self.attr_docs is None:
self.parse() self.parse()
return self.attr_docs return self.attr_docs
def find_tags(self): def find_tags(self) -> Dict[str, Tuple[str, int, int]]:
# type: () -> Dict[str, Tuple[str, int, int]]
"""Find class, function and method definitions and their location.""" """Find class, function and method definitions and their location."""
if self.tags is None: if self.tags is None:
self.parse() self.parse()

119
sphinx/pycode/parser.py
View File

@ -15,10 +15,8 @@ import sys
import tokenize import tokenize
from token import NAME, NEWLINE, INDENT, DEDENT, NUMBER, OP, STRING from token import NAME, NEWLINE, INDENT, DEDENT, NUMBER, OP, STRING
from tokenize import COMMENT, NL from tokenize import COMMENT, NL
from typing import Any, Dict, List, Tuple
if False:
# For type annotation
from typing import Any, Dict, List, Tuple # NOQA
comment_re = re.compile('^\\s*#: ?(.*)\r?\n?$') comment_re = re.compile('^\\s*#: ?(.*)\r?\n?$')
indent_re = re.compile('^\\s*$') indent_re = re.compile('^\\s*$')
@ -31,13 +29,11 @@ else:
ASSIGN_NODES = (ast.Assign) ASSIGN_NODES = (ast.Assign)
def filter_whitespace(code): def filter_whitespace(code: str) -> str:
# type: (str) -> str
return code.replace('\f', ' ') # replace FF (form feed) with whitespace return code.replace('\f', ' ') # replace FF (form feed) with whitespace
def get_assign_targets(node): def get_assign_targets(node: ast.AST) -> List[ast.expr]:
# type: (ast.AST) -> List[ast.expr]
"""Get list of targets from Assign and AnnAssign node.""" """Get list of targets from Assign and AnnAssign node."""
if isinstance(node, ast.Assign): if isinstance(node, ast.Assign):
return node.targets return node.targets
@ -45,8 +41,7 @@ def get_assign_targets(node):
return [node.target] # type: ignore return [node.target] # type: ignore
def get_lvar_names(node, self=None): def get_lvar_names(node: ast.AST, self: ast.arg = None) -> List[str]:
# type: (ast.AST, ast.arg) -> List[str]
"""Convert assignment-AST to variable names. """Convert assignment-AST to variable names.
This raises `TypeError` if the assignment does not create new variable:: This raises `TypeError` if the assignment does not create new variable::
@ -88,11 +83,9 @@ def get_lvar_names(node, self=None):
raise NotImplementedError('Unexpected node name %r' % node_name) raise NotImplementedError('Unexpected node name %r' % node_name)
def dedent_docstring(s): def dedent_docstring(s: str) -> str:
# type: (str) -> str
"""Remove common leading indentation from docstring.""" """Remove common leading indentation from docstring."""
def dummy(): def dummy() -> None:
# type: () -> None
# dummy function to mock `inspect.getdoc`. # dummy function to mock `inspect.getdoc`.
pass pass
@ -104,16 +97,15 @@ def dedent_docstring(s):
class Token: class Token:
"""Better token wrapper for tokenize module.""" """Better token wrapper for tokenize module."""
def __init__(self, kind, value, start, end, source): def __init__(self, kind: int, value: Any, start: Tuple[int, int], end: Tuple[int, int],
# type: (int, Any, Tuple[int, int], Tuple[int, int], str) -> None source: str) -> None:
self.kind = kind self.kind = kind
self.value = value self.value = value
self.start = start self.start = start
self.end = end self.end = end
self.source = source self.source = source
def __eq__(self, other): def __eq__(self, other: Any) -> bool:
# type: (Any) -> bool
if isinstance(other, int): if isinstance(other, int):
return self.kind == other return self.kind == other
elif isinstance(other, str): elif isinstance(other, str):
@ -125,32 +117,27 @@ class Token:
else: else:
raise ValueError('Unknown value: %r' % other) raise ValueError('Unknown value: %r' % other)
def match(self, *conditions): def match(self, *conditions) -> bool:
# type: (Any) -> bool
return any(self == candidate for candidate in conditions) return any(self == candidate for candidate in conditions)
def __repr__(self): def __repr__(self) -> str:
# type: () -> str
return '<Token kind=%r value=%r>' % (tokenize.tok_name[self.kind], return '<Token kind=%r value=%r>' % (tokenize.tok_name[self.kind],
self.value.strip()) self.value.strip())
class TokenProcessor: class TokenProcessor:
def __init__(self, buffers): def __init__(self, buffers: List[str]) -> None:
# type: (List[str]) -> None
lines = iter(buffers) lines = iter(buffers)
self.buffers = buffers self.buffers = buffers
self.tokens = tokenize.generate_tokens(lambda: next(lines)) self.tokens = tokenize.generate_tokens(lambda: next(lines))
self.current = None # type: Token self.current = None # type: Token
self.previous = None # type: Token self.previous = None # type: Token
def get_line(self, lineno): def get_line(self, lineno: int) -> str:
# type: (int) -> str
"""Returns specified line.""" """Returns specified line."""
return self.buffers[lineno - 1] return self.buffers[lineno - 1]
def fetch_token(self): def fetch_token(self) -> Token:
# type: () -> Token
"""Fetch a next token from source code. """Fetch a next token from source code.
Returns ``False`` if sequence finished. Returns ``False`` if sequence finished.
@ -163,8 +150,7 @@ class TokenProcessor:
return self.current return self.current
def fetch_until(self, condition): def fetch_until(self, condition: Any) -> List[Token]:
# type: (Any) -> List[Token]
"""Fetch tokens until specified token appeared. """Fetch tokens until specified token appeared.
.. note:: This also handles parenthesis well. .. note:: This also handles parenthesis well.
@ -191,13 +177,11 @@ class AfterCommentParser(TokenProcessor):
and returns the comments for variable if exists. and returns the comments for variable if exists.
""" """
def __init__(self, lines): def __init__(self, lines: List[str]) -> None:
# type: (List[str]) -> None
super().__init__(lines) super().__init__(lines)
self.comment = None # type: str self.comment = None # type: str
def fetch_rvalue(self): def fetch_rvalue(self) -> List[Token]:
# type: () -> List[Token]
"""Fetch right-hand value of assignment.""" """Fetch right-hand value of assignment."""
tokens = [] tokens = []
while self.fetch_token(): while self.fetch_token():
@ -217,8 +201,7 @@ class AfterCommentParser(TokenProcessor):
return tokens return tokens
def parse(self): def parse(self) -> None:
# type: () -> None
"""Parse the code and obtain comment after assignment.""" """Parse the code and obtain comment after assignment."""
# skip lvalue (or whole of AnnAssign) # skip lvalue (or whole of AnnAssign)
while not self.fetch_token().match([OP, '='], NEWLINE, COMMENT): while not self.fetch_token().match([OP, '='], NEWLINE, COMMENT):
@ -235,8 +218,7 @@ class AfterCommentParser(TokenProcessor):
class VariableCommentPicker(ast.NodeVisitor): class VariableCommentPicker(ast.NodeVisitor):
"""Python source code parser to pick up variable comments.""" """Python source code parser to pick up variable comments."""
def __init__(self, buffers, encoding): def __init__(self, buffers: List[str], encoding: str) -> None:
# type: (List[str], str) -> None
self.counter = itertools.count() self.counter = itertools.count()
self.buffers = buffers self.buffers = buffers
self.encoding = encoding self.encoding = encoding
@ -248,8 +230,7 @@ class VariableCommentPicker(ast.NodeVisitor):
self.deforders = {} # type: Dict[str, int] self.deforders = {} # type: Dict[str, int]
super().__init__() super().__init__()
def add_entry(self, name): def add_entry(self, name: str) -> None:
# type: (str) -> None
if self.current_function: if self.current_function:
if self.current_classes and self.context[-1] == "__init__": if self.current_classes and self.context[-1] == "__init__":
# store variable comments inside __init__ method of classes # store variable comments inside __init__ method of classes
@ -261,8 +242,7 @@ class VariableCommentPicker(ast.NodeVisitor):
self.deforders[".".join(definition)] = next(self.counter) self.deforders[".".join(definition)] = next(self.counter)
def add_variable_comment(self, name, comment): def add_variable_comment(self, name: str, comment: str) -> None:
# type: (str, str) -> None
if self.current_function: if self.current_function:
if self.current_classes and self.context[-1] == "__init__": if self.current_classes and self.context[-1] == "__init__":
# store variable comments inside __init__ method of classes # store variable comments inside __init__ method of classes
@ -274,27 +254,23 @@ class VariableCommentPicker(ast.NodeVisitor):
self.comments[(context, name)] = comment self.comments[(context, name)] = comment
def get_self(self): def get_self(self) -> ast.arg:
# type: () -> ast.arg
"""Returns the name of first argument if in function.""" """Returns the name of first argument if in function."""
if self.current_function and self.current_function.args.args: if self.current_function and self.current_function.args.args:
return self.current_function.args.args[0] return self.current_function.args.args[0]
else: else:
return None return None
def get_line(self, lineno): def get_line(self, lineno: int) -> str:
# type: (int) -> str
"""Returns specified line.""" """Returns specified line."""
return self.buffers[lineno - 1] return self.buffers[lineno - 1]
def visit(self, node): def visit(self, node: ast.AST) -> None:
# type: (ast.AST) -> None
"""Updates self.previous to .""" """Updates self.previous to ."""
super().visit(node) super().visit(node)
self.previous = node self.previous = node
def visit_Assign(self, node): def visit_Assign(self, node: ast.Assign) -> None:
# type: (ast.Assign) -> None
"""Handles Assign node and pick up a variable comment.""" """Handles Assign node and pick up a variable comment."""
try: try:
targets = get_assign_targets(node) targets = get_assign_targets(node)
@ -334,13 +310,11 @@ class VariableCommentPicker(ast.NodeVisitor):
for varname in varnames: for varname in varnames:
self.add_entry(varname) self.add_entry(varname)
def visit_AnnAssign(self, node): def visit_AnnAssign(self, node: ast.AST) -> None: # Note: ast.AnnAssign not found in py35
# type: (ast.AST) -> None
"""Handles AnnAssign node and pick up a variable comment.""" """Handles AnnAssign node and pick up a variable comment."""
self.visit_Assign(node) # type: ignore self.visit_Assign(node) # type: ignore
def visit_Expr(self, node): def visit_Expr(self, node: ast.Expr) -> None:
# type: (ast.Expr) -> None
"""Handles Expr node and pick up a comment if string.""" """Handles Expr node and pick up a comment if string."""
if (isinstance(self.previous, ASSIGN_NODES) and isinstance(node.value, ast.Str)): if (isinstance(self.previous, ASSIGN_NODES) and isinstance(node.value, ast.Str)):
try: try:
@ -357,8 +331,7 @@ class VariableCommentPicker(ast.NodeVisitor):
except TypeError: except TypeError:
pass # this assignment is not new definition! pass # this assignment is not new definition!
def visit_Try(self, node): def visit_Try(self, node: ast.Try) -> None:
# type: (ast.Try) -> None
"""Handles Try node and processes body and else-clause. """Handles Try node and processes body and else-clause.
.. note:: pycode parser ignores objects definition in except-clause. .. note:: pycode parser ignores objects definition in except-clause.
@ -368,8 +341,7 @@ class VariableCommentPicker(ast.NodeVisitor):
for subnode in node.orelse: for subnode in node.orelse:
self.visit(subnode) self.visit(subnode)
def visit_ClassDef(self, node): def visit_ClassDef(self, node: ast.ClassDef) -> None:
# type: (ast.ClassDef) -> None
"""Handles ClassDef node and set context.""" """Handles ClassDef node and set context."""
self.current_classes.append(node.name) self.current_classes.append(node.name)
self.add_entry(node.name) self.add_entry(node.name)
@ -380,8 +352,7 @@ class VariableCommentPicker(ast.NodeVisitor):
self.context.pop() self.context.pop()
self.current_classes.pop() self.current_classes.pop()
def visit_FunctionDef(self, node): def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
# type: (ast.FunctionDef) -> None
"""Handles FunctionDef node and set context.""" """Handles FunctionDef node and set context."""
if self.current_function is None: if self.current_function is None:
self.add_entry(node.name) # should be called before setting self.current_function self.add_entry(node.name) # should be called before setting self.current_function
@ -392,8 +363,7 @@ class VariableCommentPicker(ast.NodeVisitor):
self.context.pop() self.context.pop()
self.current_function = None self.current_function = None
def visit_AsyncFunctionDef(self, node): def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
# type: (ast.AsyncFunctionDef) -> None
"""Handles AsyncFunctionDef node and set context.""" """Handles AsyncFunctionDef node and set context."""
self.visit_FunctionDef(node) # type: ignore self.visit_FunctionDef(node) # type: ignore
@ -403,16 +373,14 @@ class DefinitionFinder(TokenProcessor):
classes and methods. classes and methods.
""" """
def __init__(self, lines): def __init__(self, lines: List[str]) -> None:
# type: (List[str]) -> None
super().__init__(lines) super().__init__(lines)
self.decorator = None # type: Token self.decorator = None # type: Token
self.context = [] # type: List[str] self.context = [] # type: List[str]
self.indents = [] # type: List self.indents = [] # type: List
self.definitions = {} # type: Dict[str, Tuple[str, int, int]] self.definitions = {} # type: Dict[str, Tuple[str, int, int]]
def add_definition(self, name, entry): def add_definition(self, name: str, entry: Tuple[str, int, int]) -> None:
# type: (str, Tuple[str, int, int]) -> None
"""Add a location of definition.""" """Add a location of definition."""
if self.indents and self.indents[-1][0] == 'def' and entry[0] == 'def': if self.indents and self.indents[-1][0] == 'def' and entry[0] == 'def':
# ignore definition of inner function # ignore definition of inner function
@ -420,8 +388,7 @@ class DefinitionFinder(TokenProcessor):
else: else:
self.definitions[name] = entry self.definitions[name] = entry
def parse(self): def parse(self) -> None:
# type: () -> None
"""Parse the code to obtain location of definitions.""" """Parse the code to obtain location of definitions."""
while True: while True:
token = self.fetch_token() token = self.fetch_token()
@ -442,8 +409,7 @@ class DefinitionFinder(TokenProcessor):
elif token == DEDENT: elif token == DEDENT:
self.finalize_block() self.finalize_block()
def parse_definition(self, typ): def parse_definition(self, typ: str) -> None:
# type: (str) -> None
"""Parse AST of definition.""" """Parse AST of definition."""
name = self.fetch_token() name = self.fetch_token()
self.context.append(name.value) self.context.append(name.value)
@ -464,8 +430,7 @@ class DefinitionFinder(TokenProcessor):
self.add_definition(funcname, (typ, start_pos, name.end[0])) self.add_definition(funcname, (typ, start_pos, name.end[0]))
self.context.pop() self.context.pop()
def finalize_block(self): def finalize_block(self) -> None:
# type: () -> None
"""Finalize definition block.""" """Finalize definition block."""
definition = self.indents.pop() definition = self.indents.pop()
if definition[0] != 'other': if definition[0] != 'other':
@ -484,22 +449,19 @@ class Parser:
This is a better wrapper for ``VariableCommentPicker``. This is a better wrapper for ``VariableCommentPicker``.
""" """
def __init__(self, code, encoding='utf-8'): def __init__(self, code: str, encoding: str = 'utf-8') -> None:
# type: (str, str) -> None
self.code = filter_whitespace(code) self.code = filter_whitespace(code)
self.encoding = encoding self.encoding = encoding
self.comments = {} # type: Dict[Tuple[str, str], str] self.comments = {} # type: Dict[Tuple[str, str], str]
self.deforders = {} # type: Dict[str, int] self.deforders = {} # type: Dict[str, int]
self.definitions = {} # type: Dict[str, Tuple[str, int, int]] self.definitions = {} # type: Dict[str, Tuple[str, int, int]]
def parse(self): def parse(self) -> None:
# type: () -> None
"""Parse the source code.""" """Parse the source code."""
self.parse_comments() self.parse_comments()
self.parse_definition() self.parse_definition()
def parse_comments(self): def parse_comments(self) -> None:
# type: () -> None
"""Parse the code and pick up comments.""" """Parse the code and pick up comments."""
tree = ast.parse(self.code.encode()) tree = ast.parse(self.code.encode())
picker = VariableCommentPicker(self.code.splitlines(True), self.encoding) picker = VariableCommentPicker(self.code.splitlines(True), self.encoding)
@ -507,8 +469,7 @@ class Parser:
self.comments = picker.comments self.comments = picker.comments
self.deforders = picker.deforders self.deforders = picker.deforders
def parse_definition(self): def parse_definition(self) -> None:
# type: () -> None
"""Parse the location of definitions from the code.""" """Parse the location of definitions from the code."""
parser = DefinitionFinder(self.code.splitlines(True)) parser = DefinitionFinder(self.code.splitlines(True))
parser.parse() parser.parse()