2008-12-29 13:22:18 -06:00
|
|
|
"""
|
|
|
|
sphinx.pycode
|
|
|
|
~~~~~~~~~~~~~
|
|
|
|
|
|
|
|
Utilities parsing and analyzing Python code.
|
|
|
|
|
2019-12-31 20:15:42 -06:00
|
|
|
:copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
|
2008-12-29 13:22:18 -06:00
|
|
|
:license: BSD, see LICENSE for details.
|
|
|
|
"""
|
|
|
|
|
2018-09-06 07:30:56 -05:00
|
|
|
import re
|
2019-12-16 09:58:11 -06:00
|
|
|
import tokenize
|
|
|
|
import warnings
|
2019-12-26 10:37:13 -06:00
|
|
|
from importlib import import_module
|
2018-12-15 13:39:14 -06:00
|
|
|
from io import StringIO
|
2019-01-12 06:35:18 -06:00
|
|
|
from os import path
|
2019-12-25 13:29:20 -06:00
|
|
|
from typing import Any, Dict, IO, List, Tuple, Optional
|
2018-09-06 07:30:56 -05:00
|
|
|
from zipfile import ZipFile
|
|
|
|
|
2019-12-16 09:58:11 -06:00
|
|
|
from sphinx.deprecation import RemovedInSphinx40Warning
|
2010-01-13 16:43:43 -06:00
|
|
|
from sphinx.errors import PycodeError
|
2017-07-09 11:20:19 -05:00
|
|
|
from sphinx.pycode.parser import Parser
|
2008-12-29 13:22:18 -06:00
|
|
|
|
|
|
|
|
2018-09-11 08:48:35 -05:00
|
|
|
class ModuleAnalyzer:
|
2008-12-30 05:42:26 -06:00
|
|
|
# cache for analyzer objects -- caches both by module and file name
|
2018-12-14 12:14:11 -06:00
|
|
|
cache = {} # type: Dict[Tuple[str, str], Any]
|
2008-12-29 13:22:18 -06:00
|
|
|
|
2019-12-25 13:29:20 -06:00
|
|
|
@staticmethod
|
|
|
|
def get_module_source(modname: str) -> Tuple[Optional[str], Optional[str]]:
|
|
|
|
"""Try to find the source code for a module.
|
|
|
|
|
|
|
|
Returns ('filename', 'source'). One of it can be None if
|
|
|
|
no filename or source found
|
|
|
|
"""
|
|
|
|
try:
|
|
|
|
mod = import_module(modname)
|
|
|
|
except Exception as err:
|
|
|
|
raise PycodeError('error importing %r' % modname, err)
|
|
|
|
loader = getattr(mod, '__loader__', None)
|
|
|
|
filename = getattr(mod, '__file__', None)
|
|
|
|
if loader and getattr(loader, 'get_source', None):
|
|
|
|
# prefer Native loader, as it respects #coding directive
|
2019-12-26 10:28:16 -06:00
|
|
|
try:
|
2019-12-25 13:29:20 -06:00
|
|
|
source = loader.get_source(modname)
|
|
|
|
if source:
|
|
|
|
# no exception and not None - it must be module source
|
|
|
|
return filename, source
|
2019-12-26 10:28:16 -06:00
|
|
|
except ImportError:
|
2019-12-25 13:29:20 -06:00
|
|
|
pass # Try other "source-mining" methods
|
|
|
|
if filename is None and loader and getattr(loader, 'get_filename', None):
|
|
|
|
# have loader, but no filename
|
|
|
|
try:
|
|
|
|
filename = loader.get_filename(modname)
|
|
|
|
except ImportError as err:
|
|
|
|
raise PycodeError('error getting filename for %r' % modname, err)
|
|
|
|
if filename is None:
|
|
|
|
# all methods for getting filename failed, so raise...
|
|
|
|
raise PycodeError('no source found for module %r' % modname)
|
|
|
|
filename = path.normpath(path.abspath(filename))
|
2019-12-26 10:29:43 -06:00
|
|
|
if filename.lower().endswith(('.pyo', '.pyc')):
|
2019-12-25 13:29:20 -06:00
|
|
|
filename = filename[:-1]
|
|
|
|
if not path.isfile(filename) and path.isfile(filename + 'w'):
|
|
|
|
filename += 'w'
|
2019-12-26 10:29:43 -06:00
|
|
|
elif not filename.lower().endswith(('.py', '.pyw')):
|
2019-12-25 13:29:20 -06:00
|
|
|
raise PycodeError('source is not a .py file: %r' % filename)
|
2019-12-26 10:28:16 -06:00
|
|
|
elif ('.egg' + path.sep) in filename:
|
|
|
|
pat = '(?<=\\.egg)' + re.escape(path.sep)
|
2019-12-25 13:29:20 -06:00
|
|
|
eggpath, _ = re.split(pat, filename, 1)
|
|
|
|
if path.isfile(eggpath):
|
|
|
|
return filename, None
|
|
|
|
|
|
|
|
if not path.isfile(filename):
|
|
|
|
raise PycodeError('source file is not present: %r' % filename)
|
|
|
|
return filename, None
|
2019-12-26 10:28:16 -06:00
|
|
|
|
2008-12-29 13:22:18 -06:00
|
|
|
@classmethod
|
2019-07-06 00:35:17 -05:00
|
|
|
def for_string(cls, string: str, modname: str, srcname: str = '<string>'
|
|
|
|
) -> "ModuleAnalyzer":
|
2018-10-15 12:17:32 -05:00
|
|
|
return cls(StringIO(string), modname, srcname, decoded=True)
|
2008-12-29 13:22:18 -06:00
|
|
|
|
|
|
|
@classmethod
|
2019-07-06 00:35:17 -05:00
|
|
|
def for_file(cls, filename: str, modname: str) -> "ModuleAnalyzer":
|
2008-12-30 05:42:26 -06:00
|
|
|
if ('file', filename) in cls.cache:
|
|
|
|
return cls.cache['file', filename]
|
2008-12-29 13:22:18 -06:00
|
|
|
try:
|
2019-12-16 09:58:11 -06:00
|
|
|
with tokenize.open(filename) as f:
|
|
|
|
obj = cls(f, modname, filename, decoded=True)
|
2017-12-15 06:11:38 -06:00
|
|
|
cls.cache['file', filename] = obj
|
2014-01-19 04:17:10 -06:00
|
|
|
except Exception as err:
|
2019-01-12 06:35:18 -06:00
|
|
|
if '.egg' + path.sep in filename:
|
2018-09-06 07:30:56 -05:00
|
|
|
obj = cls.cache['file', filename] = cls.for_egg(filename, modname)
|
|
|
|
else:
|
|
|
|
raise PycodeError('error opening %r' % filename, err)
|
2008-12-30 05:42:26 -06:00
|
|
|
return obj
|
2008-12-29 13:22:18 -06:00
|
|
|
|
2018-09-06 07:30:56 -05:00
|
|
|
@classmethod
|
2019-07-06 00:35:17 -05:00
|
|
|
def for_egg(cls, filename: str, modname: str) -> "ModuleAnalyzer":
|
2019-01-12 06:35:18 -06:00
|
|
|
SEP = re.escape(path.sep)
|
|
|
|
eggpath, relpath = re.split('(?<=\\.egg)' + SEP, filename)
|
2018-09-06 07:30:56 -05:00
|
|
|
try:
|
|
|
|
with ZipFile(eggpath) as egg:
|
2018-12-15 17:43:44 -06:00
|
|
|
code = egg.read(relpath).decode()
|
2018-09-06 07:30:56 -05:00
|
|
|
return cls.for_string(code, modname, filename)
|
|
|
|
except Exception as exc:
|
|
|
|
raise PycodeError('error opening %r' % filename, exc)
|
|
|
|
|
2008-12-29 13:22:18 -06:00
|
|
|
@classmethod
|
2019-07-06 00:35:17 -05:00
|
|
|
def for_module(cls, modname: str) -> "ModuleAnalyzer":
|
2008-12-30 05:42:26 -06:00
|
|
|
if ('module', modname) in cls.cache:
|
2009-01-04 13:02:24 -06:00
|
|
|
entry = cls.cache['module', modname]
|
|
|
|
if isinstance(entry, PycodeError):
|
|
|
|
raise entry
|
|
|
|
return entry
|
|
|
|
|
|
|
|
try:
|
2019-12-25 13:29:20 -06:00
|
|
|
filename, source = cls.get_module_source(modname)
|
2019-12-15 11:28:02 -06:00
|
|
|
if source is not None:
|
2019-12-26 10:37:13 -06:00
|
|
|
obj = cls.for_string(source, modname, filename or '<string>')
|
2019-12-15 11:28:02 -06:00
|
|
|
elif filename is not None:
|
2019-12-15 12:47:57 -06:00
|
|
|
obj = cls.for_file(filename, modname)
|
2014-01-19 04:17:10 -06:00
|
|
|
except PycodeError as err:
|
2009-01-04 13:02:24 -06:00
|
|
|
cls.cache['module', modname] = err
|
|
|
|
raise
|
2008-12-30 05:42:26 -06:00
|
|
|
cls.cache['module', modname] = obj
|
2008-12-29 19:09:29 -06:00
|
|
|
return obj
|
2008-12-29 13:22:18 -06:00
|
|
|
|
2019-07-06 00:35:17 -05:00
|
|
|
def __init__(self, source: IO, modname: str, srcname: str, decoded: bool = False) -> None:
|
2017-07-09 11:20:19 -05:00
|
|
|
self.modname = modname # name of the module
|
|
|
|
self.srcname = srcname # name of the source file
|
2008-12-30 05:42:26 -06:00
|
|
|
|
2010-01-13 16:43:43 -06:00
|
|
|
# cache the source code as well
|
2017-07-09 11:20:19 -05:00
|
|
|
pos = source.tell()
|
2011-09-19 02:03:07 -05:00
|
|
|
if not decoded:
|
2019-12-16 09:58:11 -06:00
|
|
|
warnings.warn('decode option for ModuleAnalyzer is deprecated.',
|
|
|
|
RemovedInSphinx40Warning)
|
|
|
|
self._encoding, _ = tokenize.detect_encoding(source.readline)
|
2017-07-09 11:20:19 -05:00
|
|
|
source.seek(pos)
|
2019-12-16 09:58:11 -06:00
|
|
|
self.code = source.read().decode(self._encoding)
|
2011-09-19 02:03:07 -05:00
|
|
|
else:
|
2019-12-16 09:58:11 -06:00
|
|
|
self._encoding = None
|
2017-07-09 11:20:19 -05:00
|
|
|
self.code = source.read()
|
2010-01-13 16:43:43 -06:00
|
|
|
|
2008-12-30 05:42:26 -06:00
|
|
|
# will be filled by parse()
|
2018-12-14 12:14:11 -06:00
|
|
|
self.attr_docs = None # type: Dict[Tuple[str, str], List[str]]
|
|
|
|
self.tagorder = None # type: Dict[str, int]
|
|
|
|
self.tags = None # type: Dict[str, Tuple[str, int, int]]
|
2008-12-30 05:42:26 -06:00
|
|
|
|
2019-07-06 00:35:17 -05:00
|
|
|
def parse(self) -> None:
|
2017-07-09 11:20:19 -05:00
|
|
|
"""Parse the source code."""
|
2009-01-10 13:34:26 -06:00
|
|
|
try:
|
2019-12-16 09:58:11 -06:00
|
|
|
parser = Parser(self.code, self._encoding)
|
2017-07-09 11:20:19 -05:00
|
|
|
parser.parse()
|
|
|
|
|
|
|
|
self.attr_docs = {}
|
2018-09-11 07:50:55 -05:00
|
|
|
for (scope, comment) in parser.comments.items():
|
2017-07-09 11:20:19 -05:00
|
|
|
if comment:
|
|
|
|
self.attr_docs[scope] = comment.splitlines() + ['']
|
|
|
|
else:
|
|
|
|
self.attr_docs[scope] = ['']
|
2008-12-30 05:42:26 -06:00
|
|
|
|
2017-07-09 11:20:19 -05:00
|
|
|
self.tags = parser.definitions
|
|
|
|
self.tagorder = parser.deforders
|
|
|
|
except Exception as exc:
|
2017-10-13 07:23:59 -05:00
|
|
|
raise PycodeError('parsing %r failed: %r' % (self.srcname, exc))
|
2017-07-09 11:20:19 -05:00
|
|
|
|
2019-07-06 00:35:17 -05:00
|
|
|
def find_attr_docs(self) -> Dict[Tuple[str, str], List[str]]:
|
2008-12-30 05:42:26 -06:00
|
|
|
"""Find class and module-level attributes and their documentation."""
|
2017-07-09 11:20:19 -05:00
|
|
|
if self.attr_docs is None:
|
|
|
|
self.parse()
|
|
|
|
|
|
|
|
return self.attr_docs
|
2008-12-29 13:22:18 -06:00
|
|
|
|
2019-07-06 00:35:17 -05:00
|
|
|
def find_tags(self) -> Dict[str, Tuple[str, int, int]]:
|
2008-12-30 05:42:26 -06:00
|
|
|
"""Find class, function and method definitions and their location."""
|
2017-07-09 11:20:19 -05:00
|
|
|
if self.tags is None:
|
|
|
|
self.parse()
|
2015-03-08 11:15:54 -05:00
|
|
|
|
2017-07-09 11:20:19 -05:00
|
|
|
return self.tags
|
2019-12-16 09:58:11 -06:00
|
|
|
|
|
|
|
@property
|
|
|
|
def encoding(self) -> str:
|
|
|
|
warnings.warn('ModuleAnalyzer.encoding is deprecated.',
|
|
|
|
RemovedInSphinx40Warning)
|
|
|
|
return self._encoding
|