From 0a982d5ebd1cdb660d07e476aefa0d438e71fcb0 Mon Sep 17 00:00:00 2001 From: hkm Date: Wed, 25 Dec 2019 22:29:20 +0300 Subject: [PATCH] Old get_module_source API restored, new version moved to ModuleAnalyzer class, tests updated --- sphinx/pycode/__init__.py | 54 +++++++++++++++++++++++++++++++++++++-- sphinx/util/__init__.py | 38 ++++++++++++--------------- tests/test_pycode.py | 11 ++++++++ tests/test_util.py | 2 +- 4 files changed, 81 insertions(+), 24 deletions(-) diff --git a/sphinx/pycode/__init__.py b/sphinx/pycode/__init__.py index 483eed432..2e077a618 100644 --- a/sphinx/pycode/__init__.py +++ b/sphinx/pycode/__init__.py @@ -11,8 +11,9 @@ import re from io import StringIO from os import path -from typing import Any, Dict, IO, List, Tuple +from typing import Any, Dict, IO, List, Tuple, Optional from zipfile import ZipFile +from importlib import import_module from sphinx.errors import PycodeError from sphinx.pycode.parser import Parser @@ -23,6 +24,55 @@ class ModuleAnalyzer: # cache for analyzer objects -- caches both by module and file name cache = {} # type: Dict[Tuple[str, str], Any] + @staticmethod + def get_module_source(modname: str) -> Tuple[Optional[str], Optional[str]]: + """Try to find the source code for a module. + + Returns ('filename', 'source'). One of it can be None if + no filename or source found + """ + try: + mod = import_module(modname) + except Exception as err: + raise PycodeError('error importing %r' % modname, err) + loader = getattr(mod, '__loader__', None) + filename = getattr(mod, '__file__', None) + if loader and getattr(loader, 'get_source', None): + # prefer Native loader, as it respects #coding directive + try: + source = loader.get_source(modname) + if source: + # no exception and not None - it must be module source + return filename, source + except ImportError as err: + pass # Try other "source-mining" methods + if filename is None and loader and getattr(loader, 'get_filename', None): + # have loader, but no filename + try: + filename = loader.get_filename(modname) + except ImportError as err: + raise PycodeError('error getting filename for %r' % modname, err) + if filename is None: + # all methods for getting filename failed, so raise... + raise PycodeError('no source found for module %r' % modname) + filename = path.normpath(path.abspath(filename)) + lfilename = filename.lower() + if lfilename.endswith('.pyo') or lfilename.endswith('.pyc'): + filename = filename[:-1] + if not path.isfile(filename) and path.isfile(filename + 'w'): + filename += 'w' + elif not (lfilename.endswith('.py') or lfilename.endswith('.pyw')): + raise PycodeError('source is not a .py file: %r' % filename) + elif ('.egg' + os.path.sep) in filename: + pat = '(?<=\\.egg)' + re.escape(os.path.sep) + eggpath, _ = re.split(pat, filename, 1) + if path.isfile(eggpath): + return filename, None + + if not path.isfile(filename): + raise PycodeError('source file is not present: %r' % filename) + return filename, None + @classmethod def for_string(cls, string: str, modname: str, srcname: str = '' ) -> "ModuleAnalyzer": @@ -63,7 +113,7 @@ class ModuleAnalyzer: return entry try: - filename, source = get_module_source(modname) + filename, source = cls.get_module_source(modname) if source is not None: obj = cls.for_string(source, modname, filename if filename is not None else '') elif filename is not None: diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index 81bc51254..19ffec633 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -25,7 +25,7 @@ from hashlib import md5 from importlib import import_module from os import path from time import mktime, strptime -from typing import Any, Callable, Dict, IO, Iterable, Iterator, List, Pattern, Set, Tuple, Optional +from typing import Any, Callable, Dict, IO, Iterable, Iterator, List, Pattern, Set, Tuple from urllib.parse import urlsplit, urlunsplit, quote_plus, parse_qsl, urlencode from docutils.utils import relative_path @@ -265,35 +265,31 @@ def save_traceback(app: "Sphinx") -> str: return path -def get_module_source(modname: str) -> Tuple[Optional[str], Optional[str]]: +def get_module_source(modname: str) -> Tuple[str, str]: """Try to find the source code for a module. - Returns ('filename', 'source'). One of it can be None if - no filename or source found + Can return ('file', 'filename') in which case the source is in the given + file, or ('string', 'source') which which case the source is the string. """ try: mod = import_module(modname) except Exception as err: raise PycodeError('error importing %r' % modname, err) - loader = getattr(mod, '__loader__', None) filename = getattr(mod, '__file__', None) - if loader and getattr(loader, 'get_source', None): - # prefer Native loader, as it respects #coding directive - try: - source = loader.get_source(modname) - if source: - # no exception and not None - it must be module source - return filename, source - except ImportError as err: - pass # Try other "source-mining" methods - if filename is None and loader and getattr(loader, 'get_filename', None): - # have loader, but no filename + loader = getattr(mod, '__loader__', None) + if loader and getattr(loader, 'get_filename', None): try: filename = loader.get_filename(modname) - except ImportError as err: - raise PycodeError('error getting filename for %r' % modname, err) + except Exception as err: + raise PycodeError('error getting filename for %r' % filename, err) + if filename is None and loader: + try: + filename = loader.get_source(modname) + if filename: + return 'string', filename + except Exception as err: + raise PycodeError('error getting source for %r' % modname, err) if filename is None: - # all methods for getting filename failed, so raise... raise PycodeError('no source found for module %r' % modname) filename = path.normpath(path.abspath(filename)) lfilename = filename.lower() @@ -307,11 +303,11 @@ def get_module_source(modname: str) -> Tuple[Optional[str], Optional[str]]: pat = '(?<=\\.egg)' + re.escape(os.path.sep) eggpath, _ = re.split(pat, filename, 1) if path.isfile(eggpath): - return filename, None + return 'file', filename if not path.isfile(filename): raise PycodeError('source file is not present: %r' % filename) - return filename, None + return 'file', filename def get_full_modname(modname: str, attribute: str) -> str: diff --git a/tests/test_pycode.py b/tests/test_pycode.py index cd039070c..be61d9efb 100644 --- a/tests/test_pycode.py +++ b/tests/test_pycode.py @@ -10,12 +10,23 @@ import os import sys +import pytest import sphinx from sphinx.pycode import ModuleAnalyzer +from sphinx.errors import PycodeError SPHINX_MODULE_PATH = os.path.splitext(sphinx.__file__)[0] + '.py' +def test_ModuleAnalyzer_get_module_source(): + assert ModuleAnalyzer.get_module_source('sphinx') == (sphinx.__file__, sphinx.__loader__.get_source('sphinx')) + + # failed to obtain source information from builtin modules + with pytest.raises(PycodeError): + ModuleAnalyzer.get_module_source('builtins') + with pytest.raises(PycodeError): + ModuleAnalyzer.get_module_source('itertools') + def test_ModuleAnalyzer_for_string(): analyzer = ModuleAnalyzer.for_string('print("Hello world")', 'module_name') diff --git a/tests/test_util.py b/tests/test_util.py index 4f9317df2..44a41dca1 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -62,7 +62,7 @@ def test_display_chunk(): def test_get_module_source(): - assert get_module_source('sphinx') == (sphinx.__file__, sphinx.__loader__.get_source('sphinx')) + assert get_module_source('sphinx') == ('file', sphinx.__file__) # failed to obtain source information from builtin modules with pytest.raises(PycodeError):