From 83f30712b8affef9c5bc6bcc52c4d70797856ece Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 17 Jan 2018 15:34:33 -0500 Subject: [PATCH 1/5] fix ext.apidoc to include "empty" packages that contain modules Commit 2d99648e9982325bbd670da11df5f809e3134284 changed the apidoc extension to ignore packages if the __init__.py file was empty. That breaks the toctree structure if those packages do contain submodules and subpackages. This patch adds a check to ensure that empty __init__.py modules are only skipped if there are no other python modules in the same directory. Addresses bug #654 Signed-off-by: Doug Hellmann --- sphinx/apidoc.py | 13 +++- .../test-apidoc-pep420/a/b/e/__init__.py | 0 tests/roots/test-apidoc-pep420/a/b/e/f.py | 1 + tests/test_apidoc.py | 60 +++++++++++++++++++ 4 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/roots/test-apidoc-pep420/a/b/e/__init__.py create mode 100644 tests/roots/test-apidoc-pep420/a/b/e/f.py diff --git a/sphinx/apidoc.py b/sphinx/apidoc.py index 3b51469d1..06cbfa026 100644 --- a/sphinx/apidoc.py +++ b/sphinx/apidoc.py @@ -16,6 +16,7 @@ """ from __future__ import print_function +import glob import os import sys import optparse @@ -194,7 +195,17 @@ def shall_skip(module, opts): # skip it if there is nothing (or just \n or \r\n) in the file if path.exists(module) and path.getsize(module) <= 2: - return True + skip = True + if os.path.basename(module) == '__init__.py': + pattern = path.join(path.dirname(module), '*.py') + # We only want to skip packages if they do not contain any + # .py files other than __init__.py. + other_modules = list(glob.glob(pattern)) + other_modules.remove(module) + skip = not other_modules + + if skip: + return True # skip if it has a "private" name and this is selected filename = path.basename(module) diff --git a/tests/roots/test-apidoc-pep420/a/b/e/__init__.py b/tests/roots/test-apidoc-pep420/a/b/e/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-apidoc-pep420/a/b/e/f.py b/tests/roots/test-apidoc-pep420/a/b/e/f.py new file mode 100644 index 000000000..a09affe86 --- /dev/null +++ b/tests/roots/test-apidoc-pep420/a/b/e/f.py @@ -0,0 +1 @@ +"Module f" diff --git a/tests/test_apidoc.py b/tests/test_apidoc.py index f5037ab71..e7f986df3 100644 --- a/tests/test_apidoc.py +++ b/tests/test_apidoc.py @@ -67,6 +67,7 @@ def test_pep_0420_enabled(make_app, apidoc): outdir = apidoc.outdir assert (outdir / 'conf.py').isfile() assert (outdir / 'a.b.c.rst').isfile() + assert (outdir / 'a.b.e.rst').isfile() assert (outdir / 'a.b.x.rst').isfile() with open(outdir / 'a.b.c.rst') as f: @@ -74,6 +75,10 @@ def test_pep_0420_enabled(make_app, apidoc): assert "automodule:: a.b.c.d\n" in rst assert "automodule:: a.b.c\n" in rst + with open(outdir / 'a.b.e.rst') as f: + rst = f.read() + assert "automodule:: a.b.e.f\n" in rst + with open(outdir / 'a.b.x.rst') as f: rst = f.read() assert "automodule:: a.b.x.y\n" in rst @@ -86,12 +91,67 @@ def test_pep_0420_enabled(make_app, apidoc): builddir = outdir / '_build' / 'text' assert (builddir / 'a.b.c.txt').isfile() + assert (builddir / 'a.b.e.txt').isfile() assert (builddir / 'a.b.x.txt').isfile() with open(builddir / 'a.b.c.txt') as f: txt = f.read() assert "a.b.c package\n" in txt + with open(builddir / 'a.b.e.txt') as f: + txt = f.read() + assert "a.b.e.f module\n" in txt + + with open(builddir / 'a.b.x.txt') as f: + txt = f.read() + assert "a.b.x namespace\n" in txt + + +@pytest.mark.apidoc( + coderoot='test-apidoc-pep420', + options=["--implicit-namespaces", "--separate"], +) +def test_pep_0420_enabled_separate(make_app, apidoc): + outdir = apidoc.outdir + assert (outdir / 'conf.py').isfile() + assert (outdir / 'a.b.c.rst').isfile() + assert (outdir / 'a.b.e.rst').isfile() + assert (outdir / 'a.b.e.f.rst').isfile() + assert (outdir / 'a.b.x.rst').isfile() + assert (outdir / 'a.b.x.y.rst').isfile() + + with open(outdir / 'a.b.c.rst') as f: + rst = f.read() + assert ".. toctree::\n\n a.b.c.d\n" in rst + + with open(outdir / 'a.b.e.rst') as f: + rst = f.read() + assert ".. toctree::\n\n a.b.e.f\n" in rst + + with open(outdir / 'a.b.x.rst') as f: + rst = f.read() + assert ".. toctree::\n\n a.b.x.y\n" in rst + + app = make_app('text', srcdir=outdir) + app.build() + print(app._status.getvalue()) + print(app._warning.getvalue()) + + builddir = outdir / '_build' / 'text' + assert (builddir / 'a.b.c.txt').isfile() + assert (builddir / 'a.b.e.txt').isfile() + assert (builddir / 'a.b.e.f.txt').isfile() + assert (builddir / 'a.b.x.txt').isfile() + assert (builddir / 'a.b.x.y.txt').isfile() + + with open(builddir / 'a.b.c.txt') as f: + txt = f.read() + assert "a.b.c package\n" in txt + + with open(builddir / 'a.b.e.f.txt') as f: + txt = f.read() + assert "a.b.e.f module\n" in txt + with open(builddir / 'a.b.x.txt') as f: txt = f.read() assert "a.b.x namespace\n" in txt From 1591607481469b6acf7ae2d5fbd8329d2352ab9b Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 23 Jan 2018 10:58:55 +0900 Subject: [PATCH 2/5] Fix #4477: Build fails after building specific files --- CHANGES | 1 + sphinx/builders/html.py | 103 ++++++++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 36 deletions(-) diff --git a/CHANGES b/CHANGES index 1d9a5fa82..63e4a26c9 100644 --- a/CHANGES +++ b/CHANGES @@ -21,6 +21,7 @@ Bugs fixed * #4438: math: math with labels with whitespace cause html error * #2437: make full reference for classes, aliased with "alias of" * #4434: pure numbers as link targets produce warning +* #4477: Build fails after building specific files Testing -------- diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 68f38320b..a06313e0f 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -54,9 +54,11 @@ from sphinx.environment.adapters.indexentries import IndexEntries if False: # For type annotation - from typing import Any, Dict, Iterable, Iterator, List, Type, Tuple, Union # NOQA - from sphinx.domains import Domain, Index # NOQA + from typing import Any, Dict, IO, Iterable, Iterator, List, Type, Tuple, Union # NOQA from sphinx.application import Sphinx # NOQA + from sphinx.config import Config # NOQA + from sphinx.domains import Domain, Index # NOQA + from sphinx.util.tags import Tags # NOQA # Experimental HTML5 Writer if is_html5_writer_available(): @@ -147,6 +149,56 @@ class Stylesheet(text_type): return self +class BuildInfo(object): + """buildinfo file manipulator. + + HTMLBuilder and its family are storing their own envdata to ``.buildinfo``. + This class is a manipulator for the file. + """ + + @classmethod + def load(cls, f): + # type: (IO) -> BuildInfo + try: + lines = f.readlines() + assert lines[0].rstrip() == '# Sphinx build info version 1' + assert lines[2].startswith('config: ') + assert lines[3].startswith('tags: ') + + build_info = BuildInfo() + build_info.config_hash = lines[2].split()[1].strip() + build_info.tags_hash = lines[3].split()[1].strip() + return build_info + except Exception as exc: + raise ValueError('build info file is broken: %r' % exc) + + def __init__(self, config=None, tags=None): + # type: (Config, Tags) -> None + self.config_hash = u'' + self.tags_hash = u'' + + if config: + values = dict((c.name, c.value) for c in config.filter('html')) + self.config_hash = get_stable_hash(values) + + if tags: + self.tags_hash = get_stable_hash(sorted(tags)) + + def __eq__(self, other): # type: ignore + # type: (BuildInfo) -> bool + return (self.config_hash == other.config_hash and + self.tags_hash == other.tags_hash) + + def dump(self, f): + # type: (IO) -> None + f.write('# Sphinx build info version 1\n' + '# This file hashes the configuration used when building these files.' + ' When it is not found, a full rebuild will be done.\n' + 'config: %s\n' + 'tags: %s\n' % + (self.config_hash, self.tags_hash)) + + class StandaloneHTMLBuilder(Builder): """ Builds standalone HTML docs. @@ -189,9 +241,7 @@ class StandaloneHTMLBuilder(Builder): def init(self): # type: () -> None - # a hash of all config values that, if changed, cause a full rebuild - self.config_hash = '' # type: unicode - self.tags_hash = '' # type: unicode + self.build_info = BuildInfo(self.config, self.tags) # basename of images directory self.imagedir = '_images' # section numbers for headings in the currently visited document @@ -272,32 +322,19 @@ class StandaloneHTMLBuilder(Builder): def get_outdated_docs(self): # type: () -> Iterator[unicode] - cfgdict = dict((confval.name, confval.value) for confval in self.config.filter('html')) - self.config_hash = get_stable_hash(cfgdict) - self.tags_hash = get_stable_hash(sorted(self.tags)) - old_config_hash = old_tags_hash = '' try: with open(path.join(self.outdir, '.buildinfo')) as fp: - version = fp.readline() - if version.rstrip() != '# Sphinx build info version 1': - raise ValueError - fp.readline() # skip commentary - cfg, old_config_hash = fp.readline().strip().split(': ') - if cfg != 'config': - raise ValueError - tag, old_tags_hash = fp.readline().strip().split(': ') - if tag != 'tags': - raise ValueError - except ValueError: - logger.warning('unsupported build info format in %r, building all', - path.join(self.outdir, '.buildinfo')) - except Exception: + buildinfo = BuildInfo.load(fp) + + if self.build_info != buildinfo: + for docname in self.env.found_docs: + yield docname + return + except ValueError as exc: + logger.warning('Failed to read build info file: %r', exc) + except IOError: + # ignore errors on reading pass - if old_config_hash != self.config_hash or \ - old_tags_hash != self.tags_hash: - for docname in self.env.found_docs: - yield docname - return if self.templates: template_mtime = self.templates.newest_template_mtime() @@ -775,14 +812,9 @@ class StandaloneHTMLBuilder(Builder): def write_buildinfo(self): # type: () -> None - # write build info file try: with open(path.join(self.outdir, '.buildinfo'), 'w') as fp: - fp.write('# Sphinx build info version 1\n' - '# This file hashes the configuration used when building' - ' these files. When it is not found, a full rebuild will' - ' be done.\nconfig: %s\ntags: %s\n' % - (self.config_hash, self.tags_hash)) + self.build_info.dump(fp) except IOError as exc: logger.warning('Failed to write build info file: %r', exc) @@ -1235,8 +1267,7 @@ class SerializingHTMLBuilder(StandaloneHTMLBuilder): def init(self): # type: () -> None - self.config_hash = '' - self.tags_hash = '' + self.build_info = BuildInfo(self.config, self.tags) self.imagedir = '_images' self.current_docname = None self.theme = None # no theme necessary From b6efff799069666d035932f1b413ae58627a1375 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 25 Jan 2018 01:18:02 +0900 Subject: [PATCH 3/5] Update CHANGS for PR #4449 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 63e4a26c9..b265ba2c1 100644 --- a/CHANGES +++ b/CHANGES @@ -22,6 +22,7 @@ Bugs fixed * #2437: make full reference for classes, aliased with "alias of" * #4434: pure numbers as link targets produce warning * #4477: Build fails after building specific files +* #4449: apidoc: include "empty" packages that contain modules Testing -------- From bcdea4da5cd46be8399ed979ab878cdc78469c5f Mon Sep 17 00:00:00 2001 From: Rouven Czerwinski Date: Wed, 24 Jan 2018 17:29:24 +0100 Subject: [PATCH 4/5] autodoc: prefer _MockImporter over other importers in sys.path In case we want to mock modules which are actually available in the current environment, we need to add the _MockImporter before other importers. We ran into this problem in our application, where importing the existing modules caused side effects which were not fixed by autodoc_mock_imports. --- sphinx/ext/autodoc/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/autodoc/importer.py b/sphinx/ext/autodoc/importer.py index cea1c12bd..ee43784ca 100644 --- a/sphinx/ext/autodoc/importer.py +++ b/sphinx/ext/autodoc/importer.py @@ -87,7 +87,7 @@ class _MockImporter(object): self.names = names self.mocked_modules = [] # type: List[str] # enable hook by adding itself to meta_path - sys.meta_path = sys.meta_path + [self] + sys.meta_path.insert(0, self) def disable(self): # remove `self` from `sys.meta_path` to disable import hook From 6897b80fc4357ef99d2f5398d872f35e06c15af6 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 27 Jan 2018 22:08:54 +0900 Subject: [PATCH 5/5] Update CHANGES for PR #4491 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 22a5f5936..f4d28a243 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,7 @@ Bugs fixed * #4415: autodoc classifies inherited classmethods as regular methods * #4415: autodoc classifies inherited staticmethods as regular methods * #4472: DOCUMENTATION_OPTIONS is not defined +* #4491: autodoc: prefer _MockImporter over other importers in sys.meta_path Testing --------