Add sphinx.project; a manipulator for project and documents

doc/extdev/appapi.rst

@@ -120,6 +120,10 @@ Sphinx runtime information
 
 The application object also provides runtime information as attributes.
 
+.. attribute:: Sphinx.project
+
+   Target project.  See :class:`.Project`.
+
 .. attribute:: Sphinx.srcdir
 
    Source directory.

doc/extdev/envapi.rst

@@ -15,6 +15,10 @@ Build environment API
 
       Reference to the :class:`.Config` object.
 
+   .. attribute:: project
+
+      Target project.  See :class:`.Project`.
+
    .. attribute:: srcdir
 
       Source directory.

doc/extdev/index.rst

@@ -85,6 +85,7 @@ APIs used for writing extensions
 
    tutorial
    appapi
+   projectapi
    envapi
    builderapi
    collectorapi

doc/extdev/projectapi.rst

@@ -0,0 +1,9 @@
+.. _project-api:
+
+Project API
+===========
+
+.. currentmodule:: sphinx.project
+
+.. autoclass:: Project
+   :members:

sphinx/application.py

@@ -34,6 +34,7 @@ from sphinx.environment import BuildEnvironment
 from sphinx.errors import ApplicationError, ConfigError, VersionRequirementError
 from sphinx.events import EventManager
 from sphinx.locale import __
+from sphinx.project import Project
 from sphinx.registry import SphinxComponentRegistry
 from sphinx.util import docutils
 from sphinx.util import import_object
@@ -137,6 +138,7 @@ class Sphinx:
         self._setting_up_extension = ['?']      # type: List[unicode]
         self.builder = None                     # type: Builder
         self.env = None                         # type: BuildEnvironment
+        self.project = None                     # type: Project
         self.registry = SphinxComponentRegistry()
         self.html_themes = {}                   # type: Dict[unicode, unicode]
 
@@ -249,6 +251,8 @@ class Sphinx:
         self.config.init_values()
         self.emit('config-inited', self.config)
 
+        # create the project
+        self.project = Project(self.srcdir, self.config.source_suffix)
         # create the builder
         self.builder = self.create_builder(buildername)
         # set up the build environment

sphinx/environment/__init__.py

@@ -24,11 +24,10 @@ from sphinx.environment.adapters.toctree import TocTree
 from sphinx.errors import SphinxError, BuildEnvironmentError, DocumentError, ExtensionError
 from sphinx.locale import __
 from sphinx.transforms import SphinxTransformer
-from sphinx.util import get_matching_docs, DownloadFiles, FilenameUniqDict
+from sphinx.util import DownloadFiles, FilenameUniqDict
 from sphinx.util import logging
 from sphinx.util.docutils import LoggingReporter
 from sphinx.util.i18n import find_catalog_files
-from sphinx.util.matching import compile_matchers
 from sphinx.util.nodes import is_translatable
 from sphinx.util.osutil import SEP, relpath
 from sphinx.util.websupport import is_commentable
@@ -41,6 +40,7 @@ if False:
     from sphinx.builders import Builder  # NOQA
     from sphinx.config import Config  # NOQA
     from sphinx.domains import Domain  # NOQA
+    from sphinx.project import Project  # NOQA
 
 logger = logging.getLogger(__name__)
 
@@ -106,6 +106,7 @@ class BuildEnvironment:
         self.srcdir = None          # type: unicode
         self.config = None          # type: Config
         self.config_status = None   # type: int
+        self.project = None         # type: Project
         self.version = None         # type: Dict[unicode, unicode]
 
         # the method of doctree versioning; see set_versioning_method
@@ -122,8 +123,6 @@ class BuildEnvironment:
         # All "docnames" here are /-separated and relative and exclude
         # the source suffix.
 
-        self.found_docs = set()     # type: Set[unicode]
-                                    # contains all existing docnames
         self.all_docs = {}          # type: Dict[unicode, float]
                                     # docname -> mtime at the time of reading
                                     # contains all read docnames
@@ -217,9 +216,13 @@ class BuildEnvironment:
         elif self.srcdir and self.srcdir != app.srcdir:
             raise BuildEnvironmentError(__('source directory has changed'))
 
+        if self.project:
+            app.project.restore(self.project)
+
         self.app = app
         self.doctreedir = app.doctreedir
         self.srcdir = app.srcdir
+        self.project = app.project
         self.version = app.registry.get_envversion(app)
 
         # initialize domains
@@ -386,25 +389,22 @@ class BuildEnvironment:
             enc_rel_fn = rel_fn.encode(sys.getfilesystemencoding())
             return rel_fn, path.abspath(path.join(self.srcdir, enc_rel_fn))
 
+    @property
+    def found_docs(self):
+        # type: () -> Set[unicode]
+        """contains all existing docnames."""
+        return self.project.docnames
+
     def find_files(self, config, builder):
         # type: (Config, Builder) -> None
         """Find all source files in the source dir and put them in
         self.found_docs.
         """
         try:
-            matchers = compile_matchers(
+            exclude_paths = (self.config.exclude_patterns +
-                config.exclude_patterns[:] +
+                             self.config.templates_path +
-                config.templates_path +
+                             builder.get_asset_paths())
-                builder.get_asset_paths() +
+            self.project.discovery(exclude_paths)
-                ['**/_sources', '.#*', '**/.#*', '*.lproj/**']
-            )
-            self.found_docs = set()
-            for docname in get_matching_docs(self.srcdir, config.source_suffix,  # type: ignore
-                                             exclude_matchers=matchers):
-                if os.access(self.doc2path(docname), os.R_OK):
-                    self.found_docs.add(docname)
-                else:
-                    logger.warning(__("document not readable. Ignored."), location=docname)
 
             # Current implementation is applying translated messages in the reading
             # phase.Therefore, in order to apply the updated message catalog, it is

sphinx/project.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.project
+    ~~~~~~~~~~~~~~
+
+    Utility function and classes for Sphinx projects.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+from typing import TYPE_CHECKING
+
+from sphinx.locale import __
+from sphinx.util import get_matching_files
+from sphinx.util import logging
+from sphinx.util.matching import compile_matchers
+
+if TYPE_CHECKING:
+    from typing import Dict, List, Set  # NOQA
+
+logger = logging.getLogger(__name__)
+EXCLUDE_PATHS = ['**/_sources', '.#*', '**/.#*', '*.lproj/**']  # type: List[unicode]
+
+
+class Project(object):
+    """A project is source code set of Sphinx document."""
+
+    def __init__(self, srcdir, source_suffix):
+        # type: (unicode, Dict[unicode, unicode]) -> None
+        #: Source directory.
+        self.srcdir = srcdir
+
+        #: source_suffix. Same as :confval:`source_suffix`.
+        self.source_suffix = source_suffix
+
+        #: The name of documents belongs to this project.
+        self.docnames = set()  # type: Set[unicode]
+
+    def restore(self, other):
+        # type: (Project) -> None
+        """Take over a result of last build."""
+        self.docnames = other.docnames
+
+    def discovery(self, exclude_paths=[]):
+        # type: (List[unicode]) -> Set[unicode]
+        """Find all document files in the source directory and put them in
+        :attr:`docnames`.
+        """
+        self.docnames = set()
+        excludes = compile_matchers(exclude_paths + EXCLUDE_PATHS)
+        for filename in get_matching_files(self.srcdir, excludes):  # type: ignore
+            for suffix in self.source_suffix:
+                if filename.endswith(suffix):
+                    docname = filename[:-len(suffix)]
+                    if os.access(os.path.join(self.srcdir, filename), os.R_OK):
+                        self.docnames.add(docname)
+                    else:
+                        logger.warning(__("document not readable. Ignored."), location=docname)
+
+        return self.docnames

tests/test_project.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""
+    test_project
+    ~~~~~~~~~~~~
+
+    Tests project module.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import pytest
+
+from sphinx.project import Project
+
+
+def test_project_discovery(rootdir):
+    project = Project(rootdir / 'test-root', {})
+
+    docnames = {'autodoc', 'bom', 'extapi', 'extensions', 'footnote', 'images',
+                'includes', 'index', 'lists', 'markup', 'math', 'objects',
+                'subdir/excluded', 'subdir/images', 'subdir/includes'}
+    subdir_docnames = {'subdir/excluded', 'subdir/images', 'subdir/includes'}
+
+    # basic case
+    project.source_suffix = ['.txt']
+    assert project.discovery() == docnames
+
+    # exclude_paths option
+    assert project.discovery(['subdir/*']) == docnames - subdir_docnames
+
+    # exclude_patterns
+    assert project.discovery(['.txt', 'subdir/*']) == docnames - subdir_docnames
+
+    # multiple source_suffixes
+    project.source_suffix = ['.txt', '.foo']
+    assert project.discovery() == docnames | {'otherext'}
+
+    # complicated source_suffix
+    project.source_suffix = ['.foo.png']
+    assert project.discovery() == {'img'}
+
+    # templates_path
+    project.source_suffix = ['.html']
+    assert project.discovery() == {'_templates/layout',
+                                   '_templates/customsb',
+                                   '_templates/contentssb'}
+
+    assert project.discovery(['_templates']) == set()