Merge pull request #5369 from tk0miya/refactor_env2

Add project object

CHANGES

@@ -23,6 +23,7 @@ Deprecated
 * The string style ``base`` argument of ``env.doc2path()`` is deprecated.
 * ``sphinx.ext.doctest.doctest_encode()``
 * ``sphinx.testing.util.remove_unicode_literal()``
+* ``sphinx.util.get_matching_docs()`` is deprecated
 * ``sphinx.util.osutil.walk()``
 * ``translatablestrings`` variable for LaTeX template
 

doc/extdev/appapi.rst

@@ -120,6 +120,10 @@ Sphinx runtime information
 
 The application object also provides runtime information as attributes.
 
+.. attribute:: Sphinx.project
+
+   Target project.  See :class:`.Project`.
+
 .. attribute:: Sphinx.srcdir
 
    Source directory.

doc/extdev/envapi.rst

@@ -15,6 +15,10 @@ Build environment API
 
       Reference to the :class:`.Config` object.
 
+   .. attribute:: project
+
+      Target project.  See :class:`.Project`.
+
    .. attribute:: srcdir
 
       Source directory.

doc/extdev/index.rst

@@ -85,6 +85,7 @@ APIs used for writing extensions
 
    tutorial
    appapi
+   projectapi
    envapi
    builderapi
    collectorapi
@@ -142,6 +143,11 @@ The following is a list of deprecated interfaces.
      - 4.0
      - N/A
 
+   * - ``sphinx.util.get_matching_docs()``
+     - 2.0
+     - 4.0
+     - ``sphinx.util.get_matching_files()``
+
    * - ``sphinx.util.osutil.walk()``
      - 2.0
      - 4.0

doc/extdev/projectapi.rst

@@ -0,0 +1,9 @@
+.. _project-api:
+
+Project API
+===========
+
+.. currentmodule:: sphinx.project
+
+.. autoclass:: Project
+   :members:

sphinx/application.py

@@ -34,6 +34,7 @@ from sphinx.environment import BuildEnvironment
 from sphinx.errors import ApplicationError, ConfigError, VersionRequirementError
 from sphinx.events import EventManager
 from sphinx.locale import __
+from sphinx.project import Project
 from sphinx.registry import SphinxComponentRegistry
 from sphinx.util import docutils
 from sphinx.util import import_object
@@ -137,6 +138,7 @@ class Sphinx:
         self._setting_up_extension = ['?']      # type: List[unicode]
         self.builder = None                     # type: Builder
         self.env = None                         # type: BuildEnvironment
+        self.project = None                     # type: Project
         self.registry = SphinxComponentRegistry()
         self.html_themes = {}                   # type: Dict[unicode, unicode]
 
@@ -249,6 +251,8 @@ class Sphinx:
         self.config.init_values()
         self.emit('config-inited', self.config)
 
+        # create the project
+        self.project = Project(self.srcdir, self.config.source_suffix)
         # create the builder
         self.builder = self.create_builder(buildername)
         # set up the build environment

sphinx/environment/__init__.py

@@ -24,13 +24,11 @@ from sphinx.environment.adapters.toctree import TocTree
 from sphinx.errors import SphinxError, BuildEnvironmentError, DocumentError, ExtensionError
 from sphinx.locale import __
 from sphinx.transforms import SphinxTransformer
-from sphinx.util import get_matching_docs, DownloadFiles, FilenameUniqDict
+from sphinx.util import DownloadFiles, FilenameUniqDict
 from sphinx.util import logging
 from sphinx.util.docutils import LoggingReporter
 from sphinx.util.i18n import find_catalog_files
-from sphinx.util.matching import compile_matchers
 from sphinx.util.nodes import is_translatable
-from sphinx.util.osutil import SEP, relpath
 from sphinx.util.websupport import is_commentable
 
 if False:
@@ -41,6 +39,7 @@ if False:
     from sphinx.builders import Builder  # NOQA
     from sphinx.config import Config  # NOQA
     from sphinx.domains import Domain  # NOQA
+    from sphinx.project import Project  # NOQA
 
 logger = logging.getLogger(__name__)
 
@@ -106,6 +105,7 @@ class BuildEnvironment:
         self.srcdir = None          # type: unicode
         self.config = None          # type: Config
         self.config_status = None   # type: int
+        self.project = None         # type: Project
         self.version = None         # type: Dict[unicode, unicode]
 
         # the method of doctree versioning; see set_versioning_method
@@ -122,8 +122,6 @@ class BuildEnvironment:
         # All "docnames" here are /-separated and relative and exclude
         # the source suffix.
 
-        self.found_docs = set()     # type: Set[unicode]
-                                    # contains all existing docnames
         self.all_docs = {}          # type: Dict[unicode, float]
                                     # docname -> mtime at the time of reading
                                     # contains all read docnames
@@ -217,9 +215,13 @@ class BuildEnvironment:
         elif self.srcdir and self.srcdir != app.srcdir:
             raise BuildEnvironmentError(__('source directory has changed'))
 
+        if self.project:
+            app.project.restore(self.project)
+
         self.app = app
         self.doctreedir = app.doctreedir
         self.srcdir = app.srcdir
+        self.project = app.project
         self.version = app.registry.get_envversion(app)
 
         # initialize domains
@@ -322,12 +324,7 @@ class BuildEnvironment:
 
         *filename* should be absolute or relative to the source directory.
         """
-        if filename.startswith(self.srcdir):
+        return self.project.path2doc(filename)
-            filename = relpath(filename, self.srcdir)
-        for suffix in self.config.source_suffix:
-            if filename.endswith(suffix):
-                return filename[:-len(suffix)]
-        return None
 
     def doc2path(self, docname, base=True, suffix=None):
         # type: (unicode, Union[bool, unicode], unicode) -> unicode
@@ -345,21 +342,13 @@ class BuildEnvironment:
             warnings.warn('The string style base argument for doc2path() is deprecated.',
                           RemovedInSphinx40Warning)
 
-        docname = docname.replace(SEP, path.sep)
+        pathname = self.project.doc2path(docname, base is True)
-        if suffix is None:
+        if suffix:
-            # Use first candidate if there is not a file for any suffix
+            filename, _ = path.splitext(pathname)
-            suffix = next(iter(self.config.source_suffix))
+            pathname = filename + suffix
-            for candidate_suffix in self.config.source_suffix:
+        if base and base is not True:
-                if path.isfile(path.join(self.srcdir, docname) +
+            pathname = path.join(base, pathname)  # type: ignore
-                               candidate_suffix):
+        return pathname
-                    suffix = candidate_suffix
-                    break
-        if base is True:
-            return path.join(self.srcdir, docname) + suffix
-        elif base is None:
-            return docname + suffix
-        else:
-            return path.join(base, docname) + suffix  # type: ignore
 
     def relfn2path(self, filename, docname=None):
         # type: (unicode, unicode) -> Tuple[unicode, unicode]
@@ -386,25 +375,22 @@ class BuildEnvironment:
             enc_rel_fn = rel_fn.encode(sys.getfilesystemencoding())
             return rel_fn, path.abspath(path.join(self.srcdir, enc_rel_fn))
 
+    @property
+    def found_docs(self):
+        # type: () -> Set[unicode]
+        """contains all existing docnames."""
+        return self.project.docnames
+
     def find_files(self, config, builder):
         # type: (Config, Builder) -> None
         """Find all source files in the source dir and put them in
         self.found_docs.
         """
         try:
-            matchers = compile_matchers(
+            exclude_paths = (self.config.exclude_patterns +
-                config.exclude_patterns[:] +
+                             self.config.templates_path +
-                config.templates_path +
+                             builder.get_asset_paths())
-                builder.get_asset_paths() +
+            self.project.discover(exclude_paths)
-                ['**/_sources', '.#*', '**/.#*', '*.lproj/**']
-            )
-            self.found_docs = set()
-            for docname in get_matching_docs(self.srcdir, config.source_suffix,  # type: ignore
-                                             exclude_matchers=matchers):
-                if os.access(self.doc2path(docname), os.R_OK):
-                    self.found_docs.add(docname)
-                else:
-                    logger.warning(__("document not readable. Ignored."), location=docname)
 
             # Current implementation is applying translated messages in the reading
             # phase.Therefore, in order to apply the updated message catalog, it is

sphinx/project.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.project
+    ~~~~~~~~~~~~~~
+
+    Utility function and classes for Sphinx projects.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+from typing import TYPE_CHECKING
+
+from sphinx.locale import __
+from sphinx.util import get_matching_files
+from sphinx.util import logging
+from sphinx.util.matching import compile_matchers
+from sphinx.util.osutil import SEP, relpath
+
+if TYPE_CHECKING:
+    from typing import Dict, List, Set  # NOQA
+
+logger = logging.getLogger(__name__)
+EXCLUDE_PATHS = ['**/_sources', '.#*', '**/.#*', '*.lproj/**']  # type: List[unicode]
+
+
+class Project(object):
+    """A project is source code set of Sphinx document."""
+
+    def __init__(self, srcdir, source_suffix):
+        # type: (unicode, Dict[unicode, unicode]) -> None
+        #: Source directory.
+        self.srcdir = srcdir
+
+        #: source_suffix. Same as :confval:`source_suffix`.
+        self.source_suffix = source_suffix
+
+        #: The name of documents belongs to this project.
+        self.docnames = set()  # type: Set[unicode]
+
+    def restore(self, other):
+        # type: (Project) -> None
+        """Take over a result of last build."""
+        self.docnames = other.docnames
+
+    def discover(self, exclude_paths=[]):
+        # type: (List[unicode]) -> Set[unicode]
+        """Find all document files in the source directory and put them in
+        :attr:`docnames`.
+        """
+        self.docnames = set()
+        excludes = compile_matchers(exclude_paths + EXCLUDE_PATHS)
+        for filename in get_matching_files(self.srcdir, excludes):  # type: ignore
+            docname = self.path2doc(filename)
+            if docname:
+                if os.access(os.path.join(self.srcdir, filename), os.R_OK):
+                    self.docnames.add(docname)
+                else:
+                    logger.warning(__("document not readable. Ignored."), location=docname)
+
+        return self.docnames
+
+    def path2doc(self, filename):
+        # type: (unicode) -> unicode
+        """Return the docname for the filename if the file is document.
+
+        *filename* should be absolute or relative to the source directory.
+        """
+        if filename.startswith(self.srcdir):
+            filename = relpath(filename, self.srcdir)
+        for suffix in self.source_suffix:
+            if filename.endswith(suffix):
+                return filename[:-len(suffix)]
+
+        # the file does not have docname
+        return None
+
+    def doc2path(self, docname, basedir=True):
+        # type: (unicode, bool) -> unicode
+        """Return the filename for the document name.
+
+        If *basedir* is True, return as an absolute path.
+        Else, return as a relative path to the source directory.
+        """
+        docname = docname.replace(SEP, os.path.sep)
+        basename = os.path.join(self.srcdir, docname)
+        for suffix in self.source_suffix:
+            if os.path.isfile(basename + suffix):
+                break
+        else:
+            # document does not exist
+            suffix = list(self.source_suffix)[0]
+
+        if basedir:
+            return basename + suffix
+        else:
+            return docname + suffix

sphinx/util/__init__.py

@@ -30,7 +30,7 @@ from docutils.utils import relative_path
 from six import text_type, binary_type
 from six.moves.urllib.parse import urlsplit, urlunsplit, quote_plus, parse_qsl, urlencode
 
-from sphinx.deprecation import RemovedInSphinx30Warning
+from sphinx.deprecation import RemovedInSphinx30Warning, RemovedInSphinx40Warning
 from sphinx.errors import PycodeError, SphinxParallelError, ExtensionError
 from sphinx.util import logging
 from sphinx.util.console import strip_colors, colorize, bold, term_width_line  # type: ignore
@@ -111,6 +111,8 @@ def get_matching_docs(dirname, suffixes, exclude_matchers=()):
 
     Exclude files and dirs matching a pattern in *exclude_patterns*.
     """
+    warnings.warn('get_matching_docs() is now deprecated. Use get_matching_files() instead.',
+                  RemovedInSphinx40Warning)
     suffixpatterns = ['*' + s for s in suffixes]
     for filename in get_matching_files(dirname, exclude_matchers):
         for suffixpattern in suffixpatterns:

tests/test_environment.py

@@ -69,3 +69,47 @@ def test_object_inventory(app):
 
     assert app.env.domains['py'].data is app.env.domaindata['py']
     assert app.env.domains['c'].data is app.env.domaindata['c']
+
+
+@pytest.mark.sphinx('dummy', testroot='basic')
+def test_env_relfn2path(app):
+    # relative filename and root document
+    relfn, absfn = app.env.relfn2path('logo.jpg', 'index')
+    assert relfn == 'logo.jpg'
+    assert absfn == app.srcdir / 'logo.jpg'
+
+    # absolute filename and root document
+    relfn, absfn = app.env.relfn2path('/logo.jpg', 'index')
+    assert relfn == 'logo.jpg'
+    assert absfn == app.srcdir / 'logo.jpg'
+
+    # relative filename and a document in subdir
+    relfn, absfn = app.env.relfn2path('logo.jpg', 'subdir/index')
+    assert relfn == 'subdir/logo.jpg'
+    assert absfn == app.srcdir / 'subdir' / 'logo.jpg'
+
+    # absolute filename and a document in subdir
+    relfn, absfn = app.env.relfn2path('/logo.jpg', 'subdir/index')
+    assert relfn == 'logo.jpg'
+    assert absfn == app.srcdir / 'logo.jpg'
+
+    # relative filename having subdir
+    relfn, absfn = app.env.relfn2path('images/logo.jpg', 'index')
+    assert relfn == 'images/logo.jpg'
+    assert absfn == app.srcdir / 'images' / 'logo.jpg'
+
+    # relative path traversal
+    relfn, absfn = app.env.relfn2path('../logo.jpg', 'index')
+    assert relfn == '../logo.jpg'
+    assert absfn == app.srcdir.parent / 'logo.jpg'
+
+    # omit docname (w/ current docname)
+    app.env.temp_data['docname'] = 'subdir/document'
+    relfn, absfn = app.env.relfn2path('images/logo.jpg')
+    assert relfn == 'subdir/images/logo.jpg'
+    assert absfn == app.srcdir / 'subdir' / 'images' / 'logo.jpg'
+
+    # omit docname (w/o current docname)
+    app.env.temp_data.clear()
+    with pytest.raises(KeyError):
+        app.env.relfn2path('images/logo.jpg')

tests/test_project.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+    test_project
+    ~~~~~~~~~~~~
+
+    Tests project module.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from collections import OrderedDict
+
+import pytest
+
+from sphinx.project import Project
+
+
+def test_project_discover(rootdir):
+    project = Project(rootdir / 'test-root', {})
+
+    docnames = {'autodoc', 'bom', 'extapi', 'extensions', 'footnote', 'images',
+                'includes', 'index', 'lists', 'markup', 'math', 'objects',
+                'subdir/excluded', 'subdir/images', 'subdir/includes'}
+    subdir_docnames = {'subdir/excluded', 'subdir/images', 'subdir/includes'}
+
+    # basic case
+    project.source_suffix = ['.txt']
+    assert project.discover() == docnames
+
+    # exclude_paths option
+    assert project.discover(['subdir/*']) == docnames - subdir_docnames
+
+    # exclude_patterns
+    assert project.discover(['.txt', 'subdir/*']) == docnames - subdir_docnames
+
+    # multiple source_suffixes
+    project.source_suffix = ['.txt', '.foo']
+    assert project.discover() == docnames | {'otherext'}
+
+    # complicated source_suffix
+    project.source_suffix = ['.foo.png']
+    assert project.discover() == {'img'}
+
+    # templates_path
+    project.source_suffix = ['.html']
+    assert project.discover() == {'_templates/layout',
+                                  '_templates/customsb',
+                                  '_templates/contentssb'}
+
+    assert project.discover(['_templates']) == set()
+
+
+@pytest.mark.sphinx(testroot='basic')
+def test_project_path2doc(app):
+    project = Project(app.srcdir, app.config.source_suffix)
+    assert project.path2doc('index.rst') == 'index'
+    assert project.path2doc('index.foo') is None  # unknown extension
+    assert project.path2doc('index.foo.rst') == 'index.foo'
+    assert project.path2doc('index') is None
+    assert project.path2doc('/path/to/index.rst') == '/path/to/index'
+    assert project.path2doc(app.srcdir / '/to/index.rst') == '/to/index'
+
+
+@pytest.mark.sphinx(srcdir='project_doc2path', testroot='basic')
+def test_project_doc2path(app):
+    source_suffix = OrderedDict([('.rst', 'restructuredtext'),
+                                 ('.txt', 'restructuredtext')])
+
+    project = Project(app.srcdir, source_suffix)
+    assert project.doc2path('index') == (app.srcdir / 'index.rst')
+
+    # first source_suffix is used for missing file
+    assert project.doc2path('foo') == (app.srcdir / 'foo.rst')
+
+    # matched source_suffix is used if exists
+    (app.srcdir / 'foo.txt').write_text('')
+    assert project.doc2path('foo') == (app.srcdir / 'foo.txt')
+
+    # absolute path
+    assert project.doc2path('index', basedir=True) == (app.srcdir / 'index.rst')
+
+    # relative path
+    assert project.doc2path('index', basedir=False) == 'index.rst'