Merge pull request #3529 from tk0miya/make_env.app_available_always

Make env.app available always

CHANGES

@@ -24,6 +24,7 @@ Incompatible changes
   ``True`` to ``False``. This means that some LaTeX macros for styling are
   by default defined only with ``\sphinx..`` prefixed names. (refs: #3429)
 * Footer "Continued on next page" of LaTeX longtable's now not framed (refs: #3497)
+* #3529: The arguments of ``BuildEnvironment.__init__`` is changed
 
 Features removed
 ----------------

sphinx/application.py

@@ -305,15 +305,15 @@ class Sphinx(object):
     def _init_env(self, freshenv):
         # type: (bool) -> None
         if freshenv:
-            self.env = BuildEnvironment(self.srcdir, self.doctreedir, self.config)
+            self.env = BuildEnvironment(self)
             self.env.find_files(self.config, self.buildername)
             for domain in self.domains.keys():
                 self.env.domains[domain] = self.domains[domain](self.env)
         else:
             try:
                 logger.info(bold(_('loading pickled environment... ')), nonl=True)
-                self.env = BuildEnvironment.frompickle(
+                filename = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
-                    self.srcdir, self.config, path.join(self.doctreedir, ENV_PICKLE_FILENAME))
+                self.env = BuildEnvironment.frompickle(filename, self)
                 self.env.domains = {}
                 for domain in self.domains.keys():
                     # this can raise if the data version doesn't fit

sphinx/builders/__init__.py

@@ -289,8 +289,7 @@ class Builder(object):
 
         # while reading, collect all warnings from docutils
         with logging.pending_warnings():
-            updated_docnames = set(self.env.update(self.config, self.srcdir,
+            updated_docnames = set(self.env.update(self.config, self.srcdir, self.doctreedir))
-                                                   self.doctreedir, self.app))
 
         doccount = len(updated_docnames)
         logger.info(bold('looking for now-outdated files... '), nonl=1)

sphinx/environment/__init__.py

@@ -20,7 +20,7 @@ import warnings
 from os import path
 from collections import defaultdict
 
-from six import itervalues, class_types, next
+from six import StringIO, itervalues, class_types, next
 from six.moves import cPickle as pickle
 
 from docutils import nodes
@@ -51,7 +51,7 @@ from sphinx.environment.adapters.toctree import TocTree
 
 if False:
     # For type annotation
-    from typing import Any, Callable, Dict, Iterator, List, Pattern, Set, Tuple, Type, Union  # NOQA
+    from typing import Any, Callable, Dict, IO, Iterator, List, Pattern, Set, Tuple, Type, Union  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.config import Config  # NOQA
@@ -104,52 +104,78 @@ class BuildEnvironment(object):
     # --------- ENVIRONMENT PERSISTENCE ----------------------------------------
 
     @staticmethod
-    def frompickle(srcdir, config, filename):
+    def load(f, app=None):
-        # type: (unicode, Config, unicode) -> BuildEnvironment
+        # type: (IO, Sphinx) -> BuildEnvironment
-        with open(filename, 'rb') as picklefile:
+        env = pickle.load(f)
-            env = pickle.load(picklefile)
         if env.version != ENV_VERSION:
             raise IOError('build environment version not current')
-        if env.srcdir != srcdir:
+        if app:
-            raise IOError('source directory has changed')
+            env.app = app
-        env.config.values = config.values
+            env.config.values = app.config.values
+            if env.srcdir != app.srcdir:
+                raise IOError('source directory has changed')
         return env
 
-    def topickle(self, filename):
+    @classmethod
-        # type: (unicode) -> None
+    def loads(cls, string, app=None):
+        # type: (unicode, Sphinx) -> BuildEnvironment
+        io = StringIO(string)
+        return cls.load(io, app)
+
+    @classmethod
+    def frompickle(cls, filename, app):
+        # type: (unicode, Sphinx) -> BuildEnvironment
+        with open(filename, 'rb') as f:
+            return cls.load(f, app)
+
+    @staticmethod
+    def dump(env, f):
+        # type: (BuildEnvironment, IO) -> None
         # remove unpicklable attributes
-        values = self.config.values
+        app = env.app
-        del self.config.values
+        del env.app
-        domains = self.domains
+        values = env.config.values
-        del self.domains
+        del env.config.values
+        domains = env.domains
+        del env.domains
         # remove potentially pickling-problematic values from config
-        for key, val in list(vars(self.config).items()):
+        for key, val in list(vars(env.config).items()):
             if key.startswith('_') or \
                isinstance(val, types.ModuleType) or \
                isinstance(val, types.FunctionType) or \
                isinstance(val, class_types):
-                del self.config[key]
+                del env.config[key]
-        with open(filename, 'wb') as picklefile:
+        pickle.dump(env, f, pickle.HIGHEST_PROTOCOL)
-            pickle.dump(self, picklefile, pickle.HIGHEST_PROTOCOL)
         # reset attributes
-        self.domains = domains
+        env.domains = domains
-        self.config.values = values
+        env.config.values = values
+        env.app = app
+
+    @classmethod
+    def dumps(cls, env):
+        # type: (BuildEnvironment) -> unicode
+        io = StringIO()
+        cls.dump(env, io)
+        return io.getvalue()
+
+    def topickle(self, filename):
+        # type: (unicode) -> None
+        with open(filename, 'wb') as f:
+            self.dump(self, f)
 
     # --------- ENVIRONMENT INITIALIZATION -------------------------------------
 
-    def __init__(self, srcdir, doctreedir, config):
+    def __init__(self, app):
-        # type: (unicode, unicode, Config) -> None
+        # type: (Sphinx) -> None
-        self.doctreedir = doctreedir
+        self.app = app
-        self.srcdir = srcdir  # type: unicode
+        self.doctreedir = app.doctreedir
-        self.config = config  # type: Config
+        self.srcdir = app.srcdir
+        self.config = app.config
 
         # the method of doctree versioning; see set_versioning_method
         self.versioning_condition = None  # type: Union[bool, Callable]
         self.versioning_compare = None  # type: bool
 
-        # the application object; only set while update() runs
-        self.app = None  # type: Sphinx
-
         # all the registered domains, set by the application
         self.domains = {}
 
@@ -466,8 +492,8 @@ class BuildEnvironment(object):
 
         return added, changed, removed
 
-    def update(self, config, srcdir, doctreedir, app):
+    def update(self, config, srcdir, doctreedir):
-        # type: (Config, unicode, unicode, Sphinx) -> List[unicode]
+        # type: (Config, unicode, unicode) -> List[unicode]
         """(Re-)read all files new or changed since last update.
 
         Store all environment docnames in the canonical format (ie using SEP as
@@ -495,7 +521,7 @@ class BuildEnvironment(object):
         # the source and doctree directories may have been relocated
         self.srcdir = srcdir
         self.doctreedir = doctreedir
-        self.find_files(config, app.buildername)
+        self.find_files(config, self.app.buildername)
         self.config = config
 
         # this cache also needs to be updated every time
@@ -506,7 +532,7 @@ class BuildEnvironment(object):
         added, changed, removed = self.get_outdated_files(config_changed)
 
         # allow user intervention as well
-        for docs in app.emit('env-get-outdated', self, added, changed, removed):
+        for docs in self.app.emit('env-get-outdated', self, added, changed, removed):
             changed.update(set(docs) & self.found_docs)
 
         # if files were added or removed, all documents with globbed toctrees
@@ -519,23 +545,21 @@ class BuildEnvironment(object):
                                                      len(removed))
         logger.info(msg)
 
-        self.app = app
-
         # clear all files no longer present
         for docname in removed:
-            app.emit('env-purge-doc', self, docname)
+            self.app.emit('env-purge-doc', self, docname)
             self.clear_doc(docname)
 
         # read all new and changed files
         docnames = sorted(added | changed)
         # allow changing and reordering the list of docs to read
-        app.emit('env-before-read-docs', self, docnames)
+        self.app.emit('env-before-read-docs', self, docnames)
 
         # check if we should do parallel or serial read
         par_ok = False
-        if parallel_available and len(docnames) > 5 and app.parallel > 1:
+        if parallel_available and len(docnames) > 5 and self.app.parallel > 1:
             par_ok = True
-            for extname, md in app._extension_metadata.items():
+            for extname, md in self.app._extension_metadata.items():
                 ext_ok = md.get('parallel_read_safe')
                 if ext_ok:
                     continue
@@ -551,17 +575,15 @@ class BuildEnvironment(object):
                 par_ok = False
                 break
         if par_ok:
-            self._read_parallel(docnames, app, nproc=app.parallel)
+            self._read_parallel(docnames, self.app, nproc=self.app.parallel)
         else:
-            self._read_serial(docnames, app)
+            self._read_serial(docnames, self.app)
 
         if config.master_doc not in self.all_docs:
             raise SphinxError('master file %s not found' %
                               self.doc2path(config.master_doc))
 
-        self.app = None
+        for retval in self.app.emit('env-updated', self):
-
-        for retval in app.emit('env-updated', self):
             if retval is not None:
                 docnames.extend(retval)
 
@@ -584,20 +606,17 @@ class BuildEnvironment(object):
             self.clear_doc(docname)
 
         def read_process(docs):
-            # type: (List[unicode]) -> BuildEnvironment
+            # type: (List[unicode]) -> unicode
             self.app = app
             for docname in docs:
                 self.read_doc(docname, app)
             # allow pickling self to send it back
-            del self.app
+            return BuildEnvironment.dumps(self)
-            del self.domains
-            del self.config.values
-            del self.config
-            return self
 
         def merge(docs, otherenv):
-            # type: (List[unicode], BuildEnvironment) -> None
+            # type: (List[unicode], unicode) -> None
-            self.merge_info_from(docs, otherenv, app)
+            env = BuildEnvironment.loads(otherenv)
+            self.merge_info_from(docs, env, app)
 
         tasks = ParallelTasks(nproc)
         chunks = make_chunks(docnames, nproc)

tests/test_environment.py

@@ -31,7 +31,7 @@ def teardown_module():
 # afford to not run update() in the setup but in its own test
 
 def test_first_update():
-    updated = env.update(app.config, app.srcdir, app.doctreedir, app)
+    updated = env.update(app.config, app.srcdir, app.doctreedir)
     assert set(updated) == env.found_docs == set(env.all_docs)
     # test if exclude_patterns works ok
     assert 'subdir/excluded' not in env.found_docs
@@ -71,7 +71,7 @@ def test_second_update():
     # the contents.txt toctree; otherwise section numbers would shift
     (root / 'autodoc.txt').unlink()
     (root / 'new.txt').write_text('New file\n========\n')
-    updated = env.update(app.config, app.srcdir, app.doctreedir, app)
+    updated = env.update(app.config, app.srcdir, app.doctreedir)
     # "includes" and "images" are in there because they contain references
     # to nonexisting downloadable or image files, which are given another
     # chance to exist
@@ -87,7 +87,7 @@ def test_env_read_docs():
 
     app.connect('env-before-read-docs', on_env_read_docs_1)
 
-    read_docnames = env.update(app.config, app.srcdir, app.doctreedir, app)
+    read_docnames = env.update(app.config, app.srcdir, app.doctreedir)
     assert len(read_docnames) > 2 and read_docnames == sorted(read_docnames)
 
     def on_env_read_docs_2(app, env, docnames):
@@ -95,7 +95,7 @@ def test_env_read_docs():
 
     app.connect('env-before-read-docs', on_env_read_docs_2)
 
-    read_docnames = env.update(app.config, app.srcdir, app.doctreedir, app)
+    read_docnames = env.update(app.config, app.srcdir, app.doctreedir)
     assert len(read_docnames) == 2
 
 

tests/test_intl.py

@@ -497,7 +497,7 @@ def test_gettext_buildr_ignores_only_directive(app):
 def test_gettext_dont_rebuild_mo(make_app, app_params, build_mo):
     # --- don't rebuild by .mo mtime
     def get_number_of_update_targets(app_):
-        updated = app_.env.update(app_.config, app_.srcdir, app_.doctreedir, app_)
+        updated = app_.env.update(app_.config, app_.srcdir, app_.doctreedir)
         return len(updated)
 
     # setup new directory
@@ -680,12 +680,12 @@ def test_html_rebuild_mo(app):
     app.build()
     # --- rebuild by .mo mtime
     app.builder.build_update()
-    updated = app.env.update(app.config, app.srcdir, app.doctreedir, app)
+    updated = app.env.update(app.config, app.srcdir, app.doctreedir)
     assert len(updated) == 0
 
     mtime = (app.srcdir / 'xx' / 'LC_MESSAGES' / 'bom.mo').stat().st_mtime
     (app.srcdir / 'xx' / 'LC_MESSAGES' / 'bom.mo').utime((mtime + 5, mtime + 5))
-    updated = app.env.update(app.config, app.srcdir, app.doctreedir, app)
+    updated = app.env.update(app.config, app.srcdir, app.doctreedir)
     assert len(updated) == 1