From 31452fc64d550f526d3a6c4dfbb1121873896503 Mon Sep 17 00:00:00 2001
From: Georg Brandl <georg@python.org>
Date: Mon, 22 Sep 2014 14:51:47 +0200
Subject: [PATCH] [WIP] parallel read

---
 doc/extdev/appapi.rst              |   3 +-
 doc/extdev/index.rst               |   7 +
 sphinx/application.py              |   1 +
 sphinx/builders/__init__.py        |  35 ++--
 sphinx/domains/__init__.py         |   8 +
 sphinx/domains/c.py                |   6 +
 sphinx/domains/cpp.py              |   6 +
 sphinx/domains/javascript.py       |   6 +
 sphinx/domains/python.py           |   9 +
 sphinx/domains/rst.py              |   6 +
 sphinx/domains/std.py              |  15 ++
 sphinx/environment.py              | 256 +++++++++++++++++++++++------
 sphinx/ext/autodoc.py              |   2 +-
 sphinx/ext/autosummary/__init__.py |   2 +-
 sphinx/ext/coverage.py             |   2 +-
 sphinx/ext/doctest.py              |   2 +-
 sphinx/ext/extlinks.py             |   2 +-
 sphinx/ext/graphviz.py             |   2 +-
 sphinx/ext/ifconfig.py             |   2 +-
 sphinx/ext/inheritance_diagram.py  |   2 +-
 sphinx/ext/intersphinx.py          |   2 +-
 sphinx/ext/jsmath.py               |   2 +-
 sphinx/ext/linkcode.py             |   5 +-
 sphinx/ext/mathjax.py              |   2 +-
 sphinx/ext/napoleon/__init__.py    |   2 +-
 sphinx/ext/pngmath.py              |   2 +-
 sphinx/ext/todo.py                 |   2 +-
 sphinx/ext/viewcode.py             |   2 +-
 sphinx/util/__init__.py            |   5 +
 29 files changed, 311 insertions(+), 87 deletions(-)

diff --git a/doc/extdev/appapi.rst b/doc/extdev/appapi.rst
index f9b43560f..d65f91a54 100644
--- a/doc/extdev/appapi.rst
+++ b/doc/extdev/appapi.rst
@@ -442,7 +442,8 @@ handlers to the events.  Example:
    Emitted after the environment has determined the list of all added and
    changed files and just before it reads them.  It allows extension authors to
    reorder the list of docnames (*inplace*) before processing, or add more
-   docnames that Sphinx did not consider changed.
+   docnames that Sphinx did not consider changed (but never add any docnames
+   that are not in ``env.found_docs``).
 
    You can also remove document names; do this with caution since it will make
    Sphinx treat changed files as unchanged.
diff --git a/doc/extdev/index.rst b/doc/extdev/index.rst
index 481688acf..5144c5f8b 100644
--- a/doc/extdev/index.rst
+++ b/doc/extdev/index.rst
@@ -33,6 +33,13 @@ as metadata of the extension.  Metadata keys currently recognized are:
 * ``'version'``: a string that identifies the extension version.  It is used for
   extension version requirement checking (see :confval:`needs_extensions`) and
   informational purposes.  If not given, ``"unknown version"`` is substituted.
+* ``'parallel_read_safe'``: a boolean that specifies if parallel reading of
+  source files can be used when the extension is loaded.  It defaults to
+  ``False``, i.e. you have to explicitly specify your extension to be
+  parallel-read-safe after checking that it is.
+* ``'parallel_write_safe'``: a boolean that specifies if parallel writing of
+  output files can be used when the extension is loaded.  Since extensions
+  usually don't negatively influence the process, this defaults to ``True``.
 
 APIs used for writing extensions
 --------------------------------
diff --git a/sphinx/application.py b/sphinx/application.py
index 052ad1ce2..f7ff95175 100644
--- a/sphinx/application.py
+++ b/sphinx/application.py
@@ -53,6 +53,7 @@ events = {
     'env-before-read-docs': 'env, docnames',
     'source-read': 'docname, source text',
     'doctree-read': 'the doctree before being pickled',
+    'env-merge-info': 'env, read docnames, other env instance',
     'missing-reference': 'env, node, contnode',
     'doctree-resolved': 'doctree, docname',
     'env-updated': 'env',
diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py
index d0e85ae7d..f797766b4 100644
--- a/sphinx/builders/__init__.py
+++ b/sphinx/builders/__init__.py
@@ -238,18 +238,11 @@ class Builder(object):
         # while reading, collect all warnings from docutils
         warnings = []
         self.env.set_warnfunc(lambda *args: warnings.append(args))
-        self.info(bold('updating environment: '), nonl=1)
-        msg, length, iterator = self.env.update(self.config, self.srcdir,
-                                                self.doctreedir, self.app)
-        self.info(msg)
-        for docname in self.status_iterator(iterator, 'reading sources... ',
-                                            purple, length):
-            updated_docnames.add(docname)
-            # nothing further to do, the environment has already
-            # done the reading
+        updated_docnames = self.env.update(self.config, self.srcdir,
+                                           self.doctreedir, self.app)
+        self.env.set_warnfunc(self.warn)
         for warning in warnings:
             self.warn(*warning)
-        self.env.set_warnfunc(self.warn)
 
         doccount = len(updated_docnames)
         self.info(bold('looking for now-outdated files... '), nonl=1)
@@ -325,16 +318,24 @@ class Builder(object):
         # check for prerequisites to parallel build
         # (parallel only works on POSIX, because the forking impl of
         # multiprocessing is required)
-        if not (multiprocessing and
+        if (multiprocessing and
                 self.app.parallel > 1 and
                 self.allow_parallel and
                 os.name == 'posix'):
-            self._write_serial(sorted(docnames), warnings)
-        else:
-            # number of subprocesses is parallel-1 because the main process
-            # is busy loading doctrees and doing write_doc_serialized()
-            self._write_parallel(sorted(docnames), warnings,
-                                 nproc=self.app.parallel - 1)
+            for extname, md in self.app._extension_metadata.items():
+                par_ok = md.get('parallel_write_safe', True)
+                if not par_ok:
+                    self.app.warn('the %s extension is not safe for parallel '
+                                  'writing, doing serial read' % extname)
+                    break
+            else:  # means no break, means everything is safe
+                # number of subprocesses is parallel-1 because the main process
+                # is busy loading doctrees and doing write_doc_serialized()
+                self._write_parallel(sorted(docnames), warnings,
+                                     nproc=self.app.parallel - 1)
+                self.env.set_warnfunc(self.warn)
+                return
+        self._write_serial(sorted(docnames), warnings)
         self.env.set_warnfunc(self.warn)
 
     def _write_serial(self, docnames, warnings):
diff --git a/sphinx/domains/__init__.py b/sphinx/domains/__init__.py
index cfba9e913..6ebaeb792 100644
--- a/sphinx/domains/__init__.py
+++ b/sphinx/domains/__init__.py
@@ -202,6 +202,14 @@ class Domain(object):
         """Remove traces of a document in the domain-specific inventories."""
         pass
 
+    def merge_domaindata(self, docnames, otherdata):
+        """Merge in data regarding *docnames* from a different domaindata
+        inventory.
+        """
+        raise NotImplementedError('merge_domaindata must be implemented in %s '
+                                  'to be able to do parallel builds!' %
+                                  self.__class__)
+
     def process_doc(self, env, docname, document):
         """Process a document after it is read by the environment."""
         pass
diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py
index e51441caa..0754e3172 100644
--- a/sphinx/domains/c.py
+++ b/sphinx/domains/c.py
@@ -269,6 +269,12 @@ class CDomain(Domain):
             if fn == docname:
                 del self.data['objects'][fullname]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates
+        for fullname, (fn, objtype) in otherdata['objects'].items():
+            if fn in docnames:
+                self.data['objects'][fullname] = (fn, objtype)
+
     def resolve_xref(self, env, fromdocname, builder,
                      typ, target, node, contnode):
         # strip pointer asterisk
diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py
index 10dd61a7e..d4455a222 100644
--- a/sphinx/domains/cpp.py
+++ b/sphinx/domains/cpp.py
@@ -1836,6 +1836,12 @@ class CPPDomain(Domain):
             if data[0] == docname:
                 del self.data['objects'][fullname]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates
+        for fullname, data in otherdata['objects'].items():
+            if data[0] in docnames:
+                self.data['objects'][fullname] = data
+
     def _resolve_xref_inner(self, env, fromdocname, builder,
                             target, node, contnode, warn=True):
         def _create_refnode(nameAst):
diff --git a/sphinx/domains/javascript.py b/sphinx/domains/javascript.py
index 1169036c9..af215fd6e 100644
--- a/sphinx/domains/javascript.py
+++ b/sphinx/domains/javascript.py
@@ -187,6 +187,12 @@ class JavaScriptDomain(Domain):
             if fn == docname:
                 del self.data['objects'][fullname]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates
+        for fullname, (fn, objtype) in otherdata['objects'].items():
+            if fn in docnames:
+                self.data['objects'][fullname] = (fn, objtype)
+
     def find_obj(self, env, obj, name, typ, searchorder=0):
         if name[-2:] == '()':
             name = name[:-2]
diff --git a/sphinx/domains/python.py b/sphinx/domains/python.py
index 3745aab39..4e08eba9b 100644
--- a/sphinx/domains/python.py
+++ b/sphinx/domains/python.py
@@ -627,6 +627,15 @@ class PythonDomain(Domain):
             if fn == docname:
                 del self.data['modules'][modname]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates?
+        for fullname, (fn, objtype) in otherdata['objects'].items():
+            if fn in docnames:
+                self.data['objects'][fullname] = (fn, objtype)
+        for modname, data in otherdata['modules'].items():
+            if data[0] in docnames:
+                self.data['modules'][modname] = data
+
     def find_obj(self, env, modname, classname, name, type, searchmode=0):
         """Find a Python object for "name", perhaps using the given module
         and/or classname.  Returns a list of (name, object entry) tuples.
diff --git a/sphinx/domains/rst.py b/sphinx/domains/rst.py
index 6a4e390f1..2c304d0c7 100644
--- a/sphinx/domains/rst.py
+++ b/sphinx/domains/rst.py
@@ -123,6 +123,12 @@ class ReSTDomain(Domain):
             if doc == docname:
                 del self.data['objects'][typ, name]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates
+        for (typ, name), doc in otherdata['objects'].items():
+            if doc in docnames:
+                self.data['objects'][typ, name] = doc
+
     def resolve_xref(self, env, fromdocname, builder, typ, target, node,
                      contnode):
         objects = self.data['objects']
diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py
index 5a9984829..f14f65aab 100644
--- a/sphinx/domains/std.py
+++ b/sphinx/domains/std.py
@@ -506,6 +506,21 @@ class StandardDomain(Domain):
             if fn == docname:
                 del self.data['anonlabels'][key]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX duplicates?
+        for key, data in otherdata['progoptions'].items():
+            if data[0] in docnames:
+                self.data['progoptions'][key] = data
+        for key, data in otherdata['objects'].items():
+            if data[0] in docnames:
+                self.data['objects'][key] = data
+        for key, data in otherdata['labels'].items():
+            if data[0] in docnames:
+                self.data['labels'][key] = data
+        for key, data in otherdata['anonlabels'].items():
+            if data[0] in docnames:
+                self.data['anonlabels'][key] = data
+
     def process_doc(self, env, docname, document):
         labels, anonlabels = self.data['labels'], self.data['anonlabels']
         for name, explicit in iteritems(document.nametypes):
diff --git a/sphinx/environment.py b/sphinx/environment.py
index 560756a89..2cb7adfdb 100644
--- a/sphinx/environment.py
+++ b/sphinx/environment.py
@@ -22,8 +22,14 @@ from os import path
 from glob import glob
 from itertools import groupby
 
+try:
+    import multiprocessing
+    import threading
+except ImportError:
+    multiprocessing = threading = None
+
 from six import iteritems, itervalues, text_type, class_types
-from six.moves import cPickle as pickle, zip
+from six.moves import cPickle as pickle, zip, queue
 from docutils import nodes
 from docutils.io import FileInput, NullOutput
 from docutils.core import Publisher
@@ -40,6 +46,7 @@ from sphinx.util import url_re, get_matching_docs, docname_join, split_into, \
     FilenameUniqDict
 from sphinx.util.nodes import clean_astext, make_refnode, WarningStream
 from sphinx.util.osutil import SEP, find_catalog_files, getcwd, fs_encoding
+from sphinx.util.console import bold, purple
 from sphinx.util.matching import compile_matchers
 from sphinx.util.websupport import is_commentable
 from sphinx.errors import SphinxError, ExtensionError
@@ -328,6 +335,50 @@ class BuildEnvironment:
         for domain in self.domains.values():
             domain.clear_doc(docname)
 
+    def merge_info_from(self, docnames, other, app):
+        """Merge global information gathered about *docnames* while reading them
+        from the *other* environment.
+
+        This possibly comes from a parallel build process.
+        """
+        docnames = set(docnames)
+        for docname in docnames:
+            self.all_docs[docname] = other.all_docs[docname]
+            if docname in other.reread_always:
+                self.reread_always.add(docname)
+            self.metadata[docname] = other.metadata[docname]
+            if docname in other.dependencies:
+                self.dependencies[docname] = other.dependencies[docname]
+            self.titles[docname] = other.titles[docname]
+            self.longtitles[docname] = other.longtitles[docname]
+            self.tocs[docname] = other.tocs[docname]
+            self.toc_num_entries[docname] = other.toc_num_entries[docname]
+            # toc_secnumbers is not assigned during read
+            if docname in other.toctree_includes:
+                self.toctree_includes[docname] = other.toctree_includes[docname]
+            self.indexentries[docname] = other.indexentries[docname]
+            if docname in other.glob_toctrees:
+                self.glob_toctrees.add(docname)
+            if docname in other.numbered_toctrees:
+                self.numbered_toctrees.add(docname)
+
+        self.images.merge_other(docnames, other.images)
+        self.dlfiles.merge_other(docnames, other.dlfiles)
+
+        for subfn, fnset in other.files_to_rebuild.items():
+            self.files_to_rebuild.setdefault(subfn, set()).update(fnset & docnames)
+        for key, data in other.citations.items():
+            # XXX duplicates?
+            if data[0] in docnames:
+                self.citations[key] = data
+        for version, changes in other.versionchanges.items():
+            self.versionchanges.setdefault(version, []).extend(
+                change for change in changes if change[1] in docnames)
+
+        for domainname, domain in self.domains.items():
+            domain.merge_domaindata(docnames, other.domaindata[domainname])
+        app.emit('env-merge-info', self, docnames, other)
+
     def doc2path(self, docname, base=True, suffix=None):
         """Return the filename for the document name.
 
@@ -443,13 +494,11 @@ class BuildEnvironment:
 
         return added, changed, removed
 
-    def update(self, config, srcdir, doctreedir, app=None):
+    def update(self, config, srcdir, doctreedir, app):
         """(Re-)read all files new or changed since last update.
 
-        Returns a summary, the total count of documents to reread and an
-        iterator that yields docnames as it processes them.  Store all
-        environment docnames in the canonical format (ie using SEP as a
-        separator in place of os.path.sep).
+        Store all environment docnames in the canonical format (ie using SEP as
+        a separator in place of os.path.sep).
         """
         config_changed = False
         if self.config is None:
@@ -481,6 +530,8 @@ class BuildEnvironment:
         # this cache also needs to be updated every time
         self._nitpick_ignore = set(self.config.nitpick_ignore)
 
+        app.info(bold('updating environment: '), nonl=1)
+
         added, changed, removed = self.get_outdated_files(config_changed)
 
         # allow user intervention as well
@@ -495,33 +546,145 @@ class BuildEnvironment:
 
         msg += '%s added, %s changed, %s removed' % (len(added), len(changed),
                                                      len(removed))
+        app.info(msg)
 
-        def update_generator():
+        self.app = app
+
+        # clear all files no longer present
+        for docname in removed:
+            app.emit('env-purge-doc', self, docname)
+            self.clear_doc(docname)
+
+        # read all new and changed files
+        docnames = sorted(added | changed)
+        # allow changing and reordering the list of docs to read
+        app.emit('env-before-read-docs', self, docnames)
+
+        # check if we should do parallel or serial read
+        par_ok = False
+        if (len(added | changed) > 5 and
+                multiprocessing and
+                app.parallel > 1 and
+                os.name == 'posix'):
+            par_ok = True
+            for extname, md in app._extension_metadata.items():
+                ext_ok = md.get('parallel_read_safe')
+                if ext_ok:
+                    continue
+                if ext_ok is None:
+                    app.warn('the %s extension does not declare if it '
+                             'is safe for parallel reading, assuming it '
+                             'isn\'t - please ask the extension author to '
+                             'check and make it explicit' % extname)
+                    app.warn('doing serial read')
+                else:
+                    app.warn('the %s extension is not safe for parallel '
+                             'reading, doing serial read' % extname)
+                par_ok = False
+                break
+        if par_ok:
+            self._read_parallel(docnames, app, nproc=app.parallel)
+        else:
+            self._read_serial(docnames, app)
+
+        if config.master_doc not in self.all_docs:
+            self.warn(None, 'master file %s not found' %
+                      self.doc2path(config.master_doc))
+
+        self.app = None
+        app.emit('env-updated', self)
+        return docnames
+
+    def _read_serial(self, docnames, app):
+        for docname in app.status_iterator(docnames, 'reading sources... ',
+                                           purple, len(docnames)):
+            # remove all inventory entries for that file
+            app.emit('env-purge-doc', self, docname)
+            self.clear_doc(docname)
+            self.read_doc(docname, app)
+
+    def _read_parallel(self, docnames, app, nproc):
+        def read_process(docs, pipe):
             self.app = app
+            self.warnings = []
+            self.set_warnfunc(lambda *args: self.warnings.append(args))
+            try:
+                for docname in docs:
+                    self.read_doc(docname, app)
+            except KeyboardInterrupt:
+                # XXX return None?
+                pass  # do not print a traceback on Ctrl-C
+            self.set_warnfunc(None)
+            del self.app
+            del self.domains
+            del self.config.values
+            del self.config
+            pipe.send(self)
 
-            # clear all files no longer present
-            for docname in removed:
-                if app:
-                    app.emit('env-purge-doc', self, docname)
-                self.clear_doc(docname)
+        def process_thread(docs):
+            precv, psend = multiprocessing.Pipe(False)
+            p = multiprocessing.Process(target=read_process, args=(docs, psend))
+            p.start()
+            # XXX error handling
+            new_env = precv.recv()
+            merge_queue.put((docs, new_env))
+            p.join()
+            semaphore.release()
 
-            # read all new and changed files
-            docnames = sorted(added | changed)
-            if app:
-                app.emit('env-before-read-docs', self, docnames)
-            for docname in docnames:
-                yield docname
-                self.read_doc(docname, app=app)
+        # allow only "nproc" worker processes at once
+        semaphore = threading.Semaphore(nproc)
+        # list of threads to join when waiting for completion
+        threads = []
+        # queue of other env objects to merge
+        merge_queue = queue.Queue()
 
-            if config.master_doc not in self.all_docs:
-                self.warn(None, 'master file %s not found' %
-                          self.doc2path(config.master_doc))
+        # clear all outdated docs at once
+        for docname in docnames:
+            app.emit('env-purge-doc', self, docname)
+            self.clear_doc(docname)
 
-            self.app = None
-            if app:
-                app.emit('env-updated', self)
+        # determine how many documents to read in one go
+        ndocs = len(docnames)
+        chunksize = min(ndocs // nproc, 10)
+        if chunksize == 0:
+            chunksize = 1
+        nchunks, rest = divmod(ndocs, chunksize)
+        if rest:
+            nchunks += 1
+        # partition documents in "chunks" that will be written by one Process
+        chunks = [docnames[i*chunksize:(i+1)*chunksize] for i in range(nchunks)]
 
-        return msg, len(added | changed), update_generator()
+        warnings = []
+        merged = 0
+        for chunk in app.status_iterator(chunks, 'reading sources... ',
+                                         purple, len(chunks)):
+            semaphore.acquire()
+            t = threading.Thread(target=process_thread, args=(chunk,))
+            t.setDaemon(True)
+            t.start()
+            threads.append(t)
+            try:
+                docs, other = merge_queue.get(False)
+            except queue.Empty:
+                pass
+            else:
+                warnings.extend(other.warnings)
+                self.merge_info_from(docs, other, app)
+                merged += 1
+
+        while merged < len(chunks):
+            docs, other = merge_queue.get()
+            warnings.extend(other.warnings)
+            self.merge_info_from(docs, other, app)
+            merged += 1
+
+        for warning in warnings:
+            self._warnfunc(*warning)
+
+        # make sure all threads have finished
+        app.info(bold('waiting for workers... '))
+        for t in threads:
+            t.join()
 
     def check_dependents(self, already):
         to_rewrite = self.assign_section_numbers()
@@ -590,19 +753,8 @@ class BuildEnvironment:
         directives.directive = directive
         roles.role = role
 
-    def read_doc(self, docname, src_path=None, save_parsed=True, app=None):
-        """Parse a file and add/update inventory entries for the doctree.
-
-        If srcpath is given, read from a different source file.
-        """
-        # remove all inventory entries for that file
-        if app:
-            app.emit('env-purge-doc', self, docname)
-
-        self.clear_doc(docname)
-
-        if src_path is None:
-            src_path = self.doc2path(docname)
+    def read_doc(self, docname, app=None):
+        """Parse a file and add/update inventory entries for the doctree."""
 
         self.temp_data['docname'] = docname
         # defaults to the global default, but can be re-set in a document
@@ -639,6 +791,7 @@ class BuildEnvironment:
                         destination_class=NullOutput)
         pub.set_components(None, 'restructuredtext', None)
         pub.process_programmatic_settings(None, self.settings, None)
+        src_path = self.doc2path(docname)
         source = SphinxFileInput(app, self, source=None, source_path=src_path,
                                  encoding=self.config.source_encoding)
         pub.source = source
@@ -706,20 +859,17 @@ class BuildEnvironment:
         self.ref_context.clear()
         roles._roles.pop('', None)  # if a document has set a local default role
 
-        if save_parsed:
-            # save the parsed doctree
-            doctree_filename = self.doc2path(docname, self.doctreedir,
-                                             '.doctree')
-            dirname = path.dirname(doctree_filename)
-            if not path.isdir(dirname):
-                os.makedirs(dirname)
-            f = open(doctree_filename, 'wb')
-            try:
-                pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
-            finally:
-                f.close()
-        else:
-            return doctree
+        # save the parsed doctree
+        doctree_filename = self.doc2path(docname, self.doctreedir,
+                                         '.doctree')
+        dirname = path.dirname(doctree_filename)
+        if not path.isdir(dirname):
+            os.makedirs(dirname)
+        f = open(doctree_filename, 'wb')
+        try:
+            pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
+        finally:
+            f.close()
 
     # utilities to use while reading a document
 
diff --git a/sphinx/ext/autodoc.py b/sphinx/ext/autodoc.py
index 113a36d5f..5b0bda17a 100644
--- a/sphinx/ext/autodoc.py
+++ b/sphinx/ext/autodoc.py
@@ -1515,7 +1515,7 @@ def setup(app):
     app.add_event('autodoc-process-signature')
     app.add_event('autodoc-skip-member')
 
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
 
 
 class testcls:
diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py
index aff9e005d..c37aa1f30 100644
--- a/sphinx/ext/autosummary/__init__.py
+++ b/sphinx/ext/autosummary/__init__.py
@@ -570,4 +570,4 @@ def setup(app):
     app.connect('doctree-read', process_autosummary_toc)
     app.connect('builder-inited', process_generate_options)
     app.add_config_value('autosummary_generate', [], True)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': False}
diff --git a/sphinx/ext/coverage.py b/sphinx/ext/coverage.py
index 841815e73..b62806fa4 100644
--- a/sphinx/ext/coverage.py
+++ b/sphinx/ext/coverage.py
@@ -265,4 +265,4 @@ def setup(app):
     app.add_config_value('coverage_ignore_c_items', {}, False)
     app.add_config_value('coverage_write_headline', True, False)
     app.add_config_value('coverage_skip_undoc_in_source', False, False)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py
index 7a90ee16b..216325cb8 100644
--- a/sphinx/ext/doctest.py
+++ b/sphinx/ext/doctest.py
@@ -443,4 +443,4 @@ def setup(app):
     app.add_config_value('doctest_test_doctest_blocks', 'default', False)
     app.add_config_value('doctest_global_setup', '', False)
     app.add_config_value('doctest_global_cleanup', '', False)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/extlinks.py b/sphinx/ext/extlinks.py
index 2549de90c..ae65dbb8f 100644
--- a/sphinx/ext/extlinks.py
+++ b/sphinx/ext/extlinks.py
@@ -59,4 +59,4 @@ def setup_link_roles(app):
 def setup(app):
     app.add_config_value('extlinks', {}, 'env')
     app.connect('builder-inited', setup_link_roles)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/graphviz.py b/sphinx/ext/graphviz.py
index 05550e2a7..56831c648 100644
--- a/sphinx/ext/graphviz.py
+++ b/sphinx/ext/graphviz.py
@@ -323,4 +323,4 @@ def setup(app):
     app.add_config_value('graphviz_dot', 'dot', 'html')
     app.add_config_value('graphviz_dot_args', [], 'html')
     app.add_config_value('graphviz_output_format', 'png', 'html')
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/ifconfig.py b/sphinx/ext/ifconfig.py
index f5d729f2e..a4e4a02df 100644
--- a/sphinx/ext/ifconfig.py
+++ b/sphinx/ext/ifconfig.py
@@ -73,4 +73,4 @@ def setup(app):
     app.add_node(ifconfig)
     app.add_directive('ifconfig', IfConfig)
     app.connect('doctree-resolved', process_ifconfig_nodes)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/inheritance_diagram.py b/sphinx/ext/inheritance_diagram.py
index a6537d1f8..0b2e5ce3a 100644
--- a/sphinx/ext/inheritance_diagram.py
+++ b/sphinx/ext/inheritance_diagram.py
@@ -408,4 +408,4 @@ def setup(app):
     app.add_config_value('inheritance_graph_attrs', {}, False),
     app.add_config_value('inheritance_node_attrs', {}, False),
     app.add_config_value('inheritance_edge_attrs', {}, False),
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/intersphinx.py b/sphinx/ext/intersphinx.py
index 67e8b860c..6f3d44eb4 100644
--- a/sphinx/ext/intersphinx.py
+++ b/sphinx/ext/intersphinx.py
@@ -282,4 +282,4 @@ def setup(app):
     app.add_config_value('intersphinx_cache_limit', 5, False)
     app.connect('missing-reference', missing_reference)
     app.connect('builder-inited', load_mappings)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/jsmath.py b/sphinx/ext/jsmath.py
index 765dfb33e..9bf38f62f 100644
--- a/sphinx/ext/jsmath.py
+++ b/sphinx/ext/jsmath.py
@@ -57,4 +57,4 @@ def setup(app):
     mathbase_setup(app, (html_visit_math, None), (html_visit_displaymath, None))
     app.add_config_value('jsmath_path', '', False)
     app.connect('builder-inited', builder_inited)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/linkcode.py b/sphinx/ext/linkcode.py
index a0b4d0da5..b11fff096 100644
--- a/sphinx/ext/linkcode.py
+++ b/sphinx/ext/linkcode.py
@@ -16,9 +16,11 @@ from sphinx import addnodes
 from sphinx.locale import _
 from sphinx.errors import SphinxError
 
+
 class LinkcodeError(SphinxError):
     category = "linkcode error"
 
+
 def doctree_read(app, doctree):
     env = app.builder.env
 
@@ -68,7 +70,8 @@ def doctree_read(app, doctree):
                                         classes=['viewcode-link'])
             signode += onlynode
 
+
 def setup(app):
     app.connect('doctree-read', doctree_read)
     app.add_config_value('linkcode_resolve', None, '')
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': False}
diff --git a/sphinx/ext/mathjax.py b/sphinx/ext/mathjax.py
index c86a982df..f677ff482 100644
--- a/sphinx/ext/mathjax.py
+++ b/sphinx/ext/mathjax.py
@@ -69,4 +69,4 @@ def setup(app):
     app.add_config_value('mathjax_inline', [r'\(', r'\)'], 'html')
     app.add_config_value('mathjax_display', [r'\[', r'\]'], 'html')
     app.connect('builder-inited', builder_inited)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/napoleon/__init__.py b/sphinx/ext/napoleon/__init__.py
index 133502499..9b43d8fda 100644
--- a/sphinx/ext/napoleon/__init__.py
+++ b/sphinx/ext/napoleon/__init__.py
@@ -256,7 +256,7 @@ def setup(app):
 
     for name, (default, rebuild) in iteritems(Config._config_values):
         app.add_config_value(name, default, rebuild)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
 
 
 def _process_docstring(app, what, name, obj, options, lines):
diff --git a/sphinx/ext/pngmath.py b/sphinx/ext/pngmath.py
index 342b1bd8a..51c9d0116 100644
--- a/sphinx/ext/pngmath.py
+++ b/sphinx/ext/pngmath.py
@@ -246,4 +246,4 @@ def setup(app):
     app.add_config_value('pngmath_latex_preamble', '', 'html')
     app.add_config_value('pngmath_add_tooltips', True, 'html')
     app.connect('build-finished', cleanup_tempdir)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': True}
diff --git a/sphinx/ext/todo.py b/sphinx/ext/todo.py
index 7a369911a..f40ed8938 100644
--- a/sphinx/ext/todo.py
+++ b/sphinx/ext/todo.py
@@ -172,4 +172,4 @@ def setup(app):
     app.connect('doctree-read', process_todos)
     app.connect('doctree-resolved', process_todo_nodes)
     app.connect('env-purge-doc', purge_todos)
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': False}
diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py
index 25243d3ce..9cccff6fe 100644
--- a/sphinx/ext/viewcode.py
+++ b/sphinx/ext/viewcode.py
@@ -204,4 +204,4 @@ def setup(app):
     app.connect('missing-reference', missing_reference)
     #app.add_config_value('viewcode_include_modules', [], 'env')
     #app.add_config_value('viewcode_exclude_modules', [], 'env')
-    return {'version': sphinx.__version__}
+    return {'version': sphinx.__version__, 'parallel_read_safe': False}
diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py
index 72485570f..30dc0cb0d 100644
--- a/sphinx/util/__init__.py
+++ b/sphinx/util/__init__.py
@@ -130,6 +130,11 @@ class FilenameUniqDict(dict):
                 del self[filename]
                 self._existing.discard(unique)
 
+    def merge_other(self, docnames, other):
+        for filename, (docs, unique) in other.items():
+            for doc in docs & docnames:
+                self.add_file(doc, filename)
+
     def __getstate__(self):
         return self._existing