Add a possibility to later execute finishing-up tasks in parallel.

sphinx/application.py

@@ -248,6 +248,15 @@ class Sphinx(object):
             else:
                 self.builder.compile_update_catalogs()
                 self.builder.build_update()
+
+            status = (self.statuscode == 0
+                      and 'succeeded' or 'finished with problems')
+            if self._warncount:
+                self.info(bold('build %s, %s warning%s.' %
+                               (status, self._warncount,
+                                self._warncount != 1 and 's' or '')))
+            else:
+                self.info(bold('build %s.' % status))
         except Exception as err:
             # delete the saved env to force a fresh build next time
             envfile = path.join(self.doctreedir, ENV_PICKLE_FILENAME)

sphinx/builders/__init__.py

@@ -23,7 +23,8 @@ from docutils import nodes
 from sphinx.util import i18n, path_stabilize
 from sphinx.util.osutil import SEP, relative_uri, find_catalog
 from sphinx.util.console import bold, darkgreen
-from sphinx.util.parallel import ParallelChunked, parallel_available
+from sphinx.util.parallel import ParallelChunked, ParallelTasks, SerialTasks, \
+    parallel_available
 
 # side effect: registers roles and directives
 from sphinx import roles
@@ -70,6 +71,10 @@ class Builder(object):
         # images that need to be copied over (source -> dest)
         self.images = {}
 
+        # these get set later
+        self.parallel_ok = False
+        self.finish_tasks = None
+
         # load default translator class
         self.translator_class = app._translators.get(self.name)
 
@@ -277,20 +282,33 @@ class Builder(object):
         if docnames and docnames != ['__all__']:
             docnames = set(docnames) & self.env.found_docs
 
-        # another indirection to support builders that don't build
-        # files individually
+        # determine if we can write in parallel
+        self.parallel_ok = False
+        if parallel_available and self.app.parallel > 1 and self.allow_parallel:
+            self.parallel_ok = True
+            for extname, md in self.app._extension_metadata.items():
+                par_ok = md.get('parallel_write_safe', True)
+                if not par_ok:
+                    self.app.warn('the %s extension is not safe for parallel '
+                                  'writing, doing serial read' % extname)
+                    self.parallel_ok = False
+                    break
+
+        #  create a task executor to use for misc. "finish-up" tasks
+        # if self.parallel_ok:
+        #     self.finish_tasks = ParallelTasks(self.app.parallel)
+        # else:
+        # for now, just execute them serially
+        self.finish_tasks = SerialTasks()
+
+        # write all "normal" documents (or everything for some builders)
         self.write(docnames, list(updated_docnames), method)
 
         # finish (write static files etc.)
         self.finish()
-        status = (self.app.statuscode == 0
-                  and 'succeeded' or 'finished with problems')
-        if self.app._warncount:
-            self.info(bold('build %s, %s warning%s.' %
-                           (status, self.app._warncount,
-                            self.app._warncount != 1 and 's' or '')))
-        else:
-            self.info(bold('build %s.' % status))
+
+        # wait for all tasks
+        self.finish_tasks.join()
 
     def write(self, build_docnames, updated_docnames, method='update'):
         if build_docnames is None or build_docnames == ['__all__']:
@@ -316,25 +334,13 @@ class Builder(object):
 
         warnings = []
         self.env.set_warnfunc(lambda *args: warnings.append(args))
-        # check for prerequisites to parallel build
-        # (parallel only works on POSIX, because the forking impl of
-        # multiprocessing is required)
-        if parallel_available and len(docnames) > 5 and self.app.parallel > 1 \
-           and self.allow_parallel:
-            for extname, md in self.app._extension_metadata.items():
-                par_ok = md.get('parallel_write_safe', True)
-                if not par_ok:
-                    self.app.warn('the %s extension is not safe for parallel '
-                                  'writing, doing serial read' % extname)
-                    break
-            else:  # means no break, means everything is safe
-                # number of subprocesses is parallel-1 because the main process
-                # is busy loading doctrees and doing write_doc_serialized()
-                self._write_parallel(sorted(docnames), warnings,
-                                     nproc=self.app.parallel - 1)
-                self.env.set_warnfunc(self.warn)
-                return
-        self._write_serial(sorted(docnames), warnings)
+        if self.parallel_ok:
+            # number of subprocesses is parallel-1 because the main process
+            # is busy loading doctrees and doing write_doc_serialized()
+            self._write_parallel(sorted(docnames), warnings,
+                                 nproc=self.app.parallel - 1)
+        else:
+            self._write_serial(sorted(docnames), warnings)
         self.env.set_warnfunc(self.warn)
 
     def _write_serial(self, docnames, warnings):

sphinx/builders/html.py

@@ -443,12 +443,19 @@ class StandaloneHTMLBuilder(Builder):
         self.index_page(docname, doctree, title)
 
     def finish(self):
-        self.info(bold('writing additional files...'), nonl=1)
+        self.finish_tasks.add_task(self.gen_indices)
+        self.finish_tasks.add_task(self.gen_additional_pages)
+        self.finish_tasks.add_task(self.copy_image_files)
+        self.finish_tasks.add_task(self.copy_download_files)
+        self.finish_tasks.add_task(self.copy_static_files)
+        self.finish_tasks.add_task(self.copy_extra_files)
+        self.finish_tasks.add_task(self.write_buildinfo)
 
-        # pages from extensions
-        for pagelist in self.app.emit('html-collect-pages'):
-            for pagename, context, template in pagelist:
-                self.handle_page(pagename, context, template)
+        # dump the search index
+        self.handle_finish()
+
+    def gen_indices(self):
+        self.info(bold('generating indices...'), nonl=1)
 
         # the global general index
         if self.get_builder_config('use_index', 'html'):
@@ -457,16 +464,27 @@ class StandaloneHTMLBuilder(Builder):
         # the global domain-specific indices
         self.write_domain_indices()
 
-        # the search page
-        if self.name != 'htmlhelp':
-            self.info(' search', nonl=1)
-            self.handle_page('search', {}, 'search.html')
+        self.info()
+
+    def gen_additional_pages(self):
+        self.info(bold('writing additional pages...'), nonl=1)
+
+        # pages from extensions
+        for pagelist in self.app.emit('html-collect-pages'):
+            for pagename, context, template in pagelist:
+                self.handle_page(pagename, context, template)
 
         # additional pages from conf.py
         for pagename, template in self.config.html_additional_pages.items():
             self.info(' '+pagename, nonl=1)
             self.handle_page(pagename, {}, template)
 
+        # the search page
+        if self.name != 'htmlhelp':
+            self.info(' search', nonl=1)
+            self.handle_page('search', {}, 'search.html')
+
+        # the opensearch xml file
         if self.config.html_use_opensearch and self.name != 'htmlhelp':
             self.info(' opensearch', nonl=1)
             fn = path.join(self.outdir, '_static', 'opensearch.xml')
@@ -474,15 +492,6 @@ class StandaloneHTMLBuilder(Builder):
 
         self.info()
 
-        self.copy_image_files()
-        self.copy_download_files()
-        self.copy_static_files()
-        self.copy_extra_files()
-        self.write_buildinfo()
-
-        # dump the search index
-        self.handle_finish()
-
     def write_genindex(self):
         # the total count of lines for each index letter, used to distribute
         # the entries into two columns
@@ -786,8 +795,8 @@ class StandaloneHTMLBuilder(Builder):
             copyfile(self.env.doc2path(pagename), source_name)
 
     def handle_finish(self):
-        self.dump_search_index()
-        self.dump_inventory()
+        self.finish_tasks.add_task(self.dump_search_index)
+        self.finish_tasks.add_task(self.dump_inventory)
 
     def dump_inventory(self):
         self.info(bold('dumping object inventory... '), nonl=True)