From 1d3362425b95151d8ecfacbd53667b2663013580 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 30 Dec 2017 00:54:55 +0900 Subject: [PATCH] Add app.is_parallel_allowed() --- sphinx/application.py | 30 +++++++++++++++- sphinx/builders/__init__.py | 11 ++---- sphinx/environment/__init__.py | 17 ++------- tests/roots/test-extensions/conf.py | 4 +++ tests/roots/test-extensions/read_parallel.py | 4 +++ tests/roots/test-extensions/read_serial.py | 4 +++ tests/roots/test-extensions/write_parallel.py | 4 +++ tests/roots/test-extensions/write_serial.py | 4 +++ tests/test_application.py | 36 +++++++++++++++++++ 9 files changed, 91 insertions(+), 23 deletions(-) create mode 100644 tests/roots/test-extensions/conf.py create mode 100644 tests/roots/test-extensions/read_parallel.py create mode 100644 tests/roots/test-extensions/read_serial.py create mode 100644 tests/roots/test-extensions/write_parallel.py create mode 100644 tests/roots/test-extensions/write_serial.py diff --git a/sphinx/application.py b/sphinx/application.py index b6fd7feef..2a084ff0c 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -19,7 +19,7 @@ import posixpath from os import path from collections import deque -from six import iteritems +from six import iteritems, itervalues from six.moves import cStringIO from docutils import nodes @@ -673,6 +673,34 @@ class Sphinx(object): logger.debug('[app] adding HTML theme: %r, %r', name, theme_path) self.html_themes[name] = theme_path + # ---- other methods ------------------------------------------------- + def is_parallel_allowed(self, typ): + # type: (unicode) -> bool + """Check parallel processing is allowed or not. + + ``typ`` is a type of processing; ``'read'`` or ``'write'``. + """ + if typ == 'read': + attrname = 'parallel_read_safe' + elif typ == 'write': + attrname = 'parallel_write_safe' + else: + raise ValueError('parallel type %s is not supported' % typ) + + for ext in itervalues(self.extensions): + allowed = getattr(ext, attrname, None) + if allowed is None: + logger.warning(__("the %s extension does not declare if it is safe " + "for parallel %sing, assuming it isn't - please " + "ask the extension author to check and make it " + "explicit"), ext.name, typ) + logger.warning('doing serial %s', typ) + return False + elif not allowed: + return False + + return True + class TemplateBridge(object): """ diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 8acd91729..d096ee4d7 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -371,15 +371,10 @@ class Builder(object): docnames = set(docnames) & self.env.found_docs # determine if we can write in parallel - self.parallel_ok = False if parallel_available and self.app.parallel > 1 and self.allow_parallel: - self.parallel_ok = True - for extension in itervalues(self.app.extensions): - if not extension.parallel_write_safe: - logger.warning('the %s extension is not safe for parallel ' - 'writing, doing serial write', extension.name) - self.parallel_ok = False - break + self.parallel_ok = self.app.is_parallel_allowed('write') + else: + self.parallel_ok = False # create a task executor to use for misc. "finish-up" tasks # if self.parallel_ok: diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index b4c40b608..583d41c1a 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -558,21 +558,10 @@ class BuildEnvironment(object): self.app.emit('env-before-read-docs', self, docnames) # check if we should do parallel or serial read - par_ok = False if parallel_available and len(docnames) > 5 and self.app.parallel > 1: - for ext in itervalues(self.app.extensions): - if ext.parallel_read_safe is None: - logger.warning(__('the %s extension does not declare if it is safe ' - 'for parallel reading, assuming it isn\'t - please ' - 'ask the extension author to check and make it ' - 'explicit'), ext.name) - logger.warning('doing serial read') - break - elif ext.parallel_read_safe is False: - break - else: - # all extensions support parallel-read - par_ok = True + par_ok = self.app.is_parallel_allowed('read') + else: + par_ok = False if par_ok: self._read_parallel(docnames, self.app, nproc=self.app.parallel) diff --git a/tests/roots/test-extensions/conf.py b/tests/roots/test-extensions/conf.py new file mode 100644 index 000000000..9a3cbc844 --- /dev/null +++ b/tests/roots/test-extensions/conf.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) diff --git a/tests/roots/test-extensions/read_parallel.py b/tests/roots/test-extensions/read_parallel.py new file mode 100644 index 000000000..a3e052f95 --- /dev/null +++ b/tests/roots/test-extensions/read_parallel.py @@ -0,0 +1,4 @@ +def setup(app): + return { + 'parallel_read_safe': True + } diff --git a/tests/roots/test-extensions/read_serial.py b/tests/roots/test-extensions/read_serial.py new file mode 100644 index 000000000..c55570a5c --- /dev/null +++ b/tests/roots/test-extensions/read_serial.py @@ -0,0 +1,4 @@ +def setup(app): + return { + 'parallel_read_safe': False + } diff --git a/tests/roots/test-extensions/write_parallel.py b/tests/roots/test-extensions/write_parallel.py new file mode 100644 index 000000000..ebc48ef9b --- /dev/null +++ b/tests/roots/test-extensions/write_parallel.py @@ -0,0 +1,4 @@ +def setup(app): + return { + 'parallel_write_safe': True, + } diff --git a/tests/roots/test-extensions/write_serial.py b/tests/roots/test-extensions/write_serial.py new file mode 100644 index 000000000..75494ce77 --- /dev/null +++ b/tests/roots/test-extensions/write_serial.py @@ -0,0 +1,4 @@ +def setup(app): + return { + 'parallel_write_safe': False + } diff --git a/tests/test_application.py b/tests/test_application.py index 785a78878..1a4b22e3e 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -12,6 +12,7 @@ from docutils import nodes from sphinx.application import ExtensionError from sphinx.domains import Domain +from sphinx.util import logging from sphinx.testing.util import strip_escseq import pytest @@ -86,3 +87,38 @@ def test_add_source_parser(app, status, warning): assert set(app.registry.get_source_parsers().keys()) == set(['*', '.md', '.test']) assert app.registry.get_source_parsers()['.md'].__name__ == 'DummyMarkdownParser' assert app.registry.get_source_parsers()['.test'].__name__ == 'TestSourceParser' + + +@pytest.mark.sphinx(testroot='extensions') +def test_add_is_parallel_allowed(app, status, warning): + logging.setup(app, status, warning) + + assert app.is_parallel_allowed('read') is True + assert app.is_parallel_allowed('write') is True + assert warning.getvalue() == '' + + app.setup_extension('read_parallel') + assert app.is_parallel_allowed('read') is True + assert app.is_parallel_allowed('write') is True + assert warning.getvalue() == '' + app.extensions.pop('read_parallel') + + app.setup_extension('write_parallel') + assert app.is_parallel_allowed('read') is False + assert app.is_parallel_allowed('write') is True + assert 'the write_parallel extension does not declare' in warning.getvalue() + app.extensions.pop('write_parallel') + warning.truncate(0) # reset warnings + + app.setup_extension('read_serial') + assert app.is_parallel_allowed('read') is False + assert app.is_parallel_allowed('write') is True + assert warning.getvalue() == '' + app.extensions.pop('read_serial') + + app.setup_extension('write_serial') + assert app.is_parallel_allowed('read') is False + assert app.is_parallel_allowed('write') is False + assert 'the write_serial extension does not declare' in warning.getvalue() + app.extensions.pop('write_serial') + warning.truncate(0) # reset warnings