From 1db38e0eca560b59a8276c4a80efd9e42483ffff Mon Sep 17 00:00:00 2001 From: mark floyd Date: Fri, 1 Dec 2017 18:34:33 -0800 Subject: [PATCH 1/7] sphinx-build: Add support for "-j auto" Adjust -j argument so it can accept special keyword "auto", which will set -j to the number of CPUs on machine. Change was inspired by headaches while trying to find the best way to get sphinx-build to use the max number of cores from any machine we build on (Ubuntu, Mac, AWS Linux, etc). Rather than write external scripts to set this (sysctl calls, grepping /proc/cpuinfo, etc), it made more sense to build the functionality into sphinx-build itself. Change Summary: * New custom type in sphinx.cmdline for handling -j argument * Adjusted help string for -j flag --- sphinx/cmdline.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sphinx/cmdline.py b/sphinx/cmdline.py index 11f1861d8..590aa8b0a 100644 --- a/sphinx/cmdline.py +++ b/sphinx/cmdline.py @@ -11,6 +11,7 @@ from __future__ import print_function import argparse +import multiprocessing import sys import traceback from os import path @@ -83,6 +84,19 @@ def handle_exception(app, args, exception, stderr=sys.stderr): file=stderr) +def jobs_argument(value): + # type (int, str) -> int + """ + Special type to handle 'auto' flags passed to 'sphinx-build' via -j flag. Can + be expanded to handle other special scaling requests, such as setting job count + to (cpu_count / 2) + """ + if value == 'auto': + return multiprocessing.cpu_count() + else: + return value + + def get_parser(): # type: () -> argparse.ArgumentParser parser = argparse.ArgumentParser( @@ -129,10 +143,9 @@ files can be built by specifying individual filenames. group.add_argument('-d', metavar='PATH', dest='doctreedir', help='path for the cached environment and doctree ' 'files (default: OUTPUTDIR/.doctrees)') - group.add_argument('-j', metavar='N', default=1, type=int, dest='jobs', + group.add_argument('-j', metavar='N', default=1, type=jobs_argument, dest='jobs', help='build in parallel with N processes where ' - 'possible') - + 'possible (special value "auto" will set N to number of CPUs') group = parser.add_argument_group('build configuration options') group.add_argument('-c', metavar='PATH', dest='confdir', help='path where configuration file (conf.py) is ' From 19efb4a39810f1035d9d891f26154c74c6b54a89 Mon Sep 17 00:00:00 2001 From: mark floyd Date: Fri, 1 Dec 2017 18:52:46 -0800 Subject: [PATCH 2/7] Fixing typo in help string --- sphinx/cmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/cmdline.py b/sphinx/cmdline.py index 590aa8b0a..55b0cec98 100644 --- a/sphinx/cmdline.py +++ b/sphinx/cmdline.py @@ -145,7 +145,7 @@ files can be built by specifying individual filenames. 'files (default: OUTPUTDIR/.doctrees)') group.add_argument('-j', metavar='N', default=1, type=jobs_argument, dest='jobs', help='build in parallel with N processes where ' - 'possible (special value "auto" will set N to number of CPUs') + 'possible (special value "auto" will set N to cpu-count)') group = parser.add_argument_group('build configuration options') group.add_argument('-c', metavar='PATH', dest='confdir', help='path where configuration file (conf.py) is ' From 9c1bbf19488f6bd9c1261b87fcc0a10002d642a4 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 13 Jan 2018 15:44:56 +0900 Subject: [PATCH 3/7] Fix jobs_argument() should accept string as an argument --- sphinx/cmdline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/cmdline.py b/sphinx/cmdline.py index 310760d24..c2668065d 100644 --- a/sphinx/cmdline.py +++ b/sphinx/cmdline.py @@ -85,7 +85,7 @@ def handle_exception(app, args, exception, stderr=sys.stderr): def jobs_argument(value): - # type (int, str) -> int + # type: (str) -> int """ Special type to handle 'auto' flags passed to 'sphinx-build' via -j flag. Can be expanded to handle other special scaling requests, such as setting job count @@ -94,7 +94,7 @@ def jobs_argument(value): if value == 'auto': return multiprocessing.cpu_count() else: - return value + return int(value) def get_parser(): From 840b5828e752f4d19d2c569ea4299dcc17c506e3 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 13 Jan 2018 15:50:26 +0900 Subject: [PATCH 4/7] docs: Add ``-j auto`` option --- doc/man/sphinx-build.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/man/sphinx-build.rst b/doc/man/sphinx-build.rst index 46f213989..fdd0d36c2 100644 --- a/doc/man/sphinx-build.rst +++ b/doc/man/sphinx-build.rst @@ -143,11 +143,15 @@ Options Distribute the build over *N* processes in parallel, to make building on multiprocessor machines more effective. Note that not all parts and not all - builders of Sphinx can be parallelized. + builders of Sphinx can be parallelized. If ``auto`` argument is given, + Sphinx uses the number of CPUs as *N*. .. versionadded:: 1.2 This option should be considered *experimental*. + .. versionchanged:: 1.7 + Support ``auto`` argument. + .. option:: -c path Don't look for the :file:`conf.py` in the source directory, but use the given From f8dec554476c17510208e0fe08956b759cc76d9c Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 13 Jan 2018 15:53:15 +0900 Subject: [PATCH 5/7] Update docstring --- sphinx/cmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/cmdline.py b/sphinx/cmdline.py index c2668065d..365238607 100644 --- a/sphinx/cmdline.py +++ b/sphinx/cmdline.py @@ -89,7 +89,7 @@ def jobs_argument(value): """ Special type to handle 'auto' flags passed to 'sphinx-build' via -j flag. Can be expanded to handle other special scaling requests, such as setting job count - to (cpu_count / 2) + to cpu_count. """ if value == 'auto': return multiprocessing.cpu_count() From ac200ecf7cfd1854fe4ce2e937c13f405ec65d19 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 13 Jan 2018 18:27:08 +0900 Subject: [PATCH 6/7] Update CHANGES --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index 39733a6d8..b91062f68 100644 --- a/CHANGES +++ b/CHANGES @@ -75,6 +75,8 @@ Features added * #4183: doctest: ``:pyversion:`` option also follows PEP-440 specification * #4235: html: Add :confval:`manpages_url` to make manpage roles to hyperlinks * #3570: autodoc: Do not display 'typing.' module for type hints +* #4271: sphinx-build supports an option called ``-j auto`` to adjust numbers of + processes automatically. Features removed ---------------- From ddcb45dad608c2d93c4a1da9fa17a0d27645e3ab Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 16 Jan 2018 01:03:53 +0900 Subject: [PATCH 7/7] Validate -j option is positive --- sphinx/cmdline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sphinx/cmdline.py b/sphinx/cmdline.py index 365238607..4cf5a07ef 100644 --- a/sphinx/cmdline.py +++ b/sphinx/cmdline.py @@ -94,7 +94,11 @@ def jobs_argument(value): if value == 'auto': return multiprocessing.cpu_count() else: - return int(value) + jobs = int(value) + if jobs <= 0: + raise argparse.ArgumentTypeError('job number should be a positive number') + else: + return jobs def get_parser():