From f4c8115a7e5f1a5d9ba10279b5a7634f63c18526 Mon Sep 17 00:00:00 2001
From: Jeroen Demeyer <jdemeyer@cage.ugent.be>
Date: Fri, 8 Sep 2017 11:42:35 +0200
Subject: [PATCH 01/31] Turn any environment unpickling error into IOError

---
 sphinx/environment/__init__.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py
index 536aa959e..a3a3e3a3b 100644
--- a/sphinx/environment/__init__.py
+++ b/sphinx/environment/__init__.py
@@ -110,7 +110,12 @@ class BuildEnvironment(object):
     @staticmethod
     def load(f, app=None):
         # type: (IO, Sphinx) -> BuildEnvironment
-        env = pickle.load(f)
+        try:
+            env = pickle.load(f)
+        except Exception as exc:
+            # This can happen for example when the pickle is from a
+            # different version of Sphinx.
+            raise IOError(exc)
         if env.version != ENV_VERSION:
             raise IOError('build environment version not current')
         if app:

From b90e58809fdb59d28ddded19d165fe38d57d5427 Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Sun, 1 Oct 2017 16:31:32 +0100
Subject: [PATCH 02/31] utils: Remove the 'reindent' tool

Nobody seems to be using this, probably because of the 'flake8' target,
and it hasn't been touched, some Python/flake8 updates aside in years.
Just remove it.

The Make target is not removed to both give us time to warn users that
the target is gone and, more importantly, to prevent merge conflicts
with other patches being submitted at the same time.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 Makefile          |   2 +-
 utils/reindent.py | 320 ----------------------------------------------
 2 files changed, 1 insertion(+), 321 deletions(-)
 delete mode 100755 utils/reindent.py

diff --git a/Makefile b/Makefile
index a20df8f39..879b10e7b 100644
--- a/Makefile
+++ b/Makefile
@@ -86,7 +86,7 @@ pylint:
 	@pylint --rcfile utils/pylintrc sphinx
 
 reindent:
-	@$(PYTHON) utils/reindent.py -r -n .
+	@echo "This target no longer does anything and will be removed imminently"
 
 test:
 	@cd tests; $(PYTHON) run.py --ignore py35 -v $(TEST)
diff --git a/utils/reindent.py b/utils/reindent.py
deleted file mode 100755
index b79657636..000000000
--- a/utils/reindent.py
+++ /dev/null
@@ -1,320 +0,0 @@
-#! /usr/bin/env python
-
-# Released to the public domain, by Tim Peters, 03 October 2000.
-
-"""reindent [-d][-r][-v] [ path ... ]
-
--d (--dryrun)   Dry run.   Analyze, but don't make any changes to, files.
--r (--recurse)  Recurse.   Search for all .py files in subdirectories too.
--n (--nobackup) No backup. Does not make a ".bak" file before reindenting.
--v (--verbose)  Verbose.   Print informative msgs; else no output.
--h (--help)     Help.      Print this usage information and exit.
-
-Change Python (.py) files to use 4-space indents and no hard tab characters.
-Also trim excess spaces and tabs from ends of lines, and remove empty lines
-at the end of files.  Also ensure the last line ends with a newline.
-
-If no paths are given on the command line, reindent operates as a filter,
-reading a single source file from standard input and writing the transformed
-source to standard output.  In this case, the -d, -r and -v flags are
-ignored.
-
-You can pass one or more file and/or directory paths.  When a directory
-path, all .py files within the directory will be examined, and, if the -r
-option is given, likewise recursively for subdirectories.
-
-If output is not to standard output, reindent overwrites files in place,
-renaming the originals with a .bak extension.  If it finds nothing to
-change, the file is left alone.  If reindent does change a file, the changed
-file is a fixed-point for future runs (i.e., running reindent on the
-resulting .py file won't change it again).
-
-The hard part of reindenting is figuring out what to do with comment
-lines.  So long as the input files get a clean bill of health from
-tabnanny.py, reindent should do a good job.
-
-The backup file is a copy of the one that is being reindented. The ".bak"
-file is generated with shutil.copy(), but some corner cases regarding
-user/group and permissions could leave the backup file more readable that
-you'd prefer. You can always use the --nobackup option to prevent this.
-"""
-from __future__ import print_function
-
-import os
-import sys
-import shutil
-import tokenize
-from six.ranges import range
-
-__version__ = "1"
-
-if sys.version_info >= (3, 0):
-    def tokens(readline, tokeneater):
-        for token in tokenize.tokenize(readline):
-            yield tokeneater(*token)
-else:
-    tokens = tokenize.tokenize
-
-verbose = 0
-recurse = 0
-dryrun = 0
-makebackup = True
-
-
-def usage(msg=None):
-    if msg is not None:
-        print(msg, file=sys.stderr)
-    print(__doc__, file=sys.stderr)
-
-
-def errprint(*args):
-    sep = ""
-    for arg in args:
-        sys.stderr.write(sep + str(arg))
-        sep = " "
-    sys.stderr.write("\n")
-
-
-def main():
-    import getopt
-    global verbose, recurse, dryrun, makebackup
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "drnvh",
-                                   ["dryrun", "recurse", "nobackup", "verbose", "help"])
-    except getopt.error as msg:
-        usage(msg)
-        return
-    for o, a in opts:
-        if o in ('-d', '--dryrun'):
-            dryrun += 1
-        elif o in ('-r', '--recurse'):
-            recurse += 1
-        elif o in ('-n', '--nobackup'):
-            makebackup = False
-        elif o in ('-v', '--verbose'):
-            verbose += 1
-        elif o in ('-h', '--help'):
-            usage()
-            return
-    if not args:
-        r = Reindenter(sys.stdin)
-        r.run()
-        r.write(sys.stdout)
-        return
-    for arg in args:
-        check(arg)
-
-
-def check(file):
-    if os.path.isdir(file) and not os.path.islink(file):
-        if verbose:
-            print("listing directory", file)
-        names = os.listdir(file)
-        for name in names:
-            fullname = os.path.join(file, name)
-            if ((recurse and os.path.isdir(fullname) and
-                 not os.path.islink(fullname) and
-                 not os.path.split(fullname)[1].startswith(".")) or
-               name.lower().endswith(".py")):
-                check(fullname)
-        return
-
-    if verbose:
-        print("checking", file, "...", end=' ')
-    try:
-        f = open(file)
-    except IOError as msg:
-        errprint("%s: I/O Error: %s" % (file, str(msg)))
-        return
-
-    with f:
-        r = Reindenter(f)
-    if r.run():
-        if verbose:
-            print("changed.")
-            if dryrun:
-                print("But this is a dry run, so leaving it alone.")
-        if not dryrun:
-            bak = file + ".bak"
-            if makebackup:
-                shutil.copyfile(file, bak)
-                if verbose:
-                    print("backed up", file, "to", bak)
-            with open(file, "w") as f:
-                r.write(f)
-            if verbose:
-                print("wrote new", file)
-        return True
-    else:
-        if verbose:
-            print("unchanged.")
-        return False
-
-
-def _rstrip(line, JUNK='\n \t'):
-    """Return line stripped of trailing spaces, tabs, newlines.
-
-    Note that line.rstrip() instead also strips sundry control characters,
-    but at least one known Emacs user expects to keep junk like that, not
-    mentioning Barry by name or anything <wink>.
-    """
-
-    i = len(line)
-    while i > 0 and line[i - 1] in JUNK:
-        i -= 1
-    return line[:i]
-
-
-class Reindenter:
-    def __init__(self, f):
-        self.find_stmt = 1  # next token begins a fresh stmt?
-        self.level = 0      # current indent level
-
-        # Raw file lines.
-        self.raw = f.readlines()
-
-        # File lines, rstripped & tab-expanded.  Dummy at start is so
-        # that we can use tokenize's 1-based line numbering easily.
-        # Note that a line is all-blank iff it's "\n".
-        self.lines = [_rstrip(line).expandtabs() + "\n"
-                      for line in self.raw]
-        self.lines.insert(0, None)
-        self.index = 1  # index into self.lines of next line
-
-        # List of (lineno, indentlevel) pairs, one for each stmt and
-        # comment line.  indentlevel is -1 for comment lines, as a
-        # signal that tokenize doesn't know what to do about them;
-        # indeed, they're our headache!
-        self.stats = []
-
-    def run(self):
-        tokens(self.getline, self.tokeneater)
-        # Remove trailing empty lines.
-        lines = self.lines
-        while lines and lines[-1] == "\n":
-            lines.pop()
-        # Sentinel.
-        stats = self.stats
-        stats.append((len(lines), 0))
-        # Map count of leading spaces to # we want.
-        have2want = {}
-        # Program after transformation.
-        after = self.after = []
-        # Copy over initial empty lines -- there's nothing to do until
-        # we see a line with *something* on it.
-        i = stats[0][0]
-        after.extend(lines[1:i])
-        for i in range(len(stats) - 1):
-            thisstmt, thislevel = stats[i]
-            nextstmt = stats[i + 1][0]
-            have = getlspace(lines[thisstmt])
-            want = thislevel * 4
-            if want < 0:
-                # A comment line.
-                if have:
-                    # An indented comment line.  If we saw the same
-                    # indentation before, reuse what it most recently
-                    # mapped to.
-                    want = have2want.get(have, -1)
-                    if want < 0:
-                        # Then it probably belongs to the next real stmt.
-                        for j in range(i + 1, len(stats) - 1):
-                            jline, jlevel = stats[j]
-                            if jlevel >= 0:
-                                if have == getlspace(lines[jline]):
-                                    want = jlevel * 4
-                                break
-                    if want < 0:        # Maybe it's a hanging
-                                        # comment like this one,
-                        # in which case we should shift it like its base
-                        # line got shifted.
-                        for j in range(i - 1, -1, -1):
-                            jline, jlevel = stats[j]
-                            if jlevel >= 0:
-                                want = (have + getlspace(after[jline - 1]) -
-                                        getlspace(lines[jline]))
-                                break
-                    if want < 0:
-                        # Still no luck -- leave it alone.
-                        want = have
-                else:
-                    want = 0
-            assert want >= 0
-            have2want[have] = want
-            diff = want - have
-            if diff == 0 or have == 0:
-                after.extend(lines[thisstmt:nextstmt])
-            else:
-                for line in lines[thisstmt:nextstmt]:
-                    if diff > 0:
-                        if line == "\n":
-                            after.append(line)
-                        else:
-                            after.append(" " * diff + line)
-                    else:
-                        remove = min(getlspace(line), -diff)
-                        after.append(line[remove:])
-        return self.raw != self.after
-
-    def write(self, f):
-        f.writelines(self.after)
-
-    # Line-getter for tokenize.
-    def getline(self):
-        if self.index >= len(self.lines):
-            line = ""
-        else:
-            line = self.lines[self.index]
-            self.index += 1
-        return line
-
-    # Line-eater for tokenize.
-    def tokeneater(self, type, token, position, end, line,
-                   INDENT=tokenize.INDENT,
-                   DEDENT=tokenize.DEDENT,
-                   NEWLINE=tokenize.NEWLINE,
-                   COMMENT=tokenize.COMMENT,
-                   NL=tokenize.NL):
-
-        if type == NEWLINE:
-            # A program statement, or ENDMARKER, will eventually follow,
-            # after some (possibly empty) run of tokens of the form
-            #     (NL | COMMENT)* (INDENT | DEDENT+)?
-            self.find_stmt = 1
-
-        elif type == INDENT:
-            self.find_stmt = 1
-            self.level += 1
-
-        elif type == DEDENT:
-            self.find_stmt = 1
-            self.level -= 1
-
-        elif type == COMMENT:
-            if self.find_stmt:
-                self.stats.append((position[0], -1))
-                # but we're still looking for a new stmt, so leave
-                # find_stmt alone
-
-        elif type == NL:
-            pass
-
-        elif self.find_stmt:
-            # This is the first "real token" following a NEWLINE, so it
-            # must be the first token of the next program statement, or an
-            # ENDMARKER.
-            self.find_stmt = 0
-            if line:   # not endmarker
-                self.stats.append((position[0], self.level))
-
-
-# Count number of leading blanks.
-def getlspace(line):
-    i, n = 0, len(line)
-    while i < n and line[i] == " ":
-        i += 1
-    return i
-
-
-if __name__ == '__main__':
-    main()

From bdaab9ef1be3e0d9ee594e659842cb162e147bfe Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Sun, 4 Jun 2017 11:02:31 +0100
Subject: [PATCH 03/31] Remove static scripts

Executable scripts without the '.py' extension are already created and
installed by setuptools, by way of the 'console_scripts' section. These
scripts are unnecessary noise. If people want to test this behavior
locally, use a virtualenv.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 doc/Makefile         |  2 +-
 sphinx-apidoc.py     | 15 ---------------
 sphinx-autogen.py    | 15 ---------------
 sphinx-build.py      | 15 ---------------
 sphinx-quickstart.py | 15 ---------------
 5 files changed, 1 insertion(+), 61 deletions(-)
 delete mode 100755 sphinx-apidoc.py
 delete mode 100755 sphinx-autogen.py
 delete mode 100755 sphinx-build.py
 delete mode 100755 sphinx-quickstart.py

diff --git a/doc/Makefile b/doc/Makefile
index d0e4e297b..c54236be0 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS   =
-SPHINXBUILD  = python ../sphinx-build.py
+SPHINXBUILD  = python ../sphinx/cmd/build.py
 SPHINXPROJ   = sphinx
 SOURCEDIR    = .
 BUILDDIR     = _build
diff --git a/sphinx-apidoc.py b/sphinx-apidoc.py
deleted file mode 100755
index eb86e0b12..000000000
--- a/sphinx-apidoc.py
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.ext.apidoc import main
-    sys.exit(main(sys.argv[1:]))
diff --git a/sphinx-autogen.py b/sphinx-autogen.py
deleted file mode 100755
index c9a78d158..000000000
--- a/sphinx-autogen.py
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.ext.autosummary.generate import main
-    sys.exit(main(sys.argv[1:]))
diff --git a/sphinx-build.py b/sphinx-build.py
deleted file mode 100755
index e8116fefc..000000000
--- a/sphinx-build.py
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx import main
-    sys.exit(main(sys.argv[1:]))
diff --git a/sphinx-quickstart.py b/sphinx-quickstart.py
deleted file mode 100755
index 3caa6590f..000000000
--- a/sphinx-quickstart.py
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.cmd.quickstart import main
-    sys.exit(main(sys.argv[1:]))

From 87630c8ae8bff8c0e23187676e6343d8903003a6 Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Sun, 4 Jun 2017 11:38:58 +0100
Subject: [PATCH 04/31] sphinx-apidoc: Convert to argparse

This is pretty self-explanatory, with most changes coming about as a
side-effect of argparse vs. optparse API differences.

Lowercase characters are used in help strings, per argparse conventions.
Some tests are converted because argparse natively supports unicode.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/ext/apidoc.py     | 211 ++++++++++++++++++++-------------------
 tests/test_ext_apidoc.py |   8 +-
 2 files changed, 112 insertions(+), 107 deletions(-)

diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py
index 4a5c56850..1a3fc0c4e 100644
--- a/sphinx/ext/apidoc.py
+++ b/sphinx/ext/apidoc.py
@@ -17,9 +17,9 @@
 
 from __future__ import print_function
 
+import argparse
 import os
 import sys
-import optparse
 from os import path
 from six import binary_type
 from fnmatch import fnmatch
@@ -287,103 +287,107 @@ def is_excluded(root, excludes):
 def main(argv=sys.argv[1:]):
     # type: (List[str]) -> int
     """Parse and check the command line arguments."""
-    parser = optparse.OptionParser(
-        usage="""\
-usage: %prog [options] -o <output_path> <module_path> [exclude_pattern, ...]
+    parser = argparse.ArgumentParser(
+        usage='usage: %(prog)s [OPTIONS] -o <OUTPUT_PATH> <MODULE_PATH> '
+              '[EXCLUDE_PATTERN, ...]',
+        epilog='For more information, visit <http://sphinx-doc.org/>.',
+        description="""
+Look recursively in <MODULE_PATH> for Python modules and packages and create
+one reST file with automodule directives per package in the <OUTPUT_PATH>.
 
-Look recursively in <module_path> for Python modules and packages and create
-one reST file with automodule directives per package in the <output_path>.
-
-The <exclude_pattern>s can be file and/or directory patterns that will be
+The <EXCLUDE_PATTERN>s can be file and/or directory patterns that will be
 excluded from generation.
 
 Note: By default this script will not overwrite already created files.""")
 
-    parser.add_option('-o', '--output-dir', action='store', dest='destdir',
-                      help='Directory to place all output', default='')
-    parser.add_option('-d', '--maxdepth', action='store', dest='maxdepth',
-                      help='Maximum depth of submodules to show in the TOC '
-                      '(default: 4)', type='int', default=4)
-    parser.add_option('-f', '--force', action='store_true', dest='force',
-                      help='Overwrite existing files')
-    parser.add_option('-l', '--follow-links', action='store_true',
-                      dest='followlinks', default=False,
-                      help='Follow symbolic links. Powerful when combined '
-                      'with collective.recipe.omelette.')
-    parser.add_option('-n', '--dry-run', action='store_true', dest='dryrun',
-                      help='Run the script without creating files')
-    parser.add_option('-e', '--separate', action='store_true',
-                      dest='separatemodules',
-                      help='Put documentation for each module on its own page')
-    parser.add_option('-P', '--private', action='store_true',
-                      dest='includeprivate',
-                      help='Include "_private" modules')
-    parser.add_option('-T', '--no-toc', action='store_true', dest='notoc',
-                      help='Don\'t create a table of contents file')
-    parser.add_option('-E', '--no-headings', action='store_true',
-                      dest='noheadings',
-                      help='Don\'t create headings for the module/package '
-                           'packages (e.g. when the docstrings already contain '
-                           'them)')
-    parser.add_option('-M', '--module-first', action='store_true',
-                      dest='modulefirst',
-                      help='Put module documentation before submodule '
-                      'documentation')
-    parser.add_option('--implicit-namespaces', action='store_true',
-                      dest='implicit_namespaces',
-                      help='Interpret module paths according to PEP-0420 '
-                           'implicit namespaces specification')
-    parser.add_option('-s', '--suffix', action='store', dest='suffix',
-                      help='file suffix (default: rst)', default='rst')
-    parser.add_option('-F', '--full', action='store_true', dest='full',
-                      help='Generate a full project with sphinx-quickstart')
-    parser.add_option('-a', '--append-syspath', action='store_true',
-                      dest='append_syspath',
-                      help='Append module_path to sys.path, used when --full is given')
-    parser.add_option('-H', '--doc-project', action='store', dest='header',
-                      help='Project name (default: root module name)')
-    parser.add_option('-A', '--doc-author', action='store', dest='author',
-                      type='str',
-                      help='Project author(s), used when --full is given')
-    parser.add_option('-V', '--doc-version', action='store', dest='version',
-                      help='Project version, used when --full is given')
-    parser.add_option('-R', '--doc-release', action='store', dest='release',
-                      help='Project release, used when --full is given, '
-                      'defaults to --doc-version')
-    parser.add_option('--version', action='store_true', dest='show_version',
-                      help='Show version information and exit')
-    group = parser.add_option_group('Extension options')
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
+
+    parser.add_argument('module_path',
+                        help='path to module to document')
+    parser.add_argument('exclude_pattern', nargs='*',
+                        help='fnmatch-style file and/or directory patterns '
+                        'to exclude from generation')
+
+    parser.add_argument('-o', '--output-dir', action='store', dest='destdir',
+                        required=True,
+                        help='directory to place all output')
+    parser.add_argument('-d', '--maxdepth', action='store', dest='maxdepth',
+                        type=int, default=4,
+                        help='maximum depth of submodules to show in the TOC '
+                             '(default: 4)')
+    parser.add_argument('-f', '--force', action='store_true', dest='force',
+                        help='overwrite existing files')
+    parser.add_argument('-l', '--follow-links', action='store_true',
+                        dest='followlinks', default=False,
+                        help='follow symbolic links. Powerful when combined '
+                              'with collective.recipe.omelette.')
+    parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun',
+                        help='run the script without creating files')
+    parser.add_argument('-e', '--separate', action='store_true',
+                        dest='separatemodules',
+                        help='put documentation for each module on its own page')
+    parser.add_argument('-P', '--private', action='store_true',
+                        dest='includeprivate',
+                        help='include "_private" modules')
+    parser.add_argument('-T', '--no-toc', action='store_true', dest='notoc',
+                        help="don't create a table of contents file")
+    parser.add_argument('-E', '--no-headings', action='store_true',
+                        dest='noheadings',
+                        help="don't create headings for the module/package "
+                             "packages (e.g. when the docstrings already "
+                             "contain them)")
+    parser.add_argument('-M', '--module-first', action='store_true',
+                        dest='modulefirst',
+                        help='put module documentation before submodule '
+                             'documentation')
+    parser.add_argument('--implicit-namespaces', action='store_true',
+                        dest='implicit_namespaces',
+                        help='interpret module paths according to PEP-0420 '
+                             'implicit namespaces specification')
+    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
+                        default='rst',
+                        help='file suffix (default: rst)')
+    parser.add_argument('-F', '--full', action='store_true', dest='full',
+                        help='generate a full project with sphinx-quickstart')
+    parser.add_argument('-a', '--append-syspath', action='store_true',
+                        dest='append_syspath',
+                        help='append module_path to sys.path, used when --full is given')
+    parser.add_argument('-H', '--doc-project', action='store', dest='header',
+                        help='project name (default: root module name)')
+    parser.add_argument('-A', '--doc-author', action='store', dest='author',
+                        help='project author(s), used when --full is given')
+    parser.add_argument('-V', '--doc-version', action='store', dest='version',
+                        help='project version, used when --full is given')
+    parser.add_argument('-R', '--doc-release', action='store', dest='release',
+                        help='project release, used when --full is given, '
+                             'defaults to --doc-version')
+
+    group = parser.add_argument_group('extension options')
     for ext in EXTENSIONS:
-        group.add_option('--ext-' + ext, action='store_true',
-                         dest='ext_' + ext, default=False,
-                         help='enable %s extension' % ext)
+        group.add_argument('--ext-' + ext, action='store_true',
+                           dest='ext_' + ext, default=False,
+                           help='enable %s extension' % ext)
 
-    (opts, args) = parser.parse_args(argv)
+    args = parser.parse_args(argv)
 
-    if opts.show_version:
-        print('Sphinx (sphinx-apidoc) %s' % __display_version__)
-        return 0
+    rootpath = path.abspath(args.module_path)
 
-    if not args:
-        parser.error('A package path is required.')
+    # normalize opts
 
-    rootpath, excludes = args[0], args[1:]
-    if not opts.destdir:
-        parser.error('An output directory is required.')
-    if opts.header is None:
-        opts.header = path.abspath(rootpath).split(path.sep)[-1]
-    if opts.suffix.startswith('.'):
-        opts.suffix = opts.suffix[1:]
+    if args.header is None:
+        args.header = rootpath.split(path.sep)[-1]
+    if args.suffix.startswith('.'):
+        args.suffix = args.suffix[1:]
     if not path.isdir(rootpath):
         print('%s is not a directory.' % rootpath, file=sys.stderr)
         sys.exit(1)
-    if not path.isdir(opts.destdir):
-        if not opts.dryrun:
-            os.makedirs(opts.destdir)
-    rootpath = path.abspath(rootpath)
-    excludes = normalize_excludes(rootpath, excludes)
-    modules = recurse_tree(rootpath, excludes, opts)
-    if opts.full:
+    if not path.isdir(args.destdir) and not args.dryrun:
+        os.makedirs(args.destdir)
+    excludes = normalize_excludes(rootpath, args.exclude_pattern)
+    modules = recurse_tree(rootpath, excludes, args)
+
+    if args.full:
         from sphinx.cmd import quickstart as qs
         modules.sort()
         prev_module = ''  # type: unicode
@@ -394,14 +398,14 @@ Note: By default this script will not overwrite already created files.""")
             prev_module = module
             text += '   %s\n' % module
         d = dict(
-            path = opts.destdir,
+            path = args.destdir,
             sep = False,
             dot = '_',
-            project = opts.header,
-            author = opts.author or 'Author',
-            version = opts.version or '',
-            release = opts.release or opts.version or '',
-            suffix = '.' + opts.suffix,
+            project = args.header,
+            author = args.author or 'Author',
+            version = args.version or '',
+            release = args.release or args.version or '',
+            suffix = '.' + args.suffix,
             master = 'index',
             epub = True,
             ext_autodoc = True,
@@ -409,29 +413,30 @@ Note: By default this script will not overwrite already created files.""")
             ext_todo = True,
             makefile = True,
             batchfile = True,
-            mastertocmaxdepth = opts.maxdepth,
+            mastertocmaxdepth = args.maxdepth,
             mastertoctree = text,
             language = 'en',
             module_path = rootpath,
-            append_syspath = opts.append_syspath,
+            append_syspath = args.append_syspath,
         )
-        enabled_exts = {'ext_' + ext: getattr(opts, 'ext_' + ext)
-                        for ext in EXTENSIONS if getattr(opts, 'ext_' + ext)}
+        enabled_exts = {'ext_' + ext: getattr(args, 'ext_' + ext)
+                        for ext in EXTENSIONS if getattr(args, 'ext_' + ext)}
         d.update(enabled_exts)
 
-        if isinstance(opts.header, binary_type):
+        if isinstance(args.header, binary_type):
             d['project'] = d['project'].decode('utf-8')
-        if isinstance(opts.author, binary_type):
+        if isinstance(args.author, binary_type):
             d['author'] = d['author'].decode('utf-8')
-        if isinstance(opts.version, binary_type):
+        if isinstance(args.version, binary_type):
             d['version'] = d['version'].decode('utf-8')
-        if isinstance(opts.release, binary_type):
+        if isinstance(args.release, binary_type):
             d['release'] = d['release'].decode('utf-8')
 
-        if not opts.dryrun:
-            qs.generate(d, silent=True, overwrite=opts.force)
-    elif not opts.notoc:
-        create_modules_toc_file(modules, opts)
+        if not args.dryrun:
+            qs.generate(d, silent=True, overwrite=args.force)
+    elif not args.notoc:
+        create_modules_toc_file(modules, args)
+
     return 0
 
 
diff --git a/tests/test_ext_apidoc.py b/tests/test_ext_apidoc.py
index 794591aa6..d98dbabb6 100644
--- a/tests/test_ext_apidoc.py
+++ b/tests/test_ext_apidoc.py
@@ -152,10 +152,10 @@ def test_trailing_underscore(make_app, apidoc):
 @pytest.mark.apidoc(
     coderoot='test-root',
     options=[
-        '--doc-project', u'プロジェクト名'.encode('utf-8'),
-        '--doc-author', u'著者名'.encode('utf-8'),
-        '--doc-version', u'バージョン'.encode('utf-8'),
-        '--doc-release', u'リリース'.encode('utf-8'),
+        '--doc-project', u'プロジェクト名',
+        '--doc-author', u'著者名',
+        '--doc-version', u'バージョン',
+        '--doc-release', u'リリース',
     ],
 )
 def test_multibyte_parameters(make_app, apidoc):

From fa74085afdb717aea83e8766734a65481d4f6f8d Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Thu, 28 Sep 2017 19:11:02 +0100
Subject: [PATCH 05/31] sphinx-apidoc: Move parser to a separate function

This lets us better reason about what the parser is doing and use tools
like 'sphinx-contrib.autoprogram' in our own documentation in the
future.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/ext/apidoc.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py
index 1a3fc0c4e..73d3027a3 100644
--- a/sphinx/ext/apidoc.py
+++ b/sphinx/ext/apidoc.py
@@ -284,9 +284,8 @@ def is_excluded(root, excludes):
     return False
 
 
-def main(argv=sys.argv[1:]):
-    # type: (List[str]) -> int
-    """Parse and check the command line arguments."""
+def get_parser():
+    # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(
         usage='usage: %(prog)s [OPTIONS] -o <OUTPUT_PATH> <MODULE_PATH> '
               '[EXCLUDE_PATTERN, ...]',
@@ -369,6 +368,13 @@ Note: By default this script will not overwrite already created files.""")
                            dest='ext_' + ext, default=False,
                            help='enable %s extension' % ext)
 
+    return parser
+
+
+def main(argv=sys.argv[1:]):
+    # type: (List[str]) -> int
+    """Parse and check the command line arguments."""
+    parser = get_parser()
     args = parser.parse_args(argv)
 
     rootpath = path.abspath(args.module_path)

From a3f1489544ad1b66c034e1a7edb009a916b4141f Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Sun, 4 Jun 2017 11:44:23 +0100
Subject: [PATCH 06/31] sphinx-apidoc: Remove 'normalize_excludes' function

This is a one-liner that really doesn't need to be split out, so don't
do it.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/ext/apidoc.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py
index 73d3027a3..cb020aba2 100644
--- a/sphinx/ext/apidoc.py
+++ b/sphinx/ext/apidoc.py
@@ -265,12 +265,6 @@ def recurse_tree(rootpath, excludes, opts):
     return toplevels
 
 
-def normalize_excludes(rootpath, excludes):
-    # type: (unicode, List[unicode]) -> List[unicode]
-    """Normalize the excluded directory list."""
-    return [path.abspath(exclude) for exclude in excludes]
-
-
 def is_excluded(root, excludes):
     # type: (unicode, List[unicode]) -> bool
     """Check if the directory is in the exclude list.
@@ -390,7 +384,7 @@ def main(argv=sys.argv[1:]):
         sys.exit(1)
     if not path.isdir(args.destdir) and not args.dryrun:
         os.makedirs(args.destdir)
-    excludes = normalize_excludes(rootpath, args.exclude_pattern)
+    excludes = [path.abspath(exclude) for exclude in args.exclude_pattern]
     modules = recurse_tree(rootpath, excludes, args)
 
     if args.full:

From fcf0c1247fba23ef86d6b6eb0f1b41f22106bb59 Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Sun, 4 Jun 2017 12:02:04 +0100
Subject: [PATCH 07/31] sphinx-autogen: Convert to argparse

Another mostly trivial conversion. The only odd thing here is that we
add a '--version' parameter to keep things in line with the other
applications.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/ext/autosummary/generate.py | 55 +++++++++++++++++-------------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py
index 13b463e87..2eaeb6530 100644
--- a/sphinx/ext/autosummary/generate.py
+++ b/sphinx/ext/autosummary/generate.py
@@ -19,16 +19,17 @@
 """
 from __future__ import print_function
 
+import argparse
+import codecs
 import os
+import pydoc
 import re
 import sys
-import pydoc
-import optparse
-import codecs
 
 from jinja2 import FileSystemLoader, TemplateNotFound
 from jinja2.sandbox import SandboxedEnvironment
 
+from sphinx import __display_version__
 from sphinx import package_dir
 from sphinx.ext.autosummary import import_by_name, get_documenter
 from sphinx.jinja2glue import BuiltinTemplateLoader
@@ -61,29 +62,35 @@ if False:
 
 def main(argv=sys.argv[1:]):
     # type: (List[str]) -> None
-    usage = """%prog [OPTIONS] SOURCEFILE ..."""
-    p = optparse.OptionParser(usage.strip())
-    p.add_option("-o", "--output-dir", action="store", type="string",
-                 dest="output_dir", default=None,
-                 help="Directory to place all output in")
-    p.add_option("-s", "--suffix", action="store", type="string",
-                 dest="suffix", default="rst",
-                 help="Default suffix for files (default: %default)")
-    p.add_option("-t", "--templates", action="store", type="string",
-                 dest="templates", default=None,
-                 help="Custom template directory (default: %default)")
-    p.add_option("-i", "--imported-members", action="store_true",
-                 dest="imported_members", default=False,
-                 help="Document imported members (default: %default)")
-    options, args = p.parse_args(argv)
+    parser = argparse.ArgumentParser(
+        usage='%(prog)s [OPTIONS] <SOURCE_FILE>...')
 
-    if len(args) < 1:
-        p.error('no input files given')
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
 
-    generate_autosummary_docs(args, options.output_dir,
-                              "." + options.suffix,
-                              template_dir=options.templates,
-                              imported_members=options.imported_members)
+    parser.add_argument('source_file', nargs='+',
+                        help='source files to generate rST files for')
+    parser.add_argument('-o', '--output-dir', action='store',
+                        dest='output_dir',
+                        help='directory to place all output in')
+    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
+                        default='rst',
+                        help='default suffix for files (default: '
+                              '%(default)s)')
+    parser.add_argument('-t', '--templates', action='store', dest='templates',
+                        default=None,
+                        help='custom template directory (default: '
+                              '%(default)s)')
+    parser.add_argument('-i', '--imported-members', action='store_true',
+                        dest='imported_members', default=False,
+                        help='document imported members (default: '
+                              '%(default)s)')
+
+    args = parser.parse_args(argv)
+    generate_autosummary_docs(args.source_file, args.output_dir,
+                              '.' + args.suffix,
+                              template_dir=args.templates,
+                              imported_members=args.imported_members)
 
 
 def _simple_info(msg):

From 6a88b5a6c27718aeaaecb124d7ecfdfd8c08e736 Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Thu, 28 Sep 2017 19:15:58 +0100
Subject: [PATCH 08/31] sphinx-autogen: Move parser to a separate function

For the same reasons as the 'sphinx-apidoc' move. We also take the
opportunity to add a help string and epilog while we're at it.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/ext/autosummary/generate.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py
index 2eaeb6530..731586430 100644
--- a/sphinx/ext/autosummary/generate.py
+++ b/sphinx/ext/autosummary/generate.py
@@ -60,16 +60,30 @@ if False:
     from sphinx.environment import BuildEnvironment  # NOQA
 
 
-def main(argv=sys.argv[1:]):
-    # type: (List[str]) -> None
+def get_parser():
+    # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(
-        usage='%(prog)s [OPTIONS] <SOURCE_FILE>...')
+        usage='%(prog)s [OPTIONS] <SOURCE_FILE>...',
+        epilog='For more information, visit <http://sphinx-doc.org/>.',
+        description="""
+Generate ReStructuredText using autosummary directives.
+
+sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates
+the reStructuredText files from the autosummary directives contained in the
+given input files.
+
+The format of the autosummary directive is documented in the
+``sphinx.ext.autosummary`` Python module and can be read using::
+
+  pydoc sphinx.ext.autosummary
+""")
 
     parser.add_argument('--version', action='version', dest='show_version',
                         version='%%(prog)s %s' % __display_version__)
 
     parser.add_argument('source_file', nargs='+',
                         help='source files to generate rST files for')
+
     parser.add_argument('-o', '--output-dir', action='store',
                         dest='output_dir',
                         help='directory to place all output in')
@@ -86,7 +100,12 @@ def main(argv=sys.argv[1:]):
                         help='document imported members (default: '
                               '%(default)s)')
 
-    args = parser.parse_args(argv)
+    return parser
+
+
+def main(argv=sys.argv[1:]):
+    # type: (List[str]) -> None
+    args = get_parser().parse_args(argv)
     generate_autosummary_docs(args.source_file, args.output_dir,
                               '.' + args.suffix,
                               template_dir=args.templates,

From b506d872927893f668292cbb7be290a93cfd5687 Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Sun, 4 Jun 2017 12:03:08 +0100
Subject: [PATCH 09/31] sphinx-autogen: Move main to end of file

Per Python customs.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/ext/autosummary/generate.py | 104 ++++++++++++++---------------
 1 file changed, 52 insertions(+), 52 deletions(-)

diff --git a/sphinx/ext/autosummary/generate.py b/sphinx/ext/autosummary/generate.py
index 731586430..f02c50692 100644
--- a/sphinx/ext/autosummary/generate.py
+++ b/sphinx/ext/autosummary/generate.py
@@ -60,58 +60,6 @@ if False:
     from sphinx.environment import BuildEnvironment  # NOQA
 
 
-def get_parser():
-    # type: () -> argparse.ArgumentParser
-    parser = argparse.ArgumentParser(
-        usage='%(prog)s [OPTIONS] <SOURCE_FILE>...',
-        epilog='For more information, visit <http://sphinx-doc.org/>.',
-        description="""
-Generate ReStructuredText using autosummary directives.
-
-sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates
-the reStructuredText files from the autosummary directives contained in the
-given input files.
-
-The format of the autosummary directive is documented in the
-``sphinx.ext.autosummary`` Python module and can be read using::
-
-  pydoc sphinx.ext.autosummary
-""")
-
-    parser.add_argument('--version', action='version', dest='show_version',
-                        version='%%(prog)s %s' % __display_version__)
-
-    parser.add_argument('source_file', nargs='+',
-                        help='source files to generate rST files for')
-
-    parser.add_argument('-o', '--output-dir', action='store',
-                        dest='output_dir',
-                        help='directory to place all output in')
-    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
-                        default='rst',
-                        help='default suffix for files (default: '
-                              '%(default)s)')
-    parser.add_argument('-t', '--templates', action='store', dest='templates',
-                        default=None,
-                        help='custom template directory (default: '
-                              '%(default)s)')
-    parser.add_argument('-i', '--imported-members', action='store_true',
-                        dest='imported_members', default=False,
-                        help='document imported members (default: '
-                              '%(default)s)')
-
-    return parser
-
-
-def main(argv=sys.argv[1:]):
-    # type: (List[str]) -> None
-    args = get_parser().parse_args(argv)
-    generate_autosummary_docs(args.source_file, args.output_dir,
-                              '.' + args.suffix,
-                              template_dir=args.templates,
-                              imported_members=args.imported_members)
-
-
 def _simple_info(msg):
     # type: (unicode) -> None
     print(msg)
@@ -399,5 +347,57 @@ def find_autosummary_in_lines(lines, module=None, filename=None):
     return documented
 
 
+def get_parser():
+    # type: () -> argparse.ArgumentParser
+    parser = argparse.ArgumentParser(
+        usage='%(prog)s [OPTIONS] <SOURCE_FILE>...',
+        epilog='For more information, visit <http://sphinx-doc.org/>.',
+        description="""
+Generate ReStructuredText using autosummary directives.
+
+sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates
+the reStructuredText files from the autosummary directives contained in the
+given input files.
+
+The format of the autosummary directive is documented in the
+``sphinx.ext.autosummary`` Python module and can be read using::
+
+  pydoc sphinx.ext.autosummary
+""")
+
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
+
+    parser.add_argument('source_file', nargs='+',
+                        help='source files to generate rST files for')
+
+    parser.add_argument('-o', '--output-dir', action='store',
+                        dest='output_dir',
+                        help='directory to place all output in')
+    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
+                        default='rst',
+                        help='default suffix for files (default: '
+                              '%(default)s)')
+    parser.add_argument('-t', '--templates', action='store', dest='templates',
+                        default=None,
+                        help='custom template directory (default: '
+                              '%(default)s)')
+    parser.add_argument('-i', '--imported-members', action='store_true',
+                        dest='imported_members', default=False,
+                        help='document imported members (default: '
+                              '%(default)s)')
+
+    return parser
+
+
+def main(argv=sys.argv[1:]):
+    # type: (List[str]) -> None
+    args = get_parser().parse_args(argv)
+    generate_autosummary_docs(args.source_file, args.output_dir,
+                              '.' + args.suffix,
+                              template_dir=args.templates,
+                              imported_members=args.imported_members)
+
+
 if __name__ == '__main__':
     main()

From bca566244ab65e26b854818efa5e9f2d509b58c9 Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Thu, 28 Sep 2017 18:48:54 +0100
Subject: [PATCH 10/31] sphinx-quickstart: Convert to argparse

Nothing unusual to see here.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/cmd/quickstart.py | 167 ++++++++++++++++-----------------------
 1 file changed, 69 insertions(+), 98 deletions(-)

diff --git a/sphinx/cmd/quickstart.py b/sphinx/cmd/quickstart.py
index cdd0295e4..5bd0faa39 100644
--- a/sphinx/cmd/quickstart.py
+++ b/sphinx/cmd/quickstart.py
@@ -11,13 +11,13 @@
 from __future__ import print_function
 from __future__ import absolute_import
 
-import re
+import argparse
 import os
+import re
 import sys
-import optparse
 import time
-from os import path
 from io import open
+from os import path
 
 # try to import readline, unix specific enhancement
 try:
@@ -509,23 +509,6 @@ where "builder" is one of the supported builders, e.g. html, latex or linkcheck.
 ''')
 
 
-def usage(argv, msg=None):
-    # type: (List[unicode], unicode) -> None
-    if msg:
-        print(msg, file=sys.stderr)
-        print(file=sys.stderr)
-
-
-USAGE = """\
-Sphinx v%s
-Usage: %%prog [options] [projectdir]
-""" % __display_version__
-
-EPILOG = """\
-For more information, visit <http://sphinx-doc.org/>.
-"""
-
-
 def valid_dir(d):
     # type: (Dict) -> bool
     dir = d['path']
@@ -556,100 +539,88 @@ def valid_dir(d):
     return True
 
 
-class MyFormatter(optparse.IndentedHelpFormatter):
-    def format_usage(self, usage):  # type: ignore
-        # type: (str) -> str
-        return usage
-
-    def format_help(self, formatter):
-        result = []
-        if self.description:
-            result.append(self.format_description(formatter))
-        if self.option_list:
-            result.append(self.format_option_help(formatter))
-        return "\n".join(result)
-
-
 def main(argv=sys.argv[1:]):
     # type: (List[str]) -> int
     if not color_terminal():
         nocolor()
 
-    parser = optparse.OptionParser(USAGE, epilog=EPILOG,
-                                   version='Sphinx v%s' % __display_version__,
-                                   formatter=MyFormatter())
-    parser.add_option('-q', '--quiet', action='store_true', dest='quiet',
-                      default=False,
-                      help='quiet mode')
+    parser = argparse.ArgumentParser(
+        usage='%(prog)s [OPTIONS] <PROJECT_DIR>',
+        epilog='For more information, visit <http://sphinx-doc.org/>.')
 
-    group = parser.add_option_group('Structure options')
-    group.add_option('--sep', action='store_true', dest='sep',
-                     help='if specified, separate source and build dirs')
-    group.add_option('--dot', metavar='DOT', dest='dot',
-                     help='replacement for dot in _templates etc.')
+    parser.add_argument('-q', '--quiet', action='store_true', dest='quiet',
+                        default=False,
+                        help='quiet mode')
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
 
-    group = parser.add_option_group('Project basic options')
-    group.add_option('-p', '--project', metavar='PROJECT', dest='project',
-                     help='project name')
-    group.add_option('-a', '--author', metavar='AUTHOR', dest='author',
-                     help='author names')
-    group.add_option('-v', metavar='VERSION', dest='version',
-                     help='version of project')
-    group.add_option('-r', '--release', metavar='RELEASE', dest='release',
-                     help='release of project')
-    group.add_option('-l', '--language', metavar='LANGUAGE', dest='language',
-                     help='document language')
-    group.add_option('--suffix', metavar='SUFFIX', dest='suffix',
-                     help='source file suffix')
-    group.add_option('--master', metavar='MASTER', dest='master',
-                     help='master document name')
-    group.add_option('--epub', action='store_true', dest='epub',
-                     default=False,
-                     help='use epub')
+    parser.add_argument('path', metavar='PROJECT_DIR', default='.',
+                        help='output path')
 
-    group = parser.add_option_group('Extension options')
+    group = parser.add_argument_group('Structure options')
+    group.add_argument('--sep', action='store_true',
+                       help='if specified, separate source and build dirs')
+    group.add_argument('--dot', metavar='DOT',
+                       help='replacement for dot in _templates etc.')
+
+    group = parser.add_argument_group('Project basic options')
+    group.add_argument('-p', '--project', metavar='PROJECT', dest='project',
+                       help='project name')
+    group.add_argument('-a', '--author', metavar='AUTHOR', dest='author',
+                       help='author names')
+    group.add_argument('-v', metavar='VERSION', dest='version', default='',
+                       help='version of project')
+    group.add_argument('-r', '--release', metavar='RELEASE', dest='release',
+                       help='release of project')
+    group.add_argument('-l', '--language', metavar='LANGUAGE', dest='language',
+                       help='document language')
+    group.add_argument('--suffix', metavar='SUFFIX',
+                       help='source file suffix')
+    group.add_argument('--master', metavar='MASTER',
+                       help='master document name')
+    group.add_argument('--epub', action='store_true', default=False,
+                       help='use epub')
+
+    group = parser.add_argument_group('Extension options')
     for ext in EXTENSIONS:
-        group.add_option('--ext-' + ext, action='store_true',
-                         dest='ext_' + ext, default=False,
-                         help='enable %s extension' % ext)
-    group.add_option('--extensions', metavar='EXTENSIONS', dest='extensions',
-                     action='append', help='enable extensions')
+        group.add_argument('--ext-' + ext, action='store_true',
+                           dest='ext_' + ext, default=False,
+                           help='enable %s extension' % ext)
+    group.add_argument('--extensions', metavar='EXTENSIONS', dest='extensions',
+                       action='append', help='enable extensions')
 
-    group = parser.add_option_group('Makefile and Batchfile creation')
-    group.add_option('--makefile', action='store_true', dest='makefile',
-                     default=False,
-                     help='create makefile')
-    group.add_option('--no-makefile', action='store_true', dest='no_makefile',
-                     default=False,
-                     help='not create makefile')
-    group.add_option('--batchfile', action='store_true', dest='batchfile',
-                     default=False,
-                     help='create batchfile')
-    group.add_option('--no-batchfile', action='store_true', dest='no_batchfile',
-                     default=False,
-                     help='not create batchfile')
-    group.add_option('-M', '--no-use-make-mode', action='store_false', dest='make_mode',
-                     help='not use make-mode for Makefile/make.bat')
-    group.add_option('-m', '--use-make-mode', action='store_true', dest='make_mode',
-                     default=True,
-                     help='use make-mode for Makefile/make.bat')
+    # TODO(stephenfin): Consider using mutually exclusive groups here
+    group = parser.add_argument_group('Makefile and Batchfile creation')
+    group.add_argument('--makefile', action='store_true', default=False,
+                       help='create makefile')
+    group.add_argument('--no-makefile', action='store_true', default=False,
+                       help='not create makefile')
+    group.add_argument('--batchfile', action='store_true', default=False,
+                       help='create batchfile')
+    group.add_argument('--no-batchfile', action='store_true', default=False,
+                       help='not create batchfile')
+    group.add_argument('-M', '--no-use-make-mode', action='store_false',
+                       dest='make_mode', default=False,
+                       help='not use make-mode for Makefile/make.bat')
+    group.add_argument('-m', '--use-make-mode', action='store_true',
+                       dest='make_mode', default=True,
+                       help='use make-mode for Makefile/make.bat')
 
-    group = parser.add_option_group('Project templating')
-    group.add_option('-t', '--templatedir', metavar='TEMPLATEDIR', dest='templatedir',
-                     help='template directory for template files')
-    group.add_option('-d', metavar='NAME=VALUE', action='append', dest='variables',
-                     help='define a template variable')
+    group = parser.add_argument_group('Project templating')
+    group.add_argument('-t', '--templatedir', metavar='TEMPLATEDIR',
+                       dest='templatedir',
+                       help='template directory for template files')
+    group.add_argument('-d', metavar='NAME=VALUE', action='append',
+                       dest='variables',
+                       help='define a template variable')
 
     # parse options
     try:
-        opts, args = parser.parse_args(argv)
+        args = parser.parse_args(argv)
     except SystemExit as err:
         return err.code
 
-    if len(args) > 0:
-        opts.ensure_value('path', args[0])
-
-    d = vars(opts)
+    d = vars(args)
     # delete None or False value
     d = dict((k, v) for k, v in d.items() if not (v is None or v is False))
 
@@ -707,7 +678,7 @@ def main(argv=sys.argv[1:]):
         except ValueError:
             print('Invalid template variable: %s' % variable)
 
-    generate(d, templatedir=opts.templatedir)
+    generate(d, templatedir=args.templatedir)
     return 0
 
 

From c68381bd6597f50e085b96d7f69a982db8baabda Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Thu, 28 Sep 2017 19:23:25 +0100
Subject: [PATCH 11/31] sphinx-quickstart: Move parser to a separate function

For the same reasons as the 'sphinx-apidoc' and 'sphinx-autogen' moves.
As before, we also take the opportunity to add a help string.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/cmd/quickstart.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/sphinx/cmd/quickstart.py b/sphinx/cmd/quickstart.py
index 5bd0faa39..a616b263e 100644
--- a/sphinx/cmd/quickstart.py
+++ b/sphinx/cmd/quickstart.py
@@ -539,14 +539,18 @@ def valid_dir(d):
     return True
 
 
-def main(argv=sys.argv[1:]):
-    # type: (List[str]) -> int
-    if not color_terminal():
-        nocolor()
-
+def get_parser():
+    # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(
         usage='%(prog)s [OPTIONS] <PROJECT_DIR>',
-        epilog='For more information, visit <http://sphinx-doc.org/>.')
+        epilog="For more information, visit <http://sphinx-doc.org/>.",
+        description="""
+Generate required files for a Sphinx project.
+
+sphinx-quickstart is an interactive tool that asks some questions about your
+project and then generates a complete documentation directory and sample
+Makefile to be used with sphinx-build.
+""")
 
     parser.add_argument('-q', '--quiet', action='store_true', dest='quiet',
                         default=False,
@@ -614,7 +618,16 @@ def main(argv=sys.argv[1:]):
                        dest='variables',
                        help='define a template variable')
 
+    return parser
+
+
+def main(argv=sys.argv[1:]):
+    # type: (List[str]) -> int
+    if not color_terminal():
+        nocolor()
+
     # parse options
+    parser = get_parser()
     try:
         args = parser.parse_args(argv)
     except SystemExit as err:

From b778cfe2999930d812c5e6f16142dc5d2311996d Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Thu, 28 Sep 2017 20:00:57 +0100
Subject: [PATCH 12/31] sphinx-build: Convert to argparse

Once again, not much different to the previous conversions. We make best
use of the argparse module, allowing us to remove some funky
workarounds.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/cmdline.py | 224 ++++++++++++++++++++--------------------------
 1 file changed, 98 insertions(+), 126 deletions(-)

diff --git a/sphinx/cmdline.py b/sphinx/cmdline.py
index 54e4dcb78..30bef6674 100644
--- a/sphinx/cmdline.py
+++ b/sphinx/cmdline.py
@@ -10,14 +10,13 @@
 """
 from __future__ import print_function
 
+import argparse
 import sys
-import optparse
 import traceback
 from os import path
 
-from six import text_type, binary_type
-
 from docutils.utils import SystemMessage
+from six import text_type, binary_type
 
 from sphinx import __display_version__
 from sphinx.errors import SphinxError
@@ -33,39 +32,9 @@ if False:
     from typing import Any, IO, List, Union  # NOQA
 
 
-USAGE = """\
-Sphinx v%s
-Usage: %%prog [options] sourcedir outdir [filenames...]
-
-Filename arguments:
-  without -a and without filenames, write new and changed files.
-  with -a, write all files.
-  with filenames, write these.
-""" % __display_version__
-
-EPILOG = """\
-For more information, visit <http://sphinx-doc.org/>.
-"""
-
-
-class MyFormatter(optparse.IndentedHelpFormatter):
-    def format_usage(self, usage):
-        # type: (Any) -> Any
-        return usage
-
-    def format_help(self, formatter):
-        # type: (Any) -> unicode
-        result = []  # type: List[unicode]
-        if self.description:  # type: ignore
-            result.append(self.format_description(formatter))
-        if self.option_list:  # type: ignore
-            result.append(self.format_option_help(formatter))  # type: ignore
-        return "\n".join(result)
-
-
-def handle_exception(app, opts, exception, stderr=sys.stderr):
+def handle_exception(app, args, exception, stderr=sys.stderr):
     # type: (Sphinx, Any, Union[Exception, KeyboardInterrupt], IO) -> None
-    if opts.pdb:
+    if args.pdb:
         import pdb
         print(red('Exception occurred while building, starting debugger:'),
               file=stderr)
@@ -73,7 +42,7 @@ def handle_exception(app, opts, exception, stderr=sys.stderr):
         pdb.post_mortem(sys.exc_info()[2])
     else:
         print(file=stderr)
-        if opts.verbosity or opts.traceback:
+        if args.verbosity or args.traceback:
             traceback.print_exc(None, stderr)
             print(file=stderr)
         if isinstance(exception, KeyboardInterrupt):
@@ -116,102 +85,105 @@ def handle_exception(app, opts, exception, stderr=sys.stderr):
 
 def main(argv=sys.argv[1:]):  # type: ignore
     # type: (List[unicode]) -> int
-    parser = optparse.OptionParser(USAGE, epilog=EPILOG, formatter=MyFormatter())
-    parser.add_option('--version', action='store_true', dest='version',
-                      help='show version information and exit')
+    parser = argparse.ArgumentParser(
+        usage='usage: %(prog)s [OPTIONS] SOURCEDIR OUTDIR [FILENAMES...]',
+        epilog='For more information, visit <http://sphinx-doc.org/>.')
 
-    group = parser.add_option_group('General options')
-    group.add_option('-b', metavar='BUILDER', dest='builder', default='html',
-                     help='builder to use; default is html')
-    group.add_option('-a', action='store_true', dest='force_all',
-                     help='write all files; default is to only write new and '
-                     'changed files')
-    group.add_option('-E', action='store_true', dest='freshenv',
-                     help='don\'t use a saved environment, always read '
-                     'all files')
-    group.add_option('-d', metavar='PATH', default=None, dest='doctreedir',
-                     help='path for the cached environment and doctree files '
-                     '(default: outdir/.doctrees)')
-    group.add_option('-j', metavar='N', default=1, type='int', dest='jobs',
-                     help='build in parallel with N processes where possible')
-    # this option never gets through to this point (it is intercepted earlier)
-    # group.add_option('-M', metavar='BUILDER', dest='make_mode',
-    #                 help='"make" mode -- as used by Makefile, like '
-    #                 '"sphinx-build -M html"')
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
 
-    group = parser.add_option_group('Build configuration options')
-    group.add_option('-c', metavar='PATH', dest='confdir',
-                     help='path where configuration file (conf.py) is located '
-                     '(default: same as sourcedir)')
-    group.add_option('-C', action='store_true', dest='noconfig',
-                     help='use no config file at all, only -D options')
-    group.add_option('-D', metavar='setting=value', action='append',
-                     dest='define', default=[],
-                     help='override a setting in configuration file')
-    group.add_option('-A', metavar='name=value', action='append',
-                     dest='htmldefine', default=[],
-                     help='pass a value into HTML templates')
-    group.add_option('-t', metavar='TAG', action='append',
-                     dest='tags', default=[],
-                     help='define tag: include "only" blocks with TAG')
-    group.add_option('-n', action='store_true', dest='nitpicky',
-                     help='nit-picky mode, warn about all missing references')
+    parser.add_argument('sourcedir',
+                        help='path to documentation source files')
+    parser.add_argument('outputdir',
+                        help='path to output directory')
+    parser.add_argument('filenames', nargs='*',
+                        help='a list of specific files to rebuild. Ignored '
+                        'if -a is specified')
 
-    group = parser.add_option_group('Console output options')
-    group.add_option('-v', action='count', dest='verbosity', default=0,
-                     help='increase verbosity (can be repeated)')
-    group.add_option('-q', action='store_true', dest='quiet',
-                     help='no output on stdout, just warnings on stderr')
-    group.add_option('-Q', action='store_true', dest='really_quiet',
-                     help='no output at all, not even warnings')
-    group.add_option('--color', dest='color',
-                     action='store_const', const='yes', default='auto',
-                     help='Do emit colored output (default: auto-detect)')
-    group.add_option('-N', '--no-color', dest='color',
-                     action='store_const', const='no',
-                     help='Do not emit colored output (default: auto-detect)')
-    group.add_option('-w', metavar='FILE', dest='warnfile',
-                     help='write warnings (and errors) to given file')
-    group.add_option('-W', action='store_true', dest='warningiserror',
-                     help='turn warnings into errors')
-    group.add_option('-T', action='store_true', dest='traceback',
-                     help='show full traceback on exception')
-    group.add_option('-P', action='store_true', dest='pdb',
-                     help='run Pdb on exception')
+    group = parser.add_argument_group('general options')
+    group.add_argument('-b', metavar='BUILDER', dest='builder',
+                       default='html',
+                       help='builder to use (default: html)')
+    group.add_argument('-a', action='store_true', dest='force_all',
+                       help='write all files (default: only write new and '
+                       'changed files)')
+    group.add_argument('-E', action='store_true', dest='freshenv',
+                       help='don\'t use a saved environment, always read '
+                       'all files')
+    group.add_argument('-d', metavar='PATH', dest='doctreedir',
+                       help='path for the cached environment and doctree '
+                       'files (default: outdir/.doctrees)')
+    group.add_argument('-j', metavar='N', default=1, type=int, dest='jobs',
+                       help='build in parallel with N processes where '
+                       'possible')
+
+    group = parser.add_argument_group('build configuration options')
+    group.add_argument('-c', metavar='PATH', dest='confdir',
+                       help='path where configuration file (conf.py) is '
+                       'located (default: same as sourcedir)')
+    group.add_argument('-C', action='store_true', dest='noconfig',
+                       help='use no config file at all, only -D options')
+    group.add_argument('-D', metavar='setting=value', action='append',
+                       dest='define', default=[],
+                       help='override a setting in configuration file')
+    group.add_argument('-A', metavar='name=value', action='append',
+                       dest='htmldefine', default=[],
+                       help='pass a value into HTML templates')
+    group.add_argument('-t', metavar='TAG', action='append',
+                       dest='tags', default=[],
+                       help='define tag: include "only" blocks with TAG')
+    group.add_argument('-n', action='store_true', dest='nitpicky',
+                       help='nit-picky mode, warn about all missing '
+                       'references')
+
+    group = parser.add_argument_group('console output options')
+    group.add_argument('-v', action='count', dest='verbosity', default=0,
+                       help='increase verbosity (can be repeated)')
+    group.add_argument('-q', action='store_true', dest='quiet',
+                       help='no output on stdout, just warnings on stderr')
+    group.add_argument('-Q', action='store_true', dest='really_quiet',
+                       help='no output at all, not even warnings')
+    group.add_argument('--color', action='store_const', const='yes',
+                       default='auto',
+                       help='do emit colored output (default: auto-detect)')
+    group.add_argument('-N', '--no-color', dest='color', action='store_const',
+                       const='no',
+                       help='do not emit colored output (default: '
+                       'auto-detect)')
+    group.add_argument('-w', metavar='FILE', dest='warnfile',
+                       help='write warnings (and errors) to given file')
+    group.add_argument('-W', action='store_true', dest='warningiserror',
+                       help='turn warnings into errors')
+    group.add_argument('-T', action='store_true', dest='traceback',
+                       help='show full traceback on exception')
+    group.add_argument('-P', action='store_true', dest='pdb',
+                       help='run Pdb on exception')
 
     # parse options
     try:
-        opts, args = parser.parse_args(argv)
+        args = parser.parse_args(argv)
     except SystemExit as err:
         return err.code
 
-    # handle basic options
-    if opts.version:
-        print('Sphinx (sphinx-build) %s' % __display_version__)
-        return 0
-
     # get paths (first and second positional argument)
     try:
-        srcdir = abspath(args[0])
-        confdir = abspath(opts.confdir or srcdir)
-        if opts.noconfig:
+        srcdir = abspath(args.sourcedir)
+        confdir = abspath(args.confdir or srcdir)
+        if args.noconfig:
             confdir = None
         if not path.isdir(srcdir):
             print('Error: Cannot find source directory `%s\'.' % srcdir,
                   file=sys.stderr)
             return 1
-        if not opts.noconfig and not path.isfile(path.join(confdir, 'conf.py')):
+        if not args.noconfig and not path.isfile(path.join(confdir, 'conf.py')):
             print('Error: Config directory doesn\'t contain a conf.py file.',
                   file=sys.stderr)
             return 1
-        outdir = abspath(args[1])
+        outdir = abspath(args.outputdir)
         if srcdir == outdir:
             print('Error: source directory and destination directory are same.',
                   file=sys.stderr)
             return 1
-    except IndexError:
-        parser.print_help()
-        return 1
     except UnicodeError:
         print(
             'Error: Multibyte filename not supported on this filesystem '
@@ -219,7 +191,7 @@ def main(argv=sys.argv[1:]):  # type: ignore
         return 1
 
     # handle remaining filename arguments
-    filenames = args[2:]
+    filenames = args.filenames
     errored = False
     for filename in filenames:
         if not path.isfile(filename):
@@ -235,35 +207,35 @@ def main(argv=sys.argv[1:]):  # type: ignore
     except Exception:
         likely_encoding = None
 
-    if opts.force_all and filenames:
+    if args.force_all and filenames:
         print('Error: Cannot combine -a option and filenames.', file=sys.stderr)
         return 1
 
-    if opts.color == 'no' or (opts.color == 'auto' and not color_terminal()):
+    if args.color == 'no' or (args.color == 'auto' and not color_terminal()):
         nocolor()
 
-    doctreedir = abspath(opts.doctreedir or path.join(outdir, '.doctrees'))
+    doctreedir = abspath(args.doctreedir or path.join(outdir, '.doctrees'))
 
     status = sys.stdout
     warning = sys.stderr
     error = sys.stderr
 
-    if opts.quiet:
+    if args.quiet:
         status = None
-    if opts.really_quiet:
+    if args.really_quiet:
         status = warning = None
-    if warning and opts.warnfile:
+    if warning and args.warnfile:
         try:
-            warnfp = open(opts.warnfile, 'w')
+            warnfp = open(args.warnfile, 'w')
         except Exception as exc:
             print('Error: Cannot open warning file %r: %s' %
-                  (opts.warnfile, exc), file=sys.stderr)
+                  (args.warnfile, exc), file=sys.stderr)
             sys.exit(1)
         warning = Tee(warning, warnfp)  # type: ignore
         error = warning
 
     confoverrides = {}
-    for val in opts.define:
+    for val in args.define:
         try:
             key, val = val.split('=', 1)
         except ValueError:
@@ -277,7 +249,7 @@ def main(argv=sys.argv[1:]):  # type: ignore
                 pass
         confoverrides[key] = val
 
-    for val in opts.htmldefine:
+    for val in args.htmldefine:
         try:
             key, val = val.split('=')
         except ValueError:
@@ -294,17 +266,17 @@ def main(argv=sys.argv[1:]):  # type: ignore
                     pass
         confoverrides['html_context.%s' % key] = val
 
-    if opts.nitpicky:
+    if args.nitpicky:
         confoverrides['nitpicky'] = True
 
     app = None
     try:
         with patch_docutils(), docutils_namespace():
-            app = Sphinx(srcdir, confdir, outdir, doctreedir, opts.builder,
-                         confoverrides, status, warning, opts.freshenv,
-                         opts.warningiserror, opts.tags, opts.verbosity, opts.jobs)
-            app.build(opts.force_all, filenames)
+            app = Sphinx(srcdir, confdir, outdir, doctreedir, args.builder,
+                         confoverrides, status, warning, args.freshenv,
+                         args.warningiserror, args.tags, args.verbosity, args.jobs)
+            app.build(args.force_all, filenames)
             return app.statuscode
     except (Exception, KeyboardInterrupt) as exc:
-        handle_exception(app, opts, exc, error)
+        handle_exception(app, args, exc, error)
         return 1

From aedeb2160ad07d387c9657345e42166d56d268b6 Mon Sep 17 00:00:00 2001
From: Stephen Finucane <stephen@that.guru>
Date: Thu, 28 Sep 2017 20:10:32 +0100
Subject: [PATCH 13/31] sphinx-build: Move parser to a separate function

This should be becoming passe at this point.

Signed-off-by: Stephen Finucane <stephen@that.guru>
---
 sphinx/cmdline.py | 35 +++++++++++++++++++++++++++++------
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/sphinx/cmdline.py b/sphinx/cmdline.py
index 30bef6674..11f1861d8 100644
--- a/sphinx/cmdline.py
+++ b/sphinx/cmdline.py
@@ -83,11 +83,27 @@ def handle_exception(app, args, exception, stderr=sys.stderr):
                   file=stderr)
 
 
-def main(argv=sys.argv[1:]):  # type: ignore
-    # type: (List[unicode]) -> int
+def get_parser():
+    # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(
-        usage='usage: %(prog)s [OPTIONS] SOURCEDIR OUTDIR [FILENAMES...]',
-        epilog='For more information, visit <http://sphinx-doc.org/>.')
+        usage='usage: %(prog)s [OPTIONS] SOURCEDIR OUTPUTDIR [FILENAMES...]',
+        epilog='For more information, visit <http://sphinx-doc.org/>.',
+        description="""
+Generate documentation from source files.
+
+sphinx-build generates documentation from the files in SOURCEDIR and places it
+in OUTPUTDIR. It looks for 'conf.py' in SOURCEDIR for the configuration
+settings.  The 'sphinx-quickstart' tool may be used to generate template files,
+including 'conf.py'
+
+sphinx-build can create documentation in different formats. A format is
+selected by specifying the builder name on the command line; it defaults to
+HTML. Builders can also perform other tasks related to documentation
+processing.
+
+By default, everything that is outdated is built. Output only for selected
+files can be built by specifying individual filenames.
+""")
 
     parser.add_argument('--version', action='version', dest='show_version',
                         version='%%(prog)s %s' % __display_version__)
@@ -112,7 +128,7 @@ def main(argv=sys.argv[1:]):  # type: ignore
                        'all files')
     group.add_argument('-d', metavar='PATH', dest='doctreedir',
                        help='path for the cached environment and doctree '
-                       'files (default: outdir/.doctrees)')
+                       'files (default: OUTPUTDIR/.doctrees)')
     group.add_argument('-j', metavar='N', default=1, type=int, dest='jobs',
                        help='build in parallel with N processes where '
                        'possible')
@@ -120,7 +136,7 @@ def main(argv=sys.argv[1:]):  # type: ignore
     group = parser.add_argument_group('build configuration options')
     group.add_argument('-c', metavar='PATH', dest='confdir',
                        help='path where configuration file (conf.py) is '
-                       'located (default: same as sourcedir)')
+                       'located (default: same as SOURCEDIR)')
     group.add_argument('-C', action='store_true', dest='noconfig',
                        help='use no config file at all, only -D options')
     group.add_argument('-D', metavar='setting=value', action='append',
@@ -159,6 +175,13 @@ def main(argv=sys.argv[1:]):  # type: ignore
     group.add_argument('-P', action='store_true', dest='pdb',
                        help='run Pdb on exception')
 
+    return parser
+
+
+def main(argv=sys.argv[1:]):  # type: ignore
+    # type: (List[unicode]) -> int
+
+    parser = get_parser()
     # parse options
     try:
         args = parser.parse_args(argv)

From 08f6f7ba4d1fe610e530a6e0ec11fd474160ab6a Mon Sep 17 00:00:00 2001
From: Matthew Fernandez <matthew.fernandez@gmail.com>
Date: Sat, 14 Oct 2017 13:56:07 -0700
Subject: [PATCH 14/31] fix comment typo

---
 tests/test_writer_latex.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_writer_latex.py b/tests/test_writer_latex.py
index 228161886..b026f8d17 100644
--- a/tests/test_writer_latex.py
+++ b/tests/test_writer_latex.py
@@ -27,7 +27,7 @@ def test_rstdim_to_latexdim():
     assert rstdim_to_latexdim('30%') == '0.300\\linewidth'
     assert rstdim_to_latexdim('160') == '160\\sphinxpxdimen'
 
-    # flaot values
+    # float values
     assert rstdim_to_latexdim('160.0em') == '160.0em'
     assert rstdim_to_latexdim('.5em') == '.5em'
 

From ed4949e00008311587dc57a3e84e203920f7af00 Mon Sep 17 00:00:00 2001
From: Matthew Fernandez <matthew.fernandez@gmail.com>
Date: Sun, 15 Oct 2017 17:15:36 -0700
Subject: [PATCH 15/31] Closes #1020: ext.todo todolist not linking to the page
 in pdflatex

---
 AUTHORS            |  1 +
 CHANGES            |  1 +
 sphinx/ext/todo.py | 16 ++++++++++++++--
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 580feeb32..13ce2df9f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -29,6 +29,7 @@ Other contributors, listed alphabetically, are:
 * Kevin Dunn -- MathJax extension
 * Josip Dzolonga -- coverage builder
 * Buck Evan -- dummy builder
+* Matthew Fernandez -- todo extension fix
 * Hernan Grecco -- search improvements
 * Horst Gutmann -- internationalization support
 * Martin Hans -- autodoc improvements
diff --git a/CHANGES b/CHANGES
index 40c91af68..5f381b608 100644
--- a/CHANGES
+++ b/CHANGES
@@ -31,6 +31,7 @@ Bugs fixed
 * #3692: Unable to build HTML if writing .buildinfo failed
 * #4152: HTML writer crashes if a field list is placed on top of the document
 * #4063: Sphinx crashes when labeling directive ``.. todolist::``
+* #1020: ext.todo todolist not linking to the page in pdflatex
 
 Testing
 --------
diff --git a/sphinx/ext/todo.py b/sphinx/ext/todo.py
index 0e5a74720..a58422793 100644
--- a/sphinx/ext/todo.py
+++ b/sphinx/ext/todo.py
@@ -69,6 +69,8 @@ class Todo(BaseAdmonition):
 
         env = self.state.document.settings.env
         targetid = 'index-%s' % env.new_serialno('index')
+        # Stash the target to be retrieved later in latex_visit_todo_node.
+        todo['targetref'] = '%s:%s' % (env.docname, targetid)
         targetnode = nodes.target('', '', ids=[targetid])
         return [targetnode, todo]
 
@@ -173,8 +175,12 @@ def process_todo_nodes(app, doctree, fromdocname):
             para += newnode
             para += nodes.Text(desc2, desc2)
 
-            # (Recursively) resolve references in the todo content
             todo_entry = todo_info['todo']
+            # Remove targetref from the (copied) node to avoid emitting a
+            # duplicate label of the original entry when we walk this node.
+            del todo_entry['targetref']
+
+            # (Recursively) resolve references in the todo content
             env.resolve_references(todo_entry, todo_info['docname'],
                                    app.builder)
 
@@ -216,7 +222,13 @@ def depart_todo_node(self, node):
 def latex_visit_todo_node(self, node):
     # type: (nodes.NodeVisitor, todo_node) -> None
     title = node.pop(0).astext().translate(tex_escape_map)
-    self.body.append(u'\n\\begin{sphinxadmonition}{note}{%s:}' % title)
+    self.body.append(u'\n\\begin{sphinxadmonition}{note}{')
+    # If this is the original todo node, emit a label that will be referenced by
+    # a hyperref in the todolist.
+    target = node.get('targetref')
+    if target is not None:
+        self.body.append(u'\\label{%s}' % target)
+    self.body.append('%s:}' % title)
 
 
 def latex_depart_todo_node(self, node):

From 6553b2ed32e6ccf50ae852d470fd1684d0628941 Mon Sep 17 00:00:00 2001
From: Matthew Fernandez <matthew.fernandez@gmail.com>
Date: Sat, 14 Oct 2017 14:34:48 -0700
Subject: [PATCH 16/31] add a test that confirms #1020

---
 tests/roots/test-ext-todo/conf.py |  5 +++++
 tests/test_ext_todo.py            | 28 ++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/tests/roots/test-ext-todo/conf.py b/tests/roots/test-ext-todo/conf.py
index c67a86c5a..5d5619245 100644
--- a/tests/roots/test-ext-todo/conf.py
+++ b/tests/roots/test-ext-todo/conf.py
@@ -2,3 +2,8 @@
 
 extensions = ['sphinx.ext.todo']
 master_doc = 'index'
+
+latex_documents = [
+    (master_doc, 'TodoTests.tex', 'Todo Tests Documentation',
+     'Robin Banks', 'manual'),
+]
diff --git a/tests/test_ext_todo.py b/tests/test_ext_todo.py
index 77d657adc..4f01a07ab 100644
--- a/tests/test_ext_todo.py
+++ b/tests/test_ext_todo.py
@@ -84,3 +84,31 @@ def test_todo_not_included(app, status, warning):
     # check handled event
     assert len(todos) == 2
     assert set(todo[1].astext() for todo in todos) == set(['todo in foo', 'todo in bar'])
+
+@pytest.mark.sphinx('latex', testroot='ext-todo', freshenv=True,
+                    confoverrides={'todo_include_todos': True, 'todo_emit_warnings': True})
+def test_todo_valid_link(app, status, warning):
+    """
+    Test that the inserted "original entry" links for todo items have a target
+    that exists in the LaTeX output. The target was previously incorrectly
+    omitted (GitHub issue #1020).
+    """
+
+    # Ensure the LaTeX output is built.
+    app.builder.build_all()
+
+    content = (app.outdir / 'TodoTests.tex').text()
+
+    # Look for the link to foo. We could equally well look for the link to bar.
+    link = r'\{\\hyperref\[\\detokenize\{(.*?foo.*?)}]\{\\sphinxcrossref{' \
+        r'\\sphinxstyleemphasis{original entry}}}}'
+    m = re.findall(link, content)
+    assert len(m) == 1
+    target = m[0]
+
+    # Look for the targets of this link.
+    labels = [m for m in re.findall(r'\\label\{([^}]*)}', content)
+        if m == target]
+
+    # If everything is correct we should have exactly one target.
+    assert len(labels) == 1

From a4b7927a29cb0b38c00c7e83c2130cb426105bfa Mon Sep 17 00:00:00 2001
From: Eric Wieser <wieser.eric@gmail.com>
Date: Fri, 20 Oct 2017 23:18:46 -0700
Subject: [PATCH 17/31] quickstart: fix return type of term_decode

term_decode is documented as `(unicode) -> unicode`, but actually:

* Accepts `bytes` arguments, despite not being documented to
* Returns `bytes` when it shouldn't

This is extracted from the more controversial #3584
---
 sphinx/cmd/quickstart.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/sphinx/cmd/quickstart.py b/sphinx/cmd/quickstart.py
index ac0859c31..af0333d65 100644
--- a/sphinx/cmd/quickstart.py
+++ b/sphinx/cmd/quickstart.py
@@ -138,25 +138,25 @@ def ok(x):
 
 
 def term_decode(text):
-    # type: (unicode) -> unicode
+    # type: (Union[bytes,unicode]) -> unicode
     if isinstance(text, text_type):
         return text
 
-    # for Python 2.x, try to get a Unicode string out of it
-    if text.decode('ascii', 'replace').encode('ascii', 'replace') == text:
-        return text
-
+    # Use the known encoding, if possible
     if TERM_ENCODING:
-        text = text.decode(TERM_ENCODING)
-    else:
-        print(turquoise('* Note: non-ASCII characters entered '
-                        'and terminal encoding unknown -- assuming '
-                        'UTF-8 or Latin-1.'))
-        try:
-            text = text.decode('utf-8')
-        except UnicodeDecodeError:
-            text = text.decode('latin1')
-    return text
+        return text.decode(TERM_ENCODING)
+
+    # If ascii is safe, use it with no warning
+    if text.decode('ascii', 'replace').encode('ascii', 'replace') == text:
+        return text.decode('ascii')
+    
+    print(turquoise('* Note: non-ASCII characters entered '
+                    'and terminal encoding unknown -- assuming '
+                    'UTF-8 or Latin-1.'))
+    try:
+        return text.decode('utf-8')
+    except UnicodeDecodeError:
+        return text.decode('latin1')
 
 
 def do_prompt(d, key, text, default=None, validator=nonempty):

From 58beeed235e5dddc7b040db03daa1a7fcc178d23 Mon Sep 17 00:00:00 2001
From: Eric Wieser <wieser.eric@gmail.com>
Date: Fri, 20 Oct 2017 23:53:33 -0700
Subject: [PATCH 18/31] fixup

Warning and missing typing function
---
 sphinx/cmd/quickstart.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sphinx/cmd/quickstart.py b/sphinx/cmd/quickstart.py
index af0333d65..ea28f4bd3 100644
--- a/sphinx/cmd/quickstart.py
+++ b/sphinx/cmd/quickstart.py
@@ -44,7 +44,7 @@ from sphinx.util import texescape
 
 if False:
     # For type annotation
-    from typing import Any, Callable, Dict, List, Pattern  # NOQA
+    from typing import Any, Callable, Dict, List, Pattern, Union  # NOQA
 
 TERM_ENCODING = getattr(sys.stdin, 'encoding', None)
 
@@ -149,7 +149,7 @@ def term_decode(text):
     # If ascii is safe, use it with no warning
     if text.decode('ascii', 'replace').encode('ascii', 'replace') == text:
         return text.decode('ascii')
-    
+
     print(turquoise('* Note: non-ASCII characters entered '
                     'and terminal encoding unknown -- assuming '
                     'UTF-8 or Latin-1.'))

From a5f6e97ac45726dcad9dd267279b04d9539e7793 Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Sat, 21 Oct 2017 16:22:27 +0900
Subject: [PATCH 19/31] Revert "Default SPHINXBUILD to 'python -msphinx'
 instead of 'sphinx-build'."

This reverts commit 0832aa73ab64d8cf5accbe4a5bbeaeedb7b46a52.
---
 sphinx/templates/quickstart/Makefile.new_t |  2 +-
 sphinx/templates/quickstart/Makefile_t     |  2 +-
 sphinx/templates/quickstart/make.bat.new_t | 10 +++++-----
 sphinx/templates/quickstart/make.bat_t     | 23 +++++++++++++++-------
 4 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/sphinx/templates/quickstart/Makefile.new_t b/sphinx/templates/quickstart/Makefile.new_t
index bba767a4c..c7cd62dda 100644
--- a/sphinx/templates/quickstart/Makefile.new_t
+++ b/sphinx/templates/quickstart/Makefile.new_t
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = python -msphinx
+SPHINXBUILD   = sphinx-build
 SPHINXPROJ    = {{ project_fn }}
 SOURCEDIR     = {{ rsrcdir }}
 BUILDDIR      = {{ rbuilddir }}
diff --git a/sphinx/templates/quickstart/Makefile_t b/sphinx/templates/quickstart/Makefile_t
index fdcf05691..5505f23f5 100644
--- a/sphinx/templates/quickstart/Makefile_t
+++ b/sphinx/templates/quickstart/Makefile_t
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = python -msphinx
+SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = {{ rbuilddir }}
 
diff --git a/sphinx/templates/quickstart/make.bat.new_t b/sphinx/templates/quickstart/make.bat.new_t
index a52951ebb..e49ffbe78 100644
--- a/sphinx/templates/quickstart/make.bat.new_t
+++ b/sphinx/templates/quickstart/make.bat.new_t
@@ -5,7 +5,7 @@ pushd %~dp0
 REM Command file for Sphinx documentation
 
 if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=python -msphinx
+	set SPHINXBUILD=sphinx-build
 )
 set SOURCEDIR={{ rsrcdir }}
 set BUILDDIR={{ rbuilddir }}
@@ -16,10 +16,10 @@ if "%1" == "" goto help
 %SPHINXBUILD% >NUL 2>NUL
 if errorlevel 9009 (
 	echo.
-	echo.The Sphinx module was not found. Make sure you have Sphinx installed,
-	echo.then set the SPHINXBUILD environment variable to point to the full
-	echo.path of the 'sphinx-build' executable. Alternatively you may add the
-	echo.Sphinx directory to PATH.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
 	echo.
 	echo.If you don't have Sphinx installed, grab it from
 	echo.http://sphinx-doc.org/
diff --git a/sphinx/templates/quickstart/make.bat_t b/sphinx/templates/quickstart/make.bat_t
index 03ae9d423..8438b5f7e 100644
--- a/sphinx/templates/quickstart/make.bat_t
+++ b/sphinx/templates/quickstart/make.bat_t
@@ -5,7 +5,7 @@ REM Command file for Sphinx documentation
 pushd %~dp0
 
 if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=python -msphinx
+	set SPHINXBUILD=sphinx-build
 )
 set BUILDDIR={{ rbuilddir }}
 set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% {{ rsrcdir }}
@@ -52,20 +52,29 @@ if "%1" == "clean" (
 )
 
 
-REM Check if sphinx-build is available
+REM Check if sphinx-build is available and fallback to Python version if any
 %SPHINXBUILD% 1>NUL 2>NUL
-if errorlevel 1 (
+if errorlevel 9009 goto sphinx_python
+goto sphinx_ok
+
+:sphinx_python
+
+set SPHINXBUILD=python -m sphinx.__init__
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
 	echo.
-	echo.The Sphinx module was not found. Make sure you have Sphinx installed,
-	echo.then set the SPHINXBUILD environment variable to point to the full
-	echo.path of the 'sphinx-build' executable. Alternatively you may add the
-	echo.Sphinx directory to PATH.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
 	echo.
 	echo.If you don't have Sphinx installed, grab it from
 	echo.http://sphinx-doc.org/
 	exit /b 1
 )
 
+:sphinx_ok
+
 
 if "%1" == "html" (
 	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html

From b8b4d9b1870e8e61408daf3d3455ed3da39cefb2 Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Sat, 21 Oct 2017 16:23:38 +0900
Subject: [PATCH 20/31] Update CHANGES for #3965

---
 CHANGES | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES b/CHANGES
index 97a61c5d5..1ff3c8b34 100644
--- a/CHANGES
+++ b/CHANGES
@@ -34,6 +34,7 @@ Bugs fixed
 * #4134: [doc] :file:`docutils.conf` is not documented explicitly
 * #4169: Chinese language doesn't trigger Chinese search automatically
 * #1020: ext.todo todolist not linking to the page in pdflatex
+* #3965: New quickstart generates wrong SPHINXBUILD in Makefile
 
 Testing
 --------

From 943cd788fc248c32a6f8b8296e2471b5b0e521e6 Mon Sep 17 00:00:00 2001
From: Dmitry Shachnev <mitya57@gmail.com>
Date: Thu, 28 Sep 2017 00:16:40 +0300
Subject: [PATCH 21/31] Make it easier to override variables in Makefiles
 generated by quickstart

Refs #3965.
---
 sphinx/templates/quickstart/Makefile_t | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sphinx/templates/quickstart/Makefile_t b/sphinx/templates/quickstart/Makefile_t
index 5505f23f5..4639a982b 100644
--- a/sphinx/templates/quickstart/Makefile_t
+++ b/sphinx/templates/quickstart/Makefile_t
@@ -2,9 +2,9 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
+SPHINXOPTS   ?=
+SPHINXBUILD  ?= sphinx-build
+PAPER        ?=
 BUILDDIR      = {{ rbuilddir }}
 
 # Internal variables.

From ffad4f08c7aa81f230ee6cc6205c80503bc25040 Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Sat, 21 Oct 2017 21:21:37 +0900
Subject: [PATCH 22/31] Fix #3739: ``:module:`` option is ignored at content of
 pyobjects

---
 CHANGES                                      |  1 +
 sphinx/domains/python.py                     | 10 ++++++++
 tests/roots/test-domain-py/module_option.rst | 25 ++++++++++++++++++++
 tests/test_domain_py.py                      |  9 +++++++
 4 files changed, 45 insertions(+)
 create mode 100644 tests/roots/test-domain-py/module_option.rst

diff --git a/CHANGES b/CHANGES
index 1ff3c8b34..139580a3e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -35,6 +35,7 @@ Bugs fixed
 * #4169: Chinese language doesn't trigger Chinese search automatically
 * #1020: ext.todo todolist not linking to the page in pdflatex
 * #3965: New quickstart generates wrong SPHINXBUILD in Makefile
+* #3739: ``:module:`` option is ignored at content of pyobjects
 
 Testing
 --------
diff --git a/sphinx/domains/python.py b/sphinx/domains/python.py
index eb6fe76cb..6aa00a8b0 100644
--- a/sphinx/domains/python.py
+++ b/sphinx/domains/python.py
@@ -348,6 +348,10 @@ class PyObject(ObjectDescription):
             if self.allow_nesting:
                 classes = self.env.ref_context.setdefault('py:classes', [])
                 classes.append(prefix)
+        if 'module' in self.options:
+            modules = self.env.ref_context.setdefault('py:modules', [])
+            modules.append(self.env.ref_context.get('py:module'))
+            self.env.ref_context['py:module'] = self.options['module']
 
     def after_content(self):
         # type: () -> None
@@ -368,6 +372,12 @@ class PyObject(ObjectDescription):
                 pass
         self.env.ref_context['py:class'] = (classes[-1] if len(classes) > 0
                                             else None)
+        if 'module' in self.options:
+            modules = self.env.ref_context.setdefault('py:modules', [])
+            if modules:
+                self.env.ref_context['py:module'] = modules.pop()
+            else:
+                self.env.ref_context.pop('py:module')
 
 
 class PyModulelevel(PyObject):
diff --git a/tests/roots/test-domain-py/module_option.rst b/tests/roots/test-domain-py/module_option.rst
new file mode 100644
index 000000000..1dec2ce0c
--- /dev/null
+++ b/tests/roots/test-domain-py/module_option.rst
@@ -0,0 +1,25 @@
+module_option
+=============
+
+.. py:class:: B
+   :module: test.extra
+
+   This is also a test.
+
+
+   .. py:method:: B.baz()
+      :module: test.extra
+
+      Does something similar to :meth:`foo`.
+
+
+   .. py:method:: B.foo()
+      :module: test.extra
+
+      Does something.
+
+
+   .. py:method:: B.test()
+      :module: test.extra
+
+      Does something completely unrelated to :meth:`foo`
diff --git a/tests/test_domain_py.py b/tests/test_domain_py.py
index 28743f9e1..5596950df 100644
--- a/tests/test_domain_py.py
+++ b/tests/test_domain_py.py
@@ -115,6 +115,15 @@ def test_domain_py_xrefs(app, status, warning):
     assert_refnode(refnodes[10], False, False, 'float', 'obj')
     assert len(refnodes) == 11
 
+    doctree = app.env.get_doctree('module_option')
+    refnodes = list(doctree.traverse(addnodes.pending_xref))
+    print(refnodes)
+    print(refnodes[0])
+    print(refnodes[1])
+    assert_refnode(refnodes[0], 'test.extra', 'B', 'foo', 'meth')
+    assert_refnode(refnodes[1], 'test.extra', 'B', 'foo', 'meth')
+    assert len(refnodes) == 2
+
 
 @pytest.mark.sphinx('dummy', testroot='domain-py')
 def test_domain_py_objects(app, status, warning):

From 90e3425ed751ded99e71174ece60a9fb6c63f03e Mon Sep 17 00:00:00 2001
From: jfbu <jfbu@free.fr>
Date: Sun, 22 Oct 2017 11:45:28 +0200
Subject: [PATCH 23/31] Fix #4149: Documentation: Help choosing latex_engine

---
 CHANGES        |  1 +
 doc/config.rst | 19 +++++++++++++++++++
 doc/latex.rst  |  1 +
 3 files changed, 21 insertions(+)

diff --git a/CHANGES b/CHANGES
index 139580a3e..7add2c4b0 100644
--- a/CHANGES
+++ b/CHANGES
@@ -36,6 +36,7 @@ Bugs fixed
 * #1020: ext.todo todolist not linking to the page in pdflatex
 * #3965: New quickstart generates wrong SPHINXBUILD in Makefile
 * #3739: ``:module:`` option is ignored at content of pyobjects
+* #4149: Documentation: Help choosing :confval:`latex_engine`
 
 Testing
 --------
diff --git a/doc/config.rst b/doc/config.rst
index e96c075e9..b615dc45a 100644
--- a/doc/config.rst
+++ b/doc/config.rst
@@ -1542,6 +1542,25 @@ These options influence LaTeX output. See further :doc:`latex`.
    * ``'lualatex'`` -- LuaLaTeX
    * ``'platex'`` -- pLaTeX (default if :confval:`language` is ``'ja'``)
 
+   PDFLaTeX's support for Unicode characters covers those from the document
+   language (the LaTeX ``babel`` and ``inputenc`` packages map them to glyph
+   slots in the document font, at various encodings allowing each only 256
+   characters; Sphinx uses by default (except for Cyrillic languages) the
+   ``times`` package), but stray characters from other scripts or special
+   symbols may require adding extra LaTeX packages or macros to the LaTeX
+   preamble.
+
+   If your project uses such extra Unicode characters, switching the engine to
+   XeLaTeX or LuaLaTeX often provides a quick fix. They only work with UTF-8
+   encoded sources and can (in fact, should) use OpenType fonts, either from
+   the system or the TeX install tree. Recent LaTeX releases will default with
+   these engines to the Latin Modern OpenType font, which has good coverage of
+   Latin and Cyrillic scripts (it is provided by standard LaTeX installation),
+   and Sphinx does not modify this default. Refer to the documentation of the
+   LaTeX ``polyglossia`` package to see how to instruct LaTeX to use some
+   other OpenType font if Unicode coverage proves insufficient (or use
+   directly ``\setmainfont`` et. al. as in :ref:`this example <latex-basic>`.)
+
 .. confval:: latex_documents
 
    This value determines how to group the document tree into LaTeX source files.
diff --git a/doc/latex.rst b/doc/latex.rst
index 61689bda1..eaf147c1c 100644
--- a/doc/latex.rst
+++ b/doc/latex.rst
@@ -29,6 +29,7 @@ The *latex* target does not benefit from pre-prepared themes like the
          cautionBgColor={named}{LightCyan}}
    \relax
 
+.. _latex-basic:
 
 Basic customization
 -------------------

From 5e86c1c934b0292efa79bbb51d5f36ebfa785d5b Mon Sep 17 00:00:00 2001
From: jfbu <jfbu@free.fr>
Date: Sun, 22 Oct 2017 13:24:03 +0200
Subject: [PATCH 24/31] Fix #4090: extra LaTeX macros via
 ``latex_additional_files``

---
 CHANGES       | 2 ++
 doc/latex.rst | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 7add2c4b0..b7f125b47 100644
--- a/CHANGES
+++ b/CHANGES
@@ -37,6 +37,8 @@ Bugs fixed
 * #3965: New quickstart generates wrong SPHINXBUILD in Makefile
 * #3739: ``:module:`` option is ignored at content of pyobjects
 * #4149: Documentation: Help choosing :confval:`latex_engine`
+* #4090: [doc] :confval:`latex_additional_files` with extra LaTeX macros should
+  not use ``.tex`` extension
 
 Testing
 --------
diff --git a/doc/latex.rst b/doc/latex.rst
index eaf147c1c..f349b8fc0 100644
--- a/doc/latex.rst
+++ b/doc/latex.rst
@@ -65,14 +65,14 @@ If the size of the ``'preamble'`` contents becomes inconvenient, one may move
 all needed macros into some file :file:`mystyle.tex` of the project source
 repertory, and get LaTeX to import it at run time::
 
-   'preamble': r'\input{mystyle.tex}',
+   'preamble': r'\input{mystyle.tex.txt}',
    # or, if the \ProvidesPackage LaTeX macro is used in a file mystyle.sty
    'preamble': r'\usepackage{mystyle}',
 
 It is needed to set appropriately :confval:`latex_additional_files`, for
 example::
 
-   latex_additional_files = ["mystyle.tex"]
+   latex_additional_files = ["mystyle.sty"]
 
 .. _latexsphinxsetup:
 

From 6b4a0123fa3e1b26d78adb05c46b044cc8eb3044 Mon Sep 17 00:00:00 2001
From: jfbu <jfbu@free.fr>
Date: Sun, 22 Oct 2017 13:28:46 +0200
Subject: [PATCH 25/31] Fix the fix of #4090 (sorry)

---
 doc/latex.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/latex.rst b/doc/latex.rst
index f349b8fc0..0cd91fa97 100644
--- a/doc/latex.rst
+++ b/doc/latex.rst
@@ -62,7 +62,7 @@ It is achieved via usage of the
 .. highlight:: latex
 
 If the size of the ``'preamble'`` contents becomes inconvenient, one may move
-all needed macros into some file :file:`mystyle.tex` of the project source
+all needed macros into some file :file:`mystyle.tex.txt` of the project source
 repertory, and get LaTeX to import it at run time::
 
    'preamble': r'\input{mystyle.tex.txt}',

From cfadf17379e394d5f38e50015b8729d6045985dd Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Sun, 22 Oct 2017 20:33:18 +0900
Subject: [PATCH 26/31] Fix #4132: Failed to convert reST parser error to
 warning

---
 CHANGES                 | 1 +
 sphinx/util/docutils.py | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index b7f125b47..841273d5f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -39,6 +39,7 @@ Bugs fixed
 * #4149: Documentation: Help choosing :confval:`latex_engine`
 * #4090: [doc] :confval:`latex_additional_files` with extra LaTeX macros should
   not use ``.tex`` extension
+* Failed to convert reST parser error to warning (refs: #4132)
 
 Testing
 --------
diff --git a/sphinx/util/docutils.py b/sphinx/util/docutils.py
index c984bcfaf..a8df9c59d 100644
--- a/sphinx/util/docutils.py
+++ b/sphinx/util/docutils.py
@@ -26,8 +26,7 @@ from sphinx.locale import __
 from sphinx.util import logging
 
 logger = logging.getLogger(__name__)
-report_re = re.compile('^(.+?:(?:\\d+)?): \\((DEBUG|INFO|WARNING|ERROR|SEVERE)/(\\d+)?\\) '
-                       '(.+?)\n?$')
+report_re = re.compile('^(.+?:(?:\\d+)?): \\((DEBUG|INFO|WARNING|ERROR|SEVERE)/(\\d+)?\\) ')
 
 if False:
     # For type annotation
@@ -162,7 +161,8 @@ class WarningStream(object):
         if not matched:
             logger.warning(text.rstrip("\r\n"))
         else:
-            location, type, level, message = matched.groups()
+            location, type, level = matched.groups()
+            message = report_re.sub('', text).rstrip()
             logger.log(type, message, location=location)
 
 

From ad2610a0bcbf04ae57c3addafb2d7a4c81a5b198 Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Sun, 22 Oct 2017 20:50:20 +0900
Subject: [PATCH 27/31] Fix mypy violation

---
 sphinx/util/docutils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sphinx/util/docutils.py b/sphinx/util/docutils.py
index a8df9c59d..92e6c8c22 100644
--- a/sphinx/util/docutils.py
+++ b/sphinx/util/docutils.py
@@ -162,7 +162,7 @@ class WarningStream(object):
             logger.warning(text.rstrip("\r\n"))
         else:
             location, type, level = matched.groups()
-            message = report_re.sub('', text).rstrip()
+            message = report_re.sub('', text).rstrip()  # type: ignore
             logger.log(type, message, location=location)
 
 

From 33fd46b2a5c994fd450da9b48e1d359eea34daf4 Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Mon, 23 Oct 2017 12:19:56 +0900
Subject: [PATCH 28/31] Bump to 1.6.5 final

---
 CHANGES            | 16 ++--------------
 sphinx/__init__.py |  4 ++--
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/CHANGES b/CHANGES
index 841273d5f..70652ce48 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,14 +1,5 @@
-Release 1.6.5 (in development)
-==============================
-
-Dependencies
-------------
-
-Incompatible changes
---------------------
-
-Deprecated
-----------
+Release 1.6.5 (released Oct 23, 2017)
+=====================================
 
 Features added
 --------------
@@ -41,9 +32,6 @@ Bugs fixed
   not use ``.tex`` extension
 * Failed to convert reST parser error to warning (refs: #4132)
 
-Testing
---------
-
 Release 1.6.4 (released Sep 26, 2017)
 =====================================
 
diff --git a/sphinx/__init__.py b/sphinx/__init__.py
index 1d7bcc2ba..91e1f903c 100644
--- a/sphinx/__init__.py
+++ b/sphinx/__init__.py
@@ -34,13 +34,13 @@ if 'PYTHONWARNINGS' not in os.environ:
 warnings.filterwarnings('ignore', "'U' mode is deprecated",
                         DeprecationWarning, module='docutils.io')
 
-__version__ = '1.6.5+'
+__version__ = '1.6.5'
 __released__ = '1.6.5'  # used when Sphinx builds its own docs
 
 # version info for better programmatic use
 # possible values for 3rd element: 'alpha', 'beta', 'rc', 'final'
 # 'final' has 0 as the last element
-version_info = (1, 6, 5, 'beta', 0)
+version_info = (1, 6, 5, 'final', 0)
 
 package_dir = path.abspath(path.dirname(__file__))
 

From 314831d332f73252214c090d356c6e84d6fb3795 Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Mon, 23 Oct 2017 12:22:42 +0900
Subject: [PATCH 29/31] Bump version

---
 CHANGES            | 21 +++++++++++++++++++++
 sphinx/__init__.py |  6 +++---
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index 70652ce48..a7035bad1 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,24 @@
+Release 1.6.6 (in development)
+==============================
+
+Dependencies
+------------
+
+Incompatible changes
+--------------------
+
+Deprecated
+----------
+
+Features added
+--------------
+
+Bugs fixed
+----------
+
+Testing
+--------
+
 Release 1.6.5 (released Oct 23, 2017)
 =====================================
 
diff --git a/sphinx/__init__.py b/sphinx/__init__.py
index 91e1f903c..4facec53e 100644
--- a/sphinx/__init__.py
+++ b/sphinx/__init__.py
@@ -34,13 +34,13 @@ if 'PYTHONWARNINGS' not in os.environ:
 warnings.filterwarnings('ignore', "'U' mode is deprecated",
                         DeprecationWarning, module='docutils.io')
 
-__version__ = '1.6.5'
-__released__ = '1.6.5'  # used when Sphinx builds its own docs
+__version__ = '1.6.6+'
+__released__ = '1.6.6'  # used when Sphinx builds its own docs
 
 # version info for better programmatic use
 # possible values for 3rd element: 'alpha', 'beta', 'rc', 'final'
 # 'final' has 0 as the last element
-version_info = (1, 6, 5, 'final', 0)
+version_info = (1, 6, 6, 'beta', 0)
 
 package_dir = path.abspath(path.dirname(__file__))
 

From 47c869c89ab47f5c89300442d5b4a1abe4f3543c Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Mon, 23 Oct 2017 23:01:18 +0900
Subject: [PATCH 30/31] Fix flake8 and style-check violations

---
 setup.cfg                  |  2 +-
 sphinx/domains/cpp.py      |  2 +-
 sphinx/locale/__init__.py  |  2 +-
 sphinx/testing/util.py     |  2 +-
 sphinx/theming.py          |  2 +-
 sphinx/util/images.py      |  2 +-
 sphinx/util/smartypants.py |  2 +-
 tests/test_build_latex.py  |  2 +-
 tests/test_intl.py         | 22 +++++++++++-----------
 utils/bump_version.py      |  2 +-
 10 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index e0312ce00..a5433c6b0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -25,7 +25,7 @@ universal = 1
 
 [flake8]
 max-line-length = 95
-ignore = E116,E241,E251
+ignore = E116,E241,E251,E741
 exclude = .git,.tox,tests/*,build/*,sphinx/search/*,sphinx/pycode/pgen2/*,doc/ext/example*.py
 
 [build_sphinx]
diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py
index f55018b47..b81f6531e 100644
--- a/sphinx/domains/cpp.py
+++ b/sphinx/domains/cpp.py
@@ -4258,7 +4258,7 @@ class DefinitionParser(object):
         pos = self.pos
         try:
             concept = self._parse_nested_name()
-        except:
+        except Exception:
             self.pos = pos
             return None
         self.skip_ws()
diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py
index 6c77eeb8b..68686e3fc 100644
--- a/sphinx/locale/__init__.py
+++ b/sphinx/locale/__init__.py
@@ -171,7 +171,7 @@ class _TranslationProxy(UserString, object):
         # type: () -> str
         try:
             return 'i' + repr(text_type(self.data))
-        except:
+        except Exception:
             return '<%s broken>' % self.__class__.__name__
 
 
diff --git a/sphinx/testing/util.py b/sphinx/testing/util.py
index 4ec2ee94a..c1b2ae2b2 100644
--- a/sphinx/testing/util.py
+++ b/sphinx/testing/util.py
@@ -137,7 +137,7 @@ class SphinxTestApp(application.Sphinx):
             application.Sphinx.__init__(self, srcdir, confdir, outdir, doctreedir,
                                         buildername, confoverrides, status, warning,
                                         freshenv, warningiserror, tags)
-        except:
+        except Exception:
             self.cleanup()
             raise
 
diff --git a/sphinx/theming.py b/sphinx/theming.py
index 1cd07b8dc..f787e8120 100644
--- a/sphinx/theming.py
+++ b/sphinx/theming.py
@@ -156,7 +156,7 @@ def is_archived_theme(filename):
     try:
         with ZipFile(filename) as f:  # type: ignore
             return THEMECONF in f.namelist()
-    except:
+    except Exception:
         return False
 
 
diff --git a/sphinx/util/images.py b/sphinx/util/images.py
index eba295a3c..1c2b4033a 100644
--- a/sphinx/util/images.py
+++ b/sphinx/util/images.py
@@ -64,7 +64,7 @@ def get_image_size(filename):
                 pass
 
         return size
-    except:
+    except Exception:
         return None
 
 
diff --git a/sphinx/util/smartypants.py b/sphinx/util/smartypants.py
index c368d7d0f..03771d168 100644
--- a/sphinx/util/smartypants.py
+++ b/sphinx/util/smartypants.py
@@ -139,7 +139,7 @@ def educateQuotes(text, language='en'):
     smart = smartquotes.smartchars(language)
     try:
         apostrophe = smart.apostrophe
-    except:
+    except Exception:
         apostrophe = u'’'
 
     # oldtext = text
diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py
index f36aa355b..b1ba73730 100644
--- a/tests/test_build_latex.py
+++ b/tests/test_build_latex.py
@@ -1036,6 +1036,6 @@ def test_latex_image_in_parsed_literal(app, status, warning):
     app.builder.build_all()
 
     result = (app.outdir / 'Python.tex').text(encoding='utf8')
-    assert ('{\\sphinxunactivateextrasandspace \\raisebox{-0.5\height}'
+    assert ('{\\sphinxunactivateextrasandspace \\raisebox{-0.5\\height}'
             '{\\scalebox{2.000000}{\\sphinxincludegraphics[height=1cm]{{pic}.png}}}'
             '}AFTER') in result
diff --git a/tests/test_intl.py b/tests/test_intl.py
index f3952e9a1..8eff52340 100644
--- a/tests/test_intl.py
+++ b/tests/test_intl.py
@@ -197,28 +197,28 @@ def test_text_inconsistency_warnings(app, warning):
     expected_warning_expr = (
         warning_fmt % {
             u'reftype': u'footnote references',
-            u'original': u"\[u?'\[#\]_'\]",
-            u'translated': u"\[\]"
+            u'original': u"\\[u?'\\[#\\]_'\\]",
+            u'translated': u"\\[\\]"
         } +
         warning_fmt % {
             u'reftype': u'footnote references',
-            u'original': u"\[u?'\[100\]_'\]",
-            u'translated': u"\[\]"
+            u'original': u"\\[u?'\\[100\\]_'\\]",
+            u'translated': u"\\[\\]"
         } +
         warning_fmt % {
             u'reftype': u'references',
-            u'original': u"\[u?'reference_'\]",
-            u'translated': u"\[u?'reference_', u?'reference_'\]"
+            u'original': u"\\[u?'reference_'\\]",
+            u'translated': u"\\[u?'reference_', u?'reference_'\\]"
         } +
         warning_fmt % {
             u'reftype': u'references',
-            u'original': u"\[\]",
-            u'translated': u"\[u?'`I18N WITH REFS INCONSISTENCY`_'\]"
+            u'original': u"\\[\\]",
+            u'translated': u"\\[u?'`I18N WITH REFS INCONSISTENCY`_'\\]"
         })
     assert_re_search(expected_warning_expr, warnings)
 
     expected_citation_warning_expr = (
-        u'.*/refs_inconsistency.txt:\\d+: WARNING: Citation \[ref2\] is not referenced.\n' +
+        u'.*/refs_inconsistency.txt:\\d+: WARNING: Citation \\[ref2\\] is not referenced.\n' +
         u'.*/refs_inconsistency.txt:\\d+: WARNING: citation not found: ref3')
     assert_re_search(expected_citation_warning_expr, warnings)
 
@@ -300,8 +300,8 @@ def test_text_glossary_term_inconsistencies(app, warning):
     expected_warning_expr = (
         u'.*/glossary_terms_inconsistency.txt:\\d+: '
         u'WARNING: inconsistent term references in translated message.'
-        u" original: \[u?':term:`Some term`', u?':term:`Some other term`'\],"
-        u" translated: \[u?':term:`SOME NEW TERM`'\]\n")
+        u" original: \\[u?':term:`Some term`', u?':term:`Some other term`'\\],"
+        u" translated: \\[u?':term:`SOME NEW TERM`'\\]\n")
     assert_re_search(expected_warning_expr, warnings)
 
 
diff --git a/utils/bump_version.py b/utils/bump_version.py
index da193a3de..9033aee70 100755
--- a/utils/bump_version.py
+++ b/utils/bump_version.py
@@ -81,7 +81,7 @@ def processing(message):
         yield
     except Skip as exc:
         print('skip: %s' % exc)
-    except:
+    except Exception:
         print('error')
         raise
     else:

From 5ae72bdd28075a02213ede26accd196c1a9dba05 Mon Sep 17 00:00:00 2001
From: Takeshi KOMIYA <i.tkomiya@gmail.com>
Date: Mon, 23 Oct 2017 23:25:50 +0900
Subject: [PATCH 31/31] Fix flake8 violation

---
 sphinx/util/inspect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sphinx/util/inspect.py b/sphinx/util/inspect.py
index a2928fc7e..c50e0c5ef 100644
--- a/sphinx/util/inspect.py
+++ b/sphinx/util/inspect.py
@@ -273,7 +273,7 @@ class Signature(object):
 
         try:
             self.annotations = typing.get_type_hints(subject)  # type: ignore
-        except:
+        except Exception:
             self.annotations = {}
 
         if bound_method: