Merge branch 'master' into improve_theme_sidebars

AUTHORS

@@ -29,6 +29,7 @@ Other contributors, listed alphabetically, are:
 * Kevin Dunn -- MathJax extension
 * Josip Dzolonga -- coverage builder
 * Buck Evan -- dummy builder
+* Matthew Fernandez -- todo extension fix
 * Hernan Grecco -- search improvements
 * Horst Gutmann -- internationalization support
 * Martin Hans -- autodoc improvements

CHANGES

@@ -81,7 +81,7 @@ Testing
 
 * Add support for docutils 0.14
 
-Release 1.6.5 (in development)
+Release 1.6.6 (in development)
 ==============================
 
 Dependencies
@@ -96,6 +96,18 @@ Deprecated
 Features added
 --------------
 
+Bugs fixed
+----------
+
+Testing
+--------
+
+Release 1.6.5 (released Oct 23, 2017)
+=====================================
+
+Features added
+--------------
+
 * #4107: Make searchtools.js compatible with pre-Sphinx1.5 templates
 * #4112: Don't override the smart_quotes setting if it was already set
 * #4125: Display reference texts of original and translated passages on
@@ -116,9 +128,13 @@ Bugs fixed
 * #4063: Sphinx crashes when labeling directive ``.. todolist::``
 * #4134: [doc] :file:`docutils.conf` is not documented explicitly
 * #4169: Chinese language doesn't trigger Chinese search automatically
-
+* #1020: ext.todo todolist not linking to the page in pdflatex
-Testing
+* #3965: New quickstart generates wrong SPHINXBUILD in Makefile
---------
+* #3739: ``:module:`` option is ignored at content of pyobjects
+* #4149: Documentation: Help choosing :confval:`latex_engine`
+* #4090: [doc] :confval:`latex_additional_files` with extra LaTeX macros should
+  not use ``.tex`` extension
+* Failed to convert reST parser error to warning (refs: #4132)
 
 Release 1.6.4 (released Sep 26, 2017)
 =====================================

Makefile

@@ -65,7 +65,7 @@ pylint:
 
 .PHONY: reindent
 reindent:
-	@$(PYTHON) utils/reindent.py -r -n .
+	@echo "This target no longer does anything and will be removed imminently"
 
 .PHONY: test
 test:

doc/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS   =
-SPHINXBUILD  = python ../sphinx-build.py
+SPHINXBUILD  = python ../sphinx/cmd/build.py
 SPHINXPROJ   = sphinx
 SOURCEDIR    = .
 BUILDDIR     = _build

doc/config.rst

@@ -1547,6 +1547,25 @@ These options influence LaTeX output. See further :doc:`latex`.
    * ``'lualatex'`` -- LuaLaTeX
    * ``'platex'`` -- pLaTeX (default if :confval:`language` is ``'ja'``)
 
+   PDFLaTeX's support for Unicode characters covers those from the document
+   language (the LaTeX ``babel`` and ``inputenc`` packages map them to glyph
+   slots in the document font, at various encodings allowing each only 256
+   characters; Sphinx uses by default (except for Cyrillic languages) the
+   ``times`` package), but stray characters from other scripts or special
+   symbols may require adding extra LaTeX packages or macros to the LaTeX
+   preamble.
+
+   If your project uses such extra Unicode characters, switching the engine to
+   XeLaTeX or LuaLaTeX often provides a quick fix. They only work with UTF-8
+   encoded sources and can (in fact, should) use OpenType fonts, either from
+   the system or the TeX install tree. Recent LaTeX releases will default with
+   these engines to the Latin Modern OpenType font, which has good coverage of
+   Latin and Cyrillic scripts (it is provided by standard LaTeX installation),
+   and Sphinx does not modify this default. Refer to the documentation of the
+   LaTeX ``polyglossia`` package to see how to instruct LaTeX to use some
+   other OpenType font if Unicode coverage proves insufficient (or use
+   directly ``\setmainfont`` et. al. as in :ref:`this example <latex-basic>`.)
+
 .. confval:: latex_documents
 
    This value determines how to group the document tree into LaTeX source files.

doc/latex.rst

@@ -29,6 +29,7 @@ The *latex* target does not benefit from pre-prepared themes like the
          cautionBgColor={named}{LightCyan}}
    \relax
 
+.. _latex-basic:
 
 Basic customization
 -------------------
@@ -61,17 +62,17 @@ It is achieved via usage of the
 .. highlight:: latex
 
 If the size of the ``'preamble'`` contents becomes inconvenient, one may move
-all needed macros into some file :file:`mystyle.tex` of the project source
+all needed macros into some file :file:`mystyle.tex.txt` of the project source
 repertory, and get LaTeX to import it at run time::
 
-   'preamble': r'\input{mystyle.tex}',
+   'preamble': r'\input{mystyle.tex.txt}',
    # or, if the \ProvidesPackage LaTeX macro is used in a file mystyle.sty
    'preamble': r'\usepackage{mystyle}',
 
 It is needed to set appropriately :confval:`latex_additional_files`, for
 example::
 
-   latex_additional_files = ["mystyle.tex"]
+   latex_additional_files = ["mystyle.sty"]
 
 .. _latexsphinxsetup:
 

setup.cfg

@@ -28,7 +28,7 @@ license_file = LICENSE
 
 [flake8]
 max-line-length = 95
-ignore = E116,E241,E251
+ignore = E116,E241,E251,E741
 exclude = .git,.tox,.venv,tests/*,build/*,doc/_build/*,sphinx/search/*,sphinx/pycode/pgen2/*,doc/ext/example*.py
 
 [mypy]

sphinx-apidoc.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.ext.apidoc import main
-    sys.exit(main(sys.argv[1:]))

sphinx-autogen.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.ext.autosummary.generate import main
-    sys.exit(main(sys.argv[1:]))

sphinx-build.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx import main
-    sys.exit(main(sys.argv[1:]))

sphinx-quickstart.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.cmd.quickstart import main
-    sys.exit(main(sys.argv[1:]))

sphinx/cmd/quickstart.py

@@ -11,13 +11,13 @@
 from __future__ import print_function
 from __future__ import absolute_import
 
-import re
+import argparse
 import os
+import re
 import sys
-import optparse
 import time
-from os import path
 from io import open
+from os import path
 
 # try to import readline, unix specific enhancement
 try:
@@ -44,7 +44,7 @@ from sphinx.util import texescape
 
 if False:
     # For type annotation
-    from typing import Any, Callable, Dict, List, Pattern  # NOQA
+    from typing import Any, Callable, Dict, List, Pattern, Union  # NOQA
 
 TERM_ENCODING = getattr(sys.stdin, 'encoding', None)
 
@@ -138,25 +138,25 @@ def ok(x):
 
 
 def term_decode(text):
-    # type: (unicode) -> unicode
+    # type: (Union[bytes,unicode]) -> unicode
     if isinstance(text, text_type):
         return text
 
-    # for Python 2.x, try to get a Unicode string out of it
+    # Use the known encoding, if possible
-    if text.decode('ascii', 'replace').encode('ascii', 'replace') == text:
-        return text
-
     if TERM_ENCODING:
-        text = text.decode(TERM_ENCODING)
+        return text.decode(TERM_ENCODING)
-    else:
+
+    # If ascii is safe, use it with no warning
+    if text.decode('ascii', 'replace').encode('ascii', 'replace') == text:
+        return text.decode('ascii')
+
     print(turquoise('* Note: non-ASCII characters entered '
                     'and terminal encoding unknown -- assuming '
                     'UTF-8 or Latin-1.'))
     try:
-            text = text.decode('utf-8')
+        return text.decode('utf-8')
     except UnicodeDecodeError:
-            text = text.decode('latin1')
+        return text.decode('latin1')
-    return text
 
 
 def do_prompt(d, key, text, default=None, validator=nonempty):
@@ -509,23 +509,6 @@ where "builder" is one of the supported builders, e.g. html, latex or linkcheck.
 ''')
 
 
-def usage(argv, msg=None):
-    # type: (List[unicode], unicode) -> None
-    if msg:
-        print(msg, file=sys.stderr)
-        print(file=sys.stderr)
-
-
-USAGE = """\
-Sphinx v%s
-Usage: %%prog [options] [projectdir]
-""" % __display_version__
-
-EPILOG = """\
-For more information, visit <http://sphinx-doc.org/>.
-"""
-
-
 def valid_dir(d):
     # type: (Dict) -> bool
     dir = d['path']
@@ -556,18 +539,86 @@ def valid_dir(d):
     return True
 
 
-class MyFormatter(optparse.IndentedHelpFormatter):
+def get_parser():
-    def format_usage(self, usage):  # type: ignore
+    # type: () -> argparse.ArgumentParser
-        # type: (str) -> str
+    parser = argparse.ArgumentParser(
-        return usage
+        usage='%(prog)s [OPTIONS] <PROJECT_DIR>',
+        epilog="For more information, visit <http://sphinx-doc.org/>.",
+        description="""
+Generate required files for a Sphinx project.
 
-    def format_help(self, formatter):
+sphinx-quickstart is an interactive tool that asks some questions about your
-        result = []
+project and then generates a complete documentation directory and sample
-        if self.description:
+Makefile to be used with sphinx-build.
-            result.append(self.format_description(formatter))
+""")
-        if self.option_list:
+
-            result.append(self.format_option_help(formatter))
+    parser.add_argument('-q', '--quiet', action='store_true', dest='quiet',
-        return "\n".join(result)
+                        default=False,
+                        help='quiet mode')
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
+
+    parser.add_argument('path', metavar='PROJECT_DIR', default='.',
+                        help='output path')
+
+    group = parser.add_argument_group('Structure options')
+    group.add_argument('--sep', action='store_true',
+                       help='if specified, separate source and build dirs')
+    group.add_argument('--dot', metavar='DOT',
+                       help='replacement for dot in _templates etc.')
+
+    group = parser.add_argument_group('Project basic options')
+    group.add_argument('-p', '--project', metavar='PROJECT', dest='project',
+                       help='project name')
+    group.add_argument('-a', '--author', metavar='AUTHOR', dest='author',
+                       help='author names')
+    group.add_argument('-v', metavar='VERSION', dest='version', default='',
+                       help='version of project')
+    group.add_argument('-r', '--release', metavar='RELEASE', dest='release',
+                       help='release of project')
+    group.add_argument('-l', '--language', metavar='LANGUAGE', dest='language',
+                       help='document language')
+    group.add_argument('--suffix', metavar='SUFFIX',
+                       help='source file suffix')
+    group.add_argument('--master', metavar='MASTER',
+                       help='master document name')
+    group.add_argument('--epub', action='store_true', default=False,
+                       help='use epub')
+
+    group = parser.add_argument_group('Extension options')
+    for ext in EXTENSIONS:
+        group.add_argument('--ext-' + ext, action='store_true',
+                           dest='ext_' + ext, default=False,
+                           help='enable %s extension' % ext)
+    group.add_argument('--extensions', metavar='EXTENSIONS', dest='extensions',
+                       action='append', help='enable extensions')
+
+    # TODO(stephenfin): Consider using mutually exclusive groups here
+    group = parser.add_argument_group('Makefile and Batchfile creation')
+    group.add_argument('--makefile', action='store_true', default=False,
+                       help='create makefile')
+    group.add_argument('--no-makefile', action='store_true', default=False,
+                       help='not create makefile')
+    group.add_argument('--batchfile', action='store_true', default=False,
+                       help='create batchfile')
+    group.add_argument('--no-batchfile', action='store_true', default=False,
+                       help='not create batchfile')
+    group.add_argument('-M', '--no-use-make-mode', action='store_false',
+                       dest='make_mode', default=False,
+                       help='not use make-mode for Makefile/make.bat')
+    group.add_argument('-m', '--use-make-mode', action='store_true',
+                       dest='make_mode', default=True,
+                       help='use make-mode for Makefile/make.bat')
+
+    group = parser.add_argument_group('Project templating')
+    group.add_argument('-t', '--templatedir', metavar='TEMPLATEDIR',
+                       dest='templatedir',
+                       help='template directory for template files')
+    group.add_argument('-d', metavar='NAME=VALUE', action='append',
+                       dest='variables',
+                       help='define a template variable')
+
+    return parser
 
 
 def main(argv=sys.argv[1:]):
@@ -575,81 +626,14 @@ def main(argv=sys.argv[1:]):
     if not color_terminal():
         nocolor()
 
-    parser = optparse.OptionParser(USAGE, epilog=EPILOG,
-                                   version='Sphinx v%s' % __display_version__,
-                                   formatter=MyFormatter())
-    parser.add_option('-q', '--quiet', action='store_true', dest='quiet',
-                      default=False,
-                      help='quiet mode')
-
-    group = parser.add_option_group('Structure options')
-    group.add_option('--sep', action='store_true', dest='sep',
-                     help='if specified, separate source and build dirs')
-    group.add_option('--dot', metavar='DOT', dest='dot',
-                     help='replacement for dot in _templates etc.')
-
-    group = parser.add_option_group('Project basic options')
-    group.add_option('-p', '--project', metavar='PROJECT', dest='project',
-                     help='project name')
-    group.add_option('-a', '--author', metavar='AUTHOR', dest='author',
-                     help='author names')
-    group.add_option('-v', metavar='VERSION', dest='version',
-                     help='version of project')
-    group.add_option('-r', '--release', metavar='RELEASE', dest='release',
-                     help='release of project')
-    group.add_option('-l', '--language', metavar='LANGUAGE', dest='language',
-                     help='document language')
-    group.add_option('--suffix', metavar='SUFFIX', dest='suffix',
-                     help='source file suffix')
-    group.add_option('--master', metavar='MASTER', dest='master',
-                     help='master document name')
-    group.add_option('--epub', action='store_true', dest='epub',
-                     default=False,
-                     help='use epub')
-
-    group = parser.add_option_group('Extension options')
-    for ext in EXTENSIONS:
-        group.add_option('--ext-' + ext, action='store_true',
-                         dest='ext_' + ext, default=False,
-                         help='enable %s extension' % ext)
-    group.add_option('--extensions', metavar='EXTENSIONS', dest='extensions',
-                     action='append', help='enable extensions')
-
-    group = parser.add_option_group('Makefile and Batchfile creation')
-    group.add_option('--makefile', action='store_true', dest='makefile',
-                     default=False,
-                     help='create makefile')
-    group.add_option('--no-makefile', action='store_true', dest='no_makefile',
-                     default=False,
-                     help='not create makefile')
-    group.add_option('--batchfile', action='store_true', dest='batchfile',
-                     default=False,
-                     help='create batchfile')
-    group.add_option('--no-batchfile', action='store_true', dest='no_batchfile',
-                     default=False,
-                     help='not create batchfile')
-    group.add_option('-M', '--no-use-make-mode', action='store_false', dest='make_mode',
-                     help='not use make-mode for Makefile/make.bat')
-    group.add_option('-m', '--use-make-mode', action='store_true', dest='make_mode',
-                     default=True,
-                     help='use make-mode for Makefile/make.bat')
-
-    group = parser.add_option_group('Project templating')
-    group.add_option('-t', '--templatedir', metavar='TEMPLATEDIR', dest='templatedir',
-                     help='template directory for template files')
-    group.add_option('-d', metavar='NAME=VALUE', action='append', dest='variables',
-                     help='define a template variable')
-
     # parse options
+    parser = get_parser()
     try:
-        opts, args = parser.parse_args(argv)
+        args = parser.parse_args(argv)
     except SystemExit as err:
         return err.code
 
-    if len(args) > 0:
+    d = vars(args)
-        opts.ensure_value('path', args[0])
-
-    d = vars(opts)
     # delete None or False value
     d = dict((k, v) for k, v in d.items() if not (v is None or v is False))
 
@@ -707,7 +691,7 @@ def main(argv=sys.argv[1:]):
         except ValueError:
             print('Invalid template variable: %s' % variable)
 
-    generate(d, templatedir=opts.templatedir)
+    generate(d, templatedir=args.templatedir)
     return 0
 
 

sphinx/cmdline.py

@@ -10,14 +10,13 @@
 """
 from __future__ import print_function
 
+import argparse
 import sys
-import optparse
 import traceback
 from os import path
 
-from six import text_type, binary_type
-
 from docutils.utils import SystemMessage
+from six import text_type, binary_type
 
 from sphinx import __display_version__
 from sphinx.errors import SphinxError
@@ -33,39 +32,9 @@ if False:
     from typing import Any, IO, List, Union  # NOQA
 
 
-USAGE = """\
+def handle_exception(app, args, exception, stderr=sys.stderr):
-Sphinx v%s
-Usage: %%prog [options] sourcedir outdir [filenames...]
-
-Filename arguments:
-  without -a and without filenames, write new and changed files.
-  with -a, write all files.
-  with filenames, write these.
-""" % __display_version__
-
-EPILOG = """\
-For more information, visit <http://sphinx-doc.org/>.
-"""
-
-
-class MyFormatter(optparse.IndentedHelpFormatter):
-    def format_usage(self, usage):
-        # type: (Any) -> Any
-        return usage
-
-    def format_help(self, formatter):
-        # type: (Any) -> unicode
-        result = []  # type: List[unicode]
-        if self.description:  # type: ignore
-            result.append(self.format_description(formatter))
-        if self.option_list:  # type: ignore
-            result.append(self.format_option_help(formatter))  # type: ignore
-        return "\n".join(result)
-
-
-def handle_exception(app, opts, exception, stderr=sys.stderr):
     # type: (Sphinx, Any, Union[Exception, KeyboardInterrupt], IO) -> None
-    if opts.pdb:
+    if args.pdb:
         import pdb
         print(red('Exception occurred while building, starting debugger:'),
               file=stderr)
@@ -73,7 +42,7 @@ def handle_exception(app, opts, exception, stderr=sys.stderr):
         pdb.post_mortem(sys.exc_info()[2])
     else:
         print(file=stderr)
-        if opts.verbosity or opts.traceback:
+        if args.verbosity or args.traceback:
             traceback.print_exc(None, stderr)
             print(file=stderr)
         if isinstance(exception, KeyboardInterrupt):
@@ -114,104 +83,130 @@ def handle_exception(app, opts, exception, stderr=sys.stderr):
                   file=stderr)
 
 
-def main(argv=sys.argv[1:]):  # type: ignore
+def get_parser():
-    # type: (List[unicode]) -> int
+    # type: () -> argparse.ArgumentParser
-    parser = optparse.OptionParser(USAGE, epilog=EPILOG, formatter=MyFormatter())
+    parser = argparse.ArgumentParser(
-    parser.add_option('--version', action='store_true', dest='version',
+        usage='usage: %(prog)s [OPTIONS] SOURCEDIR OUTPUTDIR [FILENAMES...]',
-                      help='show version information and exit')
+        epilog='For more information, visit <http://sphinx-doc.org/>.',
+        description="""
+Generate documentation from source files.
 
-    group = parser.add_option_group('General options')
+sphinx-build generates documentation from the files in SOURCEDIR and places it
-    group.add_option('-b', metavar='BUILDER', dest='builder', default='html',
+in OUTPUTDIR. It looks for 'conf.py' in SOURCEDIR for the configuration
-                     help='builder to use; default is html')
+settings.  The 'sphinx-quickstart' tool may be used to generate template files,
-    group.add_option('-a', action='store_true', dest='force_all',
+including 'conf.py'
-                     help='write all files; default is to only write new and '
+
-                     'changed files')
+sphinx-build can create documentation in different formats. A format is
-    group.add_option('-E', action='store_true', dest='freshenv',
+selected by specifying the builder name on the command line; it defaults to
+HTML. Builders can also perform other tasks related to documentation
+processing.
+
+By default, everything that is outdated is built. Output only for selected
+files can be built by specifying individual filenames.
+""")
+
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
+
+    parser.add_argument('sourcedir',
+                        help='path to documentation source files')
+    parser.add_argument('outputdir',
+                        help='path to output directory')
+    parser.add_argument('filenames', nargs='*',
+                        help='a list of specific files to rebuild. Ignored '
+                        'if -a is specified')
+
+    group = parser.add_argument_group('general options')
+    group.add_argument('-b', metavar='BUILDER', dest='builder',
+                       default='html',
+                       help='builder to use (default: html)')
+    group.add_argument('-a', action='store_true', dest='force_all',
+                       help='write all files (default: only write new and '
+                       'changed files)')
+    group.add_argument('-E', action='store_true', dest='freshenv',
                        help='don\'t use a saved environment, always read '
                        'all files')
-    group.add_option('-d', metavar='PATH', default=None, dest='doctreedir',
+    group.add_argument('-d', metavar='PATH', dest='doctreedir',
-                     help='path for the cached environment and doctree files '
+                       help='path for the cached environment and doctree '
-                     '(default: outdir/.doctrees)')
+                       'files (default: OUTPUTDIR/.doctrees)')
-    group.add_option('-j', metavar='N', default=1, type='int', dest='jobs',
+    group.add_argument('-j', metavar='N', default=1, type=int, dest='jobs',
-                     help='build in parallel with N processes where possible')
+                       help='build in parallel with N processes where '
-    # this option never gets through to this point (it is intercepted earlier)
+                       'possible')
-    # group.add_option('-M', metavar='BUILDER', dest='make_mode',
-    #                 help='"make" mode -- as used by Makefile, like '
-    #                 '"sphinx-build -M html"')
 
-    group = parser.add_option_group('Build configuration options')
+    group = parser.add_argument_group('build configuration options')
-    group.add_option('-c', metavar='PATH', dest='confdir',
+    group.add_argument('-c', metavar='PATH', dest='confdir',
-                     help='path where configuration file (conf.py) is located '
+                       help='path where configuration file (conf.py) is '
-                     '(default: same as sourcedir)')
+                       'located (default: same as SOURCEDIR)')
-    group.add_option('-C', action='store_true', dest='noconfig',
+    group.add_argument('-C', action='store_true', dest='noconfig',
                        help='use no config file at all, only -D options')
-    group.add_option('-D', metavar='setting=value', action='append',
+    group.add_argument('-D', metavar='setting=value', action='append',
                        dest='define', default=[],
                        help='override a setting in configuration file')
-    group.add_option('-A', metavar='name=value', action='append',
+    group.add_argument('-A', metavar='name=value', action='append',
                        dest='htmldefine', default=[],
                        help='pass a value into HTML templates')
-    group.add_option('-t', metavar='TAG', action='append',
+    group.add_argument('-t', metavar='TAG', action='append',
                        dest='tags', default=[],
                        help='define tag: include "only" blocks with TAG')
-    group.add_option('-n', action='store_true', dest='nitpicky',
+    group.add_argument('-n', action='store_true', dest='nitpicky',
-                     help='nit-picky mode, warn about all missing references')
+                       help='nit-picky mode, warn about all missing '
+                       'references')
 
-    group = parser.add_option_group('Console output options')
+    group = parser.add_argument_group('console output options')
-    group.add_option('-v', action='count', dest='verbosity', default=0,
+    group.add_argument('-v', action='count', dest='verbosity', default=0,
                        help='increase verbosity (can be repeated)')
-    group.add_option('-q', action='store_true', dest='quiet',
+    group.add_argument('-q', action='store_true', dest='quiet',
                        help='no output on stdout, just warnings on stderr')
-    group.add_option('-Q', action='store_true', dest='really_quiet',
+    group.add_argument('-Q', action='store_true', dest='really_quiet',
                        help='no output at all, not even warnings')
-    group.add_option('--color', dest='color',
+    group.add_argument('--color', action='store_const', const='yes',
-                     action='store_const', const='yes', default='auto',
+                       default='auto',
-                     help='Do emit colored output (default: auto-detect)')
+                       help='do emit colored output (default: auto-detect)')
-    group.add_option('-N', '--no-color', dest='color',
+    group.add_argument('-N', '--no-color', dest='color', action='store_const',
-                     action='store_const', const='no',
+                       const='no',
-                     help='Do not emit colored output (default: auto-detect)')
+                       help='do not emit colored output (default: '
-    group.add_option('-w', metavar='FILE', dest='warnfile',
+                       'auto-detect)')
+    group.add_argument('-w', metavar='FILE', dest='warnfile',
                        help='write warnings (and errors) to given file')
-    group.add_option('-W', action='store_true', dest='warningiserror',
+    group.add_argument('-W', action='store_true', dest='warningiserror',
                        help='turn warnings into errors')
-    group.add_option('-T', action='store_true', dest='traceback',
+    group.add_argument('-T', action='store_true', dest='traceback',
                        help='show full traceback on exception')
-    group.add_option('-P', action='store_true', dest='pdb',
+    group.add_argument('-P', action='store_true', dest='pdb',
                        help='run Pdb on exception')
 
+    return parser
+
+
+def main(argv=sys.argv[1:]):  # type: ignore
+    # type: (List[unicode]) -> int
+
+    parser = get_parser()
     # parse options
     try:
-        opts, args = parser.parse_args(argv)
+        args = parser.parse_args(argv)
     except SystemExit as err:
         return err.code
 
-    # handle basic options
-    if opts.version:
-        print('Sphinx (sphinx-build) %s' % __display_version__)
-        return 0
-
     # get paths (first and second positional argument)
     try:
-        srcdir = abspath(args[0])
+        srcdir = abspath(args.sourcedir)
-        confdir = abspath(opts.confdir or srcdir)
+        confdir = abspath(args.confdir or srcdir)
-        if opts.noconfig:
+        if args.noconfig:
             confdir = None
         if not path.isdir(srcdir):
             print('Error: Cannot find source directory `%s\'.' % srcdir,
                   file=sys.stderr)
             return 1
-        if not opts.noconfig and not path.isfile(path.join(confdir, 'conf.py')):
+        if not args.noconfig and not path.isfile(path.join(confdir, 'conf.py')):
             print('Error: Config directory doesn\'t contain a conf.py file.',
                   file=sys.stderr)
             return 1
-        outdir = abspath(args[1])
+        outdir = abspath(args.outputdir)
         if srcdir == outdir:
             print('Error: source directory and destination directory are same.',
                   file=sys.stderr)
             return 1
-    except IndexError:
-        parser.print_help()
-        return 1
     except UnicodeError:
         print(
             'Error: Multibyte filename not supported on this filesystem '
@@ -219,7 +214,7 @@ def main(argv=sys.argv[1:]):  # type: ignore
         return 1
 
     # handle remaining filename arguments
-    filenames = args[2:]
+    filenames = args.filenames
     errored = False
     for filename in filenames:
         if not path.isfile(filename):
@@ -235,35 +230,35 @@ def main(argv=sys.argv[1:]):  # type: ignore
     except Exception:
         likely_encoding = None
 
-    if opts.force_all and filenames:
+    if args.force_all and filenames:
         print('Error: Cannot combine -a option and filenames.', file=sys.stderr)
         return 1
 
-    if opts.color == 'no' or (opts.color == 'auto' and not color_terminal()):
+    if args.color == 'no' or (args.color == 'auto' and not color_terminal()):
         nocolor()
 
-    doctreedir = abspath(opts.doctreedir or path.join(outdir, '.doctrees'))
+    doctreedir = abspath(args.doctreedir or path.join(outdir, '.doctrees'))
 
     status = sys.stdout
     warning = sys.stderr
     error = sys.stderr
 
-    if opts.quiet:
+    if args.quiet:
         status = None
-    if opts.really_quiet:
+    if args.really_quiet:
         status = warning = None
-    if warning and opts.warnfile:
+    if warning and args.warnfile:
         try:
-            warnfp = open(opts.warnfile, 'w')
+            warnfp = open(args.warnfile, 'w')
         except Exception as exc:
             print('Error: Cannot open warning file %r: %s' %
-                  (opts.warnfile, exc), file=sys.stderr)
+                  (args.warnfile, exc), file=sys.stderr)
             sys.exit(1)
         warning = Tee(warning, warnfp)  # type: ignore
         error = warning
 
     confoverrides = {}
-    for val in opts.define:
+    for val in args.define:
         try:
             key, val = val.split('=', 1)
         except ValueError:
@@ -277,7 +272,7 @@ def main(argv=sys.argv[1:]):  # type: ignore
                 pass
         confoverrides[key] = val
 
-    for val in opts.htmldefine:
+    for val in args.htmldefine:
         try:
             key, val = val.split('=')
         except ValueError:
@@ -294,17 +289,17 @@ def main(argv=sys.argv[1:]):  # type: ignore
                     pass
         confoverrides['html_context.%s' % key] = val
 
-    if opts.nitpicky:
+    if args.nitpicky:
         confoverrides['nitpicky'] = True
 
     app = None
     try:
         with patch_docutils(), docutils_namespace():
-            app = Sphinx(srcdir, confdir, outdir, doctreedir, opts.builder,
+            app = Sphinx(srcdir, confdir, outdir, doctreedir, args.builder,
-                         confoverrides, status, warning, opts.freshenv,
+                         confoverrides, status, warning, args.freshenv,
-                         opts.warningiserror, opts.tags, opts.verbosity, opts.jobs)
+                         args.warningiserror, args.tags, args.verbosity, args.jobs)
-            app.build(opts.force_all, filenames)
+            app.build(args.force_all, filenames)
             return app.statuscode
     except (Exception, KeyboardInterrupt) as exc:
-        handle_exception(app, opts, exc, error)
+        handle_exception(app, args, exc, error)
         return 1

sphinx/domains/cpp.py

@@ -4862,7 +4862,7 @@ class DefinitionParser(object):
         pos = self.pos
         try:
             concept = self._parse_nested_name()
-        except:
+        except Exception:
             self.pos = pos
             return None
         self.skip_ws()

sphinx/domains/python.py

@@ -348,6 +348,10 @@ class PyObject(ObjectDescription):
             if self.allow_nesting:
                 classes = self.env.ref_context.setdefault('py:classes', [])
                 classes.append(prefix)
+        if 'module' in self.options:
+            modules = self.env.ref_context.setdefault('py:modules', [])
+            modules.append(self.env.ref_context.get('py:module'))
+            self.env.ref_context['py:module'] = self.options['module']
 
     def after_content(self):
         # type: () -> None
@@ -368,6 +372,12 @@ class PyObject(ObjectDescription):
                 pass
         self.env.ref_context['py:class'] = (classes[-1] if len(classes) > 0
                                             else None)
+        if 'module' in self.options:
+            modules = self.env.ref_context.setdefault('py:modules', [])
+            if modules:
+                self.env.ref_context['py:module'] = modules.pop()
+            else:
+                self.env.ref_context.pop('py:module')
 
 
 class PyModulelevel(PyObject):

sphinx/environment/__init__.py

@@ -110,7 +110,12 @@ class BuildEnvironment(object):
     @staticmethod
     def load(f, app=None):
         # type: (IO, Sphinx) -> BuildEnvironment
+        try:
             env = pickle.load(f)
+        except Exception as exc:
+            # This can happen for example when the pickle is from a
+            # different version of Sphinx.
+            raise IOError(exc)
         if env.version != ENV_VERSION:
             raise IOError('build environment version not current')
         if app:

sphinx/ext/apidoc.py

@@ -17,9 +17,9 @@
 
 from __future__ import print_function
 
+import argparse
 import os
 import sys
-import optparse
 from os import path
 from six import binary_type
 from fnmatch import fnmatch
@@ -265,12 +265,6 @@ def recurse_tree(rootpath, excludes, opts):
     return toplevels
 
 
-def normalize_excludes(rootpath, excludes):
-    # type: (unicode, List[unicode]) -> List[unicode]
-    """Normalize the excluded directory list."""
-    return [path.abspath(exclude) for exclude in excludes]
-
-
 def is_excluded(root, excludes):
     # type: (unicode, List[unicode]) -> bool
     """Check if the directory is in the exclude list.
@@ -284,106 +278,116 @@ def is_excluded(root, excludes):
     return False
 
 
-def main(argv=sys.argv[1:]):
+def get_parser():
-    # type: (List[str]) -> int
+    # type: () -> argparse.ArgumentParser
-    """Parse and check the command line arguments."""
+    parser = argparse.ArgumentParser(
-    parser = optparse.OptionParser(
+        usage='usage: %(prog)s [OPTIONS] -o <OUTPUT_PATH> <MODULE_PATH> '
-        usage="""\
+              '[EXCLUDE_PATTERN, ...]',
-usage: %prog [options] -o <output_path> <module_path> [exclude_pattern, ...]
+        epilog='For more information, visit <http://sphinx-doc.org/>.',
+        description="""
+Look recursively in <MODULE_PATH> for Python modules and packages and create
+one reST file with automodule directives per package in the <OUTPUT_PATH>.
 
-Look recursively in <module_path> for Python modules and packages and create
+The <EXCLUDE_PATTERN>s can be file and/or directory patterns that will be
-one reST file with automodule directives per package in the <output_path>.
-
-The <exclude_pattern>s can be file and/or directory patterns that will be
 excluded from generation.
 
 Note: By default this script will not overwrite already created files.""")
 
-    parser.add_option('-o', '--output-dir', action='store', dest='destdir',
+    parser.add_argument('--version', action='version', dest='show_version',
-                      help='Directory to place all output', default='')
+                        version='%%(prog)s %s' % __display_version__)
-    parser.add_option('-d', '--maxdepth', action='store', dest='maxdepth',
+
-                      help='Maximum depth of submodules to show in the TOC '
+    parser.add_argument('module_path',
-                      '(default: 4)', type='int', default=4)
+                        help='path to module to document')
-    parser.add_option('-f', '--force', action='store_true', dest='force',
+    parser.add_argument('exclude_pattern', nargs='*',
-                      help='Overwrite existing files')
+                        help='fnmatch-style file and/or directory patterns '
-    parser.add_option('-l', '--follow-links', action='store_true',
+                        'to exclude from generation')
+
+    parser.add_argument('-o', '--output-dir', action='store', dest='destdir',
+                        required=True,
+                        help='directory to place all output')
+    parser.add_argument('-d', '--maxdepth', action='store', dest='maxdepth',
+                        type=int, default=4,
+                        help='maximum depth of submodules to show in the TOC '
+                             '(default: 4)')
+    parser.add_argument('-f', '--force', action='store_true', dest='force',
+                        help='overwrite existing files')
+    parser.add_argument('-l', '--follow-links', action='store_true',
                         dest='followlinks', default=False,
-                      help='Follow symbolic links. Powerful when combined '
+                        help='follow symbolic links. Powerful when combined '
                               'with collective.recipe.omelette.')
-    parser.add_option('-n', '--dry-run', action='store_true', dest='dryrun',
+    parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun',
-                      help='Run the script without creating files')
+                        help='run the script without creating files')
-    parser.add_option('-e', '--separate', action='store_true',
+    parser.add_argument('-e', '--separate', action='store_true',
                         dest='separatemodules',
-                      help='Put documentation for each module on its own page')
+                        help='put documentation for each module on its own page')
-    parser.add_option('-P', '--private', action='store_true',
+    parser.add_argument('-P', '--private', action='store_true',
                         dest='includeprivate',
-                      help='Include "_private" modules')
+                        help='include "_private" modules')
-    parser.add_option('-T', '--no-toc', action='store_true', dest='notoc',
+    parser.add_argument('-T', '--no-toc', action='store_true', dest='notoc',
-                      help='Don\'t create a table of contents file')
+                        help="don't create a table of contents file")
-    parser.add_option('-E', '--no-headings', action='store_true',
+    parser.add_argument('-E', '--no-headings', action='store_true',
                         dest='noheadings',
-                      help='Don\'t create headings for the module/package '
+                        help="don't create headings for the module/package "
-                           'packages (e.g. when the docstrings already contain '
+                             "packages (e.g. when the docstrings already "
-                           'them)')
+                             "contain them)")
-    parser.add_option('-M', '--module-first', action='store_true',
+    parser.add_argument('-M', '--module-first', action='store_true',
                         dest='modulefirst',
-                      help='Put module documentation before submodule '
+                        help='put module documentation before submodule '
                              'documentation')
-    parser.add_option('--implicit-namespaces', action='store_true',
+    parser.add_argument('--implicit-namespaces', action='store_true',
                         dest='implicit_namespaces',
-                      help='Interpret module paths according to PEP-0420 '
+                        help='interpret module paths according to PEP-0420 '
                              'implicit namespaces specification')
-    parser.add_option('-s', '--suffix', action='store', dest='suffix',
+    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
-                      help='file suffix (default: rst)', default='rst')
+                        default='rst',
-    parser.add_option('-F', '--full', action='store_true', dest='full',
+                        help='file suffix (default: rst)')
-                      help='Generate a full project with sphinx-quickstart')
+    parser.add_argument('-F', '--full', action='store_true', dest='full',
-    parser.add_option('-a', '--append-syspath', action='store_true',
+                        help='generate a full project with sphinx-quickstart')
+    parser.add_argument('-a', '--append-syspath', action='store_true',
                         dest='append_syspath',
-                      help='Append module_path to sys.path, used when --full is given')
+                        help='append module_path to sys.path, used when --full is given')
-    parser.add_option('-H', '--doc-project', action='store', dest='header',
+    parser.add_argument('-H', '--doc-project', action='store', dest='header',
-                      help='Project name (default: root module name)')
+                        help='project name (default: root module name)')
-    parser.add_option('-A', '--doc-author', action='store', dest='author',
+    parser.add_argument('-A', '--doc-author', action='store', dest='author',
-                      type='str',
+                        help='project author(s), used when --full is given')
-                      help='Project author(s), used when --full is given')
+    parser.add_argument('-V', '--doc-version', action='store', dest='version',
-    parser.add_option('-V', '--doc-version', action='store', dest='version',
+                        help='project version, used when --full is given')
-                      help='Project version, used when --full is given')
+    parser.add_argument('-R', '--doc-release', action='store', dest='release',
-    parser.add_option('-R', '--doc-release', action='store', dest='release',
+                        help='project release, used when --full is given, '
-                      help='Project release, used when --full is given, '
                              'defaults to --doc-version')
-    parser.add_option('--version', action='store_true', dest='show_version',
+
-                      help='Show version information and exit')
+    group = parser.add_argument_group('extension options')
-    group = parser.add_option_group('Extension options')
     for ext in EXTENSIONS:
-        group.add_option('--ext-' + ext, action='store_true',
+        group.add_argument('--ext-' + ext, action='store_true',
                            dest='ext_' + ext, default=False,
                            help='enable %s extension' % ext)
 
-    (opts, args) = parser.parse_args(argv)
+    return parser
 
-    if opts.show_version:
-        print('Sphinx (sphinx-apidoc) %s' % __display_version__)
-        return 0
 
-    if not args:
+def main(argv=sys.argv[1:]):
-        parser.error('A package path is required.')
+    # type: (List[str]) -> int
+    """Parse and check the command line arguments."""
+    parser = get_parser()
+    args = parser.parse_args(argv)
 
-    rootpath, excludes = args[0], args[1:]
+    rootpath = path.abspath(args.module_path)
-    if not opts.destdir:
+
-        parser.error('An output directory is required.')
+    # normalize opts
-    if opts.header is None:
+
-        opts.header = path.abspath(rootpath).split(path.sep)[-1]
+    if args.header is None:
-    if opts.suffix.startswith('.'):
+        args.header = rootpath.split(path.sep)[-1]
-        opts.suffix = opts.suffix[1:]
+    if args.suffix.startswith('.'):
+        args.suffix = args.suffix[1:]
     if not path.isdir(rootpath):
         print('%s is not a directory.' % rootpath, file=sys.stderr)
         sys.exit(1)
-    if not path.isdir(opts.destdir):
+    if not path.isdir(args.destdir) and not args.dryrun:
-        if not opts.dryrun:
+        os.makedirs(args.destdir)
-            os.makedirs(opts.destdir)
+    excludes = [path.abspath(exclude) for exclude in args.exclude_pattern]
-    rootpath = path.abspath(rootpath)
+    modules = recurse_tree(rootpath, excludes, args)
-    excludes = normalize_excludes(rootpath, excludes)
+
-    modules = recurse_tree(rootpath, excludes, opts)
+    if args.full:
-    if opts.full:
         from sphinx.cmd import quickstart as qs
         modules.sort()
         prev_module = ''  # type: unicode
@@ -394,14 +398,14 @@ Note: By default this script will not overwrite already created files.""")
             prev_module = module
             text += '   %s\n' % module
         d = dict(
-            path = opts.destdir,
+            path = args.destdir,
             sep = False,
             dot = '_',
-            project = opts.header,
+            project = args.header,
-            author = opts.author or 'Author',
+            author = args.author or 'Author',
-            version = opts.version or '',
+            version = args.version or '',
-            release = opts.release or opts.version or '',
+            release = args.release or args.version or '',
-            suffix = '.' + opts.suffix,
+            suffix = '.' + args.suffix,
             master = 'index',
             epub = True,
             ext_autodoc = True,
@@ -409,29 +413,30 @@ Note: By default this script will not overwrite already created files.""")
             ext_todo = True,
             makefile = True,
             batchfile = True,
-            mastertocmaxdepth = opts.maxdepth,
+            mastertocmaxdepth = args.maxdepth,
             mastertoctree = text,
             language = 'en',
             module_path = rootpath,
-            append_syspath = opts.append_syspath,
+            append_syspath = args.append_syspath,
         )
-        enabled_exts = {'ext_' + ext: getattr(opts, 'ext_' + ext)
+        enabled_exts = {'ext_' + ext: getattr(args, 'ext_' + ext)
-                        for ext in EXTENSIONS if getattr(opts, 'ext_' + ext)}
+                        for ext in EXTENSIONS if getattr(args, 'ext_' + ext)}
         d.update(enabled_exts)
 
-        if isinstance(opts.header, binary_type):
+        if isinstance(args.header, binary_type):
             d['project'] = d['project'].decode('utf-8')
-        if isinstance(opts.author, binary_type):
+        if isinstance(args.author, binary_type):
             d['author'] = d['author'].decode('utf-8')
-        if isinstance(opts.version, binary_type):
+        if isinstance(args.version, binary_type):
             d['version'] = d['version'].decode('utf-8')
-        if isinstance(opts.release, binary_type):
+        if isinstance(args.release, binary_type):
             d['release'] = d['release'].decode('utf-8')
 
-        if not opts.dryrun:
+        if not args.dryrun:
-            qs.generate(d, silent=True, overwrite=opts.force)
+            qs.generate(d, silent=True, overwrite=args.force)
-    elif not opts.notoc:
+    elif not args.notoc:
-        create_modules_toc_file(modules, opts)
+        create_modules_toc_file(modules, args)
+
     return 0
 
 

sphinx/ext/autosummary/generate.py

@@ -19,16 +19,17 @@
 """
 from __future__ import print_function
 
+import argparse
+import codecs
 import os
+import pydoc
 import re
 import sys
-import pydoc
-import optparse
-import codecs
 
 from jinja2 import FileSystemLoader, TemplateNotFound
 from jinja2.sandbox import SandboxedEnvironment
 
+from sphinx import __display_version__
 from sphinx import package_dir
 from sphinx.ext.autosummary import import_by_name, get_documenter
 from sphinx.jinja2glue import BuiltinTemplateLoader
@@ -59,33 +60,6 @@ if False:
     from sphinx.environment import BuildEnvironment  # NOQA
 
 
-def main(argv=sys.argv[1:]):
-    # type: (List[str]) -> None
-    usage = """%prog [OPTIONS] SOURCEFILE ..."""
-    p = optparse.OptionParser(usage.strip())
-    p.add_option("-o", "--output-dir", action="store", type="string",
-                 dest="output_dir", default=None,
-                 help="Directory to place all output in")
-    p.add_option("-s", "--suffix", action="store", type="string",
-                 dest="suffix", default="rst",
-                 help="Default suffix for files (default: %default)")
-    p.add_option("-t", "--templates", action="store", type="string",
-                 dest="templates", default=None,
-                 help="Custom template directory (default: %default)")
-    p.add_option("-i", "--imported-members", action="store_true",
-                 dest="imported_members", default=False,
-                 help="Document imported members (default: %default)")
-    options, args = p.parse_args(argv)
-
-    if len(args) < 1:
-        p.error('no input files given')
-
-    generate_autosummary_docs(args, options.output_dir,
-                              "." + options.suffix,
-                              template_dir=options.templates,
-                              imported_members=options.imported_members)
-
-
 def _simple_info(msg):
     # type: (unicode) -> None
     print(msg)
@@ -373,5 +347,57 @@ def find_autosummary_in_lines(lines, module=None, filename=None):
     return documented
 
 
+def get_parser():
+    # type: () -> argparse.ArgumentParser
+    parser = argparse.ArgumentParser(
+        usage='%(prog)s [OPTIONS] <SOURCE_FILE>...',
+        epilog='For more information, visit <http://sphinx-doc.org/>.',
+        description="""
+Generate ReStructuredText using autosummary directives.
+
+sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates
+the reStructuredText files from the autosummary directives contained in the
+given input files.
+
+The format of the autosummary directive is documented in the
+``sphinx.ext.autosummary`` Python module and can be read using::
+
+  pydoc sphinx.ext.autosummary
+""")
+
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
+
+    parser.add_argument('source_file', nargs='+',
+                        help='source files to generate rST files for')
+
+    parser.add_argument('-o', '--output-dir', action='store',
+                        dest='output_dir',
+                        help='directory to place all output in')
+    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
+                        default='rst',
+                        help='default suffix for files (default: '
+                              '%(default)s)')
+    parser.add_argument('-t', '--templates', action='store', dest='templates',
+                        default=None,
+                        help='custom template directory (default: '
+                              '%(default)s)')
+    parser.add_argument('-i', '--imported-members', action='store_true',
+                        dest='imported_members', default=False,
+                        help='document imported members (default: '
+                              '%(default)s)')
+
+    return parser
+
+
+def main(argv=sys.argv[1:]):
+    # type: (List[str]) -> None
+    args = get_parser().parse_args(argv)
+    generate_autosummary_docs(args.source_file, args.output_dir,
+                              '.' + args.suffix,
+                              template_dir=args.templates,
+                              imported_members=args.imported_members)
+
+
 if __name__ == '__main__':
     main()

sphinx/ext/todo.py

@@ -69,6 +69,8 @@ class Todo(BaseAdmonition):
 
         env = self.state.document.settings.env
         targetid = 'index-%s' % env.new_serialno('index')
+        # Stash the target to be retrieved later in latex_visit_todo_node.
+        todo['targetref'] = '%s:%s' % (env.docname, targetid)
         targetnode = nodes.target('', '', ids=[targetid])
         return [targetnode, todo]
 
@@ -173,8 +175,12 @@ def process_todo_nodes(app, doctree, fromdocname):
             para += newnode
             para += nodes.Text(desc2, desc2)
 
-            # (Recursively) resolve references in the todo content
             todo_entry = todo_info['todo']
+            # Remove targetref from the (copied) node to avoid emitting a
+            # duplicate label of the original entry when we walk this node.
+            del todo_entry['targetref']
+
+            # (Recursively) resolve references in the todo content
             env.resolve_references(todo_entry, todo_info['docname'],
                                    app.builder)
 
@@ -216,7 +222,13 @@ def depart_todo_node(self, node):
 def latex_visit_todo_node(self, node):
     # type: (nodes.NodeVisitor, todo_node) -> None
     title = node.pop(0).astext().translate(tex_escape_map)
-    self.body.append(u'\n\\begin{sphinxadmonition}{note}{%s:}' % title)
+    self.body.append(u'\n\\begin{sphinxadmonition}{note}{')
+    # If this is the original todo node, emit a label that will be referenced by
+    # a hyperref in the todolist.
+    target = node.get('targetref')
+    if target is not None:
+        self.body.append(u'\\label{%s}' % target)
+    self.body.append('%s:}' % title)
 
 
 def latex_depart_todo_node(self, node):

sphinx/locale/__init__.py

@@ -171,7 +171,7 @@ class _TranslationProxy(UserString, object):
         # type: () -> str
         try:
             return 'i' + repr(text_type(self.data))
-        except:
+        except Exception:
             return '<%s broken>' % self.__class__.__name__
 
 

sphinx/templates/quickstart/Makefile.new_t

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = python -msphinx
+SPHINXBUILD   = sphinx-build
 SPHINXPROJ    = {{ project_fn }}
 SOURCEDIR     = {{ rsrcdir }}
 BUILDDIR      = {{ rbuilddir }}

sphinx/templates/quickstart/Makefile_t

@@ -2,9 +2,9 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS   ?=
-SPHINXBUILD   = python -msphinx
+SPHINXBUILD  ?= sphinx-build
-PAPER         =
+PAPER        ?=
 BUILDDIR      = {{ rbuilddir }}
 
 # Internal variables.

sphinx/templates/quickstart/make.bat.new_t

@@ -5,7 +5,7 @@ pushd %~dp0
 REM Command file for Sphinx documentation
 
 if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=python -msphinx
+	set SPHINXBUILD=sphinx-build
 )
 set SOURCEDIR={{ rsrcdir }}
 set BUILDDIR={{ rbuilddir }}
@@ -16,10 +16,10 @@ if "%1" == "" goto help
 %SPHINXBUILD% >NUL 2>NUL
 if errorlevel 9009 (
 	echo.
-	echo.The Sphinx module was not found. Make sure you have Sphinx installed,
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.then set the SPHINXBUILD environment variable to point to the full
+	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.path of the 'sphinx-build' executable. Alternatively you may add the
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.Sphinx directory to PATH.
+	echo.may add the Sphinx directory to PATH.
 	echo.
 	echo.If you don't have Sphinx installed, grab it from
 	echo.http://sphinx-doc.org/

sphinx/templates/quickstart/make.bat_t

@@ -5,7 +5,7 @@ REM Command file for Sphinx documentation
 pushd %~dp0
 
 if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=python -msphinx
+	set SPHINXBUILD=sphinx-build
 )
 set BUILDDIR={{ rbuilddir }}
 set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% {{ rsrcdir }}
@@ -52,20 +52,29 @@ if "%1" == "clean" (
 )
 
 
-REM Check if sphinx-build is available
+REM Check if sphinx-build is available and fallback to Python version if any
 %SPHINXBUILD% 1>NUL 2>NUL
-if errorlevel 1 (
+if errorlevel 9009 goto sphinx_python
+goto sphinx_ok
+
+:sphinx_python
+
+set SPHINXBUILD=python -m sphinx.__init__
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
 	echo.
-	echo.The Sphinx module was not found. Make sure you have Sphinx installed,
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.then set the SPHINXBUILD environment variable to point to the full
+	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.path of the 'sphinx-build' executable. Alternatively you may add the
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.Sphinx directory to PATH.
+	echo.may add the Sphinx directory to PATH.
 	echo.
 	echo.If you don't have Sphinx installed, grab it from
 	echo.http://sphinx-doc.org/
 	exit /b 1
 )
 
+:sphinx_ok
+
 
 if "%1" == "html" (
 	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html

sphinx/testing/util.py

@@ -132,7 +132,7 @@ class SphinxTestApp(application.Sphinx):
             application.Sphinx.__init__(self, srcdir, confdir, outdir, doctreedir,
                                         buildername, confoverrides, status, warning,
                                         freshenv, warningiserror, tags)
-        except:
+        except Exception:
             self.cleanup()
             raise
 

sphinx/theming.py

@@ -157,7 +157,7 @@ def is_archived_theme(filename):
     try:
         with ZipFile(filename) as f:  # type: ignore
             return THEMECONF in f.namelist()
-    except:
+    except Exception:
         return False
 
 

sphinx/util/docutils.py

@@ -26,8 +26,7 @@ from sphinx.locale import __
 from sphinx.util import logging
 
 logger = logging.getLogger(__name__)
-report_re = re.compile('^(.+?:(?:\\d+)?): \\((DEBUG|INFO|WARNING|ERROR|SEVERE)/(\\d+)?\\) '
+report_re = re.compile('^(.+?:(?:\\d+)?): \\((DEBUG|INFO|WARNING|ERROR|SEVERE)/(\\d+)?\\) ')
-                       '(.+?)\n?$')
 
 if False:
     # For type annotation
@@ -162,7 +161,8 @@ class WarningStream(object):
         if not matched:
             logger.warning(text.rstrip("\r\n"))
         else:
-            location, type, level, message = matched.groups()
+            location, type, level = matched.groups()
+            message = report_re.sub('', text).rstrip()  # type: ignore
             logger.log(type, message, location=location)
 
 

sphinx/util/images.py

@@ -64,7 +64,7 @@ def get_image_size(filename):
                 pass
 
         return size
-    except:
+    except Exception:
         return None
 
 

sphinx/util/inspect.py

@@ -273,7 +273,7 @@ class Signature(object):
 
         try:
             self.annotations = typing.get_type_hints(subject)  # type: ignore
-        except:
+        except Exception:
             self.annotations = {}
 
         if bound_method:

sphinx/util/smartypants.py

@@ -139,7 +139,7 @@ def educateQuotes(text, language='en'):
     smart = smartquotes.smartchars(language)
     try:
         apostrophe = smart.apostrophe
-    except:
+    except Exception:
         apostrophe = u'’'
 
     # oldtext = text

tests/roots/test-domain-py/module_option.rst

@@ -0,0 +1,25 @@
+module_option
+=============
+
+.. py:class:: B
+   :module: test.extra
+
+   This is also a test.
+
+
+   .. py:method:: B.baz()
+      :module: test.extra
+
+      Does something similar to :meth:`foo`.
+
+
+   .. py:method:: B.foo()
+      :module: test.extra
+
+      Does something.
+
+
+   .. py:method:: B.test()
+      :module: test.extra
+
+      Does something completely unrelated to :meth:`foo`

tests/roots/test-ext-todo/conf.py

@@ -2,3 +2,8 @@
 
 extensions = ['sphinx.ext.todo']
 master_doc = 'index'
+
+latex_documents = [
+    (master_doc, 'TodoTests.tex', 'Todo Tests Documentation',
+     'Robin Banks', 'manual'),
+]

tests/test_build_latex.py

@@ -1035,6 +1035,6 @@ def test_latex_image_in_parsed_literal(app, status, warning):
     app.builder.build_all()
 
     result = (app.outdir / 'Python.tex').text(encoding='utf8')
-    assert ('{\\sphinxunactivateextrasandspace \\raisebox{-0.5\height}'
+    assert ('{\\sphinxunactivateextrasandspace \\raisebox{-0.5\\height}'
             '{\\scalebox{2.000000}{\\sphinxincludegraphics[height=1cm]{{pic}.png}}}'
             '}AFTER') in result

tests/test_domain_py.py

@@ -116,6 +116,15 @@ def test_domain_py_xrefs(app, status, warning):
     assert_refnode(refnodes[11], False, False, 'list', 'class')
     assert len(refnodes) == 12
 
+    doctree = app.env.get_doctree('module_option')
+    refnodes = list(doctree.traverse(addnodes.pending_xref))
+    print(refnodes)
+    print(refnodes[0])
+    print(refnodes[1])
+    assert_refnode(refnodes[0], 'test.extra', 'B', 'foo', 'meth')
+    assert_refnode(refnodes[1], 'test.extra', 'B', 'foo', 'meth')
+    assert len(refnodes) == 2
+
 
 @pytest.mark.sphinx('dummy', testroot='domain-py')
 def test_domain_py_objects(app, status, warning):

tests/test_ext_apidoc.py

@@ -152,10 +152,10 @@ def test_trailing_underscore(make_app, apidoc):
 @pytest.mark.apidoc(
     coderoot='test-root',
     options=[
-        '--doc-project', u'プロジェクト名'.encode('utf-8'),
+        '--doc-project', u'プロジェクト名',
-        '--doc-author', u'著者名'.encode('utf-8'),
+        '--doc-author', u'著者名',
-        '--doc-version', u'バージョン'.encode('utf-8'),
+        '--doc-version', u'バージョン',
-        '--doc-release', u'リリース'.encode('utf-8'),
+        '--doc-release', u'リリース',
     ],
 )
 def test_multibyte_parameters(make_app, apidoc):

tests/test_ext_todo.py

@@ -84,3 +84,31 @@ def test_todo_not_included(app, status, warning):
     # check handled event
     assert len(todos) == 2
     assert set(todo[1].astext() for todo in todos) == set(['todo in foo', 'todo in bar'])
+
+@pytest.mark.sphinx('latex', testroot='ext-todo', freshenv=True,
+                    confoverrides={'todo_include_todos': True, 'todo_emit_warnings': True})
+def test_todo_valid_link(app, status, warning):
+    """
+    Test that the inserted "original entry" links for todo items have a target
+    that exists in the LaTeX output. The target was previously incorrectly
+    omitted (GitHub issue #1020).
+    """
+
+    # Ensure the LaTeX output is built.
+    app.builder.build_all()
+
+    content = (app.outdir / 'TodoTests.tex').text()
+
+    # Look for the link to foo. We could equally well look for the link to bar.
+    link = r'\{\\hyperref\[\\detokenize\{(.*?foo.*?)}]\{\\sphinxcrossref{' \
+        r'\\sphinxstyleemphasis{original entry}}}}'
+    m = re.findall(link, content)
+    assert len(m) == 1
+    target = m[0]
+
+    # Look for the targets of this link.
+    labels = [m for m in re.findall(r'\\label\{([^}]*)}', content)
+        if m == target]
+
+    # If everything is correct we should have exactly one target.
+    assert len(labels) == 1

tests/test_intl.py

@@ -197,28 +197,28 @@ def test_text_inconsistency_warnings(app, warning):
     expected_warning_expr = (
         warning_fmt % {
             u'reftype': u'footnote references',
-            u'original': u"\[u?'\[#\]_'\]",
+            u'original': u"\\[u?'\\[#\\]_'\\]",
-            u'translated': u"\[\]"
+            u'translated': u"\\[\\]"
         } +
         warning_fmt % {
             u'reftype': u'footnote references',
-            u'original': u"\[u?'\[100\]_'\]",
+            u'original': u"\\[u?'\\[100\\]_'\\]",
-            u'translated': u"\[\]"
+            u'translated': u"\\[\\]"
         } +
         warning_fmt % {
             u'reftype': u'references',
-            u'original': u"\[u?'reference_'\]",
+            u'original': u"\\[u?'reference_'\\]",
-            u'translated': u"\[u?'reference_', u?'reference_'\]"
+            u'translated': u"\\[u?'reference_', u?'reference_'\\]"
         } +
         warning_fmt % {
             u'reftype': u'references',
-            u'original': u"\[\]",
+            u'original': u"\\[\\]",
-            u'translated': u"\[u?'`I18N WITH REFS INCONSISTENCY`_'\]"
+            u'translated': u"\\[u?'`I18N WITH REFS INCONSISTENCY`_'\\]"
         })
     assert_re_search(expected_warning_expr, warnings)
 
     expected_citation_warning_expr = (
-        u'.*/refs_inconsistency.txt:\\d+: WARNING: Citation \[ref2\] is not referenced.\n' +
+        u'.*/refs_inconsistency.txt:\\d+: WARNING: Citation \\[ref2\\] is not referenced.\n' +
         u'.*/refs_inconsistency.txt:\\d+: WARNING: citation not found: ref3')
     assert_re_search(expected_citation_warning_expr, warnings)
 
@@ -300,8 +300,8 @@ def test_text_glossary_term_inconsistencies(app, warning):
     expected_warning_expr = (
         u'.*/glossary_terms_inconsistency.txt:\\d+: '
         u'WARNING: inconsistent term references in translated message.'
-        u" original: \[u?':term:`Some term`', u?':term:`Some other term`'\],"
+        u" original: \\[u?':term:`Some term`', u?':term:`Some other term`'\\],"
-        u" translated: \[u?':term:`SOME NEW TERM`'\]\n")
+        u" translated: \\[u?':term:`SOME NEW TERM`'\\]\n")
     assert_re_search(expected_warning_expr, warnings)
 
 

tests/test_writer_latex.py

@@ -27,7 +27,7 @@ def test_rstdim_to_latexdim():
     assert rstdim_to_latexdim('30%') == '0.300\\linewidth'
     assert rstdim_to_latexdim('160') == '160\\sphinxpxdimen'
 
-    # flaot values
+    # float values
     assert rstdim_to_latexdim('160.0em') == '160.0em'
     assert rstdim_to_latexdim('.5em') == '.5em'
 

utils/bump_version.py

@@ -81,7 +81,7 @@ def processing(message):
         yield
     except Skip as exc:
         print('skip: %s' % exc)
-    except:
+    except Exception:
         print('error')
         raise
     else:

utils/reindent.py

@@ -1,320 +0,0 @@
-#! /usr/bin/env python
-
-# Released to the public domain, by Tim Peters, 03 October 2000.
-
-"""reindent [-d][-r][-v] [ path ... ]
-
--d (--dryrun)   Dry run.   Analyze, but don't make any changes to, files.
--r (--recurse)  Recurse.   Search for all .py files in subdirectories too.
--n (--nobackup) No backup. Does not make a ".bak" file before reindenting.
--v (--verbose)  Verbose.   Print informative msgs; else no output.
--h (--help)     Help.      Print this usage information and exit.
-
-Change Python (.py) files to use 4-space indents and no hard tab characters.
-Also trim excess spaces and tabs from ends of lines, and remove empty lines
-at the end of files.  Also ensure the last line ends with a newline.
-
-If no paths are given on the command line, reindent operates as a filter,
-reading a single source file from standard input and writing the transformed
-source to standard output.  In this case, the -d, -r and -v flags are
-ignored.
-
-You can pass one or more file and/or directory paths.  When a directory
-path, all .py files within the directory will be examined, and, if the -r
-option is given, likewise recursively for subdirectories.
-
-If output is not to standard output, reindent overwrites files in place,
-renaming the originals with a .bak extension.  If it finds nothing to
-change, the file is left alone.  If reindent does change a file, the changed
-file is a fixed-point for future runs (i.e., running reindent on the
-resulting .py file won't change it again).
-
-The hard part of reindenting is figuring out what to do with comment
-lines.  So long as the input files get a clean bill of health from
-tabnanny.py, reindent should do a good job.
-
-The backup file is a copy of the one that is being reindented. The ".bak"
-file is generated with shutil.copy(), but some corner cases regarding
-user/group and permissions could leave the backup file more readable that
-you'd prefer. You can always use the --nobackup option to prevent this.
-"""
-from __future__ import print_function
-
-import os
-import sys
-import shutil
-import tokenize
-from six.ranges import range
-
-__version__ = "1"
-
-if sys.version_info >= (3, 0):
-    def tokens(readline, tokeneater):
-        for token in tokenize.tokenize(readline):
-            yield tokeneater(*token)
-else:
-    tokens = tokenize.tokenize
-
-verbose = 0
-recurse = 0
-dryrun = 0
-makebackup = True
-
-
-def usage(msg=None):
-    if msg is not None:
-        print(msg, file=sys.stderr)
-    print(__doc__, file=sys.stderr)
-
-
-def errprint(*args):
-    sep = ""
-    for arg in args:
-        sys.stderr.write(sep + str(arg))
-        sep = " "
-    sys.stderr.write("\n")
-
-
-def main():
-    import getopt
-    global verbose, recurse, dryrun, makebackup
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "drnvh",
-                                   ["dryrun", "recurse", "nobackup", "verbose", "help"])
-    except getopt.error as msg:
-        usage(msg)
-        return
-    for o, a in opts:
-        if o in ('-d', '--dryrun'):
-            dryrun += 1
-        elif o in ('-r', '--recurse'):
-            recurse += 1
-        elif o in ('-n', '--nobackup'):
-            makebackup = False
-        elif o in ('-v', '--verbose'):
-            verbose += 1
-        elif o in ('-h', '--help'):
-            usage()
-            return
-    if not args:
-        r = Reindenter(sys.stdin)
-        r.run()
-        r.write(sys.stdout)
-        return
-    for arg in args:
-        check(arg)
-
-
-def check(file):
-    if os.path.isdir(file) and not os.path.islink(file):
-        if verbose:
-            print("listing directory", file)
-        names = os.listdir(file)
-        for name in names:
-            fullname = os.path.join(file, name)
-            if ((recurse and os.path.isdir(fullname) and
-                 not os.path.islink(fullname) and
-                 not os.path.split(fullname)[1].startswith(".")) or
-               name.lower().endswith(".py")):
-                check(fullname)
-        return
-
-    if verbose:
-        print("checking", file, "...", end=' ')
-    try:
-        f = open(file)
-    except IOError as msg:
-        errprint("%s: I/O Error: %s" % (file, str(msg)))
-        return
-
-    with f:
-        r = Reindenter(f)
-    if r.run():
-        if verbose:
-            print("changed.")
-            if dryrun:
-                print("But this is a dry run, so leaving it alone.")
-        if not dryrun:
-            bak = file + ".bak"
-            if makebackup:
-                shutil.copyfile(file, bak)
-                if verbose:
-                    print("backed up", file, "to", bak)
-            with open(file, "w") as f:
-                r.write(f)
-            if verbose:
-                print("wrote new", file)
-        return True
-    else:
-        if verbose:
-            print("unchanged.")
-        return False
-
-
-def _rstrip(line, JUNK='\n \t'):
-    """Return line stripped of trailing spaces, tabs, newlines.
-
-    Note that line.rstrip() instead also strips sundry control characters,
-    but at least one known Emacs user expects to keep junk like that, not
-    mentioning Barry by name or anything <wink>.
-    """
-
-    i = len(line)
-    while i > 0 and line[i - 1] in JUNK:
-        i -= 1
-    return line[:i]
-
-
-class Reindenter:
-    def __init__(self, f):
-        self.find_stmt = 1  # next token begins a fresh stmt?
-        self.level = 0      # current indent level
-
-        # Raw file lines.
-        self.raw = f.readlines()
-
-        # File lines, rstripped & tab-expanded.  Dummy at start is so
-        # that we can use tokenize's 1-based line numbering easily.
-        # Note that a line is all-blank iff it's "\n".
-        self.lines = [_rstrip(line).expandtabs() + "\n"
-                      for line in self.raw]
-        self.lines.insert(0, None)
-        self.index = 1  # index into self.lines of next line
-
-        # List of (lineno, indentlevel) pairs, one for each stmt and
-        # comment line.  indentlevel is -1 for comment lines, as a
-        # signal that tokenize doesn't know what to do about them;
-        # indeed, they're our headache!
-        self.stats = []
-
-    def run(self):
-        tokens(self.getline, self.tokeneater)
-        # Remove trailing empty lines.
-        lines = self.lines
-        while lines and lines[-1] == "\n":
-            lines.pop()
-        # Sentinel.
-        stats = self.stats
-        stats.append((len(lines), 0))
-        # Map count of leading spaces to # we want.
-        have2want = {}
-        # Program after transformation.
-        after = self.after = []
-        # Copy over initial empty lines -- there's nothing to do until
-        # we see a line with *something* on it.
-        i = stats[0][0]
-        after.extend(lines[1:i])
-        for i in range(len(stats) - 1):
-            thisstmt, thislevel = stats[i]
-            nextstmt = stats[i + 1][0]
-            have = getlspace(lines[thisstmt])
-            want = thislevel * 4
-            if want < 0:
-                # A comment line.
-                if have:
-                    # An indented comment line.  If we saw the same
-                    # indentation before, reuse what it most recently
-                    # mapped to.
-                    want = have2want.get(have, -1)
-                    if want < 0:
-                        # Then it probably belongs to the next real stmt.
-                        for j in range(i + 1, len(stats) - 1):
-                            jline, jlevel = stats[j]
-                            if jlevel >= 0:
-                                if have == getlspace(lines[jline]):
-                                    want = jlevel * 4
-                                break
-                    if want < 0:        # Maybe it's a hanging
-                                        # comment like this one,
-                        # in which case we should shift it like its base
-                        # line got shifted.
-                        for j in range(i - 1, -1, -1):
-                            jline, jlevel = stats[j]
-                            if jlevel >= 0:
-                                want = (have + getlspace(after[jline - 1]) -
-                                        getlspace(lines[jline]))
-                                break
-                    if want < 0:
-                        # Still no luck -- leave it alone.
-                        want = have
-                else:
-                    want = 0
-            assert want >= 0
-            have2want[have] = want
-            diff = want - have
-            if diff == 0 or have == 0:
-                after.extend(lines[thisstmt:nextstmt])
-            else:
-                for line in lines[thisstmt:nextstmt]:
-                    if diff > 0:
-                        if line == "\n":
-                            after.append(line)
-                        else:
-                            after.append(" " * diff + line)
-                    else:
-                        remove = min(getlspace(line), -diff)
-                        after.append(line[remove:])
-        return self.raw != self.after
-
-    def write(self, f):
-        f.writelines(self.after)
-
-    # Line-getter for tokenize.
-    def getline(self):
-        if self.index >= len(self.lines):
-            line = ""
-        else:
-            line = self.lines[self.index]
-            self.index += 1
-        return line
-
-    # Line-eater for tokenize.
-    def tokeneater(self, type, token, position, end, line,
-                   INDENT=tokenize.INDENT,
-                   DEDENT=tokenize.DEDENT,
-                   NEWLINE=tokenize.NEWLINE,
-                   COMMENT=tokenize.COMMENT,
-                   NL=tokenize.NL):
-
-        if type == NEWLINE:
-            # A program statement, or ENDMARKER, will eventually follow,
-            # after some (possibly empty) run of tokens of the form
-            #     (NL | COMMENT)* (INDENT | DEDENT+)?
-            self.find_stmt = 1
-
-        elif type == INDENT:
-            self.find_stmt = 1
-            self.level += 1
-
-        elif type == DEDENT:
-            self.find_stmt = 1
-            self.level -= 1
-
-        elif type == COMMENT:
-            if self.find_stmt:
-                self.stats.append((position[0], -1))
-                # but we're still looking for a new stmt, so leave
-                # find_stmt alone
-
-        elif type == NL:
-            pass
-
-        elif self.find_stmt:
-            # This is the first "real token" following a NEWLINE, so it
-            # must be the first token of the next program statement, or an
-            # ENDMARKER.
-            self.find_stmt = 0
-            if line:   # not endmarker
-                self.stats.append((position[0], self.level))
-
-
-# Count number of leading blanks.
-def getlspace(line):
-    i, n = 0, len(line)
-    while i < n and line[i] == " ":
-        i += 1
-    return i
-
-
-if __name__ == '__main__':
-    main()