Merge branch '1.7' into relax-numref-on-sections

.appveyor.yml

@@ -6,20 +6,16 @@ environment:
 
   matrix:
     - PYTHON: 27
-      DOCUTILS: 0.13.1
-      TEST_IGNORE: --ignore py35
-    - PYTHON: 27
-      DOCUTILS: 0.14
       TEST_IGNORE: --ignore py35
     - PYTHON: 36
-      DOCUTILS: 0.14
     - PYTHON: 36-x64
-      DOCUTILS: 0.14
 
 install:
   - C:\Python%PYTHON%\python.exe -m pip install -U pip setuptools
-  - C:\Python%PYTHON%\python.exe -m pip install docutils==%DOCUTILS%
+  - C:\Python%PYTHON%\python.exe -m pip install .[test,websupport]
-  - C:\Python%PYTHON%\python.exe -m pip install -r test-reqs.txt
+
+cache:
+  - '%LOCALAPPDATA%\pip\Cache'
 
 # No automatic build, just run python tests
 build: off
@@ -39,10 +35,24 @@ test_script:
       if (-not $test_ignore) { $test_ignore = '' }
       $tests = $env:TEST
       if (-not $tests) { $tests = '' }
-      & "C:\Python$($env:PYTHON)\python.exe" run.py $test_ignore.Split(' ') --junitxml .junit.xml $tests.Split(' ')
+      & "C:\Python$($env:PYTHON)\python.exe" -m pytest $test_ignore.Split(' ') --junitxml .junit.xml $tests.Split(' ')
       Pop-Location
       if ($LastExitCode -eq 1) { Write-Host "Test Failures Occurred, leaving for test result parsing" }
       elseif ($LastExitCode -ne 0) { Write-Host "Other Error Occurred, aborting"; exit $LastExitCode }
 
 after_test:
   - ps: (New-Object System.Net.WebClient).UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path (Join-Path tests .junit.xml)))
+
+after_build:
+  # Remove old or huge cache files to hopefully not exceed the 1GB cache limit.
+  #
+  # If the cache limit is reached, the cache will not be updated (of not even
+  # created in the first run). So this is a trade of between keeping the cache
+  # current and having a cache at all.
+  # NB: This is done only `on_success` since the cache in uploaded only on
+  # success anyway.
+  - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -type f -mtime +360 -delete
+  - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -type f -size +10M -delete
+  - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -empty -delete
+  # Show size of cache
+  - C:\cygwin\bin\du -hs "%LOCALAPPDATA%\pip\Cache"

.circleci/config.yml

@@ -6,4 +6,6 @@ jobs:
     working_directory: /sphinx
     steps:
       - checkout
+      - run: /python3.4/bin/pip install -U pip setuptools
+      - run: /python3.4/bin/pip install -U .[test,websupport]
       - run: make test PYTHON=/python3.4/bin/python

.codecov.yml

@@ -0,0 +1,10 @@
+coverage:
+  status:
+    project:
+      default:
+        # allowed to drop X% and still result in a "success" commit status
+        threshold: 0.05
+    patch:
+      default:
+        # allowed to drop X% and still result in a "success" commit status
+        threshold: 0.05

.gitignore

@@ -7,11 +7,15 @@
 .cache/
 .idea
 .mypy_cache/
+.pytest_cache/
 .ropeproject/
 TAGS
 .tags
-.tox
+.tox/
-.venv
+.tx/
+.venv/
+.coverage
+htmlcov
 .DS_Store
 sphinx/pycode/Grammar*pickle
 distribute-*

.travis.yml

@@ -1,48 +1,43 @@
 language: python
 sudo: false
 dist: trusty
-cache:
+cache: pip
-  directories:
+
-    - $HOME/.cache/pip
-python:
-  - "pypy-5.4.1"
-  - "3.6"
-  - "3.5"
-  - "3.4"
-  - "2.7"
-  - "nightly"
 env:
   global:
-    - TEST='-v --durations 25'
     - PYTHONFAULTHANDLER=x
-    - PYTHONWARNINGS=all
     - SKIP_LATEX_BUILD=1
-  matrix:
+
-    - DOCUTILS=0.13.1
-    - DOCUTILS=0.14
 matrix:
-  exclude:
+  include:
-    - python: "3.4"
+    - python: 'pypy'
-      env: DOCUTILS=0.13.1
+      env: TOXENV=pypy
-    - python: "3.5"
+    - python: '2.7'
-      env: DOCUTILS=0.13.1
+      env:
-    - python: "3.6"
+        - TOXENV=du13
-      env: DOCUTILS=0.13.1
+        - PYTEST_ADDOPTS="--cov ./ --cov-append --cov-config setup.cfg"
-    - python: nightly
+    - python: '3.4'
-      env: DOCUTILS=0.13.1
+      env: TOXENV=py34
-    - python: "pypy-5.4.1"
+    - python: '3.5'
-      env: DOCUTILS=0.13.1
+      env: TOXENV=py35
-addons:
+    - python: '3.6'
-  apt:
+      env:
-    packages:
+        - TOXENV=py36
-      - graphviz
+        - PYTEST_ADDOPTS="--cov ./ --cov-append --cov-config setup.cfg"
-      - imagemagick
+    - python: 'nightly'
+      env: TOXENV=py37
+    - python: '3.6'
+      env: TOXENV=docs
+    - python: '3.6'
+      env: TOXENV=mypy
+    - python: '2.7'
+      env: TOXENV=flake8
+
 install:
-  - pip install -U pip setuptools
+  - pip install -U tox codecov
-  - pip install docutils==$DOCUTILS
+
-  - pip install -r test-reqs.txt
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then python3.6 -m pip install mypy typed-ast; fi
 script:
-  - flake8
+  - tox -- -v
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then make style-check type-check test-async; fi
+
-  - if [[ $TRAVIS_PYTHON_VERSION != '3.6' ]]; then make test; fi
+after_success:
+  - if [[ -e .coverage ]]; then codecov -e $TOXENV; fi

AUTHORS

@@ -18,6 +18,7 @@ Other co-maintainers:
 Other contributors, listed alphabetically, are:
 
 * Alastair Houghton -- Apple Help builder
+* Alexander Todorov -- inheritance_diagram tests and improvements
 * Andi Albrecht -- agogo theme
 * Jakob Lykke Andersen -- Rewritten C++ domain
 * Henrique Bastos -- SVG support for graphviz extension
@@ -29,10 +30,13 @@ Other contributors, listed alphabetically, are:
 * Kevin Dunn -- MathJax extension
 * Josip Dzolonga -- coverage builder
 * Buck Evan -- dummy builder
+* Matthew Fernandez -- todo extension fix
 * Hernan Grecco -- search improvements
 * Horst Gutmann -- internationalization support
 * Martin Hans -- autodoc improvements
+* Zac Hatfield-Dodds -- doctest reporting improvements
 * Doug Hellmann -- graphviz improvements
+* Tim Hoffmann -- theme improvements
 * Timotheus Kampik - JS theme & search enhancements
 * Dave Kuhlman -- original LaTeX writer
 * Blaise Laflamme -- pyramid theme
@@ -66,9 +70,11 @@ Other contributors, listed alphabetically, are:
 * Barry Warsaw -- setup command improvements
 * Sebastian Wiesner -- image handling, distutils support
 * Michael Wilson -- Intersphinx HTTP basic auth support
+* Matthew Woodcraft -- text output improvements
 * Joel Wurtz -- cellspanning support in LaTeX
 * Hong Xu -- svg support in imgmath extension and various bug fixes
 * Stephen Finucane -- setup command improvements and documentation
+* Daniel Pizetta -- inheritance diagram improvements
 
 Many thanks for all contributions!
 

CHANGES

@@ -1,9 +1,85 @@
-Release 1.7 (in development)
+Release 1.7.2 (in development)
-============================
+==============================
+
+Dependencies
+------------
 
 Incompatible changes
 --------------------
 
+Deprecated
+----------
+
+Features added
+--------------
+
+Bugs fixed
+----------
+
+* #4669: sphinx.build_main and sphinx.make_main throw NameError
+* #4685: autosummary emits meaningless warnings
+* autodoc: crashed when invalid options given
+* pydomain: always strip parenthesis if empty (refs: #1042)
+* #4689: autosummary: unexpectedly strips docstrings containing "i.e."
+* #4701: viewcode: Misplaced ``<div>`` in viewcode html output
+
+Testing
+--------
+
+Release 1.7.1 (released Feb 23, 2018)
+=====================================
+
+Deprecated
+----------
+
+* #4623: ``sphinx.build_main()`` is deprecated.
+* autosummary: The interface of ``sphinx.ext.autosummary.get_documenter()`` has
+  been changed (Since 1.7.0)
+* #4664: ``sphinx.ext.intersphinx.debug()`` is deprecated.
+
+For more details, see `deprecation APIs list
+<http://www.sphinx-doc.org/en/master/extdev/index.html#deprecated-apis>`_
+
+Bugs fixed
+----------
+
+* #4608: epub: Invalid meta tag is generated
+* #4260: autodoc: keyword only argument separator is not disappeared if it is
+  appeared at top of the argument list
+* #4622: epub: :confval:`epub_scheme` does not effect to content.opf
+* #4627: graphviz: Fit graphviz images to page
+* #4617: quickstart: PROJECT_DIR argument is required
+* #4623: sphinx.build_main no longer exists in 1.7.0
+* #4615: The argument of ``sphinx.build`` has been changed in 1.7.0
+* autosummary: The interface of ``sphinx.ext.autosummary.get_documenter()`` has
+  been changed
+* #4630: Have order on msgids in sphinx.pot deterministic
+* #4563: autosummary: Incorrect end of line punctuation detection
+* #4577: Enumerated sublists with explicit start with wrong number
+* #4641: A external link in TOC cannot contain "?" with ``:glob:`` option
+* C++, add missing parsing of explicit casts and typeid in expression parsing.
+* C++, add missing parsing of ``this`` in expression parsing.
+* #4655: Fix incomplete localization strings in Polish
+* #4653: Fix error reporting for parameterless ImportErrors
+* #4664: Reading objects.inv fails again
+* #4662: ``any`` refs with ``term`` targets crash when an ambiguity is
+  encountered
+
+Release 1.7.0 (released Feb 12, 2018)
+=====================================
+
+Dependencies
+------------
+
+1.7.0b1
+
+* Add ``packaging`` package
+
+Incompatible changes
+--------------------
+
+1.7.0b1
+
 * #3668: The arguments has changed of main functions for each command
 * #3893: Unknown html_theme_options throw warnings instead of errors
 * #3927: Python parameter/variable types should match classes, not all objects
@@ -11,17 +87,50 @@ Incompatible changes
   package when ``--implicit-namespaces`` option given, not subdirectories of
   given directory.
 * #3929: apidoc: Move sphinx.apidoc to sphinx.ext.apidoc
+* #4226: apidoc: Generate new style makefile (make-mode)
+* #4274: sphinx-build returns 2 as an exit code on argument error
+* #4389: output directory will be created after loading extensions
+* autodoc does not generate warnings messages to the generated document even if
+  :confval:`keep_warnings` is True.  They are only emitted to stderr.
+* shebang line is removed from generated conf.py
+* #2557: autodoc: :confval:`autodoc_mock_imports` only mocks specified modules
+  with their descendants.  It does not mock their ancestors.  If you want to
+  mock them, please specify the name of ancestors explicitly.
+* #3620: html theme: move DOCUMENTATION_OPTIONS to independent JavaScript file
+  (refs: #4295)
+* #4246: Limit width of text body for all themes. Conifigurable via theme
+  options ``body_min_width`` and ``body_max_width``.
+
+1.7.0b2
+
+* #4467: html theme: Rename ``csss`` block to ``css``
 
 Deprecated
 ----------
 
+1.7.0b1
+
+* using a string value for :confval:`html_sidebars` is deprecated and only list
+  values will be accepted at 2.0.
+* ``format_annotation()`` and ``formatargspec()`` is deprecated.  Please use
+  ``sphinx.util.inspect.Signature`` instead.
+* ``sphinx.ext.autodoc.AutodocReporter`` is replaced by ``sphinx.util.docutils.
+  switch_source_input()`` and now deprecated.  It will be removed in Sphinx-2.0.
+* ``sphinx.ext.autodoc.add_documenter()`` and ``AutoDirective._register`` is now
+  deprecated.  Please use ``app.add_autodocumenter()`` instead.
+* ``AutoDirective._special_attrgetters`` is now deprecated.  Please use
+  ``app.add_autodoc_attrgetter()`` instead.
+
 Features added
 --------------
 
+1.7.0b1
+
 * C++, handle ``decltype(auto)``.
 * #2406: C++, add proper parsing of expressions, including linking of identifiers.
 * C++, add a ``cpp:expr`` role for inserting inline C++ expressions or types.
-* #4094: C++, allow empty template argument lists.
+* C++, support explicit member instantiations with shorthand ``template`` prefix.
+* C++, make function parameters linkable, like template params.
 * #3638: Allow to change a label of reference to equation using
   ``math_eqref_format``
 
@@ -36,11 +145,43 @@ Features added
 * #3972: epub: Sort manifest entries by filename
 * #4052: viewcode: Sort before highlighting module code
 * #1448: qthelp: Add new config value; :confval:`qthelp_namespace`
+* #4140: html themes: Make body tag inheritable
+* #4168: improve zh search with jieba
+* HTML themes can set up default sidebars through ``theme.conf``
+* #3160: html: Use ``<kdb>`` to represent ``:kbd:`` role
+* #4212: autosummary: catch all exceptions when importing modules
+* #4166: Add :confval:`math_numfig` for equation numbering by section (refs:
+  #3991, #4080). Thanks to Oliver Jahn.
+* #4311: Let LaTeX obey :confval:`numfig_secnum_depth` for figures, tables, and
+  code-blocks
+* #947: autodoc now supports ignore-module-all to ignore a module's ``__all__``
+* #4332: Let LaTeX obey :confval:`math_numfig` for equation numbering
+* #4093: sphinx-build creates empty directories for unknown targets/builders
+* Add ``top-classes`` option for the ``sphinx.ext.inheritance_diagram``
+  extension to limit the scope of inheritance graphs.
+* #4183: doctest: ``:pyversion:`` option also follows PEP-440 specification
+* #4235: html: Add :confval:`manpages_url` to make manpage roles to hyperlinks
+* #3570: autodoc: Do not display 'typing.' module for type hints
+* #4354: sphinx-build now emits finish message.  Builders can modify it through
+  ``Builder.epilog`` attribute
+* #4245: html themes: Add ``language`` to javascript vars list
+* #4079: html: Add ``notranslate`` class to each code-blocks, literals and maths
+  to let Google Translate know they are not translatable
+* #4137: doctest: doctest block is always highlighted as python console (pycon)
+* #4137: doctest: testcode block is always highlighted as python
+* #3998: text: Assign section numbers by default.  You can control it using
+  :confval:`text_add_secnumbers` and :confval:`text_secnumber_suffix`
 
+1.7.0b2
+
+* #4271: sphinx-build supports an option called ``-j auto`` to adjust numbers of
+  processes automatically.
 
 Features removed
 ----------------
 
+1.7.0b1
+
 * Configuration variables
 
   - html_use_smartypants
@@ -66,35 +207,141 @@ Features removed
 * ``sphinx.util.nodes.process_only_nodes()``
 * LaTeX environment ``notice``, use ``sphinxadmonition`` instead
 * LaTeX ``\sphinxstylethead``, use ``\sphinxstyletheadfamily``
+* C++, support of function concepts. Thanks to mickk-on-cpp.
+* Not used and previously not documented LaTeX macros ``\shortversion``
+  and ``\setshortversion``
+
 
 Bugs fixed
 ----------
 
+1.7.0b1
+
 * #3882: Update the order of files for HTMLHelp and QTHelp
 * #3962: sphinx-apidoc does not recognize implicit namespace packages correctly
+* #4094: C++, allow empty template argument lists.
+* C++, also hyperlink types in the name of declarations with qualified names.
+* C++, do not add index entries for declarations inside concepts.
+* C++, support the template disambiguator for dependent names.
+* #4314: For PDF 'howto' documents, numbering of code-blocks differs from the
+  one of figures and tables
+* #4330: PDF 'howto' documents have an incoherent default LaTeX tocdepth counter
+  setting
+* #4198: autosummary emits multiple 'autodoc-process-docstring' event. Thanks
+  to Joel Nothman.
+* #4081: Warnings and errors colored the same when building
+* latex: Do not display 'Release' label if :confval:`release` is not set
+
+1.7.0b2
+
+* #4415: autodoc classifies inherited classmethods as regular methods
+* #4415: autodoc classifies inherited staticmethods as regular methods
+* #4472: DOCUMENTATION_OPTIONS is not defined
+* #4491: autodoc: prefer _MockImporter over other importers in sys.meta_path
+* #4490: autodoc: type annotation is broken with python 3.7.0a4+
+* utils package is no longer installed
+* #3952: apidoc: module header is too escaped
+* #4275: Formats accepted by sphinx.util.i18n.format_date are limited
+* #4493: recommonmark raises AttributeError if AutoStructify enabled
+* #4209: intersphinx: In link title, "v" should be optional if target has no
+  version
+* #4230: slowdown in writing pages with sphinx 1.6
+* #4522: epub: document is not rebuilt even if config changed
+
+1.7.0b3
+
+* #4019: inheritance_diagram AttributeError stoping make process
+* #4531: autosummary: methods are not treated as attributes
+* #4538: autodoc: ``sphinx.ext.autodoc.Options`` has been moved
+* #4539: autodoc emits warnings for partialmethods
+* #4223: doctest: failing tests reported in wrong file, at wrong line
+* i18n: message catalogs are not compiled if specific filenames are given for
+  ``sphinx-build`` as arguments (refs: #4560)
+* #4027: sphinx.ext.autosectionlabel now expects labels to be the same as they
+  are in the raw source; no smart quotes, nothig fancy.
+* #4581: apidoc: Excluded modules still included
+
 
 Testing
 --------
 
+1.7.0b1
+
 * Add support for docutils 0.14
+* Add tests for the ``sphinx.ext.inheritance_diagram`` extension.
 
-Release 1.6.5 (in development)
+Release 1.6.7 (released Feb 04, 2018)
-==============================
+=====================================
 
-Dependencies
+Bugs fixed
-------------
-
-Incompatible changes
---------------------
-
-Deprecated
 ----------
 
+* #1922: html search: Upper characters problem in French
+* #4412: Updated jQuery version from 3.1.0 to 3.2.1
+* #4438: math: math with labels with whitespace cause html error
+* #2437: make full reference for classes, aliased with "alias of"
+* #4434: pure numbers as link targets produce warning
+* #4477: Build fails after building specific files
+* #4449: apidoc: include "empty" packages that contain modules
+* #3917: citation labels are tranformed to ellipsis
+* #4501: graphviz: epub3 validation error caused if graph is not clickable
+* #4514: graphviz: workaround for wrong map ID which graphviz generates
+* #4525: autosectionlabel does not support parallel build
+* #3953: Do not raise warning when there is a working intersphinx inventory
+* #4487: math: ValueError is raised on parallel build. Thanks to jschueller.
+* #2372: autosummary: invalid signatures are shown for type annotated functions
+* #3942: html: table is not aligned to center even if ``:align: center``
+
+Release 1.6.6 (released Jan 08, 2018)
+=====================================
+
+Features added
+--------------
+
+* #4181: autodoc: Sort dictionary keys when possible
+* ``VerbatimHighlightColor`` is a new
+  :ref:`LaTeX 'sphinxsetup' <latexsphinxsetup>` key (refs: #4285)
+* Easier customizability of LaTeX macros involved in rendering of code-blocks
+* Show traceback if conf.py raises an exception (refs: #4369)
+* Add :confval:`smartquotes` to disable smart quotes through ``conf.py``
+  (refs: #3967)
+* Add :confval:`smartquotes_action` and :confval:`smartquotes_excludes`
+  (refs: #4142, #4357)
+
+Bugs fixed
+----------
+
+* #4334: sphinx-apidoc: Don't generate references to non-existing files in TOC
+* #4206: latex: reST label between paragraphs loses paragraph break
+* #4231: html: Apply fixFirefoxAnchorBug only under Firefox
+* #4221: napoleon depends on autodoc, but users need to load it manually
+* #2298: automodule fails to document a class attribute
+* #4099: C++: properly link class reference to class from inside constructor
+* #4267: PDF build broken by Unicode U+2116 NUMERO SIGN character
+* #4249: PDF output: Pygments error highlighting increases line spacing in
+  code blocks
+* #1238: Support ``:emphasize-lines:`` in PDF output
+* #4279: Sphinx crashes with pickling error when run with multiple processes and
+  remote image
+* #1421: Respect the quiet flag in sphinx-quickstart
+* #4281: Race conditions when creating output directory
+* #4315: For PDF 'howto' documents, ``latex_toplevel_sectioning='part'`` generates
+  ``\chapter`` commands
+* #4214: Two todolist directives break sphinx-1.6.5
+* Fix links to external option docs with intersphinx (refs: #3769)
+* #4091: Private members not documented without :undoc-members:
+
+Release 1.6.5 (released Oct 23, 2017)
+=====================================
+
 Features added
 --------------
 
 * #4107: Make searchtools.js compatible with pre-Sphinx1.5 templates
 * #4112: Don't override the smart_quotes setting if it was already set
+* #4125: Display reference texts of original and translated passages on
+  i18n warning message
+* #4147: Include the exception when logging PO/MO file read/write
 
 Bugs fixed
 ----------
@@ -105,9 +352,18 @@ Bugs fixed
 * #4096: C++, don't crash when using the wrong role type. Thanks to mitya57.
 * #4070, #4111: crashes when the warning message contains format strings (again)
 * #4108: Search word highlighting breaks SVG images
-
+* #3692: Unable to build HTML if writing .buildinfo failed
-Testing
+* #4152: HTML writer crashes if a field list is placed on top of the document
---------
+* #4063: Sphinx crashes when labeling directive ``.. todolist::``
+* #4134: [doc] :file:`docutils.conf` is not documented explicitly
+* #4169: Chinese language doesn't trigger Chinese search automatically
+* #1020: ext.todo todolist not linking to the page in pdflatex
+* #3965: New quickstart generates wrong SPHINXBUILD in Makefile
+* #3739: ``:module:`` option is ignored at content of pyobjects
+* #4149: Documentation: Help choosing :confval:`latex_engine`
+* #4090: [doc] :confval:`latex_additional_files` with extra LaTeX macros should
+  not use ``.tex`` extension
+* Failed to convert reST parser error to warning (refs: #4132)
 
 Release 1.6.4 (released Sep 26, 2017)
 =====================================

CONTRIBUTING.rst

@@ -33,10 +33,10 @@ Bug Reports and Feature Requests
 
 If you have encountered a problem with Sphinx or have an idea for a new
 feature, please submit it to the `issue tracker`_ on GitHub or discuss it
-on the sphinx-dev mailing list.
+on the `sphinx-dev`_ mailing list.
 
 For bug reports, please include the output produced during the build process
-and also the log file Sphinx creates after it encounters an un-handled
+and also the log file Sphinx creates after it encounters an unhandled
 exception.  The location of this file should be shown towards the end of the
 error message.
 
@@ -45,6 +45,7 @@ issue.  If possible, try to create a minimal project that produces the error
 and post that instead.
 
 .. _`issue tracker`: https://github.com/sphinx-doc/sphinx/issues
+.. _`sphinx-dev`: mailto:sphinx-dev@googlegroups.com
 
 
 Contributing to Sphinx
@@ -58,10 +59,10 @@ of the core developers before it is merged into the main repository.
 #. Check for open issues or open a fresh issue to start a discussion around a
    feature idea or a bug.
 #. If you feel uncomfortable or uncertain about an issue or your changes, feel
-   free to email sphinx-dev@googlegroups.com.
+   free to email the *sphinx-dev* mailing list.
 #. Fork `the repository`_ on GitHub to start making your changes to the
-   **master** branch for next major version, or **stable** branch for next
+   ``master`` branch for next major version, or ``X.Y`` branch for next
-   minor version.
+   minor version (see `Branch Model`_).
 #. Write a test which shows that the bug was fixed or that the feature works
    as expected.
 #. Send a pull request and bug the maintainer until it gets merged and
@@ -91,17 +92,22 @@ These are the basic steps needed to start developing on Sphinx.
 #. Checkout the appropriate branch.
 
    For changes that should be included in the next minor release (namely bug
-   fixes), use the ``stable`` branch. ::
+   fixes), use the ``X.Y`` branch. ::
 
-       git checkout stable
+       git checkout X.Y
 
    For new features or other substantial changes that should wait until the
-   next major release, use the ``master`` branch.
+   next major release, use the ``master`` branch  (see `Branch Model`_ for
+   detail).
 
-#. Optional: setup a virtual environment. ::
+#. Setup a virtual environment.
 
-       virtualenv ~/sphinxenv
+   This is not necessary for unit testing, thanks to ``tox``, but it is
-       . ~/sphinxenv/bin/activate
+   necessary if you wish to run ``sphinx-build`` locally or run unit tests
+   without the help of ``tox``. ::
+
+       virtualenv ~/.venv
+       . ~/.venv/bin/activate
        pip install -e .
 
 #. Create a new working branch.  Choose any name you like. ::
@@ -112,40 +118,53 @@ These are the basic steps needed to start developing on Sphinx.
 
    For tips on working with the code, see the `Coding Guide`_.
 
-#. Test, test, test.  Possible steps:
+#. Test, test, test.
 
-   * Run the unit tests::
+   Testing is best done through ``tox``, which provides a number of targets and
+   allows testing against multiple different Python environments:
 
-       pip install -r test-reqs.txt
+   * To list all possible targets::
-       make test
 
-   * Again, it's useful to turn on deprecation warnings on so they're shown in
+         tox -av
-     the test output::
 
-       PYTHONWARNINGS=all make test
+   * To run unit tests for a specific Python version, such as 3.6::
 
-   * Build the documentation and check the output for different builders::
+         tox -e py36
 
-       cd doc
+   * To run unit tests for a specific Python version and turn on deprecation
-       make clean html latexpdf
+     warnings on so they're shown in the test output::
 
-   * Run code style checks and type checks (type checks require mypy)::
+         PYTHONWARNINGS=all tox -e py36
 
-       make style-check
+   * To run code style and type checks::
-       make type-check
 
-   * Run the unit tests under different Python environments using
+         tox -e mypy
-     :program:`tox`::
+         tox -e flake8
 
-       pip install tox
+   * Arguments to ``pytest`` can be passed via ``tox``, e.g. in order to run a
-       tox -v
+     particular test::
 
-   * Add a new unit test in the ``tests`` directory if you can.
+       tox -e py36 tests/test_module.py::test_new_feature
+
+   * To build the documentation::
+
+         tox -e docs
+
+   * To build the documentation in multiple formats::
+
+         tox -e docs -- -b html,latexpdf
+
+   You can also test by installing dependencies in your local environment. ::
+
+       pip install .[test]
+
+   New unit tests should be included in the ``tests`` directory where
+   necessary:
 
    * For bug fixes, first add a test that fails without your changes and passes
      after they are applied.
 
-   * Tests that need a sphinx-build run should be integrated in one of the
+   * Tests that need a ``sphinx-build`` run should be integrated in one of the
      existing test modules if possible.  New tests that to ``@with_app`` and
      then ``build_all`` for a few assertions are not good since *the test suite
      should not take more than a minute to run*.
@@ -169,7 +188,7 @@ These are the basic steps needed to start developing on Sphinx.
        git push origin feature-xyz
 
 #. Submit a pull request from your branch to the respective branch (``master``
-   or ``stable``) on ``sphinx-doc/sphinx`` using the GitHub interface.
+   or ``X.Y``).
 
 #. Wait for a core developer to review your changes.
 
@@ -262,7 +281,7 @@ Debugging Tips
   code by running the command ``make clean`` or using the
   :option:`sphinx-build -E` option.
 
-* Use the :option:`sphinx-build -P` option to run Pdb on exceptions.
+* Use the :option:`sphinx-build -P` option to run ``pdb`` on exceptions.
 
 * Use ``node.pformat()`` and ``node.asdom().toxml()`` to generate a printable
   representation of the document structure.
@@ -286,6 +305,24 @@ Debugging Tips
      $ npm install
      $ node_modules/.bin/grunt build # -> dest/*.global.js
 
+
+Branch Model
+------------
+
+Sphinx project uses following branches for developing.
+
+``master``
+    Used for main development.  All improvement and refactoring, bug fixes
+    are allowed.
+
+``X.Y``
+    Where ``X.Y`` is the ``MAJOR.MINOR`` release.  Used to maintain current
+    stable release.  Only bug fixes and stable changes are allowed.  Only the
+    most recent stable release is currently retained. When a new version is
+    released, the old release branch will be deleted and replaced by an
+    equivalent tag.
+
+
 Deprecating a feature
 ---------------------
 
@@ -299,14 +336,17 @@ There are a couple reasons that code in Sphinx might be deprecated:
   no longer needs to support the older version of Python that doesn't include
   the library, the library will be deprecated in Sphinx.
 
-As the :ref:`deprecation-policy` describes,
+As the :ref:`deprecation-policy` describes, the first release of Sphinx that
-the first release of Sphinx that deprecates a feature (``A.B``) should raise a
+deprecates a feature (``A.B``) should raise a ``RemovedInSphinxXXWarning``
-``RemovedInSphinxXXWarning`` (where XX is the Sphinx version where the feature
+(where ``XX`` is the Sphinx version where the feature will be removed) when the
-will be removed) when the deprecated feature is invoked. Assuming we have good
+deprecated feature is invoked. Assuming we have good test coverage, these
-test coverage, these warnings are converted to errors when running the test
+warnings are converted to errors when running the test suite with warnings
-suite with warnings enabled: ``python -Wall tests/run.py``. Thus, when adding
+enabled::
-a ``RemovedInSphinxXXWarning`` you need to eliminate or silence any warnings
+
-generated when running the tests.
+    pytest -Wall
+
+Thus, when adding a ``RemovedInSphinxXXWarning`` you need to eliminate or
+silence any warnings generated when running the tests.
 
 .. _deprecation-policy:
 

EXAMPLES

@@ -34,6 +34,8 @@ Documentation using the alabaster theme
 * pytest: https://docs.pytest.org/ (customized)
 * python-apt: https://apt.alioth.debian.org/python-apt-doc/
 * PyVisfile: https://documen.tician.de/pyvisfile/
+* Requests: http://www.python-requests.org/
+* searx: https://asciimoo.github.io/searx/
 * Tablib: http://docs.python-tablib.org/
 * urllib3: https://urllib3.readthedocs.io/ (customized)
 * Werkzeug: http://werkzeug.pocoo.org/docs/ (customized)
@@ -46,6 +48,7 @@ Documentation using the classic theme
 * APSW: https://rogerbinns.github.io/apsw/
 * Arb: http://arblib.org/
 * Bazaar: http://doc.bazaar.canonical.com/ (customized)
+* Beautiful Soup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/
 * Blender: https://docs.blender.org/api/current/
 * Bugzilla: https://bugzilla.readthedocs.io/
 * Buildbot: https://docs.buildbot.net/latest/
@@ -79,6 +82,8 @@ Documentation using the classic theme
 * Pyevolve: http://pyevolve.sourceforge.net/
 * Pygame: https://www.pygame.org/docs/ (customized)
 * PyMQI: https://pythonhosted.org/pymqi/
+* PyQt4: http://pyqt.sourceforge.net/Docs/PyQt4/ (customized)
+* PyQt5: http://pyqt.sourceforge.net/Docs/PyQt5/ (customized)
 * Python 2: https://docs.python.org/2/
 * Python 3: https://docs.python.org/3/ (customized)
 * Python Packaging Authority: https://www.pypa.io/ (customized)
@@ -88,7 +93,7 @@ Documentation using the classic theme
 * simuPOP: http://simupop.sourceforge.net/manual_release/build/userGuide.html (customized)
 * Sprox: http://sprox.org/ (customized)
 * SymPy: http://docs.sympy.org/
-* TurboGears: https://turbogears.readthedocs.org/ (customized)
+* TurboGears: https://turbogears.readthedocs.io/ (customized)
 * tvtk: http://docs.enthought.com/mayavi/tvtk/
 * Varnish: https://www.varnish-cache.org/docs/ (customized, alabaster for index)
 * Waf: https://waf.io/apidocs/
@@ -121,11 +126,16 @@ Documentation using the nature theme
 
 * Alembic: http://alembic.zzzcomputing.com/
 * Cython: http://docs.cython.org/
+* easybuild: https://easybuild.readthedocs.io/
 * jsFiddle: http://doc.jsfiddle.net/
 * libLAS: https://www.liblas.org/ (customized)
+* Lmod: https://lmod.readthedocs.io/
 * MapServer: http://mapserver.org/ (customized)
+* Pandas: https://pandas.pydata.org/pandas-docs/stable/
+* pyglet: https://pyglet.readthedocs.io/ (customized)
 * Setuptools: https://setuptools.readthedocs.io/
 * Spring Python: https://docs.spring.io/spring-python/1.2.x/sphinx/html/
+* StatsModels: http://www.statsmodels.org/ (customized)
 * Sylli: http://sylli.sourceforge.net/
 
 Documentation using another builtin theme
@@ -133,6 +143,7 @@ Documentation using another builtin theme
 
 * Breathe: https://breathe.readthedocs.io/ (haiku)
 * MPipe: https://vmlaker.github.io/mpipe/ (sphinx13)
+* NLTK: http://www.nltk.org/ (agogo)
 * Programmieren mit PyGTK und Glade (German):
   http://www.florian-diesch.de/doc/python-und-glade/online/ (agogo, customized)
 * PyPubSub: https://pypubsub.readthedocs.io/ (bizstyle)
@@ -147,11 +158,15 @@ Documentation using sphinx_rtd_theme
 
 * Annotator: http://docs.annotatorjs.org/
 * Ansible: https://docs.ansible.com/ (customized)
+* Arcade: http://arcade.academy/
+* aria2: https://aria2.github.io/manual/en/html/
 * ASE: https://wiki.fysik.dtu.dk/ase/
 * Autofac: http://docs.autofac.org/
 * BigchainDB: https://docs.bigchaindb.com/
+* Blocks: https://blocks.readthedocs.io/
 * bootstrap-datepicker: https://bootstrap-datepicker.readthedocs.io/
 * Certbot: https://letsencrypt.readthedocs.io/
+* Chainer: https://docs.chainer.org/ (customized)
 * CherryPy: http://docs.cherrypy.org/
 * Chainer: https://docs.chainer.org/
 * CodeIgniter: https://www.codeigniter.com/user_guide/
@@ -178,14 +193,18 @@ Documentation using sphinx_rtd_theme
 * Idris: http://docs.idris-lang.org/
 * javasphinx: https://bronto-javasphinx.readthedocs.io/
 * Julia: https://julia.readthedocs.io/
+* Jupyter Notebook: https://jupyter-notebook.readthedocs.io/
+* Lasagne: https://lasagne.readthedocs.io/
 * Linguistica: https://linguistica-uchicago.github.io/lxa5/
 * Linux kernel: https://www.kernel.org/doc/html/latest/index.html
 * MathJax: https://docs.mathjax.org/
 * MDTraj: http://mdtraj.org/latest/ (customized)
 * MICrobial Community Analysis (micca): http://micca.org/docs/latest/
 * MicroPython: https://docs.micropython.org/
+* Minds: https://www.minds.org/docs/ (customized)
 * Mink: http://mink.behat.org/
 * Mockery: http://docs.mockery.io/
+* mod_wsgi: https://modwsgi.readthedocs.io/
 * MoinMoin: https://moin-20.readthedocs.io/
 * Mopidy: https://docs.mopidy.com/
 * MyHDL: http://docs.myhdl.org/
@@ -224,13 +243,16 @@ Documentation using sphinx_rtd_theme
 * Sylius: http://docs.sylius.org/
 * Tango Controls: https://tango-controls.readthedocs.io/ (customized)
 * Topshelf: http://docs.topshelf-project.com/
+* Theano: http://www.deeplearning.net/software/theano/
 * ThreatConnect: https://docs.threatconnect.com/
 * Tuleap: https://tuleap.net/doc/en/
 * TYPO3: https://docs.typo3.org/ (customized)
+* uWSGI: https://uwsgi-docs.readthedocs.io/
 * Wagtail: http://docs.wagtail.io/
 * Web Application Attack and Audit Framework (w3af): http://docs.w3af.org/
 * Weblate: https://docs.weblate.org/
 * x265: https://x265.readthedocs.io/
+* ZeroNet: https://zeronet.readthedocs.io/
 
 Documentation using sphinx_bootstrap_theme
 ------------------------------------------
@@ -238,19 +260,21 @@ Documentation using sphinx_bootstrap_theme
 * Bootstrap Theme: https://ryan-roemer.github.io/sphinx-bootstrap-theme/
 * C/C++ Software Development with Eclipse: http://eclipsebook.in/
 * Dataverse: http://guides.dataverse.org/
-* e-cidadania: http://e-cidadania.readthedocs.org/
+* e-cidadania: https://e-cidadania.readthedocs.io/
 * Hangfire: http://docs.hangfire.io/
 * Hedge: https://documen.tician.de/hedge/
 * ObsPy: https://docs.obspy.org/
 * Open Dylan: https://opendylan.org/documentation/
 * Pootle: http://docs.translatehouse.org/projects/pootle/
 * PyUblas: https://documen.tician.de/pyublas/
+* seaborn: https://seaborn.pydata.org/
 
 Documentation using a custom theme or integrated in a website
 -------------------------------------------------------------
 
 * Apache Cassandra: https://cassandra.apache.org/doc/
 * Astropy: http://docs.astropy.org/
+* Bokeh: https://bokeh.pydata.org/
 * Boto 3: https://boto3.readthedocs.io/
 * CakePHP: https://book.cakephp.org/
 * CasperJS: http://docs.casperjs.org/
@@ -263,6 +287,7 @@ Documentation using a custom theme or integrated in a website
 * Enterprise Toolkit for Acrobat products:
   https://www.adobe.com/devnet-docs/acrobatetk/
 * Gameduino: http://excamera.com/sphinx/gameduino/
+* gensim: https://radimrehurek.com/gensim/
 * GeoServer: http://docs.geoserver.org/
 * gevent: http://www.gevent.org/
 * GHC - Glasgow Haskell Compiler: http://downloads.haskell.org/~ghc/master/users-guide/
@@ -307,6 +332,7 @@ Documentation using a custom theme or integrated in a website
 * Sulu: http://docs.sulu.io/
 * SQLAlchemy: https://docs.sqlalchemy.org/
 * tinyTiM: http://tinytim.sourceforge.net/docs/2.0/
+* Twisted: http://twistedmatrix.com/documents/current/
 * Ubuntu Packaging Guide: http://packaging.ubuntu.com/html/
 * WebFaction: https://docs.webfaction.com/
 * WTForms: https://wtforms.readthedocs.io/
@@ -320,8 +346,10 @@ Homepages and other non-documentation sites
 * Benoit Boissinot: https://bboissin.appspot.com/ (classic, customized)
 * Computer Networks, Parallelization, and Simulation Laboratory (CNPSLab):
   https://lab.miletic.net/ (sphinx_rtd_theme)
+* Deep Learning Tutorials: http://www.deeplearning.net/tutorial/ (sphinxdoc)
 * Loyola University Chicago COMP 339-439 Distributed Systems course:
   http://books.cs.luc.edu/distributedsystems/ (sphinx_bootstrap_theme)
+* Pylearn2: http://www.deeplearning.net/software/pylearn2/ (sphinxdoc, customized)
 * SciPy Cookbook: https://scipy-cookbook.readthedocs.io/ (sphinx_rtd_theme)
 * The Wine Cellar Book: https://www.thewinecellarbook.com/doc/en/ (sphinxdoc)
 * Thomas Cokelaer's Python, Sphinx and reStructuredText tutorials:

LICENSE

@@ -1,7 +1,7 @@
 License for Sphinx
 ==================
 
-Copyright (c) 2007-2017 by the Sphinx team (see AUTHORS file).
+Copyright (c) 2007-2018 by the Sphinx team (see AUTHORS file).
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -31,64 +31,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 Licenses for incorporated software
 ==================================
 
-The pgen2 package, included in this distribution under the name
-sphinx.pycode.pgen2, is available in the Python 2.6 distribution under
-the PSF license agreement for Python:
-
-----------------------------------------------------------------------
-Copyright © 2001-2008 Python Software Foundation; All Rights Reserved.
-
-PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
---------------------------------------------
-
-1. This LICENSE AGREEMENT is between the Python Software Foundation
-   ("PSF"), and the Individual or Organization ("Licensee") accessing
-   and otherwise using Python 2.6 software in source or binary form
-   and its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, PSF
-   hereby grants Licensee a nonexclusive, royalty-free, world-wide
-   license to reproduce, analyze, test, perform and/or display
-   publicly, prepare derivative works, distribute, and otherwise use
-   Python 2.6 alone or in any derivative version, provided, however,
-   that PSF's License Agreement and PSF's notice of copyright, i.e.,
-   "Copyright © 2001-2008 Python Software Foundation; All Rights
-   Reserved" are retained in Python 2.6 alone or in any derivative
-   version prepared by Licensee.
-
-3. In the event Licensee prepares a derivative work that is based on
-   or incorporates Python 2.6 or any part thereof, and wants to make
-   the derivative work available to others as provided herein, then
-   Licensee hereby agrees to include in any such work a brief summary
-   of the changes made to Python 2.6.
-
-4. PSF is making Python 2.6 available to Licensee on an "AS IS" basis.
-   PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.  BY
-   WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY
-   REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY
-   PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 2.6 WILL NOT INFRINGE
-   ANY THIRD PARTY RIGHTS.
-
-5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-   2.6 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
-   AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON
-   2.6, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY
-   THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-   breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any
-   relationship of agency, partnership, or joint venture between PSF
-   and Licensee.  This License Agreement does not grant permission to
-   use PSF trademarks or trade name in a trademark sense to endorse or
-   promote products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using Python 2.6, Licensee
-   agrees to be bound by the terms and conditions of this License
-   Agreement.
-----------------------------------------------------------------------
-
 The included smartypants module, included as sphinx.util.smartypants,
 is available under the following license:
 

MANIFEST.in

@@ -12,7 +12,6 @@ include sphinx-autogen.py
 include sphinx-build.py
 include sphinx-quickstart.py
 include sphinx-apidoc.py
-include test-reqs.txt
 include tox.ini
 include sphinx/locale/.tx/config
 

Makefile

@@ -1,67 +1,41 @@
 PYTHON ?= python
 
-.PHONY: all style-check type-check clean clean-pyc clean-patchfiles clean-backupfiles \
+.PHONY: all
-        clean-generated pylint reindent test covertest build
-
-DONT_CHECK = -i .ropeproject \
-             -i .tox \
-             -i build \
-             -i dist \
-             -i doc/_build \
-             -i sphinx/pycode/pgen2 \
-             -i sphinx/search/da.py \
-             -i sphinx/search/de.py \
-             -i sphinx/search/en.py \
-             -i sphinx/search/es.py \
-             -i sphinx/search/fi.py \
-             -i sphinx/search/fr.py \
-             -i sphinx/search/hu.py \
-             -i sphinx/search/it.py \
-             -i sphinx/search/ja.py \
-             -i sphinx/search/nl.py \
-             -i sphinx/search/no.py \
-             -i sphinx/search/pt.py \
-             -i sphinx/search/ro.py \
-             -i sphinx/search/ru.py \
-             -i sphinx/search/sv.py \
-             -i sphinx/search/tr.py \
-             -i sphinx/style/jquery.js \
-             -i sphinx/util/smartypants.py \
-             -i tests/build \
-             -i tests/path.py \
-             -i tests/roots/test-directive-code/target.py \
-             -i tests/roots/test-warnings/undecodable.rst \
-             -i tests/test_autodoc_py35.py \
-             -i tests/typing_test_data.py \
-             -i utils/convert.py
-
 all: clean-pyc clean-backupfiles style-check type-check test
 
+.PHONY: style-check
 style-check:
-	@PYTHONWARNINGS=all $(PYTHON) utils/check_sources.py $(DONT_CHECK) .
+	@flake8
 
+.PHONY: type-check
 type-check:
 	mypy sphinx/
 
+.PHONY: clean
 clean: clean-pyc clean-pycache clean-patchfiles clean-backupfiles clean-generated clean-testfiles clean-buildfiles clean-mypyfiles
 
+.PHONY: clean-pyc
 clean-pyc:
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 
+.PHONY: clean-pycache
 clean-pycache:
 	find . -name __pycache__ -exec rm -rf {} +
 
+.PHONY: clean-patchfiles
 clean-patchfiles:
 	find . -name '*.orig' -exec rm -f {} +
 	find . -name '*.rej' -exec rm -f {} +
 
+.PHONY: clean-backupfiles
 clean-backupfiles:
 	find . -name '*~' -exec rm -f {} +
 	find . -name '*.bak' -exec rm -f {} +
 	find . -name '*.swp' -exec rm -f {} +
 	find . -name '*.swo' -exec rm -f {} +
 
+.PHONY: clean-generated
 clean-generated:
 	find . -name '.DS_Store' -exec rm -f {} +
 	rm -rf Sphinx.egg-info/
@@ -70,32 +44,48 @@ clean-generated:
 	rm -f utils/*3.py*
 	rm -f utils/regression_test.js
 
+.PHONY: clean-testfiles
 clean-testfiles:
 	rm -rf tests/.coverage
 	rm -rf tests/build
 	rm -rf .tox/
 	rm -rf .cache/
 
+.PHONY: clean-buildfiles
 clean-buildfiles:
 	rm -rf build
 
+.PHONY: clean-mypyfiles
 clean-mypyfiles:
 	rm -rf .mypy_cache/
 
+.PHONY: pylint
 pylint:
 	@pylint --rcfile utils/pylintrc sphinx
 
+.PHONY: reindent
 reindent:
-	@$(PYTHON) utils/reindent.py -r -n .
+	@echo "This target no longer does anything and will be removed imminently"
 
+.PHONY: test
 test:
-	@cd tests; $(PYTHON) run.py --ignore py35 -v $(TEST)
+	@$(PYTHON) -m pytest -v $(TEST)
 
+.PHONY: test-async
 test-async:
-	@cd tests; $(PYTHON) run.py -v $(TEST)
+	@echo "This target no longer does anything and will be removed imminently"
 
+.PHONY: covertest
 covertest:
-	@cd tests; $(PYTHON) run.py -v --cov=sphinx --junitxml=.junit.xml $(TEST)
+	@$(PYTHON) -m pytest -v --cov=sphinx --junitxml=.junit.xml $(TEST)
 
+.PHONY: build
 build:
 	@$(PYTHON) setup.py build
+
+.PHONY: docs
+docs:
+ifndef target
+	  $(info You need to give a provide a target variable, e.g. `make docs target=html`.)
+endif
+	  $(MAKE) -C doc $(target)

README.rst

@@ -1,45 +1,106 @@
+========
+ Sphinx
+========
+
 .. image:: https://img.shields.io/pypi/v/sphinx.svg
    :target: https://pypi.python.org/pypi/Sphinx
+   :alt: Package on PyPi
+
 .. image:: https://readthedocs.org/projects/sphinx/badge/
    :target: http://www.sphinx-doc.org/
    :alt: Documentation Status
+
 .. image:: https://travis-ci.org/sphinx-doc/sphinx.svg?branch=master
    :target: https://travis-ci.org/sphinx-doc/sphinx
+   :alt: Build Status (Travis CI)
 
-=================
+.. image:: https://ci.appveyor.com/api/projects/status/github/sphinx-doc/sphinx?branch=master&svg=true
-README for Sphinx
+   :target: https://ci.appveyor.com/project/sphinxdoc/sphinx
-=================
+   :alt: Build Status (AppVeyor)
 
-This is the Sphinx documentation generator, see http://www.sphinx-doc.org/.
+.. image:: https://circleci.com/gh/sphinx-doc/sphinx.svg?style=shield
+   :target: https://circleci.com/gh/sphinx-doc/sphinx
+   :alt: Build Status (CircleCI)
 
+.. image:: https://codecov.io/gh/sphinx-doc/sphinx/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/sphinx-doc/sphinx
+   :alt: Code Coverage Status (Codecov)
 
-Installing
+Sphinx is a tool that makes it easy to create intelligent and beautiful
-==========
+documentation for Python projects (or other documents consisting of multiple
+reStructuredText sources), written by Georg Brandl.  It was originally created
+for the new Python documentation, and has excellent facilities for Python
+project documentation, but C/C++ is supported as well, and more languages are
+planned.
 
-Install from PyPI to use stable version::
+Sphinx uses reStructuredText as its markup language, and many of its strengths
+come from the power and straightforwardness of reStructuredText and its parsing
+and translating suite, the Docutils.
+
+Among its features are the following:
+
+* Output formats: HTML (including derivative formats such as HTML Help, Epub
+  and Qt Help), plain text, manual pages and LaTeX or direct PDF output
+  using rst2pdf
+* Extensive cross-references: semantic markup and automatic links
+  for functions, classes, glossary terms and similar pieces of information
+* Hierarchical structure: easy definition of a document tree, with automatic
+  links to siblings, parents and children
+* Automatic indices: general index as well as a module index
+* Code handling: automatic highlighting using the Pygments highlighter
+* Flexible HTML output using the Jinja 2 templating engine
+* Various extensions are available, e.g. for automatic testing of snippets
+  and inclusion of appropriately formatted docstrings
+* Setuptools integration
+
+For more information, refer to the `the documentation`__.
+
+.. __: http://www.sphinx-doc.org/
+
+Installation
+============
+
+Sphinx is published on `PyPI`__ and can be installed from there::
 
    pip install -U sphinx
 
-Install from PyPI to use beta version::
+We also publish beta releases::
 
    pip install -U --pre sphinx
 
-Install from newest dev version in stable branch::
+If you wish to install `Sphinx` for development purposes, refer to `the
+contributors guide`__.
 
-   pip install git+https://github.com/sphinx-doc/sphinx@stable
+__ https://pypi.python.org/pypi/Sphinx
+__ http://www.sphinx-doc.org/en/master/devguide.html
 
-Install from newest dev version in master branch::
+Documentation
+=============
 
-   pip install git+https://github.com/sphinx-doc/sphinx
+Documentation is available from `sphinx-doc.org`__.
 
-Install from cloned source::
+__ http://www.sphinx-doc.org/
 
-   pip install .
+Testing
+=======
 
-Install from cloned source as editable::
+Continuous testing is provided by `Travis`__ (for unit tests and style checks
+on Linux), `AppVeyor`__ (for unit tests on Windows), and `CircleCI`__ (for
+large processes like TeX compilation).
 
-   pip install -e .
+For information on running tests locally, refer to `the contributors guide`__.
 
+__ https://travis-ci.org/sphinx-doc/sphinx
+__ https://ci.appveyor.com/project/sphinxdoc/sphinx
+__ https://circleci.com/gh/sphinx-doc/sphinx
+__ http://www.sphinx-doc.org/en/master/devguide.html
+
+Contributing
+============
+
+Refer to `the contributors guide`__.
+
+__ http://www.sphinx-doc.org/en/master/devguide.html
 
 Release signatures
 ==================
@@ -48,37 +109,3 @@ Releases are signed with following keys:
 
 * `498D6B9E <https://pgp.mit.edu/pks/lookup?op=vindex&search=0x102C2C17498D6B9E>`_
 * `5EBA0E07 <https://pgp.mit.edu/pks/lookup?op=vindex&search=0x1425F8CE5EBA0E07>`_
-
-Reading the docs
-================
-
-You can read them online at <http://www.sphinx-doc.org/>.
-
-Or, after installing::
-
-   cd doc
-   make html
-
-Then, direct your browser to ``_build/html/index.html``.
-
-Testing
-=======
-
-To run the tests with the interpreter available as ``python``, use::
-
-    make test
-
-If you want to use a different interpreter, e.g. ``python3``, use::
-
-    PYTHON=python3 make test
-
-Continuous testing runs on travis: https://travis-ci.org/sphinx-doc/sphinx
-
-
-Contributing
-============
-
-See `CONTRIBUTING.rst`__
-
-.. __: CONTRIBUTING.rst
-

doc/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS   =
-SPHINXBUILD  = python ../sphinx-build.py
+SPHINXBUILD  = python ../sphinx/cmd/build.py
 SPHINXPROJ   = sphinx
 SOURCEDIR    = .
 BUILDDIR     = _build

doc/_templates/index.html

@@ -74,9 +74,9 @@
 
   <p>{%trans%}
     You can also download PDF/EPUB versions of the Sphinx documentation:
-    a <a href="http://readthedocs.org/projects/sphinx/downloads/pdf/stable/">PDF version</a> generated from
+    a <a href="https://media.readthedocs.org/pdf/sphinx/stable/sphinx.pdf">PDF version</a> generated from
     the LaTeX Sphinx produces, and
-    a <a href="http://readthedocs.org/projects/sphinx/downloads/epub/stable/">EPUB version</a>.
+    a <a href="https://media.readthedocs.org/epub/sphinx/stable/sphinx.epub">EPUB version</a>.
    {%endtrans%}
   </p>
 
@@ -106,7 +106,7 @@
   <h2>{%trans%}Hosting{%endtrans%}</h2>
 
   <p>{%trans%}Need a place to host your Sphinx docs?
-    <a href="http://readthedocs.org">readthedocs.org</a> hosts a lot of Sphinx docs
+    <a href="https://readthedocs.org/">readthedocs.org</a> hosts a lot of Sphinx docs
     already, and integrates well with projects' source control.  It also features a
     powerful built-in search that exceeds the possibilities of Sphinx' JavaScript-based
     offline search.{%endtrans%}</p>

doc/_templates/indexsidebar.html

@@ -20,12 +20,14 @@ Index</a>, or install it with:{%endtrans%}</p>
 <h3>{%trans%}Questions? Suggestions?{%endtrans%}</h3>
 
 <p>{%trans%}Join the <a href="http://groups.google.com/group/sphinx-users">sphinx-users</a> mailing list on Google Groups:{%endtrans%}</p>
+<div class="subscribeformwrapper">
 <form action="http://groups.google.com/group/sphinx-users/boxsubscribe"
-      style="padding-left: 0.5em">
+      class="subscribeform">
-  <input type="text" name="email" value="your@email" style="font-size: 90%; width: 120px"
+  <input type="text" name="email" value="your@email"
-         onfocus="$(this).val('');"/>
+         onfocus="$(this).val('');" />
-  <input type="submit" name="sub" value="Subscribe" style="font-size: 90%; width: 70px"/>
+  <input type="submit" name="sub" value="Subscribe" />
 </form>
+</div>
 <p>{%trans%}or come to the <tt>#sphinx-doc</tt> channel on FreeNode.{%endtrans%}</p>
 <p>{%trans%}You can also open an issue at the
   <a href="https://github.com/sphinx-doc/sphinx/issues">tracker</a>.{%endtrans%}</p>

doc/_themes/sphinx13/layout.html

@@ -4,7 +4,7 @@
 
     Sphinx layout template for the sphinxdoc theme.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 #}
 {%- extends "basic/layout.html" %}

doc/_themes/sphinx13/static/sphinx13.css

@@ -4,7 +4,7 @@
  *
  * Sphinx stylesheet -- sphinx13 theme.
  *
- * :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */
@@ -140,11 +140,37 @@ div.sphinxsidebar .logo img {
     vertical-align: middle;
 }
 
+div.subscribeformwrapper {
+    display: block;
+    overflow: auto;
+    margin-bottom: 1.2em;
+}
+
 div.sphinxsidebar input {
     border: 1px solid #aaa;
     font-family: 'Open Sans', 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva',
                  'Verdana', sans-serif;
-    font-size: 1em;
+}
+
+div.sphinxsidebar .subscribeform {
+    margin-top: 0;
+}
+
+div.sphinxsidebar .subscribeform input {
+    border: 1px solid #aaa;
+    font-size: 0.9em;
+    float: left;
+    padding: 0.25em 0.5em;
+    box-sizing: border-box;
+}
+
+div.sphinxsidebar .subscribeform input[type="text"] {
+    width: 60%;
+}
+
+div.sphinxsidebar .subscribeform input[type="submit"] {
+    width: 40%;
+    border-left: none;
 }
 
 div.sphinxsidebar h3 {
@@ -281,7 +307,7 @@ tt {
     border: 1px solid #ddd;
     border-radius: 2px;
     color: #333;
-    padding: 1px;
+    padding: 1px 0.2em;
 }
 
 tt.descname, tt.descclassname, tt.xref {

doc/conf.py

@@ -3,6 +3,7 @@
 # Sphinx documentation build configuration file
 
 import re
+
 import sphinx
 
 
@@ -15,7 +16,7 @@ templates_path = ['_templates']
 exclude_patterns = ['_build']
 
 project = 'Sphinx'
-copyright = '2007-2017, Georg Brandl and the Sphinx team'
+copyright = '2007-2018, Georg Brandl and the Sphinx team'
 version = sphinx.__display_version__
 release = version
 show_authors = True

doc/config.rst

@@ -9,8 +9,17 @@ The build configuration file
    :synopsis: Build configuration file.
 
 The :term:`configuration directory` must contain a file named :file:`conf.py`.
-This file (containing Python code) is called the "build configuration file" and
+This file (containing Python code) is called the "build configuration file"
-contains all configuration needed to customize Sphinx input and output behavior.
+and contains (almost) all configuration needed to customize Sphinx input
+and output behavior.
+
+  An optional file `docutils.conf`_ can be added to the configuration
+  directory to adjust `Docutils`_ configuration if not otherwise overriden or
+  set by Sphinx.
+
+  .. _`docutils`: http://docutils.sourceforge.net/
+
+  .. _`docutils.conf`: http://docutils.sourceforge.net/docs/user/config.html
 
 The configuration file is executed as Python code at build time (using
 :func:`execfile`, and with the current directory set to its containing
@@ -129,12 +138,10 @@ General configuration
 
    - ``'library/xml.rst'`` -- ignores the ``library/xml.rst`` file (replaces
      entry in :confval:`unused_docs`)
-   - ``'library/xml'`` -- ignores the ``library/xml`` directory (replaces entry
+   - ``'library/xml'`` -- ignores the ``library/xml`` directory
-     in :confval:`exclude_trees`)
    - ``'library/xml*'`` -- ignores all files and directories starting with
      ``library/xml``
-   - ``'**/.svn'`` -- ignores all ``.svn`` directories (replaces entry in
+   - ``'**/.svn'`` -- ignores all ``.svn`` directories
-     :confval:`exclude_dirnames`)
 
    :confval:`exclude_patterns` is also consulted when looking for static files
    in :confval:`html_static_path` and :confval:`html_extra_path`.
@@ -286,6 +293,24 @@ General configuration
 
    .. versionadded:: 1.3
 
+.. confval:: manpages_url
+
+   A URL to cross-reference :rst:role:`manpage` directives. If this is
+   defined to ``https://manpages.debian.org/{path}``, the
+   :literal:`:manpage:`man(1)`` role will like to
+   <https://manpages.debian.org/man(1)>. The patterns available are:
+
+     * ``page`` - the manual page (``man``)
+     * ``section`` - the manual section (``1``)
+     * ``path`` - the original manual page and section specified (``man(1)``)
+
+   This also supports manpages specified as ``man.1``.
+
+   .. note:: This currently affects only HTML writers but could be
+             expanded in the future.
+
+   .. versionadded:: 1.7
+
 .. confval:: nitpicky
 
    If true, Sphinx will warn about *all* references where the target cannot be
@@ -306,8 +331,8 @@ General configuration
 .. confval:: numfig
 
    If true, figures, tables and code-blocks are automatically numbered if they
-   have a caption.  At same time, the `numref` role is enabled.  For now, it
+   have a caption.  The :rst:role:`numref` role is enabled.
-   works only with the HTML builder and LaTeX builder. Default is ``False``.
+   Obeyed so far only by HTML and LaTeX builders. Default is ``False``.
 
    .. note::
 
@@ -330,13 +355,80 @@ General configuration
 
 .. confval:: numfig_secnum_depth
 
-   The scope of figure numbers, that is, the numfig feature numbers figures
+   - if set to ``0``, figures, tables and code-blocks are continuously numbered
-   in which scope. ``0`` means "whole document". ``1`` means "in a section".
+     starting at ``1``.
-   Sphinx numbers like x.1, x.2, x.3... ``2`` means "in a subsection". Sphinx
+   - if ``1`` (default) numbers will be ``x.1``, ``x.2``, ... with ``x``
-   numbers like x.x.1, x.x.2, x.x.3..., and so on. Default is ``1``.
+     the section number (top level sectioning; no ``x.`` if no section).
+     This naturally applies only if section numbering has been activated via
+     the ``:numbered:`` option of the :rst:dir:`toctree` directive.
+   - ``2`` means that numbers will be ``x.y.1``, ``x.y.2``, ... if located in
+     a sub-section (but still ``x.1``, ``x.2``, ... if located directly under a
+     section and ``1``, ``2``, ... if not in any top level section.)
+   - etc...
 
    .. versionadded:: 1.3
 
+   .. versionchanged:: 1.7
+      The LaTeX builder obeys this setting (if :confval:`numfig` is set to
+      ``True``).
+
+.. confval:: smartquotes
+
+   If true, the `Docutils Smart Quotes transform`__, originally based on
+   `SmartyPants`__ (limited to English) and currently applying to many
+   languages, will be used to convert quotes and dashes to typographically
+   correct entities.  Default: ``True``.
+
+   __ http://docutils.sourceforge.net/docs/user/smartquotes.html
+   __ https://daringfireball.net/projects/smartypants/
+
+   .. versionadded:: 1.6.6
+      It replaces deprecated :confval:`html_use_smartypants`.
+      It applies by default to all builders except ``man`` and ``text``
+      (see :confval:`smartquotes_excludes`.)
+
+   A `docutils.conf`__ file located in the configuration directory (or a
+   global :file:`~/.docutils` file) is obeyed unconditionally if it
+   *deactivates* smart quotes via the corresponding `Docutils option`__.  But
+   if it *activates* them, then :confval:`smartquotes` does prevail.
+
+   __ http://docutils.sourceforge.net/docs/user/config.html
+   __ http://docutils.sourceforge.net/docs/user/config.html#smart-quotes
+
+.. confval:: smartquotes_action
+
+   This string, for use with Docutils ``0.14`` or later, customizes the Smart
+   Quotes transform.  See the file :file:`smartquotes.py` at the `Docutils
+   repository`__ for details.  The default ``'qDe'`` educates normal **q**\
+   uote characters ``"``, ``'``, em- and en-**D**\ ashes ``---``, ``--``, and
+   **e**\ llipses ``...``.
+
+   .. versionadded:: 1.6.6
+
+   __ https://sourceforge.net/p/docutils/code/HEAD/tree/trunk/docutils/
+
+.. confval:: smartquotes_excludes
+
+   This is a ``dict`` whose default is::
+
+     {'languages': ['ja'], 'builders': ['man', 'text']}
+
+   Each entry gives a sufficient condition to ignore the
+   :confval:`smartquotes` setting and deactivate the Smart Quotes transform.
+   Accepted keys are as above ``'builders'`` or ``'languages'``.
+   The values are lists.
+
+   .. note:: Currently, in case of invocation of :program:`make` with multiple
+      targets, the first target name is the only one which is tested against
+      the ``'builders'`` entry and it decides for all.  Also, a ``make text``
+      following ``make html`` needs to be issued in the form ``make text
+      O="-E"`` to force re-parsing of source files, as the cached ones are
+      already transformed.  On the other hand the issue does not arise with
+      direct usage of :program:`sphinx-build` as it caches
+      (in its default usage) the parsed source files in per builder locations.
+
+   .. versionadded:: 1.6.6
+
 .. confval:: tls_verify
 
    If true, Sphinx verifies server certifications.  Default is ``True``.
@@ -766,6 +858,14 @@ that use Sphinx's HTMLWriter class.
    The empty string is equivalent to ``'%b %d, %Y'`` (or a
    locale-dependent equivalent).
 
+.. confval:: html_use_smartypants
+
+   If true, quotes and dashes are converted to typographically correct
+   entities.  Default: ``True``.
+
+   .. deprecated:: 1.6
+      To disable smart quotes, use rather :confval:`smartquotes`.
+
 .. confval:: html_add_permalinks
 
    Sphinx will add "permalinks" for each heading and description environment as
@@ -796,14 +896,19 @@ that use Sphinx's HTMLWriter class.
      to include.  If all or some of the default sidebars are to be included,
      they must be put into this list as well.
 
-     The default sidebars (for documents that don't match any pattern) are:
+     The default sidebars (for documents that don't match any pattern) are
-     ``['localtoc.html', 'relations.html', 'sourcelink.html',
+     defined by theme itself.  Builtin themes are using these templates by
+     default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
      'searchbox.html']``.
 
    * If a value is a single string, it specifies a custom sidebar to be added
      between the ``'sourcelink.html'`` and ``'searchbox.html'`` entries.  This
      is for compatibility with Sphinx versions before 1.0.
 
+   .. deprecated:: 1.7
+
+      a single string value for ``html_sidebars`` will be removed in 2.0
+
    Builtin sidebar templates that can be rendered are:
 
    * **localtoc.html** -- a fine-grained table of contents of the current
@@ -1424,10 +1529,6 @@ the `Dublin Core metadata <http://dublincore.org/>`_.
    a chapter, but can be confusing because it mixes entries of different
    depth in one list.  The default value is ``True``.
 
-   .. note::
-
-      ``epub3`` builder ignores ``epub_tocdup`` option(always ``False``)
-
 .. confval:: epub_tocscope
 
    This setting control the scope of the epub table of contents.  The setting
@@ -1522,6 +1623,25 @@ These options influence LaTeX output. See further :doc:`latex`.
    * ``'lualatex'`` -- LuaLaTeX
    * ``'platex'`` -- pLaTeX (default if :confval:`language` is ``'ja'``)
 
+   PDFLaTeX's support for Unicode characters covers those from the document
+   language (the LaTeX ``babel`` and ``inputenc`` packages map them to glyph
+   slots in the document font, at various encodings allowing each only 256
+   characters; Sphinx uses by default (except for Cyrillic languages) the
+   ``times`` package), but stray characters from other scripts or special
+   symbols may require adding extra LaTeX packages or macros to the LaTeX
+   preamble.
+
+   If your project uses such extra Unicode characters, switching the engine to
+   XeLaTeX or LuaLaTeX often provides a quick fix. They only work with UTF-8
+   encoded sources and can (in fact, should) use OpenType fonts, either from
+   the system or the TeX install tree. Recent LaTeX releases will default with
+   these engines to the Latin Modern OpenType font, which has good coverage of
+   Latin and Cyrillic scripts (it is provided by standard LaTeX installation),
+   and Sphinx does not modify this default. Refer to the documentation of the
+   LaTeX ``polyglossia`` package to see how to instruct LaTeX to use some
+   other OpenType font if Unicode coverage proves insufficient (or use
+   directly ``\setmainfont`` et. al. as in :ref:`this example <latex-basic>`.)
+
 .. confval:: latex_documents
 
    This value determines how to group the document tree into LaTeX source files.
@@ -1570,10 +1690,15 @@ These options influence LaTeX output. See further :doc:`latex`.
 .. confval:: latex_toplevel_sectioning
 
    This value determines the topmost sectioning unit. It should be chosen from
-   ``part``, ``chapter`` or ``section``. The default is ``None``; the topmost
+   ``'part'``, ``'chapter'`` or ``'section'``. The default is ``None``;
-   sectioning unit is switched by documentclass. ``section`` is used if
+   the topmost
+   sectioning unit is switched by documentclass: ``section`` is used if
    documentclass will be ``howto``, otherwise ``chapter`` will be used.
 
+   Note that if LaTeX uses ``\part`` command, then the numbering of sectioning
+   units one level deep gets off-sync with HTML numbering, because LaTeX
+   numbers continuously ``\chapter`` (or ``\section`` for ``howto``.)
+
    .. versionadded:: 1.4
 
 .. confval:: latex_appendices
@@ -1914,6 +2039,20 @@ These options influence text output.
 
    .. versionadded:: 1.1
 
+.. confval:: text_add_secnumbers
+
+   A boolean that decides whether section numbers are included in text output.
+   Default is ``True``.
+
+   .. versionadded:: 1.7
+
+.. confval:: text_secnumber_suffix
+
+   Suffix for section numbers in text output.  Default: ``". "``. Set to ``" "``
+   to suppress the final dot on section numbers.
+
+   .. versionadded:: 1.7
+
 
 .. _man-options:
 

doc/develop.rst

@@ -138,7 +138,7 @@ own extensions.
 .. _cmakedomain: https://bitbucket.org/klorenz/sphinxcontrib-cmakedomain
 .. _GNU Make: http://www.gnu.org/software/make/
 .. _makedomain: https://bitbucket.org/klorenz/sphinxcontrib-makedomain
-.. _inlinesyntaxhighlight: http://sphinxcontrib-inlinesyntaxhighlight.readthedocs.org
+.. _inlinesyntaxhighlight: https://sphinxcontrib-inlinesyntaxhighlight.readthedocs.io/
 .. _CMake: https://cmake.org
 .. _domaintools: https://bitbucket.org/klorenz/sphinxcontrib-domaintools
 .. _restbuilder: https://pypi.python.org/pypi/sphinxcontrib-restbuilder

doc/domains.rst

@@ -720,13 +720,13 @@ a visibility statement (``public``, ``private`` or ``protected``).
 
 
 .. rst:directive:: .. cpp:concept:: template-parameter-list name
-                   .. cpp:concept:: template-parameter-list name()
 
    .. warning:: The support for concepts is experimental. It is based on the
-      Concepts Technical Specification, and the features may change as the TS evolves.
+      current draft standard and the Concepts Technical Specification.
+      The features may change as they evolve.
 
-   Describe a variable concept or a function concept. Both must have exactly 1
+   Describe a concept. It must have exactly 1 template parameter list. The name may be a
-   template parameter list. The name may be a nested name. Examples::
+   nested name. Example::
 
       .. cpp:concept:: template<typename It> std::Iterator
 
@@ -744,12 +744,7 @@ a visibility statement (``public``, ``private`` or ``protected``).
          - :cpp:expr:`*r`, when :cpp:expr:`r` is dereferenceable.
          - :cpp:expr:`++r`, with return type :cpp:expr:`It&`, when :cpp:expr:`r` is incrementable.
 
-      .. cpp:concept:: template<typename Cont> std::Container()
+   This will render as follows:
-
-         Holder of elements, to which it can provide access via
-         :cpp:concept:`Iterator` s.
-
-   They will render as follows:
 
    .. cpp:concept:: template<typename It> std::Iterator
 
@@ -767,11 +762,6 @@ a visibility statement (``public``, ``private`` or ``protected``).
       - :cpp:expr:`*r`, when :cpp:expr:`r` is dereferenceable.
       - :cpp:expr:`++r`, with return type :cpp:expr:`It&`, when :cpp:expr:`r` is incrementable.
 
-   .. cpp:concept:: template<typename Cont> std::Container()
-
-      Holder of elements, to which it can provide access via
-      :cpp:concept:`Iterator` s.
-
 Options
 .......
 
@@ -785,8 +775,9 @@ Some directives support options:
 Constrained Templates
 ~~~~~~~~~~~~~~~~~~~~~
 
-.. warning:: The support for constrained templates is experimental. It is based on the
+.. warning:: The support for concepts is experimental. It is based on the
-  Concepts Technical Specification, and the features may change as the TS evolves.
+  current draft standard and the Concepts Technical Specification.
+  The features may change as they evolve.
 
 .. note:: Sphinx does not currently support ``requires`` clauses.
 

doc/ext/autodoc.rst

@@ -103,8 +103,10 @@ inserting them into the page source under a suitable :rst:dir:`py:module`,
      will document all non-private member functions and properties (that is,
      those whose name doesn't start with ``_``).
 
-     For modules, ``__all__`` will be respected when looking for members; the
+     For modules, ``__all__`` will be respected when looking for members unless
-     order of the members will also be the order in ``__all__``.
+     you give the ``ignore-module-all`` flag option.  Without
+     ``ignore-module-all``, the order of the members will also be the order in
+     ``__all__``.
 
      You can also give an explicit list of members; only these will then be
      documented::
@@ -339,7 +341,7 @@ There are also new config values that you can set:
    This value is a list of autodoc directive flags that should be automatically
    applied to all autodoc directives.  The supported flags are ``'members'``,
    ``'undoc-members'``, ``'private-members'``, ``'special-members'``,
-   ``'inherited-members'`` and ``'show-inheritance'``.
+   ``'inherited-members'``, ``'show-inheritance'`` and ``'ignore-module-all'``.
 
    If you set one of these flags in this config value, you can use a negated
    form, :samp:`'no-{flag}'`, in an autodoc directive, to disable it once.

doc/ext/doctest.rst

@@ -80,12 +80,24 @@ a comma-separated list of group names.
          .. doctest::
             :pyversion: > 3.3
 
-     The supported operands are ``<``, ``<=``, ``==``, ``>=``, ``>``, and
+     The following operands are supported:
-     comparison is performed by `distutils.version.LooseVersion
+
-     <https://www.python.org/dev/peps/pep-0386/#distutils>`__.
+     * ``~=``: Compatible release clause
+     * ``==``: Version matching clause
+     * ``!=``: Version exclusion clause
+     * ``<=``, ``>=``: Inclusive ordered comparison clause
+     * ``<``, ``>``: Exclusive ordered comparison clause
+     * ``===``: Arbitrary equality clause.
+
+     ``pyversion`` option is followed `PEP-440: Version Specifiers
+     <https://www.python.org/dev/peps/pep-0440/#version-specifiers>`__.
 
      .. versionadded:: 1.6
 
+     .. versionchanged:: 1.7
+
+        Supported PEP-440 operands and notations
+
    Note that like with standard doctests, you have to use ``<BLANKLINE>`` to
    signal a blank line in the expected output.  The ``<BLANKLINE>`` is removed
    when building presentation output (HTML, LaTeX etc.).

doc/ext/inheritance.rst

@@ -42,6 +42,54 @@ It adds this directive:
    .. versionchanged:: 1.5
       Added ``caption`` option
 
+   It also supports a ``top-classes`` option which requires one or more class
+   names separated by comma. If specified inheritance traversal will stop at the
+   specified class names. Given the following Python module::
+
+        """
+               A
+              / \
+             B   C
+            / \ / \
+           E   D   F
+        """
+
+        class A(object):
+            pass
+
+        class B(A):
+            pass
+
+        class C(A):
+            pass
+
+        class D(B, C):
+            pass
+
+        class E(B):
+            pass
+
+        class F(C):
+            pass
+
+   If you have specified a module in the inheritance diagram like this::
+
+        .. inheritance-diagram:: dummy.test
+           :top-classes: dummy.test.B, dummy.test.C
+
+   any base classes which are ancestors to ``top-classes`` and are also defined
+   in the same module will be rendered as stand alone nodes. In this example
+   class A will be rendered as stand alone node in the graph. This is a known
+   issue due to how this extension works internally.
+
+   If you don't want class A (or any other ancestors) to be visible then specify
+   only the classes you would like to generate the diagram for like this::
+
+        .. inheritance-diagram:: dummy.test.D dummy.test.E dummy.test.F
+           :top-classes: dummy.test.B, dummy.test.C
+
+   .. versionchanged:: 1.7
+      Added ``top-classes`` option to limit the scope of inheritance graphs.
 
 New config values are:
 

doc/ext/math.rst

@@ -44,6 +44,15 @@ or use Python raw strings (``r"raw"``).
 
    Example: ``'Eq.{number}'`` is rendered as ``Eq.10``
 
+.. confval:: math_numfig
+
+   If ``True``, displayed math equations are numbered across pages when
+   :confval:`numfig` is enabled.  The :confval:`numfig_secnum_depth` setting
+   is respected.  The :rst:role:`eq`, not :rst:role:`numref`, role
+   must be used to reference equation numbers.  Default is ``True``.
+
+   .. versionadded:: 1.7
+
 :mod:`.mathbase` defines these new markup elements:
 
 .. rst:role:: math
@@ -85,7 +94,7 @@ or use Python raw strings (``r"raw"``).
    Normally, equations are not numbered.  If you want your equation to get a
    number, use the ``label`` option.  When given, it selects an internal label
    for the equation, by which it can be cross-referenced, and causes an equation
-   number to be issued.  See :rst:role:`eqref` for an example.  The numbering
+   number to be issued.  See :rst:role:`eq` for an example.  The numbering
    style depends on the output format.
 
    There is also an option ``nowrap`` that prevents any wrapping of the given
@@ -102,8 +111,7 @@ or use Python raw strings (``r"raw"``).
 
 .. rst:role:: eq
 
-   Role for cross-referencing equations via their label.  This currently works
+   Role for cross-referencing equations via their label.  Example::
-   only within the same document.  Example::
 
       .. math:: e^{i\pi} + 1 = 0
          :label: euler

doc/ext/napoleon.rst

@@ -68,8 +68,8 @@ Getting Started
 
        # conf.py
 
-       # Add autodoc and napoleon to the extensions list
+       # Add napoleon to the extensions list
-       extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon']
+       extensions = ['sphinx.ext.napoleon']
 
 2. Use `sphinx-apidoc` to build your API documentation::
 
@@ -246,13 +246,12 @@ Configuration
 
 Listed below are all the settings used by napoleon and their default
 values. These settings can be changed in the Sphinx `conf.py` file. Make
-sure that both "sphinx.ext.autodoc" and "sphinx.ext.napoleon" are
+sure that "sphinx.ext.napoleon" is enabled in `conf.py`::
-enabled in `conf.py`::
 
     # conf.py
 
     # Add any Sphinx extension module names here, as strings
-    extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon']
+    extensions = ['sphinx.ext.napoleon']
 
     # Napoleon settings
     napoleon_google_docstring = True

doc/ext/thirdparty.rst

@@ -6,7 +6,7 @@ repository.  It is open for anyone who wants to maintain an extension
 publicly; just send a short message asking for write permissions.
 
 There are also several extensions hosted elsewhere.  The `Sphinx extension
-survey <http://sphinxext-survey.readthedocs.org/en/latest/>`__ contains a
+survey <https://sphinxext-survey.readthedocs.io/>`__ contains a
 comprehensive list.
 
 If you write an extension that you think others will find useful or you think

doc/extdev/builderapi.rst

@@ -15,6 +15,7 @@ Builder API
 
    .. autoattribute:: name
    .. autoattribute:: format
+   .. autoattribute:: epilog
    .. autoattribute:: supported_image_types
 
    These methods are predefined and will be called from the application:

doc/extdev/index.rst

@@ -86,3 +86,129 @@ APIs used for writing extensions
    parserapi
    nodes
    logging
+
+Deprecated APIs
+---------------
+
+On developing Sphinx, we are always careful to the compatibility of our APIs.
+But, sometimes, the change of interface are needed for some reasons.  In such
+cases, we've marked thme as deprecated. And they are kept during the two
+major versions (for more details, please see :ref:`deprecation-policy`).
+
+The following is a list of deprecated interface.
+
+.. list-table:: deprecated APIs
+   :header-rows: 1
+
+   * - Target
+     - Deprecated
+     - (will be) Removed
+     - Alternatives
+
+   * - ``sphinx.build_main()``
+     - 1.7
+     - 2.0
+     - ``sphinx.cmd.build.build_main()``
+
+   * - ``sphinx.ext.intersphinx.debug()``
+     - 1.7
+     - 2.0
+     - ``sphinx.ext.intersphinx.inspect_main()``
+
+   * - ``sphinx.ext.autodoc.format_annotation()``
+     - 1.7
+     - 2.0
+     - ``sphinx.util.inspect.Signature``
+
+   * - ``sphinx.ext.autodoc.formatargspec()``
+     - 1.7
+     - 2.0
+     - ``sphinx.util.inspect.Signature``
+
+   * - ``sphinx.ext.autodoc.AutodocReporter``
+     - 1.7
+     - 2.0
+     - ``sphinx.util.docutils.switch_source_input()``
+
+   * - ``sphinx.ext.autodoc.add_documenter()``
+     - 1.7
+     - 2.0
+     - :meth:`~sphinx.application.Sphinx.add_autodocumenter()`
+
+   * - ``sphinx.ext.autodoc.AutoDirective._register``
+     - 1.7
+     - 2.0
+     - :meth:`~sphinx.application.Sphinx.add_autodocumenter()`
+
+   * - ``AutoDirective._special_attrgetters``
+     - 1.7
+     - 2.0
+     - :meth:`~sphinx.application.Sphinx.add_autodoc_attrgetter()`
+
+   * - ``Sphinx.warn()``, ``Sphinx.info()``
+     - 1.6
+     - 2.0
+     - :ref:`logging-api`
+
+   * - ``BuildEnvironment.set_warnfunc()``
+     - 1.6
+     - 2.0
+     - :ref:`logging-api`
+
+   * - ``BuildEnvironment.note_toctree()``
+     - 1.6
+     - 2.0
+     - ``Toctree.note()`` (in ``sphinx.environment.adapters.toctree``)
+
+   * - ``BuildEnvironment.get_toc_for()``
+     - 1.6
+     - 2.0
+     - ``Toctree.get_toc_for()`` (in ``sphinx.environment.adapters.toctree``)
+
+   * - ``BuildEnvironment.get_toctree_for()``
+     - 1.6
+     - 2.0
+     - ``Toctree.get_toctree_for()`` (in ``sphinx.environment.adapters.toctree``)
+
+   * - ``BuildEnvironment.create_index()``
+     - 1.6
+     - 2.0
+     - ``IndexEntries.create_index()`` (in ``sphinx.environment.adapters.indexentries``)
+
+   * - ``sphinx.websupport``
+     - 1.6
+     - 2.0
+     - `sphinxcontrib-websupport <https://pypi.python.org/pypi/sphinxcontrib-websupport>`_
+
+   * - ``StandaloneHTMLBuilder.css_files``
+     - 1.6
+     - 2.0
+     - :meth:`~sphinx.application.Sphinx.add_stylesheet()`
+
+   * - ``Sphinx.status_iterator()``
+     - 1.6
+     - 1.7
+     - ``sphinx.util.status_iterator()``
+
+   * - ``Sphinx.old_status_iterator()``
+     - 1.6
+     - 1.7
+     - ``sphinx.util.old_status_iterator()``
+
+   * - ``Sphinx._directive_helper()``
+     - 1.6
+     - 1.7
+     - ``sphinx.util.docutils.directive_helper()``
+
+   * - ``sphinx.util.compat.Directive``
+     - 1.6
+     - 1.7
+     - ``docutils.parsers.rst.Directive``
+
+   * - ``sphinx.util.compat.docutils_version``
+     - 1.6
+     - 1.7
+     - ``sphinx.util.docutils.__version_info__``
+
+.. note:: On deprecating on public APIs (internal functions and classes),
+          we also follow the policy as much as possible.

doc/extdev/markupapi.rst

@@ -117,12 +117,30 @@ Both APIs parse the content into a given node. They are used like this::
 
    node = docutils.nodes.paragraph()
    # either
-   from sphinx.ext.autodoc import AutodocReporter
-   self.state.memo.reporter = AutodocReporter(self.result, self.state.memo.reporter)  # override reporter to avoid errors from "include" directive
    nested_parse_with_titles(self.state, self.result, node)
    # or
    self.state.nested_parse(self.result, 0, node)
 
+.. note::
+
+   ``sphinx.util.docutils.switch_source_input()`` allows to change a target file
+   during nested_parse.  It is useful to mixed contents.  For example, ``sphinx.
+   ext.autodoc`` uses it to parse docstrings::
+
+       from sphinx.util.docutils import switch_source_input
+
+       # Switch source_input between parsing content.
+       # Inside this context, all parsing errors and warnings are reported as
+       # happened in new source_input (in this case, ``self.result``).
+       with switch_source_input(self.state, self.result):
+           node = docutils.nodes.paragraph()
+           self.state.nested_parse(self.result, 0, node)
+
+   .. deprecated:: 1.7
+
+      Until Sphinx-1.6, ``sphinx.ext.autodoc.AutodocReporter`` is used for this purpose.
+      For now, it is replaced by ``switch_source_input()``.
+
 If you don't need the wrapping node, you can use any concrete node type and
 return ``node.children`` from the Directive.
 

doc/faq.rst

@@ -58,7 +58,7 @@ Read the Docs
     Sphinx. They will host sphinx documentation, along with supporting a number
     of other features including version support, PDF generation, and more. The
     `Getting Started
-    <http://read-the-docs.readthedocs.org/en/latest/getting_started.html>`_
+    <https://read-the-docs.readthedocs.io/en/latest/getting_started.html>`_
     guide is a good place to start.
 
 Epydoc

doc/intro.rst

@@ -3,7 +3,7 @@ Introduction
 
 This is the documentation for the Sphinx documentation builder.  Sphinx is a
 tool that translates a set of reStructuredText_ source files into various output
-formats, automatically producing cross-references, indices etc.  That is, if
+formats, automatically producing cross-references, indices, etc.  That is, if
 you have a directory containing a bunch of reST-formatted documents (and
 possibly subdirectories of docs in there as well), Sphinx can generate a
 nicely-organized arrangement of HTML files (in some other directory) for easy
@@ -17,7 +17,7 @@ docs have a look at `Epydoc <http://epydoc.sourceforge.net/>`_, which also
 understands reST.
 
 For a great "introduction" to writing docs in general -- the whys and hows, see
-also `Write the docs <http://write-the-docs.readthedocs.org/>`_, written by Eric
+also `Write the docs <https://write-the-docs.readthedocs.io/>`_, written by Eric
 Holscher.
 
 .. _rinohtype: https://github.com/brechtm/rinohtype
@@ -38,7 +38,7 @@ to reStructuredText/Sphinx from other documentation systems.
   code to convert Python-doc-style LaTeX markup to Sphinx reST.
 
 * Marcin Wojdyr has written a script to convert Docbook to reST with Sphinx
-  markup; it is at `Google Code <https://github.com/wojdyr/db2rst>`_.
+  markup; it is at `GitHub <https://github.com/wojdyr/db2rst>`_.
 
 * Christophe de Vienne wrote a tool to convert from Open/LibreOffice documents
   to Sphinx: `odt2sphinx <https://pypi.python.org/pypi/odt2sphinx/>`_.

doc/latex.rst

@@ -29,6 +29,7 @@ The *latex* target does not benefit from pre-prepared themes like the
          cautionBgColor={named}{LightCyan}}
    \relax
 
+.. _latex-basic:
 
 Basic customization
 -------------------
@@ -61,17 +62,17 @@ It is achieved via usage of the
 .. highlight:: latex
 
 If the size of the ``'preamble'`` contents becomes inconvenient, one may move
-all needed macros into some file :file:`mystyle.tex` of the project source
+all needed macros into some file :file:`mystyle.tex.txt` of the project source
 repertory, and get LaTeX to import it at run time::
 
-   'preamble': r'\input{mystyle.tex}',
+   'preamble': r'\input{mystyle.tex.txt}',
    # or, if the \ProvidesPackage LaTeX macro is used in a file mystyle.sty
    'preamble': r'\usepackage{mystyle}',
 
 It is needed to set appropriately :confval:`latex_additional_files`, for
 example::
 
-   latex_additional_files = ["mystyle.tex"]
+   latex_additional_files = ["mystyle.sty"]
 
 .. _latexsphinxsetup:
 
@@ -266,6 +267,16 @@ The available styling options
 ``VerbatimBorderColor``
     default ``{rgb}{0,0,0}``. The frame color, defaults to black.
 
+``VerbatimHighlightColor``
+    default ``{rgb}{0.878,1,1}``. The color for highlighted lines.
+
+    .. versionadded:: 1.6.6
+
+.. note::
+
+   Starting with this colour key, and for all others coming next, the actual
+   names declared to "color" or "xcolor" are prefixed with "sphinx".
+
 ``verbatimsep``
     default ``\fboxsep``. The separation between code lines and the frame.
 
@@ -287,11 +298,6 @@ The available styling options
     default ``{rgb}{0,0,0}`` (black). The colour for the two horizontal rules
     used by Sphinx in LaTeX for styling a :dudir:`note` type admonition.
 
-.. note::
-
-   The actual colour names declared to "color" or "xcolor" are prefixed with
-   "sphinx".
-
 ``noteborder``, ``hintborder``, ``importantborder``, ``tipborder``
     default ``0.5pt``. The width of the two horizontal rules.
 
@@ -440,6 +446,11 @@ Environments
   .. versionadded:: 1.5
      options ``verbatimwithframe``, ``verbatimwrapslines``,
      ``verbatimsep``, ``verbatimborder``.
+  .. versionadded:: 1.6.6
+     support for ``:emphasize-lines:`` option
+  .. versionadded:: 1.6.6
+     easier customizability of the formatting via exposed to user LaTeX macros
+     such as ``\sphinxVerbatimHighlightLine``.
 - the bibliography uses ``sphinxthebibliography`` and the Python Module index
   as well as the general index both use ``sphinxtheindex``; these environments
   are wrappers of the ``thebibliography`` and respectively ``theindex``

doc/man/sphinx-apidoc.rst

@@ -91,7 +91,7 @@ Options
 
    Interpret paths recursively according to PEP-0420.
 
-.. option:: -M
+.. option:: -M, --module-first
 
    Put module documentation before submodule documentation.
 
@@ -118,6 +118,14 @@ These options are used when :option:`--full` is specified:
 
    Sets the project release to put in generated files (see :confval:`release`).
 
+Environment
+-----------
+
+.. envvar:: SPHINX_APIDOC_OPTIONS
+
+   A comma-separated list of option to append to generated ``automodule``
+   directives. Defaults to ``members,undoc-members,show-inheritance``.
+
 See also
 --------
 

doc/man/sphinx-build.rst

@@ -99,17 +99,16 @@ Options
    :ref:`builders <builders>`, the following build pipelines are available:
 
    **latexpdf**
-     Build LaTeX files and run them through :program:`pdflatex`.
+     Build LaTeX files and run them through :program:`pdflatex`, or as per
-
+     :confval:`latex_engine` setting.
-   **latexpdfja**
+     If :confval:`language` is set to ``'ja'``, will use automatically
-     Build LaTeX files and run them through :program:`platex/dvipdfmx`.
+     the :program:`platex/dvipdfmx` latex to PDF pipeline.
-     We recommend using ``latexpdf`` instead.
 
    **info**
      Build Texinfo files and run them through :program:`makeinfo`.
 
    .. important::
-     Sphinx only recognizes the ``-M`` option if it is placed first.
+      Sphinx only recognizes the ``-M`` option if it is placed first.
 
    .. versionadded:: 1.2.1
 
@@ -144,11 +143,15 @@ Options
 
    Distribute the build over *N* processes in parallel, to make building on
    multiprocessor machines more effective.  Note that not all parts and not all
-   builders of Sphinx can be parallelized.
+   builders of Sphinx can be parallelized.  If ``auto`` argument is given,
+   Sphinx uses the number of CPUs as *N*.
 
    .. versionadded:: 1.2
       This option should be considered *experimental*.
 
+   .. versionchanged:: 1.7
+      Support ``auto`` argument.
+
 .. option:: -c path
 
    Don't look for the :file:`conf.py` in the source directory, but use the given

doc/markup/code.rst

@@ -121,6 +121,8 @@ emphasize particular lines::
 .. versionchanged:: 1.3
    ``lineno-start`` has been added.
 
+.. versionchanged:: 1.6.6
+   LaTeX supports the ``emphasize-lines`` option.
 
 Includes
 ^^^^^^^^
@@ -188,8 +190,8 @@ Includes
    ``lines``, the first allowed line having by convention the line number ``1``.
 
    When lines have been selected in any of the ways described above, the
-   line numbers in ``emphasize-lines`` also refer to the selection, with the
+   line numbers in ``emphasize-lines`` refer to those selected lines, counted
-   first selected line having number ``1``.
+   consecutively starting at ``1``.
 
    When specifying particular parts of a file to display, it can be useful to
    display the original line numbers. This can be done using the

doc/markup/inline.rst

@@ -63,7 +63,7 @@ Cross-referencing anything
      by :rst:role:`doc`, :rst:role:`ref` or :rst:role:`option`.
 
      Custom objects added to the standard domain by extensions (see
-     :meth:`.add_object_type`) are also searched.
+     :meth:`Sphinx.add_object_type`) are also searched.
 
    * Then, it looks for objects (targets) in all loaded domains.  It is up to
      the domains how specific a match must be.  For example, in the Python
@@ -227,15 +227,15 @@ Cross-referencing figures by figure number
    reST labels are used.  When you use this role, it will insert a reference to
    the figure with link text by its figure number like "Fig. 1.1".
 
-   If an explicit link text is given (like usual: ``:numref:`Image of Sphinx (Fig.
+   If an explicit link text is given (as usual: ``:numref:`Image of Sphinx (Fig.
-   %s) <my-figure>```), the link caption will be the title of the reference.
+   %s) <my-figure>```), the link caption will serve as title of the reference.
-   As a special character, `%s` and `{number}` will be replaced to figure
+   As placeholders, `%s` and `{number}` get replaced by the figure
-   number.  `{name}` will be replaced to figure caption.
+   number and  `{name}` by the figure caption.
-   If no explicit link text is given, the value of :confval:`numfig_format` is
+   If no explicit link text is given, the :confval:`numfig_format` setting is
-   used to default value of link text.
+   used as fall-back default.
 
-   If :confval:`numfig` is ``False``, figures are not numbered.
+   If :confval:`numfig` is ``False``, figures are not numbered,
-   so this role inserts not a reference but labels or link text.
+   so this role inserts not a reference but the label or the link text.
 
 Cross-referencing other items of interest
 -----------------------------------------
@@ -355,7 +355,8 @@ in a different style:
 .. rst:role:: manpage
 
    A reference to a Unix manual page including the section,
-   e.g. ``:manpage:`ls(1)```.
+   e.g. ``:manpage:`ls(1)```. Creates a hyperlink to an external site
+   rendering the manpage if :confval:`manpages_url` is defined.
 
 .. rst:role:: menuselection
 

doc/markup/misc.rst

@@ -323,6 +323,11 @@ following directive exists:
       Sphinx's merged cells interact well with ``p{width}``, ``\X{a}{b}``, ``Y{f}``
       and tabulary's columns.
 
+   .. note::
+
+      :rst:dir:`tabularcolumns` conflicts with ``:widths:`` option of table
+      directives.  If both are specified, ``:widths:`` option will be ignored.
+
 Math
 ----
 

doc/theming.rst

@@ -114,8 +114,19 @@ These themes are:
   - **nosidebar** (true or false): Don't include the sidebar.  Defaults to
     ``False``.
 
-  - **sidebarwidth** (an integer): Width of the sidebar in pixels.  (Do not
+  - **sidebarwidth** (int or str): Width of the sidebar in pixels.
-    include ``px`` in the value.)  Defaults to 230 pixels.
+    This can be an int, which is interpreted as pixels or a valid CSS
+    dimension string such as '70em' or '50%'.  Defaults to 230 pixels.
+
+  - **body_min_width** (int or str): Minimal width of the document body.
+    This can be an int, which is interpreted as pixels or a valid CSS
+    dimension string such as '70em' or '50%'. Use 0 if you don't want
+    a width limit. Defaults may depend on the theme (often 450px).
+
+  - **body_max_width** (int or str): Maximal width of the document body.
+    This can be an int, which is interpreted as pixels or a valid CSS
+    dimension string such as '70em' or '50%'. Use 'none' if you don't
+    want a width limit. Defaults may depend on the theme (often 800px).
 
 * **alabaster** -- `Alabaster theme`_ is a modified "Kr" Sphinx theme from @kennethreitz
   (especially as used in his Requests project), which was itself originally based on
@@ -276,6 +287,7 @@ Python :mod:`ConfigParser` module) and has the following structure:
     inherit = base theme
     stylesheet = main CSS name
     pygments_style = stylename
+    sidebars = localtoc.html, relations.html, sourcelink.html, searchbox.html
 
     [options]
     variable = default value
@@ -295,10 +307,16 @@ Python :mod:`ConfigParser` module) and has the following structure:
   highlighting.  This can be overridden by the user in the
   :confval:`pygments_style` config value.
 
+* The **sidebars** setting gives the comma separated list of sidebar templates
+  for constructing sidebars.  This can be overridden by the user in the
+  :confval:`html_sidebars` config value.
+
 * The **options** section contains pairs of variable names and default values.
   These options can be overridden by the user in :confval:`html_theme_options`
   and are accessible from all templates as ``theme_<name>``.
 
+.. versionadded:: 1.7
+   sidebar settings
 
 .. _distribute-your-theme:
 

setup.cfg

@@ -1,11 +1,20 @@
+[metadata]
+license_file = LICENSE
+
 [egg_info]
 tag_build = .dev
 tag_date = true
 
+[bdist_wheel]
+universal = 1
+
 [aliases]
 release = egg_info -Db ''
 upload = upload --sign --identity=36580288
 
+[build_sphinx]
+warning-is-error = 1
+
 [extract_messages]
 mapping_file = babel.cfg
 output_file = sphinx/locale/sphinx.pot
@@ -20,13 +29,18 @@ output_dir = sphinx/locale/
 domain = sphinx
 directory = sphinx/locale/
 
-[wheel]
-universal = 1
-
 [flake8]
 max-line-length = 95
-ignore = E116,E241,E251
+ignore = E116,E241,E251,E741,I101
-exclude = .git,.tox,.venv,tests/*,build/*,sphinx/search/*,sphinx/pycode/pgen2/*,doc/ext/example*.py
+exclude = .git,.tox,.venv,tests/*,build/*,doc/_build/*,sphinx/search/*,doc/ext/example*.py
+application-import-names = sphinx
+import-order-style = smarkets
+
+[flake8:local-plugins]
+extension =
+    X101 = utils.checks:sphinx_has_header
+paths =
+    .
 
 [mypy]
 python_version = 2.7
@@ -38,5 +52,20 @@ incremental = True
 check_untyped_defs = True
 warn_unused_ignores = True
 
-[build_sphinx]
+[tool:pytest]
-warning-is-error = 1
+filterwarnings =
+    ignore::DeprecationWarning:docutils.io
+
+[coverage:run]
+branch = True
+source = sphinx
+
+[coverage:report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+    # Don't complain if tests don't hit defensive assertion code:
+    raise NotImplementedError
+    # Don't complain if non-runnable code isn't run:
+    if __name__ == .__main__.:
+ignore_errors = True

setup.py

@@ -1,47 +1,20 @@
 # -*- coding: utf-8 -*-
-from setuptools import setup, find_packages
-
 import os
 import sys
 from distutils import log
-from distutils.cmd import Command
+
+from setuptools import find_packages, setup
 
 import sphinx
 
-long_desc = '''
+with open('README.rst') as f:
-Sphinx is a tool that makes it easy to create intelligent and beautiful
+    long_desc = f.read()
-documentation for Python projects (or other documents consisting of multiple
-reStructuredText sources), written by Georg Brandl.  It was originally created
-for the new Python documentation, and has excellent facilities for Python
-project documentation, but C/C++ is supported as well, and more languages are
-planned.
-
-Sphinx uses reStructuredText as its markup language, and many of its strengths
-come from the power and straightforwardness of reStructuredText and its parsing
-and translating suite, the Docutils.
-
-Among its features are the following:
-
-* Output formats: HTML (including derivative formats such as HTML Help, Epub
-  and Qt Help), plain text, manual pages and LaTeX or direct PDF output
-  using rst2pdf
-* Extensive cross-references: semantic markup and automatic links
-  for functions, classes, glossary terms and similar pieces of information
-* Hierarchical structure: easy definition of a document tree, with automatic
-  links to siblings, parents and children
-* Automatic indices: general index as well as a module index
-* Code handling: automatic highlighting using the Pygments highlighter
-* Flexible HTML output using the Jinja 2 templating engine
-* Various extensions are available, e.g. for automatic testing of snippets
-  and inclusion of appropriately formatted docstrings
-* Setuptools integration
-'''
 
 if sys.version_info < (2, 7) or (3, 0) <= sys.version_info < (3, 4):
     print('ERROR: Sphinx requires at least Python 2.7 or 3.4 to run.')
     sys.exit(1)
 
-requires = [
+install_requires = [
     'six>=1.5',
     'Jinja2>=2.3',
     'Pygments>=2.0',
@@ -52,6 +25,7 @@ requires = [
     'imagesize',
     'requests>=2.0.0',
     'setuptools',
+    'packaging',
     'sphinxcontrib-websupport',
 ]
 
@@ -68,20 +42,22 @@ extras_require = {
         'whoosh>=2.0',
     ],
     'test': [
+        'mock',
         'pytest',
-        'mock',  # it would be better for 'test:python_version in 2.7'
+        'pytest-cov',
-        'simplejson',  # better: 'test:platform_python_implementation=="PyPy"'
         'html5lib',
+        'flake8>=3.5.0',
+        'flake8-import-order',
+    ],
+    'test:python_version<"3"': [
+        'enum34',
+    ],
+    'test:python_version>="3"': [
+        'mypy',
+        'typed_ast',
     ],
 }
 
-# for sdist installation with pip-1.5.6
-if sys.platform == 'win32':
-    requires.append('colorama>=0.3.5')
-
-if sys.version_info < (3, 5):
-    requires.append('typing')
-
 # Provide a "compile_catalog" command that also creates the translated
 # JavaScript files if Babel is available.
 
@@ -90,10 +66,7 @@ cmdclass = {}
 try:
     from babel.messages.pofile import read_po
     from babel.messages.frontend import compile_catalog
-    try:
+    from json import dump
-        from simplejson import dump
-    except ImportError:
-        from json import dump
 except ImportError:
     pass
 else:
@@ -178,32 +151,6 @@ else:
     cmdclass['compile_catalog'] = compile_catalog_plusjs
 
 
-class CompileGrammarCommand(Command):
-    description = 'Compile python grammar file for pycode'
-    user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        from sphinx.pycode.pgen2.driver import compile_grammar
-
-        compile_grammar('sphinx/pycode/Grammar-py2.txt')
-        print('sphinx/pycode/Grammar-py2.txt ... done')
-
-        compile_grammar('sphinx/pycode/Grammar-py3.txt')
-        print('sphinx/pycode/Grammar-py3.txt ... done')
-
-    def sub_commands(self):
-        pass
-
-
-cmdclass['compile_grammar'] = CompileGrammarCommand
-
-
 setup(
     name='Sphinx',
     version=sphinx.__version__,
@@ -225,7 +172,13 @@ setup(
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy',
         'Framework :: Sphinx',
         'Framework :: Sphinx :: Extension',
         'Framework :: Sphinx :: Theme',
@@ -235,7 +188,7 @@ setup(
         'Topic :: Utilities',
     ],
     platforms='any',
-    packages=find_packages(exclude=['tests']),
+    packages=find_packages(exclude=['tests', 'utils']),
     include_package_data=True,
     entry_points={
         'console_scripts': [
@@ -248,7 +201,7 @@ setup(
             'build_sphinx = sphinx.setup_command:BuildDoc',
         ],
     },
-    install_requires=requires,
+    install_requires=install_requires,
     extras_require=extras_require,
     cmdclass=cmdclass,
 )

sphinx-apidoc.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.ext.apidoc import main
-    sys.exit(main(sys.argv[1:]))

sphinx-autogen.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.ext.autosummary.generate import main
-    sys.exit(main(sys.argv[1:]))

sphinx-build.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx import main
-    sys.exit(main(sys.argv[1:]))

sphinx-quickstart.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Sphinx - Python documentation toolchain
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import sys
-
-if __name__ == '__main__':
-    from sphinx.cmd.quickstart import main
-    sys.exit(main(sys.argv[1:]))

sphinx/__init__.py

@@ -5,7 +5,7 @@
 
     The Sphinx documentation toolchain.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -15,10 +15,10 @@
 from __future__ import absolute_import
 
 import os
+import sys
 import warnings
 from os import path
 
-from .cmd import build
 from .deprecation import RemovedInNextVersionWarning
 from .deprecation import RemovedInSphinx20Warning
 
@@ -31,13 +31,13 @@ if 'PYTHONWARNINGS' not in os.environ:
 warnings.filterwarnings('ignore', "'U' mode is deprecated",
                         DeprecationWarning, module='docutils.io')
 
-__version__ = '1.7+'
+__version__ = '1.7.2+'
-__released__ = '1.7'  # used when Sphinx builds its own docs
+__released__ = '1.7.2'  # used when Sphinx builds its own docs
 
 # version info for better programmatic use
 # possible values for 3rd element: 'alpha', 'beta', 'rc', 'final'
 # 'final' has 0 as the last element
-version_info = (1, 7, 0, 'beta', 0)
+version_info = (1, 7, 2, 'beta', 0)
 
 package_dir = path.abspath(path.dirname(__file__))
 
@@ -61,15 +61,40 @@ if __version__.endswith('+'):
 
 
 def main(*args, **kwargs):
+    from .cmd import build
     warnings.warn(
         '`sphinx.main()` has moved to `sphinx.cmd.build.main()`.',
         RemovedInSphinx20Warning,
         stacklevel=2,
     )
-    build.main(*args, **kwargs)
+    args = args[1:]  # skip first argument to adjust arguments (refs: #4615)
+    return build.main(*args, **kwargs)
+
+
+def build_main(argv=sys.argv):
+    """Sphinx build "main" command-line entry."""
+    from .cmd import build
+    warnings.warn(
+        '`sphinx.build_main()` has moved to `sphinx.cmd.build.build_main()`.',
+        RemovedInSphinx20Warning,
+        stacklevel=2,
+    )
+    return build.build_main(argv[1:])  # skip first argument to adjust arguments (refs: #4615)
+
+
+def make_main(argv=sys.argv):
+    """Sphinx build "make mode" entry."""
+    from .cmd import build
+    warnings.warn(
+        '`sphinx.build_main()` has moved to `sphinx.cmd.build.make_main()`.',
+        RemovedInSphinx20Warning,
+        stacklevel=2,
+    )
+    return build.make_main(argv[1:])  # skip first argument to adjust arguments (refs: #4615)
 
 
 if __name__ == '__main__':
+    from .cmd import build
     warnings.warn(
         '`sphinx` has moved to `sphinx.build`.',
         RemovedInSphinx20Warning,

sphinx/__main__.py

@@ -5,10 +5,12 @@
 
     The Sphinx documentation toolchain.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
+
 import sys
-from sphinx import main
+
+from sphinx.cmd.build import main
 
 sys.exit(main(sys.argv[1:]))

sphinx/addnodes.py

@@ -5,14 +5,15 @@
 
     Additional docutils nodes.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from typing import TYPE_CHECKING
+
 from docutils import nodes
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import List, Sequence  # NOQA
 
 

sphinx/apidoc.py

@@ -5,7 +5,7 @@
 
     This file has moved to :py:mod:`sphinx.ext.apidoc`.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -21,6 +21,7 @@ def main(*args, **kwargs):
         RemovedInSphinx20Warning,
         stacklevel=2,
     )
+    args = args[1:]  # skip first argument to adjust arguments (refs: #4615)
     _main(*args, **kwargs)
 
 

sphinx/application.py

@@ -7,46 +7,44 @@
 
     Gracefully adapted from the TextPress system by Armin.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 from __future__ import print_function
 
 import os
+import posixpath
 import sys
 import warnings
-import posixpath
-from os import path
 from collections import deque
-
+from os import path
-from six import iteritems
+from typing import TYPE_CHECKING
-from six.moves import cStringIO
 
 from docutils import nodes
 from docutils.parsers.rst import directives, roles
+from six import iteritems, itervalues
+from six.moves import cStringIO
 
 import sphinx
 from sphinx import package_dir, locale
 from sphinx.config import Config
-from sphinx.errors import ConfigError, ExtensionError, VersionRequirementError
 from sphinx.deprecation import RemovedInSphinx20Warning
 from sphinx.environment import BuildEnvironment
+from sphinx.errors import ConfigError, ExtensionError, VersionRequirementError
 from sphinx.events import EventManager
 from sphinx.extension import verify_required_extensions
-from sphinx.io import SphinxStandaloneReader
 from sphinx.locale import __
 from sphinx.registry import SphinxComponentRegistry
-from sphinx.util import pycompat  # noqa: F401
 from sphinx.util import import_object
 from sphinx.util import logging
-from sphinx.util.tags import Tags
+from sphinx.util import pycompat  # noqa: F401
-from sphinx.util.osutil import ENOENT
 from sphinx.util.console import bold  # type: ignore
 from sphinx.util.docutils import is_html5_writer_available, directive_helper
 from sphinx.util.i18n import find_catalog_source_files
+from sphinx.util.osutil import ENOENT, ensuredir
+from sphinx.util.tags import Tags
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Callable, Dict, IO, Iterable, Iterator, List, Tuple, Type, Union  # NOQA
     from docutils.parsers import Parser  # NOQA
     from docutils.transform import Transform  # NOQA
@@ -54,7 +52,9 @@ if False:
     from sphinx.domains import Domain, Index  # NOQA
     from sphinx.environment.collectors import EnvironmentCollector  # NOQA
     from sphinx.extension import Extension  # NOQA
+    from sphinx.roles import XRefRole  # NOQA
     from sphinx.theming import Theme  # NOQA
+    from sphinx.util.typing import RoleFunction  # NOQA
 
 builtin_extensions = (
     'sphinx.builders.applehelp',
@@ -83,10 +83,12 @@ builtin_extensions = (
     'sphinx.directives.code',
     'sphinx.directives.other',
     'sphinx.directives.patches',
+    'sphinx.io',
     'sphinx.parsers',
     'sphinx.roles',
     'sphinx.transforms.post_transforms',
     'sphinx.transforms.post_transforms.images',
+    'sphinx.util.compat',
     # collectors should be loaded by specific order
     'sphinx.environment.collectors.dependencies',
     'sphinx.environment.collectors.asset',
@@ -119,10 +121,9 @@ class Sphinx(object):
         self.env = None                         # type: BuildEnvironment
         self.registry = SphinxComponentRegistry()
         self.enumerable_nodes = {}              # type: Dict[nodes.Node, Tuple[unicode, Callable]]  # NOQA
-        self.post_transforms = []               # type: List[Transform]
         self.html_themes = {}                   # type: Dict[unicode, unicode]
 
-        self.srcdir = srcdir
+        self.srcdir = srcdir                    # type: unicode
         self.confdir = confdir
         self.outdir = outdir
         self.doctreedir = doctreedir
@@ -156,10 +157,6 @@ class Sphinx(object):
         # status code for command-line application
         self.statuscode = 0
 
-        if not path.isdir(outdir):
-            logger.info('making output directory...')
-            os.makedirs(outdir)
-
         # read config
         self.tags = Tags(tags)
         self.config = Config(confdir, CONFIG_FILENAME,
@@ -196,6 +193,10 @@ class Sphinx(object):
         # preload builder module (before init config values)
         self.preload_builder(buildername)
 
+        if not path.isdir(outdir):
+            logger.info('making output directory...')
+            ensuredir(outdir)
+
         # the config file itself can be an extension
         if self.config.setup:
             self._setting_up_extension = ['conf.py']
@@ -337,6 +338,13 @@ class Sphinx(object):
                                  (status, self._warncount)))
             else:
                 logger.info(bold(__('build %s.') % status))
+
+            if self.statuscode == 0 and self.builder.epilog:
+                logger.info('')
+                logger.info(self.builder.epilog % {
+                    'outdir': path.relpath(self.outdir),
+                    'project': self.config.project
+                })
         except Exception as err:
             # delete the saved env to force a fresh build next time
             envfile = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
@@ -443,7 +451,6 @@ class Sphinx(object):
 
     def add_builder(self, builder):
         # type: (Type[Builder]) -> None
-        logger.debug('[app] adding builder: %r', builder)
         self.registry.add_builder(builder)
 
     def add_config_value(self, name, default, rebuild, types=()):
@@ -463,7 +470,6 @@ class Sphinx(object):
 
     def set_translator(self, name, translator_class):
         # type: (unicode, Type[nodes.NodeVisitor]) -> None
-        logger.info(bold(__('Change of translator for the %s builder.') % name))
         self.registry.add_translator(name, translator_class)
 
     def add_node(self, node, **kwds):
@@ -552,39 +558,30 @@ class Sphinx(object):
 
     def add_domain(self, domain):
         # type: (Type[Domain]) -> None
-        logger.debug('[app] adding domain: %r', domain)
         self.registry.add_domain(domain)
 
     def override_domain(self, domain):
         # type: (Type[Domain]) -> None
-        logger.debug('[app] overriding domain: %r', domain)
         self.registry.override_domain(domain)
 
     def add_directive_to_domain(self, domain, name, obj,
                                 has_content=None, argument_spec=None, **option_spec):
         # type: (unicode, unicode, Any, bool, Any, Any) -> None
-        logger.debug('[app] adding directive to domain: %r',
-                     (domain, name, obj, has_content, argument_spec, option_spec))
         self.registry.add_directive_to_domain(domain, name, obj,
                                               has_content, argument_spec, **option_spec)
 
     def add_role_to_domain(self, domain, name, role):
-        # type: (unicode, unicode, Any) -> None
+        # type: (unicode, unicode, Union[RoleFunction, XRefRole]) -> None
-        logger.debug('[app] adding role to domain: %r', (domain, name, role))
         self.registry.add_role_to_domain(domain, name, role)
 
     def add_index_to_domain(self, domain, index):
         # type: (unicode, Type[Index]) -> None
-        logger.debug('[app] adding index to domain: %r', (domain, index))
         self.registry.add_index_to_domain(domain, index)
 
     def add_object_type(self, directivename, rolename, indextemplate='',
                         parse_node=None, ref_nodeclass=None, objname='',
                         doc_field_types=[]):
         # type: (unicode, unicode, unicode, Callable, nodes.Node, unicode, List) -> None
-        logger.debug('[app] adding object type: %r',
-                     (directivename, rolename, indextemplate, parse_node,
-                      ref_nodeclass, objname, doc_field_types))
         self.registry.add_object_type(directivename, rolename, indextemplate, parse_node,
                                       ref_nodeclass, objname, doc_field_types)
 
@@ -601,21 +598,16 @@ class Sphinx(object):
     def add_crossref_type(self, directivename, rolename, indextemplate='',
                           ref_nodeclass=None, objname=''):
         # type: (unicode, unicode, unicode, nodes.Node, unicode) -> None
-        logger.debug('[app] adding crossref type: %r',
-                     (directivename, rolename, indextemplate, ref_nodeclass,
-                      objname))
         self.registry.add_crossref_type(directivename, rolename,
                                         indextemplate, ref_nodeclass, objname)
 
     def add_transform(self, transform):
         # type: (Type[Transform]) -> None
-        logger.debug('[app] adding transform: %r', transform)
+        self.registry.add_transform(transform)
-        SphinxStandaloneReader.transforms.append(transform)
 
     def add_post_transform(self, transform):
         # type: (Type[Transform]) -> None
-        logger.debug('[app] adding post transform: %r', transform)
+        self.registry.add_post_transform(transform)
-        self.post_transforms.append(transform)
 
     def add_javascript(self, filename):
         # type: (unicode) -> None
@@ -657,15 +649,14 @@ class Sphinx(object):
     def add_autodocumenter(self, cls):
         # type: (Any) -> None
         logger.debug('[app] adding autodocumenter: %r', cls)
-        from sphinx.ext import autodoc
+        from sphinx.ext.autodoc.directive import AutodocDirective
-        autodoc.add_documenter(cls)
+        self.registry.add_documenter(cls.objtype, cls)
-        self.add_directive('auto' + cls.objtype, autodoc.AutoDirective)
+        self.add_directive('auto' + cls.objtype, AutodocDirective)
 
-    def add_autodoc_attrgetter(self, type, getter):
+    def add_autodoc_attrgetter(self, typ, getter):
-        # type: (Any, Callable) -> None
+        # type: (Type, Callable[[Any, unicode, Any], Any]) -> None
-        logger.debug('[app] adding autodoc attrgetter: %r', (type, getter))
+        logger.debug('[app] adding autodoc attrgetter: %r', (typ, getter))
-        from sphinx.ext import autodoc
+        self.registry.add_autodoc_attrgetter(typ, getter)
-        autodoc.AutoDirective._special_attrgetters[type] = getter
 
     def add_search_language(self, cls):
         # type: (Any) -> None
@@ -676,7 +667,6 @@ class Sphinx(object):
 
     def add_source_parser(self, suffix, parser):
         # type: (unicode, Parser) -> None
-        logger.debug('[app] adding search source_parser: %r, %r', suffix, parser)
         self.registry.add_source_parser(suffix, parser)
 
     def add_env_collector(self, collector):
@@ -689,6 +679,39 @@ class Sphinx(object):
         logger.debug('[app] adding HTML theme: %r, %r', name, theme_path)
         self.html_themes[name] = theme_path
 
+    # ---- other methods -------------------------------------------------
+    def is_parallel_allowed(self, typ):
+        # type: (unicode) -> bool
+        """Check parallel processing is allowed or not.
+
+        ``typ`` is a type of processing; ``'read'`` or ``'write'``.
+        """
+        if typ == 'read':
+            attrname = 'parallel_read_safe'
+            message = __("the %s extension does not declare if it is safe "
+                         "for parallel reading, assuming it isn't - please "
+                         "ask the extension author to check and make it "
+                         "explicit")
+        elif typ == 'write':
+            attrname = 'parallel_write_safe'
+            message = __("the %s extension does not declare if it is safe "
+                         "for parallel writing, assuming it isn't - please "
+                         "ask the extension author to check and make it "
+                         "explicit")
+        else:
+            raise ValueError('parallel type %s is not supported' % typ)
+
+        for ext in itervalues(self.extensions):
+            allowed = getattr(ext, attrname, None)
+            if allowed is None:
+                logger.warning(message, ext.name)
+                logger.warning('doing serial %s', typ)
+                return False
+            elif not allowed:
+                return False
+
+        return True
+
 
 class TemplateBridge(object):
     """

sphinx/builders/__init__.py

@@ -5,28 +5,22 @@
 
     Builder superclass for all builders.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-import os
-from os import path
 import warnings
+from os import path
+from typing import TYPE_CHECKING
 
-try:
-    import multiprocessing
-except ImportError:
-    multiprocessing = None
-
-from six import itervalues
 from docutils import nodes
 
 from sphinx.deprecation import RemovedInSphinx20Warning
 from sphinx.environment.adapters.asset import ImageAdapter
-from sphinx.util import i18n, path_stabilize, logging, status_iterator
+from sphinx.util import i18n, logging, status_iterator
-from sphinx.util.osutil import SEP, relative_uri
-from sphinx.util.i18n import find_catalog
 from sphinx.util.console import bold  # type: ignore
+from sphinx.util.i18n import find_catalog
+from sphinx.util.osutil import SEP, ensuredir, relative_uri
 from sphinx.util.parallel import ParallelTasks, SerialTasks, make_chunks, \
     parallel_available
 
@@ -34,8 +28,12 @@ from sphinx.util.parallel import ParallelTasks, SerialTasks, make_chunks, \
 from sphinx import roles       # noqa
 from sphinx import directives  # noqa
 
-if False:
+try:
-    # For type annotation
+    import multiprocessing
+except ImportError:
+    multiprocessing = None
+
+if TYPE_CHECKING:
     from typing import Any, Callable, Dict, Iterable, List, Sequence, Set, Tuple, Union  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.config import Config  # NOQA
@@ -56,6 +54,11 @@ class Builder(object):
     name = ''  # type: unicode
     #: The builder's output format, or '' if no document output is produced.
     format = ''  # type: unicode
+    #: The message emitted upon successful build completion. This can be a
+    #: printf-style template string with the following keys: ``outdir``,
+    #: ``project``
+    epilog = ''  # type: unicode
+
     # default translator class for the builder.  This will be overrided by
     # ``app.set_translator()``.
     default_translator_class = None  # type: nodes.NodeVisitor
@@ -79,8 +82,7 @@ class Builder(object):
         self.confdir = app.confdir
         self.outdir = app.outdir
         self.doctreedir = app.doctreedir
-        if not path.isdir(self.doctreedir):
+        ensuredir(self.doctreedir)
-            os.makedirs(self.doctreedir)
 
         self.app = app              # type: Sphinx
         self.env = None             # type: BuildEnvironment
@@ -253,11 +255,14 @@ class Builder(object):
         # type: (List[unicode]) -> None
         def to_domain(fpath):
             # type: (unicode) -> unicode
-            docname, _ = path.splitext(path_stabilize(fpath))
+            docname = self.env.path2doc(path.abspath(fpath))
-            dom = find_catalog(docname, self.config.gettext_compact)
+            if docname:
-            return dom
+                return find_catalog(docname, self.config.gettext_compact)
+            else:
+                return None
 
         specified_domains = set(map(to_domain, specified_files))
+        specified_domains.discard(None)
         catalogs = i18n.find_catalog_source_files(
             [path.join(self.srcdir, x) for x in self.config.locale_dirs],
             self.config.language,
@@ -373,15 +378,10 @@ class Builder(object):
             docnames = set(docnames) & self.env.found_docs
 
         # determine if we can write in parallel
-        self.parallel_ok = False
         if parallel_available and self.app.parallel > 1 and self.allow_parallel:
-            self.parallel_ok = True
+            self.parallel_ok = self.app.is_parallel_allowed('write')
-            for extension in itervalues(self.app.extensions):
+        else:
-                if not extension.parallel_write_safe:
+            self.parallel_ok = False
-                    logger.warning('the %s extension is not safe for parallel '
-                                   'writing, doing serial write', extension.name)
-                    self.parallel_ok = False
-                    break
 
         #  create a task executor to use for misc. "finish-up" tasks
         # if self.parallel_ok:

sphinx/builders/_epub_base.py

@@ -5,16 +5,27 @@
 
     Base class of epub2/epub3 builders.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import os
 import re
-from os import path
-from sphinx.util.i18n import format_date
-from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
 from collections import namedtuple
+from os import path
+from typing import TYPE_CHECKING
+from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
+
+from docutils import nodes
+from docutils.utils import smartquotes
+
+from sphinx import addnodes
+from sphinx.builders.html import BuildInfo, StandaloneHTMLBuilder
+from sphinx.util import logging
+from sphinx.util import status_iterator
+from sphinx.util.fileutil import copy_asset_file
+from sphinx.util.i18n import format_date
+from sphinx.util.osutil import ensuredir, copyfile
 
 try:
     from PIL import Image
@@ -24,18 +35,7 @@ except ImportError:
     except ImportError:
         Image = None
 
-from docutils import nodes
+if TYPE_CHECKING:
-from docutils.utils import smartquotes
-
-from sphinx import addnodes
-from sphinx.builders.html import StandaloneHTMLBuilder
-from sphinx.util import logging
-from sphinx.util import status_iterator
-from sphinx.util.osutil import ensuredir, copyfile
-from sphinx.util.fileutil import copy_asset_file
-
-if False:
-    # For type annotation
     from typing import Any, Dict, List, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -159,6 +159,10 @@ class EpubBuilder(StandaloneHTMLBuilder):
         self.id_cache = {}  # type: Dict[unicode, unicode]
         self.use_index = self.get_builder_config('use_index', 'epub')
 
+    def create_build_info(self):
+        # type: () -> BuildInfo
+        return BuildInfo(self.config, self.tags, ['html', 'epub'])
+
     def get_theme_config(self):
         # type: () -> Tuple[unicode, Dict]
         return self.config.epub_theme, self.config.epub_theme_options

sphinx/builders/applehelp.py

@@ -5,32 +5,30 @@
 
     Build Apple help books.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 from __future__ import print_function
 
 import codecs
 import pipes
-
+import plistlib
-from os import path, environ
 import shlex
+import subprocess
+from os import path, environ
+from typing import TYPE_CHECKING
 
 from sphinx.builders.html import StandaloneHTMLBuilder
 from sphinx.config import string_classes
+from sphinx.errors import SphinxError
 from sphinx.util import logging
-from sphinx.util.osutil import copyfile, ensuredir, make_filename
 from sphinx.util.console import bold  # type: ignore
 from sphinx.util.fileutil import copy_asset
-from sphinx.util.pycompat import htmlescape
 from sphinx.util.matching import Matcher
-from sphinx.errors import SphinxError
+from sphinx.util.osutil import copyfile, ensuredir, make_filename
+from sphinx.util.pycompat import htmlescape
 
-import plistlib
+if TYPE_CHECKING:
-import subprocess
-
-if False:
-    # For type annotation
     from typing import Any, Dict  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -75,6 +73,10 @@ class AppleHelpBuilder(StandaloneHTMLBuilder):
     on the ``hiutil`` command line tool.
     """
     name = 'applehelp'
+    epilog = ('The help book is in %(outdir)s.\n'
+              'Note that won\'t be able to view it unless you put it in '
+              '~/Library/Documentation/Help or install it in your application '
+              'bundle.')
 
     # don't copy the reST source
     copysource = False

sphinx/builders/changes.py

@@ -5,27 +5,27 @@
 
     Changelog builder.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import codecs
 from os import path
+from typing import TYPE_CHECKING
 
 from six import iteritems
 
 from sphinx import package_dir
+from sphinx.builders import Builder
 from sphinx.locale import _
 from sphinx.theming import HTMLThemeFactory
-from sphinx.builders import Builder
 from sphinx.util import logging
-from sphinx.util.osutil import ensuredir, os_path
 from sphinx.util.console import bold  # type: ignore
 from sphinx.util.fileutil import copy_asset_file
+from sphinx.util.osutil import ensuredir, os_path
 from sphinx.util.pycompat import htmlescape
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -38,6 +38,7 @@ class ChangesBuilder(Builder):
     Write a summary with all versionadded/changed directives.
     """
     name = 'changes'
+    epilog = 'The overview file is in %(outdir)s.'
 
     def init(self):
         # type: () -> None

sphinx/builders/devhelp.py

@@ -7,30 +7,30 @@
 
     .. _Devhelp: http://live.gnome.org/devhelp
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 from __future__ import absolute_import
 
-import re
 import gzip
+import re
 from os import path
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 
 from sphinx import addnodes
-from sphinx.util import logging
-from sphinx.util.osutil import make_filename
 from sphinx.builders.html import StandaloneHTMLBuilder
 from sphinx.environment.adapters.indexentries import IndexEntries
+from sphinx.util import logging
+from sphinx.util.osutil import make_filename
 
 try:
     import xml.etree.ElementTree as etree
 except ImportError:
     import lxml.etree as etree  # type: ignore
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -43,6 +43,10 @@ class DevhelpBuilder(StandaloneHTMLBuilder):
     Builder that also outputs GNOME Devhelp file.
     """
     name = 'devhelp'
+    epilog = ('To view the help file:\n'
+              '$ mkdir -p $HOME/.local/share/devhelp/%(project)s\n'
+              '$ ln -s %(outdir)s $HOME/.local/share/devhelp/%(project)s\n'
+              '$ devhelp')
 
     # don't copy the reST source
     copysource = False

sphinx/builders/dummy.py

@@ -9,11 +9,11 @@
     :license: BSD, see LICENSE for details.
 """
 
+from typing import TYPE_CHECKING
 
 from sphinx.builders import Builder
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Set  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.application import Sphinx  # NOQA
@@ -21,6 +21,8 @@ if False:
 
 class DummyBuilder(Builder):
     name = 'dummy'
+    epilog = 'The dummy builder generates no files.'
+
     allow_parallel = True
 
     def init(self):

sphinx/builders/epub3.py

@@ -10,19 +10,19 @@
     :license: BSD, see LICENSE for details.
 """
 
-from os import path
 from collections import namedtuple
+from os import path
+from typing import TYPE_CHECKING
 
 from sphinx import package_dir
-from sphinx.config import string_classes, ENUM
 from sphinx.builders import _epub_base
+from sphinx.config import string_classes, ENUM
 from sphinx.util import logging, xmlname_checker
 from sphinx.util.fileutil import copy_asset_file
 from sphinx.util.i18n import format_date
 from sphinx.util.osutil import make_filename
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Iterable, List  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.application import Sphinx  # NOQA
@@ -63,6 +63,7 @@ class Epub3Builder(_epub_base.EpubBuilder):
     an epub file.
     """
     name = 'epub'
+    epilog = 'The ePub file is in %(outdir)s.'
 
     supported_remote_images = False
     template_dir = path.join(package_dir, 'templates', 'epub3')
@@ -132,6 +133,7 @@ class Epub3Builder(_epub_base.EpubBuilder):
         metadata['ibook_scroll_axis'] = IBOOK_SCROLL_AXIS.get(writing_mode)
         metadata['date'] = self.esc(format_date("%Y-%m-%dT%H:%M:%SZ"))
         metadata['version'] = self.esc(self.config.version)
+        metadata['epub_version'] = self.config.epub_version
         return metadata
 
     def prepare_writing(self, docnames):
@@ -142,6 +144,7 @@ class Epub3Builder(_epub_base.EpubBuilder):
         self.globalcontext['theme_writing_mode'] = THEME_WRITING_MODES.get(writing_mode)
         self.globalcontext['html_tag'] = self.html_tag
         self.globalcontext['use_meta_charset'] = self.use_meta_charset
+        self.globalcontext['skip_ua_compatible'] = True
 
     def build_navlist(self, navnodes):
         # type: (List[nodes.Node]) -> List[NavPoint]
@@ -227,15 +230,16 @@ def setup(app):
 
     # config values
     app.add_config_value('epub_basename', lambda self: make_filename(self.project), None)
-    app.add_config_value('epub_theme', 'epub', 'html')
+    app.add_config_value('epub_version', 3.0, 'epub')  # experimental
-    app.add_config_value('epub_theme_options', {}, 'html')
+    app.add_config_value('epub_theme', 'epub', 'epub')
-    app.add_config_value('epub_title', lambda self: self.html_title, 'html')
+    app.add_config_value('epub_theme_options', {}, 'epub')
-    app.add_config_value('epub_author', 'unknown', 'html')
+    app.add_config_value('epub_title', lambda self: self.html_title, 'epub')
-    app.add_config_value('epub_language', lambda self: self.language or 'en', 'html')
+    app.add_config_value('epub_author', 'unknown', 'epub')
-    app.add_config_value('epub_publisher', 'unknown', 'html')
+    app.add_config_value('epub_language', lambda self: self.language or 'en', 'epub')
-    app.add_config_value('epub_copyright', lambda self: self.copyright, 'html')
+    app.add_config_value('epub_publisher', 'unknown', 'epub')
-    app.add_config_value('epub_identifier', 'unknown', 'html')
+    app.add_config_value('epub_copyright', lambda self: self.copyright, 'epub')
-    app.add_config_value('epub_scheme', 'unknown', 'html')
+    app.add_config_value('epub_identifier', 'unknown', 'epub')
+    app.add_config_value('epub_scheme', 'unknown', 'epub')
     app.add_config_value('epub_uid', 'unknown', 'env')
     app.add_config_value('epub_cover', (), 'env')
     app.add_config_value('epub_guide', (), 'env')
@@ -247,11 +251,11 @@ def setup(app):
     app.add_config_value('epub_tocscope', 'default', 'env')
     app.add_config_value('epub_fix_images', False, 'env')
     app.add_config_value('epub_max_image_width', 0, 'env')
-    app.add_config_value('epub_show_urls', 'inline', 'html')
+    app.add_config_value('epub_show_urls', 'inline', 'epub')
-    app.add_config_value('epub_use_index', lambda self: self.html_use_index, 'html')
+    app.add_config_value('epub_use_index', lambda self: self.html_use_index, 'epub')
-    app.add_config_value('epub_description', 'unknown', 'epub3', string_classes)
+    app.add_config_value('epub_description', 'unknown', 'epub', string_classes)
-    app.add_config_value('epub_contributor', 'unknown', 'epub3', string_classes)
+    app.add_config_value('epub_contributor', 'unknown', 'epub', string_classes)
-    app.add_config_value('epub_writing_mode', 'horizontal', 'epub3',
+    app.add_config_value('epub_writing_mode', 'horizontal', 'epub',
                          ENUM('horizontal', 'vertical'))
 
     return {

sphinx/builders/gettext.py

@@ -5,32 +5,32 @@
 
     The MessageCatalogBuilder class.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from __future__ import unicode_literals
 
-from os import path, walk, getenv
 from codecs import open
-from time import time
+from collections import defaultdict, OrderedDict
 from datetime import datetime, tzinfo, timedelta
-from collections import defaultdict
+from os import path, walk, getenv
+from time import time
+from typing import TYPE_CHECKING
 from uuid import uuid4
 
 from six import iteritems, StringIO
 
 from sphinx.builders import Builder
+from sphinx.locale import pairindextypes
 from sphinx.util import split_index_msg, logging, status_iterator
-from sphinx.util.tags import Tags
+from sphinx.util.console import bold  # type: ignore
+from sphinx.util.i18n import find_catalog
 from sphinx.util.nodes import extract_messages, traverse_translatable_index
 from sphinx.util.osutil import safe_relpath, ensuredir, canon_path
-from sphinx.util.i18n import find_catalog
+from sphinx.util.tags import Tags
-from sphinx.util.console import bold  # type: ignore
-from sphinx.locale import pairindextypes
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, DefaultDict, Dict, Iterable, List, Set, Tuple  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.util.i18n import CatalogInfo  # NOQA
@@ -68,8 +68,8 @@ class Catalog(object):
         # type: () -> None
         self.messages = []  # type: List[unicode]
                             # retain insertion order, a la OrderedDict
-        self.metadata = {}  # type: Dict[unicode, List[Tuple[unicode, int, unicode]]]
+        self.metadata = OrderedDict()  # type: Dict[unicode, List[Tuple[unicode, int, unicode]]]  # NOQA
-                            # msgid -> file, line, uid
+                                        # msgid -> file, line, uid
 
     def add(self, msg, origin):
         # type: (unicode, MsgOrigin) -> None
@@ -214,6 +214,7 @@ class MessageCatalogBuilder(I18nBuilder):
     Builds gettext-style message catalogs (.pot files).
     """
     name = 'gettext'
+    epilog = 'The message catalogs are in %(outdir)s.'
 
     def init(self):
         # type: () -> None
@@ -235,7 +236,8 @@ class MessageCatalogBuilder(I18nBuilder):
 
     def _extract_from_template(self):
         # type: () -> None
-        files = self._collect_templates()
+        files = list(self._collect_templates())
+        files.sort()
         logger.info(bold('building [%s]: ' % self.name), nonl=1)
         logger.info('targets for %d template files', len(files))
 

sphinx/builders/html.py

@@ -5,58 +5,59 @@
 
     Several HTML builders.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import codecs
+import posixpath
 import re
 import sys
-import codecs
 import warnings
-import posixpath
-from os import path
 from hashlib import md5
-
+from os import path
-from six import iteritems, text_type, string_types
+from typing import TYPE_CHECKING
-from six.moves import cPickle as pickle
 
 import docutils
 from docutils import nodes
-from docutils.io import DocTreeInput, StringOutput
 from docutils.core import Publisher
-from docutils.utils import new_document, relative_path
 from docutils.frontend import OptionParser
+from docutils.io import DocTreeInput, StringOutput
 from docutils.readers.doctree import Reader as DoctreeReader
+from docutils.utils import relative_path
+from six import iteritems, text_type, string_types
+from six.moves import cPickle as pickle
 
 from sphinx import package_dir, __display_version__
-from sphinx.util import jsonimpl, logging, status_iterator
+from sphinx.application import ENV_PICKLE_FILENAME
-from sphinx.util.i18n import format_date
+from sphinx.builders import Builder
-from sphinx.util.inventory import InventoryFile
-from sphinx.util.osutil import SEP, os_path, relative_uri, ensuredir, \
-    movefile, copyfile
-from sphinx.util.nodes import inline_all_toctrees
-from sphinx.util.docutils import is_html5_writer_available
-from sphinx.util.fileutil import copy_asset
-from sphinx.util.matching import patmatch, Matcher, DOTFILES
 from sphinx.config import string_classes
 from sphinx.deprecation import RemovedInSphinx20Warning
+from sphinx.environment.adapters.asset import ImageAdapter
+from sphinx.environment.adapters.indexentries import IndexEntries
+from sphinx.environment.adapters.toctree import TocTree
+from sphinx.highlighting import PygmentsBridge
 from sphinx.locale import _, l_
 from sphinx.search import js_index
 from sphinx.theming import HTMLThemeFactory
-from sphinx.builders import Builder
+from sphinx.util import jsonimpl, logging, status_iterator
-from sphinx.application import ENV_PICKLE_FILENAME
-from sphinx.highlighting import PygmentsBridge
 from sphinx.util.console import bold, darkgreen  # type: ignore
+from sphinx.util.docutils import is_html5_writer_available, new_document
+from sphinx.util.fileutil import copy_asset
+from sphinx.util.i18n import format_date
+from sphinx.util.inventory import InventoryFile
+from sphinx.util.matching import patmatch, Matcher, DOTFILES
+from sphinx.util.nodes import inline_all_toctrees
+from sphinx.util.osutil import SEP, os_path, relative_uri, ensuredir, \
+    movefile, copyfile
 from sphinx.writers.html import HTMLWriter, HTMLTranslator
-from sphinx.environment.adapters.asset import ImageAdapter
-from sphinx.environment.adapters.toctree import TocTree
-from sphinx.environment.adapters.indexentries import IndexEntries
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
+    from typing import Any, Dict, IO, Iterable, Iterator, List, Type, Tuple, Union  # NOQA
-    from typing import Any, Dict, Iterable, Iterator, List, Type, Tuple, Union  # NOQA
-    from sphinx.domains import Domain, Index  # NOQA
     from sphinx.application import Sphinx  # NOQA
+    from sphinx.config import Config  # NOQA
+    from sphinx.domains import Domain, Index  # NOQA
+    from sphinx.util.tags import Tags  # NOQA
 
 # Experimental HTML5 Writer
 if is_html5_writer_available():
@@ -147,12 +148,68 @@ class Stylesheet(text_type):
         return self
 
 
+class BuildInfo(object):
+    """buildinfo file manipulator.
+
+    HTMLBuilder and its family are storing their own envdata to ``.buildinfo``.
+    This class is a manipulator for the file.
+    """
+
+    @classmethod
+    def load(cls, f):
+        # type: (IO) -> BuildInfo
+        try:
+            lines = f.readlines()
+            assert lines[0].rstrip() == '# Sphinx build info version 1'
+            assert lines[2].startswith('config: ')
+            assert lines[3].startswith('tags: ')
+
+            build_info = BuildInfo()
+            build_info.config_hash = lines[2].split()[1].strip()
+            build_info.tags_hash = lines[3].split()[1].strip()
+            return build_info
+        except Exception as exc:
+            raise ValueError('build info file is broken: %r' % exc)
+
+    def __init__(self, config=None, tags=None, config_categories=[]):
+        # type: (Config, Tags, List[unicode]) -> None
+        self.config_hash = u''
+        self.tags_hash = u''
+
+        if config:
+            values = dict((c.name, c.value) for c in config.filter(config_categories))
+            self.config_hash = get_stable_hash(values)
+
+        if tags:
+            self.tags_hash = get_stable_hash(sorted(tags))
+
+    def __eq__(self, other):  # type: ignore
+        # type: (BuildInfo) -> bool
+        return (self.config_hash == other.config_hash and
+                self.tags_hash == other.tags_hash)
+
+    def __ne__(self, other):  # type: ignore
+        # type: (BuildInfo) -> bool
+        return not (self == other)  # for py27
+
+    def dump(self, f):
+        # type: (IO) -> None
+        f.write('# Sphinx build info version 1\n'
+                '# This file hashes the configuration used when building these files.'
+                ' When it is not found, a full rebuild will be done.\n'
+                'config: %s\n'
+                'tags: %s\n' %
+                (self.config_hash, self.tags_hash))
+
+
 class StandaloneHTMLBuilder(Builder):
     """
     Builds standalone HTML docs.
     """
     name = 'html'
     format = 'html'
+    epilog = 'The HTML pages are in %(outdir)s.'
+
     copysource = True
     allow_parallel = True
     out_suffix = '.html'
@@ -189,9 +246,7 @@ class StandaloneHTMLBuilder(Builder):
 
     def init(self):
         # type: () -> None
-        # a hash of all config values that, if changed, cause a full rebuild
+        self.build_info = self.create_build_info()
-        self.config_hash = ''  # type: unicode
-        self.tags_hash = ''  # type: unicode
         # basename of images directory
         self.imagedir = '_images'
         # section numbers for headings in the currently visited document
@@ -219,6 +274,10 @@ class StandaloneHTMLBuilder(Builder):
                            'is old. Docutils\' version should be 0.13 or newer, but %s.') %
                           docutils.__version__)
 
+    def create_build_info(self):
+        # type: () -> BuildInfo
+        return BuildInfo(self.config, self.tags, ['html'])
+
     def _get_translations_js(self):
         # type: () -> unicode
         candidates = [path.join(dir, self.config.language,
@@ -272,32 +331,19 @@ class StandaloneHTMLBuilder(Builder):
 
     def get_outdated_docs(self):
         # type: () -> Iterator[unicode]
-        cfgdict = dict((confval.name, confval.value) for confval in self.config.filter('html'))
-        self.config_hash = get_stable_hash(cfgdict)
-        self.tags_hash = get_stable_hash(sorted(self.tags))  # type: ignore
-        old_config_hash = old_tags_hash = ''
         try:
             with open(path.join(self.outdir, '.buildinfo')) as fp:
-                version = fp.readline()
+                buildinfo = BuildInfo.load(fp)
-                if version.rstrip() != '# Sphinx build info version 1':
+
-                    raise ValueError
+            if self.build_info != buildinfo:
-                fp.readline()  # skip commentary
+                for docname in self.env.found_docs:
-                cfg, old_config_hash = fp.readline().strip().split(': ')
+                    yield docname
-                if cfg != 'config':
+                return
-                    raise ValueError
+        except ValueError as exc:
-                tag, old_tags_hash = fp.readline().strip().split(': ')
+            logger.warning('Failed to read build info file: %r', exc)
-                if tag != 'tags':
+        except IOError:
-                    raise ValueError
+            # ignore errors on reading
-        except ValueError:
-            logger.warning('unsupported build info format in %r, building all',
-                           path.join(self.outdir, '.buildinfo'))
-        except Exception:
             pass
-        if old_config_hash != self.config_hash or \
-           old_tags_hash != self.tags_hash:
-            for docname in self.env.found_docs:
-                yield docname
-            return
 
         if self.templates:
             template_mtime = self.templates.newest_template_mtime()
@@ -548,9 +594,9 @@ class StandaloneHTMLBuilder(Builder):
         doctree.settings = self.docsettings
 
         self.secnumbers = self.env.toc_secnumbers.get(docname, {})
-        self.fignumbers = self.env.toc_fignumbers.get(docname, {})
+        self.fignumbers = self.env.toc_fignumbers.get(docname, {})  # type: Dict[unicode, Dict[unicode, Tuple[int, ...]]]  # NOQA
         self.imgpath = relative_uri(self.get_target_uri(docname), '_images')
-        self.dlpath = relative_uri(self.get_target_uri(docname), '_downloads')
+        self.dlpath = relative_uri(self.get_target_uri(docname), '_downloads')  # type: unicode
         self.current_docname = docname
         self.docwriter.write(doctree, destination)
         self.docwriter.assemble_parts()
@@ -775,13 +821,11 @@ class StandaloneHTMLBuilder(Builder):
 
     def write_buildinfo(self):
         # type: () -> None
-        # write build info file
+        try:
-        with open(path.join(self.outdir, '.buildinfo'), 'w') as fp:
+            with open(path.join(self.outdir, '.buildinfo'), 'w') as fp:
-            fp.write('# Sphinx build info version 1\n'
+                self.build_info.dump(fp)
-                     '# This file hashes the configuration used when building'
+        except IOError as exc:
-                     ' these files. When it is not found, a full rebuild will'
+            logger.warning('Failed to write build info file: %r', exc)
-                     ' be done.\nconfig: %s\ntags: %s\n' %
-                     (self.config_hash, self.tags_hash))
 
     def cleanup(self):
         # type: () -> None
@@ -861,9 +905,21 @@ class StandaloneHTMLBuilder(Builder):
         def has_wildcard(pattern):
             # type: (unicode) -> bool
             return any(char in pattern for char in '*?[')
-        sidebars = None
+        sidebars = self.theme.get_config('theme', 'sidebars', None)
         matched = None
         customsidebar = None
+
+        # default sidebars settings for selected theme
+        theme_default_sidebars = self.theme.get_config('theme', 'sidebars', None)
+        if theme_default_sidebars:
+            sidebars = [name.strip() for name in theme_default_sidebars.split(',')]
+        elif self.theme.name == 'alabaster':
+            # provide default settings for alabaster (for compatibility)
+            # Note: this will be removed before Sphinx-2.0
+            sidebars = ['about.html', 'navigation.html', 'relation.html',
+                        'searchbox.html', 'donate.html']
+
+        # user sidebar settings
         for pattern, patsidebars in iteritems(self.config.html_sidebars):
             if patmatch(pagename, pattern):
                 if matched:
@@ -878,6 +934,7 @@ class StandaloneHTMLBuilder(Builder):
                         continue
                 matched = pattern
                 sidebars = patsidebars
+
         if sidebars is None:
             # keep defaults
             pass
@@ -885,6 +942,11 @@ class StandaloneHTMLBuilder(Builder):
             # 0.x compatible mode: insert custom sidebar before searchbox
             customsidebar = sidebars
             sidebars = None
+            warnings.warn('Now html_sidebars only allows list of sidebar '
+                          'templates as a value. Support for a string value '
+                          'will be removed at Sphinx-2.0.',
+                          RemovedInSphinx20Warning)
+
         ctx['sidebars'] = sidebars
         ctx['customsidebar'] = customsidebar
 
@@ -1045,6 +1107,8 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
     HTML page.
     """
     name = 'singlehtml'
+    epilog = 'The HTML page is in %(outdir)s.'
+
     copysource = False
 
     def get_outdated_docs(self):  # type: ignore
@@ -1232,8 +1296,7 @@ class SerializingHTMLBuilder(StandaloneHTMLBuilder):
 
     def init(self):
         # type: () -> None
-        self.config_hash = ''
+        self.build_info = BuildInfo(self.config, self.tags)
-        self.tags_hash = ''
         self.imagedir = '_images'
         self.current_docname = None
         self.theme = None       # no theme necessary
@@ -1307,12 +1370,14 @@ class PickleHTMLBuilder(SerializingHTMLBuilder):
     """
     A Builder that dumps the generated HTML into pickle files.
     """
+    name = 'pickle'
+    epilog = 'You can now process the pickle files in %(outdir)s.'
+
     implementation = pickle
     implementation_dumps_unicode = False
     additional_dump_args = (pickle.HIGHEST_PROTOCOL,)
     indexer_format = pickle
     indexer_dumps_unicode = False
-    name = 'pickle'
     out_suffix = '.fpickle'
     globalcontext_filename = 'globalcontext.pickle'
     searchindex_filename = 'searchindex.pickle'
@@ -1326,11 +1391,13 @@ class JSONHTMLBuilder(SerializingHTMLBuilder):
     """
     A builder that dumps the generated HTML into JSON files.
     """
+    name = 'json'
+    epilog = 'You can now process the JSON files in %(outdir)s.'
+
     implementation = jsonimpl
     implementation_dumps_unicode = True
     indexer_format = jsonimpl
     indexer_dumps_unicode = True
-    name = 'json'
     out_suffix = '.fjson'
     globalcontext_filename = 'globalcontext.json'
     searchindex_filename = 'searchindex.json'

sphinx/builders/htmlhelp.py

@@ -6,14 +6,15 @@
     Build HTML help support files.
     Parts adapted from Python's Doc/tools/prechm.py.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 from __future__ import print_function
 
-import os
 import codecs
+import os
 from os import path
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 
@@ -24,8 +25,7 @@ from sphinx.util import logging
 from sphinx.util.osutil import make_filename
 from sphinx.util.pycompat import htmlescape
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, IO, List, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -174,6 +174,8 @@ class HTMLHelpBuilder(StandaloneHTMLBuilder):
     index files.  Adapted from the original Doc/tools/prechm.py.
     """
     name = 'htmlhelp'
+    epilog = ('You can now run HTML Help Workshop with the .htp file in '
+              '%(outdir)s.')
 
     # don't copy the reST source
     copysource = False

sphinx/builders/latex.py

@@ -5,36 +5,35 @@
 
     LaTeX builder.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import os
 from os import path
-
+from typing import TYPE_CHECKING
-from six import text_type
 
 from docutils import nodes
-from docutils.io import FileOutput
-from docutils.utils import new_document
 from docutils.frontend import OptionParser
+from docutils.io import FileOutput
+from six import text_type
 
 from sphinx import package_dir, addnodes, highlighting
-from sphinx.config import string_classes, ENUM
-from sphinx.errors import SphinxError, ConfigError
-from sphinx.locale import _
 from sphinx.builders import Builder
+from sphinx.config import string_classes, ENUM
 from sphinx.environment import NoUri
 from sphinx.environment.adapters.asset import ImageAdapter
+from sphinx.errors import SphinxError, ConfigError
+from sphinx.locale import _
 from sphinx.util import texescape, logging, status_iterator
-from sphinx.util.nodes import inline_all_toctrees
-from sphinx.util.fileutil import copy_asset_file
-from sphinx.util.osutil import SEP, make_filename
 from sphinx.util.console import bold, darkgreen  # type: ignore
+from sphinx.util.docutils import new_document
+from sphinx.util.fileutil import copy_asset_file
+from sphinx.util.nodes import inline_all_toctrees
+from sphinx.util.osutil import SEP, make_filename
 from sphinx.writers.latex import LaTeXWriter, LaTeXTranslator
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Iterable, List, Tuple, Union  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.config import Config  # NOQA
@@ -49,6 +48,12 @@ class LaTeXBuilder(Builder):
     """
     name = 'latex'
     format = 'latex'
+    epilog = 'The LaTeX files are in %(outdir)s.'
+    if os.name == 'posix':
+        epilog += ("\nRun 'make' in that directory to run these through "
+                   "(pdf)latex\n"
+                   "(use `make latexpdf' here to do that automatically).")
+
     supported_image_types = ['application/pdf', 'image/png', 'image/jpeg']
     supported_remote_images = False
     default_translator_class = LaTeXTranslator

sphinx/builders/linkcheck.py

@@ -5,20 +5,21 @@
 
     The CheckExternalLinksBuilder class.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import codecs
 import re
 import socket
-import codecs
 import threading
 from os import path
+from typing import TYPE_CHECKING
 
+from docutils import nodes
 from requests.exceptions import HTTPError
 from six.moves import queue, html_parser
 from six.moves.urllib.parse import unquote
-from docutils import nodes
 
 # 2015-06-25 barry@python.org.  This exception was deprecated in Python 3.3 and
 # removed in Python 3.5, however for backward compatibility reasons, we're not
@@ -37,8 +38,7 @@ from sphinx.util.console import (  # type: ignore
 )
 from sphinx.util.requests import is_ssl_error
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List, Set, Tuple, Union  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.util.requests.requests import Response  # NOQA
@@ -90,6 +90,8 @@ class CheckExternalLinksBuilder(Builder):
     Checks for broken external links.
     """
     name = 'linkcheck'
+    epilog = ('Look for any errors in the above output or in '
+              '%(outdir)s/output.txt')
 
     def init(self):
         # type: () -> None

sphinx/builders/manpage.py

@@ -5,28 +5,27 @@
 
     Manual pages builder.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from os import path
+from typing import TYPE_CHECKING
 
-from six import string_types
-
-from docutils.io import FileOutput
 from docutils.frontend import OptionParser
+from docutils.io import FileOutput
+from six import string_types
 
 from sphinx import addnodes
 from sphinx.builders import Builder
 from sphinx.environment import NoUri
 from sphinx.util import logging
+from sphinx.util.console import bold, darkgreen  # type: ignore
 from sphinx.util.nodes import inline_all_toctrees
 from sphinx.util.osutil import make_filename
-from sphinx.util.console import bold, darkgreen  # type: ignore
 from sphinx.writers.manpage import ManualPageWriter, ManualPageTranslator
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List, Set, Union  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -40,6 +39,8 @@ class ManualPageBuilder(Builder):
     """
     name = 'man'
     format = 'man'
+    epilog = 'The manual pages are in %(outdir)s.'
+
     default_translator_class = ManualPageTranslator
     supported_image_types = []  # type: List[unicode]
 

sphinx/builders/qthelp.py

@@ -5,19 +5,19 @@
 
     Build input files for the Qt collection generator.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-import os
-import re
 import codecs
+import os
 import posixpath
+import re
 from os import path
-
+from typing import TYPE_CHECKING
-from six import text_type
 
 from docutils import nodes
+from six import text_type
 
 from sphinx import addnodes
 from sphinx.builders.html import StandaloneHTMLBuilder
@@ -27,8 +27,7 @@ from sphinx.util import force_decode, logging
 from sphinx.util.osutil import make_filename
 from sphinx.util.pycompat import htmlescape
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -108,6 +107,11 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
     Builder that also outputs Qt help project, contents and index files.
     """
     name = 'qthelp'
+    epilog = ('You can now run "qcollectiongenerator" with the .qhcp '
+              'project file in %(outdir)s, like this:\n'
+              '$ qcollectiongenerator %(outdir)s/%(project)s.qhcp\n'
+              'To view the help file:\n'
+              '$ assistant -collectionFile %(outdir)s/%(project)s.qhc')
 
     # don't copy the reST source
     copysource = False
@@ -269,7 +273,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
             link = node['refuri']
             title = htmlescape(node.astext()).replace('"', '"')
             item = section_template % {'title': title, 'ref': link}
-            item = u' ' * 4 * indentlevel + item  # type: ignore
+            item = u' ' * 4 * indentlevel + item
             parts.append(item.encode('ascii', 'xmlcharrefreplace'))
         elif isinstance(node, nodes.bullet_list):
             for subnode in node:

sphinx/builders/texinfo.py

@@ -5,32 +5,33 @@
 
     Texinfo builder.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import os
 from os import path
+from typing import TYPE_CHECKING
 
 from docutils import nodes
-from docutils.io import FileOutput
-from docutils.utils import new_document
 from docutils.frontend import OptionParser
+from docutils.io import FileOutput
 
 from sphinx import addnodes
-from sphinx.locale import _
 from sphinx.builders import Builder
 from sphinx.environment import NoUri
 from sphinx.environment.adapters.asset import ImageAdapter
+from sphinx.locale import _
 from sphinx.util import logging
 from sphinx.util import status_iterator
+from sphinx.util.console import bold, darkgreen  # type: ignore
+from sphinx.util.docutils import new_document
 from sphinx.util.fileutil import copy_asset_file
 from sphinx.util.nodes import inline_all_toctrees
 from sphinx.util.osutil import SEP, make_filename
-from sphinx.util.console import bold, darkgreen  # type: ignore
 from sphinx.writers.texinfo import TexinfoWriter, TexinfoTranslator
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from sphinx.application import Sphinx  # NOQA
     from typing import Any, Dict, Iterable, List, Tuple, Union  # NOQA
 
@@ -97,6 +98,12 @@ class TexinfoBuilder(Builder):
     """
     name = 'texinfo'
     format = 'texinfo'
+    epilog = 'The Texinfo files are in %(outdir)s.'
+    if os.name == 'posix':
+        epilog += ("\nRun 'make' in that directory to run these through "
+                   "makeinfo\n"
+                   "(use 'make info' here to do that automatically).")
+
     supported_image_types = ['image/png', 'image/jpeg',
                              'image/gif']
     default_translator_class = TexinfoTranslator

sphinx/builders/text.py

@@ -5,12 +5,13 @@
 
     Plain-text Sphinx builder.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import codecs
 from os import path
+from typing import TYPE_CHECKING
 
 from docutils.io import StringOutput
 
@@ -19,9 +20,8 @@ from sphinx.util import logging
 from sphinx.util.osutil import ensuredir, os_path
 from sphinx.writers.text import TextWriter, TextTranslator
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
+    from typing import Any, Dict, Iterator, Set, Tuple  # NOQA
-    from typing import Any, Dict, Iterator, Set  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -31,6 +31,8 @@ logger = logging.getLogger(__name__)
 class TextBuilder(Builder):
     name = 'text'
     format = 'text'
+    epilog = 'The text files are in %(outdir)s.'
+
     out_suffix = '.txt'
     allow_parallel = True
     default_translator_class = TextTranslator
@@ -39,7 +41,8 @@ class TextBuilder(Builder):
 
     def init(self):
         # type: () -> None
-        pass
+        # section numbers for headings in the currently visited document
+        self.secnumbers = {}  # type: Dict[unicode, Tuple[int, ...]]
 
     def get_outdated_docs(self):
         # type: () -> Iterator[unicode]
@@ -72,6 +75,7 @@ class TextBuilder(Builder):
     def write_doc(self, docname, doctree):
         # type: (unicode, nodes.Node) -> None
         self.current_docname = docname
+        self.secnumbers = self.env.toc_secnumbers.get(docname, {})
         destination = StringOutput(encoding='utf-8')
         self.writer.write(doctree, destination)
         outfilename = path.join(self.outdir, os_path(docname) + self.out_suffix)
@@ -93,6 +97,8 @@ def setup(app):
 
     app.add_config_value('text_sectionchars', '*=-~"+`', 'env')
     app.add_config_value('text_newlines', 'unix', 'env')
+    app.add_config_value('text_add_secnumbers', True, 'env')
+    app.add_config_value('text_secnumber_suffix', '. ', 'env')
 
     return {
         'version': 'builtin',

sphinx/builders/websupport.py

@@ -5,12 +5,13 @@
 
     Builder for the web support package.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-if False:
+from typing import TYPE_CHECKING
-    # For type annotation
+
+if TYPE_CHECKING:
     from typing import Any, Dict  # NOQA
     from sphinx.application import Sphinx  # NOQA
 

sphinx/builders/xml.py

@@ -5,12 +5,13 @@
 
     Docutils-native XML and pseudo-XML builders.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import codecs
 from os import path
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 from docutils.io import StringOutput
@@ -21,8 +22,7 @@ from sphinx.util import logging
 from sphinx.util.osutil import ensuredir, os_path
 from sphinx.writers.xml import XMLWriter, PseudoXMLWriter
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Iterator, Set  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -35,6 +35,8 @@ class XMLBuilder(Builder):
     """
     name = 'xml'
     format = 'xml'
+    epilog = 'The XML files are in %(outdir)s.'
+
     out_suffix = '.xml'
     allow_parallel = True
 
@@ -108,6 +110,8 @@ class PseudoXMLBuilder(XMLBuilder):
     """
     name = 'pseudoxml'
     format = 'pseudoxml'
+    epilog = 'The pseudo-XML files are in %(outdir)s.'
+
     out_suffix = '.pseudoxml'
 
     _writer_class = PseudoXMLWriter

sphinx/cmd/__init__.py

@@ -5,6 +5,6 @@
 
     Modules for command line executables.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

sphinx/cmd/build.py

@@ -5,14 +5,14 @@
 
     Build documentation from a provided source.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import sys
+from typing import TYPE_CHECKING
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import List  # NOQA
 
 

sphinx/cmd/quickstart.py

@@ -5,19 +5,21 @@
 
     Quickly setup documentation source to work with Sphinx.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-from __future__ import print_function
 from __future__ import absolute_import
+from __future__ import print_function
 
-import re
+import argparse
 import os
+import re
 import sys
-import optparse
 import time
-from os import path
+from collections import OrderedDict
 from io import open
+from os import path
+from typing import TYPE_CHECKING
 
 # try to import readline, unix specific enhancement
 try:
@@ -29,26 +31,40 @@ try:
 except ImportError:
     pass
 
+from docutils.utils import column_width
 from six import PY2, PY3, text_type, binary_type
 from six.moves import input
 from six.moves.urllib.parse import quote as urlquote
-from docutils.utils import column_width
 
 from sphinx import __display_version__, package_dir
-from sphinx.util.osutil import make_filename
+from sphinx.util import texescape
 from sphinx.util.console import (  # type: ignore
     purple, bold, red, turquoise, nocolor, color_terminal
 )
+from sphinx.util.osutil import ensuredir, make_filename
 from sphinx.util.template import SphinxRenderer
-from sphinx.util import texescape
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
+    from typing import Any, Callable, Dict, List, Pattern, Union  # NOQA
-    from typing import Any, Callable, Dict, List, Pattern  # NOQA
 
 TERM_ENCODING = getattr(sys.stdin, 'encoding', None)
 
-DEFAULT_VALUE = {
+EXTENSIONS = OrderedDict([
+    ('autodoc', 'automatically insert docstrings from modules'),
+    ('doctest', 'automatically test code snippets in doctest blocks'),
+    ('intersphinx', 'link between Sphinx documentation of different projects'),
+    ('todo', 'write "todo" entries that can be shown or hidden on build'),
+    ('coverage', 'checks for documentation coverage'),
+    ('imgmath', 'include math, rendered as PNG or SVG images'),
+    ('mathjax', 'include math, rendered in the browser by MathJax'),
+    ('ifconfig', 'conditional inclusion of content based on config values'),
+    ('viewcode',
+     'include links to the source code of documented Python objects'),
+    ('githubpages',
+     'create .nojekyll file to publish the document on GitHub pages'),
+])
+
+DEFAULTS = {
     'path': '.',
     'sep': False,
     'dot': '_',
@@ -56,26 +72,13 @@ DEFAULT_VALUE = {
     'suffix': '.rst',
     'master': 'index',
     'epub': False,
-    'ext_autodoc': False,
-    'ext_doctest': False,
-    'ext_todo': False,
     'makefile': True,
     'batchfile': True,
 }
 
-EXTENSIONS = ('autodoc', 'doctest', 'intersphinx', 'todo', 'coverage',
-              'imgmath', 'mathjax', 'ifconfig', 'viewcode', 'githubpages')
-
 PROMPT_PREFIX = '> '
 
 
-def mkdir_p(dir):
-    # type: (unicode) -> None
-    if path.isdir(dir):
-        return
-    os.makedirs(dir)
-
-
 # function to get input from terminal -- overridden by the test suite
 def term_input(prompt):
     # type: (unicode) -> unicode
@@ -138,29 +141,29 @@ def ok(x):
 
 
 def term_decode(text):
-    # type: (unicode) -> unicode
+    # type: (Union[bytes,unicode]) -> unicode
     if isinstance(text, text_type):
         return text
 
-    # for Python 2.x, try to get a Unicode string out of it
+    # Use the known encoding, if possible
-    if text.decode('ascii', 'replace').encode('ascii', 'replace') == text:
-        return text
-
     if TERM_ENCODING:
-        text = text.decode(TERM_ENCODING)
+        return text.decode(TERM_ENCODING)
-    else:
+
-        print(turquoise('* Note: non-ASCII characters entered '
+    # If ascii is safe, use it with no warning
-                        'and terminal encoding unknown -- assuming '
+    if text.decode('ascii', 'replace').encode('ascii', 'replace') == text:
-                        'UTF-8 or Latin-1.'))
+        return text.decode('ascii')
-        try:
+
-            text = text.decode('utf-8')
+    print(turquoise('* Note: non-ASCII characters entered '
-        except UnicodeDecodeError:
+                    'and terminal encoding unknown -- assuming '
-            text = text.decode('latin1')
+                    'UTF-8 or Latin-1.'))
-    return text
+    try:
+        return text.decode('utf-8')
+    except UnicodeDecodeError:
+        return text.decode('latin1')
 
 
-def do_prompt(d, key, text, default=None, validator=nonempty):
+def do_prompt(text, default=None, validator=nonempty):
-    # type: (Dict, unicode, unicode, unicode, Callable[[unicode], Any]) -> None
+    # type: (unicode, unicode, Callable[[unicode], Any]) -> Union[unicode, bool]
     while True:
         if default is not None:
             prompt = PROMPT_PREFIX + '%s [%s]: ' % (text, default)  # type: unicode
@@ -191,7 +194,7 @@ def do_prompt(d, key, text, default=None, validator=nonempty):
             print(red('* ' + str(err)))
             continue
         break
-    d[key] = x
+    return x
 
 
 def convert_python_source(source, rex=re.compile(r"[uU]('.*?')")):
@@ -235,7 +238,7 @@ def ask_user(d):
     * suffix:    source file suffix
     * master:    master document name
     * epub:      use epub (bool)
-    * ext_*:     extensions to use (bools)
+    * extensions:  extensions to use (list)
     * makefile:  make Makefile
     * batchfile: make command file
     """
@@ -251,7 +254,7 @@ Selected root path: %s''' % d['path']))
     else:
         print('''
 Enter the root path for documentation.''')
-        do_prompt(d, 'path', 'Root path for the documentation', '.', is_path)
+        d['path'] = do_prompt('Root path for the documentation', '.', is_path)
 
     while path.isfile(path.join(d['path'], 'conf.py')) or \
             path.isfile(path.join(d['path'], 'source', 'conf.py')):
@@ -260,8 +263,8 @@ Enter the root path for documentation.''')
                    'selected root path.'))
         print('sphinx-quickstart will not overwrite existing Sphinx projects.')
         print()
-        do_prompt(d, 'path', 'Please enter a new root path (or just Enter '
+        d['path'] = do_prompt('Please enter a new root path (or just Enter '
-                  'to exit)', '', is_path)
+                              'to exit)', '', is_path)
         if not d['path']:
             sys.exit(1)
 
@@ -270,22 +273,22 @@ Enter the root path for documentation.''')
 You have two options for placing the build directory for Sphinx output.
 Either, you use a directory "_build" within the root path, or you separate
 "source" and "build" directories within the root path.''')
-        do_prompt(d, 'sep', 'Separate source and build directories (y/n)', 'n',
+        d['sep'] = do_prompt('Separate source and build directories (y/n)',
-                  boolean)
+                             'n', boolean)
 
     if 'dot' not in d:
         print('''
 Inside the root directory, two more directories will be created; "_templates"
 for custom HTML templates and "_static" for custom stylesheets and other static
 files. You can enter another prefix (such as ".") to replace the underscore.''')
-        do_prompt(d, 'dot', 'Name prefix for templates and static dir', '_', ok)
+        d['dot'] = do_prompt('Name prefix for templates and static dir', '_', ok)
 
     if 'project' not in d:
         print('''
 The project name will occur in several places in the built documentation.''')
-        do_prompt(d, 'project', 'Project name')
+        d['project'] = do_prompt('Project name')
     if 'author' not in d:
-        do_prompt(d, 'author', 'Author name(s)')
+        d['author'] = do_prompt('Author name(s)')
 
     if 'version' not in d:
         print('''
@@ -294,9 +297,9 @@ software. Each version can have multiple releases. For example, for
 Python the version is something like 2.5 or 3.0, while the release is
 something like 2.5.1 or 3.0a1.  If you don't need this dual structure,
 just set both to the same value.''')
-        do_prompt(d, 'version', 'Project version', '', allow_empty)
+        d['version'] = do_prompt('Project version', '', allow_empty)
     if 'release' not in d:
-        do_prompt(d, 'release', 'Project release', d['version'], allow_empty)
+        d['release'] = do_prompt('Project release', d['version'], allow_empty)
 
     if 'language' not in d:
         print('''
@@ -306,7 +309,7 @@ translate text that it generates into that language.
 
 For a list of supported codes, see
 http://sphinx-doc.org/config.html#confval-language.''')
-        do_prompt(d, 'language', 'Project language', 'en')
+        d['language'] = do_prompt('Project language', 'en')
         if d['language'] == 'en':
             d['language'] = None
 
@@ -314,7 +317,7 @@ http://sphinx-doc.org/config.html#confval-language.''')
         print('''
 The file name suffix for source files. Commonly, this is either ".txt"
 or ".rst".  Only files with this suffix are considered documents.''')
-        do_prompt(d, 'suffix', 'Source file suffix', '.rst', suffix)
+        d['suffix'] = do_prompt('Source file suffix', '.rst', suffix)
 
     if 'master' not in d:
         print('''
@@ -322,8 +325,8 @@ One document is special in that it is considered the top node of the
 "contents tree", that is, it is the root of the hierarchical structure
 of the documents. Normally, this is "index", but if your "index"
 document is a custom template, you can also set this to another filename.''')
-        do_prompt(d, 'master', 'Name of your master document (without suffix)',
+        d['master'] = do_prompt('Name of your master document (without suffix)',
-                  'index')
+                                'index')
 
     while path.isfile(path.join(d['path'], d['master'] + d['suffix'])) or \
             path.isfile(path.join(d['path'], 'source', d['master'] + d['suffix'])):
@@ -332,65 +335,40 @@ document is a custom template, you can also set this to another filename.''')
                    'selected root path.' % (d['master'] + d['suffix'])))
         print('sphinx-quickstart will not overwrite the existing file.')
         print()
-        do_prompt(d, 'master', 'Please enter a new file name, or rename the '
+        d['master'] = do_prompt('Please enter a new file name, or rename the '
-                  'existing file and press Enter', d['master'])
+                                'existing file and press Enter', d['master'])
 
     if 'epub' not in d:
         print('''
 Sphinx can also add configuration for epub output:''')
-        do_prompt(d, 'epub', 'Do you want to use the epub builder (y/n)',
+        d['epub'] = do_prompt('Do you want to use the epub builder (y/n)',
-                  'n', boolean)
+                              'n', boolean)
 
-    if 'ext_autodoc' not in d:
+    if 'extensions' not in d:
-        print('''
+        print('Indicate which of the following Sphinx extensions should be '
-Please indicate if you want to use one of the following Sphinx extensions:''')
+              'enabled:')
-        do_prompt(d, 'ext_autodoc', 'autodoc: automatically insert docstrings '
+        d['extensions'] = []
-                  'from modules (y/n)', 'n', boolean)
+        for name, description in EXTENSIONS.items():
-    if 'ext_doctest' not in d:
+            if do_prompt('%s: %s (y/n)' % (name, description), 'n', boolean):
-        do_prompt(d, 'ext_doctest', 'doctest: automatically test code snippets '
+                d['extensions'].append('sphinx.ext.%s' % name)
-                  'in doctest blocks (y/n)', 'n', boolean)
-    if 'ext_intersphinx' not in d:
-        do_prompt(d, 'ext_intersphinx', 'intersphinx: link between Sphinx '
-                  'documentation of different projects (y/n)', 'n', boolean)
-    if 'ext_todo' not in d:
-        do_prompt(d, 'ext_todo', 'todo: write "todo" entries '
-                  'that can be shown or hidden on build (y/n)', 'n', boolean)
-    if 'ext_coverage' not in d:
-        do_prompt(d, 'ext_coverage', 'coverage: checks for documentation '
-                  'coverage (y/n)', 'n', boolean)
-    if 'ext_imgmath' not in d:
-        do_prompt(d, 'ext_imgmath', 'imgmath: include math, rendered '
-                  'as PNG or SVG images (y/n)', 'n', boolean)
-    if 'ext_mathjax' not in d:
-        do_prompt(d, 'ext_mathjax', 'mathjax: include math, rendered in the '
-                  'browser by MathJax (y/n)', 'n', boolean)
-    if d['ext_imgmath'] and d['ext_mathjax']:
-        print('''Note: imgmath and mathjax cannot be enabled at the same time.
-imgmath has been deselected.''')
-        d['ext_imgmath'] = False
-    if 'ext_ifconfig' not in d:
-        do_prompt(d, 'ext_ifconfig', 'ifconfig: conditional inclusion of '
-                  'content based on config values (y/n)', 'n', boolean)
-    if 'ext_viewcode' not in d:
-        do_prompt(d, 'ext_viewcode', 'viewcode: include links to the source '
-                  'code of documented Python objects (y/n)', 'n', boolean)
-    if 'ext_githubpages' not in d:
-        do_prompt(d, 'ext_githubpages', 'githubpages: create .nojekyll file '
-                  'to publish the document on GitHub pages (y/n)', 'n', boolean)
 
-    if 'no_makefile' in d:
+        # Handle conflicting options
-        d['makefile'] = False
+        if set(['sphinx.ext.imgmath', 'sphinx.ext.mathjax']).issubset(
-    elif 'makefile' not in d:
+                d['extensions']):
+            print('Note: imgmath and mathjax cannot be enabled at the same '
+                  'time. imgmath has been deselected.')
+            d['extensions'].remove('sphinx.ext.imgmath')
+
+    if 'makefile' not in d:
         print('''
 A Makefile and a Windows command file can be generated for you so that you
 only have to run e.g. `make html' instead of invoking sphinx-build
 directly.''')
-        do_prompt(d, 'makefile', 'Create Makefile? (y/n)', 'y', boolean)
+        d['makefile'] = do_prompt('Create Makefile? (y/n)', 'y', boolean)
-    if 'no_batchfile' in d:
+
-        d['batchfile'] = False
+    if 'batchfile' not in d:
-    elif 'batchfile' not in d:
+        d['batchfile'] = do_prompt('Create Windows command file? (y/n)',
-        do_prompt(d, 'batchfile', 'Create Windows command file? (y/n)',
+                                   'y', boolean)
-                  'y', boolean)
     print()
 
 
@@ -400,7 +378,6 @@ def generate(d, overwrite=True, silent=False, templatedir=None):
     template = QuickstartRenderer(templatedir=templatedir)
 
     texescape.init()
-    indent = ' ' * 4
 
     if 'mastertoctree' not in d:
         d['mastertoctree'] = ''
@@ -414,10 +391,6 @@ def generate(d, overwrite=True, silent=False, templatedir=None):
     d['now'] = time.asctime()
     d['project_underline'] = column_width(d['project']) * '='
     d.setdefault('extensions', [])
-    for name in EXTENSIONS:
-        if d.get('ext_' + name):
-            d['extensions'].append('sphinx.ext.' + name)
-    d['extensions'] = (',\n' + indent).join(repr(name) for name in d['extensions'])
     d['copyright'] = time.strftime('%Y') + ', ' + d['author']
     d['author_texescaped'] = text_type(d['author']).\
         translate(texescape.tex_escape_map)
@@ -433,11 +406,11 @@ def generate(d, overwrite=True, silent=False, templatedir=None):
         d[key + '_str'] = d[key].replace('\\', '\\\\').replace("'", "\\'")
 
     if not path.isdir(d['path']):
-        mkdir_p(d['path'])
+        ensuredir(d['path'])
 
     srcdir = d['sep'] and path.join(d['path'], 'source') or d['path']
 
-    mkdir_p(srcdir)
+    ensuredir(srcdir)
     if d['sep']:
         builddir = path.join(d['path'], 'build')
         d['exclude_patterns'] = ''
@@ -448,18 +421,20 @@ def generate(d, overwrite=True, silent=False, templatedir=None):
             'Thumbs.db', '.DS_Store',
         ])
         d['exclude_patterns'] = ', '.join(exclude_patterns)
-    mkdir_p(builddir)
+    ensuredir(builddir)
-    mkdir_p(path.join(srcdir, d['dot'] + 'templates'))
+    ensuredir(path.join(srcdir, d['dot'] + 'templates'))
-    mkdir_p(path.join(srcdir, d['dot'] + 'static'))
+    ensuredir(path.join(srcdir, d['dot'] + 'static'))
 
     def write_file(fpath, content, newline=None):
         # type: (unicode, unicode, unicode) -> None
         if overwrite or not path.isfile(fpath):
-            print('Creating file %s.' % fpath)
+            if 'quiet' not in d:
+                print('Creating file %s.' % fpath)
             with open(fpath, 'wt', encoding='utf-8', newline=newline) as f:
                 f.write(content)
         else:
-            print('File %s already exists, skipping.' % fpath)
+            if 'quiet' not in d:
+                print('File %s already exists, skipping.' % fpath)
 
     conf_path = os.path.join(templatedir, 'conf.py_t') if templatedir else None
     if not conf_path or not path.isfile(conf_path):
@@ -509,23 +484,6 @@ where "builder" is one of the supported builders, e.g. html, latex or linkcheck.
 ''')
 
 
-def usage(argv, msg=None):
-    # type: (List[unicode], unicode) -> None
-    if msg:
-        print(msg, file=sys.stderr)
-        print(file=sys.stderr)
-
-
-USAGE = """\
-Sphinx v%s
-Usage: %%prog [options] [projectdir]
-""" % __display_version__
-
-EPILOG = """\
-For more information, visit <http://sphinx-doc.org/>.
-"""
-
-
 def valid_dir(d):
     # type: (Dict) -> bool
     dir = d['path']
@@ -556,18 +514,86 @@ def valid_dir(d):
     return True
 
 
-class MyFormatter(optparse.IndentedHelpFormatter):
+def get_parser():
-    def format_usage(self, usage):  # type: ignore
+    # type: () -> argparse.ArgumentParser
-        # type: (str) -> str
+    parser = argparse.ArgumentParser(
-        return usage
+        usage='%(prog)s [OPTIONS] <PROJECT_DIR>',
+        epilog="For more information, visit <http://sphinx-doc.org/>.",
+        description="""
+Generate required files for a Sphinx project.
 
-    def format_help(self, formatter):
+sphinx-quickstart is an interactive tool that asks some questions about your
-        result = []
+project and then generates a complete documentation directory and sample
-        if self.description:
+Makefile to be used with sphinx-build.
-            result.append(self.format_description(formatter))
+""")
-        if self.option_list:
+
-            result.append(self.format_option_help(formatter))
+    parser.add_argument('-q', '--quiet', action='store_true', dest='quiet',
-        return "\n".join(result)
+                        default=False,
+                        help='quiet mode')
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
+
+    parser.add_argument('path', metavar='PROJECT_DIR', default='.', nargs='?',
+                        help='output path')
+
+    group = parser.add_argument_group('Structure options')
+    group.add_argument('--sep', action='store_true',
+                       help='if specified, separate source and build dirs')
+    group.add_argument('--dot', metavar='DOT',
+                       help='replacement for dot in _templates etc.')
+
+    group = parser.add_argument_group('Project basic options')
+    group.add_argument('-p', '--project', metavar='PROJECT', dest='project',
+                       help='project name')
+    group.add_argument('-a', '--author', metavar='AUTHOR', dest='author',
+                       help='author names')
+    group.add_argument('-v', metavar='VERSION', dest='version', default='',
+                       help='version of project')
+    group.add_argument('-r', '--release', metavar='RELEASE', dest='release',
+                       help='release of project')
+    group.add_argument('-l', '--language', metavar='LANGUAGE', dest='language',
+                       help='document language')
+    group.add_argument('--suffix', metavar='SUFFIX',
+                       help='source file suffix')
+    group.add_argument('--master', metavar='MASTER',
+                       help='master document name')
+    group.add_argument('--epub', action='store_true', default=False,
+                       help='use epub')
+
+    group = parser.add_argument_group('Extension options')
+    for ext in EXTENSIONS:
+        group.add_argument('--ext-%s' % ext, action='append_const',
+                           const='sphinx.ext.%s' % ext, dest='extensions',
+                           help='enable %s extension' % ext)
+    group.add_argument('--extensions', metavar='EXTENSIONS', dest='extensions',
+                       action='append', help='enable arbitrary extensions')
+
+    group = parser.add_argument_group('Makefile and Batchfile creation')
+    group.add_argument('--makefile', action='store_true', dest='makefile',
+                       help='create makefile')
+    group.add_argument('--no-makefile', action='store_false', dest='makefile',
+                       help='do not create makefile')
+    group.add_argument('--batchfile', action='store_true', dest='batchfile',
+                       help='create batchfile')
+    group.add_argument('--no-batchfile', action='store_false',
+                       dest='batchfile',
+                       help='do not create batchfile')
+    group.add_argument('-m', '--use-make-mode', action='store_true',
+                       dest='make_mode', default=True,
+                       help='use make-mode for Makefile/make.bat')
+    group.add_argument('-M', '--no-use-make-mode', action='store_false',
+                       dest='make_mode',
+                       help='do not use make-mode for Makefile/make.bat')
+
+    group = parser.add_argument_group('Project templating')
+    group.add_argument('-t', '--templatedir', metavar='TEMPLATEDIR',
+                       dest='templatedir',
+                       help='template directory for template files')
+    group.add_argument('-d', metavar='NAME=VALUE', action='append',
+                       dest='variables',
+                       help='define a template variable')
+
+    return parser
 
 
 def main(argv=sys.argv[1:]):
@@ -575,81 +601,14 @@ def main(argv=sys.argv[1:]):
     if not color_terminal():
         nocolor()
 
-    parser = optparse.OptionParser(USAGE, epilog=EPILOG,
-                                   version='Sphinx v%s' % __display_version__,
-                                   formatter=MyFormatter())
-    parser.add_option('-q', '--quiet', action='store_true', dest='quiet',
-                      default=False,
-                      help='quiet mode')
-
-    group = parser.add_option_group('Structure options')
-    group.add_option('--sep', action='store_true', dest='sep',
-                     help='if specified, separate source and build dirs')
-    group.add_option('--dot', metavar='DOT', dest='dot',
-                     help='replacement for dot in _templates etc.')
-
-    group = parser.add_option_group('Project basic options')
-    group.add_option('-p', '--project', metavar='PROJECT', dest='project',
-                     help='project name')
-    group.add_option('-a', '--author', metavar='AUTHOR', dest='author',
-                     help='author names')
-    group.add_option('-v', metavar='VERSION', dest='version',
-                     help='version of project')
-    group.add_option('-r', '--release', metavar='RELEASE', dest='release',
-                     help='release of project')
-    group.add_option('-l', '--language', metavar='LANGUAGE', dest='language',
-                     help='document language')
-    group.add_option('--suffix', metavar='SUFFIX', dest='suffix',
-                     help='source file suffix')
-    group.add_option('--master', metavar='MASTER', dest='master',
-                     help='master document name')
-    group.add_option('--epub', action='store_true', dest='epub',
-                     default=False,
-                     help='use epub')
-
-    group = parser.add_option_group('Extension options')
-    for ext in EXTENSIONS:
-        group.add_option('--ext-' + ext, action='store_true',
-                         dest='ext_' + ext, default=False,
-                         help='enable %s extension' % ext)
-    group.add_option('--extensions', metavar='EXTENSIONS', dest='extensions',
-                     action='append', help='enable extensions')
-
-    group = parser.add_option_group('Makefile and Batchfile creation')
-    group.add_option('--makefile', action='store_true', dest='makefile',
-                     default=False,
-                     help='create makefile')
-    group.add_option('--no-makefile', action='store_true', dest='no_makefile',
-                     default=False,
-                     help='not create makefile')
-    group.add_option('--batchfile', action='store_true', dest='batchfile',
-                     default=False,
-                     help='create batchfile')
-    group.add_option('--no-batchfile', action='store_true', dest='no_batchfile',
-                     default=False,
-                     help='not create batchfile')
-    group.add_option('-M', '--no-use-make-mode', action='store_false', dest='make_mode',
-                     help='not use make-mode for Makefile/make.bat')
-    group.add_option('-m', '--use-make-mode', action='store_true', dest='make_mode',
-                     default=True,
-                     help='use make-mode for Makefile/make.bat')
-
-    group = parser.add_option_group('Project templating')
-    group.add_option('-t', '--templatedir', metavar='TEMPLATEDIR', dest='templatedir',
-                     help='template directory for template files')
-    group.add_option('-d', metavar='NAME=VALUE', action='append', dest='variables',
-                     help='define a template variable')
-
     # parse options
+    parser = get_parser()
     try:
-        opts, args = parser.parse_args(argv)
+        args = parser.parse_args(argv)
     except SystemExit as err:
         return err.code
 
-    if len(args) > 0:
+    d = vars(args)
-        opts.ensure_value('path', args[0])
-
-    d = vars(opts)
     # delete None or False value
     d = dict((k, v) for k, v in d.items() if not (v is None or v is False))
 
@@ -664,14 +623,9 @@ def main(argv=sys.argv[1:]):
             # quiet mode with all required params satisfied, use default
             d.setdefault('version', '')
             d.setdefault('release', d['version'])
-            d2 = DEFAULT_VALUE.copy()
+            d2 = DEFAULTS.copy()
-            d2.update(dict(("ext_" + ext, False) for ext in EXTENSIONS))
             d2.update(d)
             d = d2
-            if 'no_makefile' in d:
-                d['makefile'] = False
-            if 'no_batchfile' in d:
-                d['batchfile'] = False
 
             if not valid_dir(d):
                 print()
@@ -692,13 +646,12 @@ def main(argv=sys.argv[1:]):
         if isinstance(value, binary_type):
             d[key] = term_decode(value)
 
-    # parse extensions list
+    # handle use of CSV-style extension values
     d.setdefault('extensions', [])
     for ext in d['extensions'][:]:
         if ',' in ext:
             d['extensions'].remove(ext)
-            for modname in ext.split(','):
+            d['extensions'].extend(ext.split(','))
-                d['extensions'].append(modname)
 
     for variable in d.get('variables', []):
         try:
@@ -707,7 +660,7 @@ def main(argv=sys.argv[1:]):
         except ValueError:
             print('Invalid template variable: %s' % variable)
 
-    generate(d, templatedir=opts.templatedir)
+    generate(d, templatedir=args.templatedir)
     return 0
 
 

sphinx/cmdline.py

@@ -5,67 +5,37 @@
 
     sphinx-build command-line handling.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 from __future__ import print_function
 
+import argparse
+import multiprocessing
 import sys
-import optparse
 import traceback
 from os import path
-
+from typing import TYPE_CHECKING
-from six import text_type, binary_type
 
 from docutils.utils import SystemMessage
+from six import text_type, binary_type
 
 from sphinx import __display_version__
-from sphinx.errors import SphinxError
 from sphinx.application import Sphinx
+from sphinx.errors import SphinxError
 from sphinx.util import Tee, format_exception_cut_frames, save_traceback
 from sphinx.util.console import red, nocolor, color_terminal  # type: ignore
 from sphinx.util.docutils import docutils_namespace, patch_docutils
 from sphinx.util.osutil import abspath, fs_encoding
 from sphinx.util.pycompat import terminal_safe
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, IO, List, Union  # NOQA
 
 
-USAGE = """\
+def handle_exception(app, args, exception, stderr=sys.stderr):
-Sphinx v%s
-Usage: %%prog [options] sourcedir outdir [filenames...]
-
-Filename arguments:
-  without -a and without filenames, write new and changed files.
-  with -a, write all files.
-  with filenames, write these.
-""" % __display_version__
-
-EPILOG = """\
-For more information, visit <http://sphinx-doc.org/>.
-"""
-
-
-class MyFormatter(optparse.IndentedHelpFormatter):
-    def format_usage(self, usage):
-        # type: (Any) -> Any
-        return usage
-
-    def format_help(self, formatter):
-        # type: (Any) -> unicode
-        result = []  # type: List[unicode]
-        if self.description:  # type: ignore
-            result.append(self.format_description(formatter))
-        if self.option_list:  # type: ignore
-            result.append(self.format_option_help(formatter))  # type: ignore
-        return "\n".join(result)
-
-
-def handle_exception(app, opts, exception, stderr=sys.stderr):
     # type: (Sphinx, Any, Union[Exception, KeyboardInterrupt], IO) -> None
-    if opts.pdb:
+    if args.pdb:
         import pdb
         print(red('Exception occurred while building, starting debugger:'),
               file=stderr)
@@ -73,7 +43,7 @@ def handle_exception(app, opts, exception, stderr=sys.stderr):
         pdb.post_mortem(sys.exc_info()[2])
     else:
         print(file=stderr)
-        if opts.verbosity or opts.traceback:
+        if args.verbosity or args.traceback:
             traceback.print_exc(None, stderr)
             print(file=stderr)
         if isinstance(exception, KeyboardInterrupt):
@@ -114,119 +84,151 @@ def handle_exception(app, opts, exception, stderr=sys.stderr):
                   file=stderr)
 
 
+def jobs_argument(value):
+    # type: (str) -> int
+    """
+    Special type to handle 'auto' flags passed to 'sphinx-build' via -j flag. Can
+    be expanded to handle other special scaling requests, such as setting job count
+    to cpu_count.
+    """
+    if value == 'auto':
+        return multiprocessing.cpu_count()
+    else:
+        jobs = int(value)
+        if jobs <= 0:
+            raise argparse.ArgumentTypeError('job number should be a positive number')
+        else:
+            return jobs
+
+
+def get_parser():
+    # type: () -> argparse.ArgumentParser
+    parser = argparse.ArgumentParser(
+        usage='usage: %(prog)s [OPTIONS] SOURCEDIR OUTPUTDIR [FILENAMES...]',
+        epilog='For more information, visit <http://sphinx-doc.org/>.',
+        description="""
+Generate documentation from source files.
+
+sphinx-build generates documentation from the files in SOURCEDIR and places it
+in OUTPUTDIR. It looks for 'conf.py' in SOURCEDIR for the configuration
+settings.  The 'sphinx-quickstart' tool may be used to generate template files,
+including 'conf.py'
+
+sphinx-build can create documentation in different formats. A format is
+selected by specifying the builder name on the command line; it defaults to
+HTML. Builders can also perform other tasks related to documentation
+processing.
+
+By default, everything that is outdated is built. Output only for selected
+files can be built by specifying individual filenames.
+""")
+
+    parser.add_argument('--version', action='version', dest='show_version',
+                        version='%%(prog)s %s' % __display_version__)
+
+    parser.add_argument('sourcedir',
+                        help='path to documentation source files')
+    parser.add_argument('outputdir',
+                        help='path to output directory')
+    parser.add_argument('filenames', nargs='*',
+                        help='a list of specific files to rebuild. Ignored '
+                        'if -a is specified')
+
+    group = parser.add_argument_group('general options')
+    group.add_argument('-b', metavar='BUILDER', dest='builder',
+                       default='html',
+                       help='builder to use (default: html)')
+    group.add_argument('-a', action='store_true', dest='force_all',
+                       help='write all files (default: only write new and '
+                       'changed files)')
+    group.add_argument('-E', action='store_true', dest='freshenv',
+                       help='don\'t use a saved environment, always read '
+                       'all files')
+    group.add_argument('-d', metavar='PATH', dest='doctreedir',
+                       help='path for the cached environment and doctree '
+                       'files (default: OUTPUTDIR/.doctrees)')
+    group.add_argument('-j', metavar='N', default=1, type=jobs_argument, dest='jobs',
+                       help='build in parallel with N processes where '
+                       'possible (special value "auto" will set N to cpu-count)')
+    group = parser.add_argument_group('build configuration options')
+    group.add_argument('-c', metavar='PATH', dest='confdir',
+                       help='path where configuration file (conf.py) is '
+                       'located (default: same as SOURCEDIR)')
+    group.add_argument('-C', action='store_true', dest='noconfig',
+                       help='use no config file at all, only -D options')
+    group.add_argument('-D', metavar='setting=value', action='append',
+                       dest='define', default=[],
+                       help='override a setting in configuration file')
+    group.add_argument('-A', metavar='name=value', action='append',
+                       dest='htmldefine', default=[],
+                       help='pass a value into HTML templates')
+    group.add_argument('-t', metavar='TAG', action='append',
+                       dest='tags', default=[],
+                       help='define tag: include "only" blocks with TAG')
+    group.add_argument('-n', action='store_true', dest='nitpicky',
+                       help='nit-picky mode, warn about all missing '
+                       'references')
+
+    group = parser.add_argument_group('console output options')
+    group.add_argument('-v', action='count', dest='verbosity', default=0,
+                       help='increase verbosity (can be repeated)')
+    group.add_argument('-q', action='store_true', dest='quiet',
+                       help='no output on stdout, just warnings on stderr')
+    group.add_argument('-Q', action='store_true', dest='really_quiet',
+                       help='no output at all, not even warnings')
+    group.add_argument('--color', action='store_const', const='yes',
+                       default='auto',
+                       help='do emit colored output (default: auto-detect)')
+    group.add_argument('-N', '--no-color', dest='color', action='store_const',
+                       const='no',
+                       help='do not emit colored output (default: '
+                       'auto-detect)')
+    group.add_argument('-w', metavar='FILE', dest='warnfile',
+                       help='write warnings (and errors) to given file')
+    group.add_argument('-W', action='store_true', dest='warningiserror',
+                       help='turn warnings into errors')
+    group.add_argument('-T', action='store_true', dest='traceback',
+                       help='show full traceback on exception')
+    group.add_argument('-P', action='store_true', dest='pdb',
+                       help='run Pdb on exception')
+
+    return parser
+
+
 def main(argv=sys.argv[1:]):  # type: ignore
     # type: (List[unicode]) -> int
-    parser = optparse.OptionParser(USAGE, epilog=EPILOG, formatter=MyFormatter())
-    parser.add_option('--version', action='store_true', dest='version',
-                      help='show version information and exit')
 
-    group = parser.add_option_group('General options')
+    parser = get_parser()
-    group.add_option('-b', metavar='BUILDER', dest='builder', default='html',
+    args = parser.parse_args(argv)
-                     help='builder to use; default is html')
-    group.add_option('-a', action='store_true', dest='force_all',
-                     help='write all files; default is to only write new and '
-                     'changed files')
-    group.add_option('-E', action='store_true', dest='freshenv',
-                     help='don\'t use a saved environment, always read '
-                     'all files')
-    group.add_option('-d', metavar='PATH', default=None, dest='doctreedir',
-                     help='path for the cached environment and doctree files '
-                     '(default: outdir/.doctrees)')
-    group.add_option('-j', metavar='N', default=1, type='int', dest='jobs',
-                     help='build in parallel with N processes where possible')
-    # this option never gets through to this point (it is intercepted earlier)
-    # group.add_option('-M', metavar='BUILDER', dest='make_mode',
-    #                 help='"make" mode -- as used by Makefile, like '
-    #                 '"sphinx-build -M html"')
-
-    group = parser.add_option_group('Build configuration options')
-    group.add_option('-c', metavar='PATH', dest='confdir',
-                     help='path where configuration file (conf.py) is located '
-                     '(default: same as sourcedir)')
-    group.add_option('-C', action='store_true', dest='noconfig',
-                     help='use no config file at all, only -D options')
-    group.add_option('-D', metavar='setting=value', action='append',
-                     dest='define', default=[],
-                     help='override a setting in configuration file')
-    group.add_option('-A', metavar='name=value', action='append',
-                     dest='htmldefine', default=[],
-                     help='pass a value into HTML templates')
-    group.add_option('-t', metavar='TAG', action='append',
-                     dest='tags', default=[],
-                     help='define tag: include "only" blocks with TAG')
-    group.add_option('-n', action='store_true', dest='nitpicky',
-                     help='nit-picky mode, warn about all missing references')
-
-    group = parser.add_option_group('Console output options')
-    group.add_option('-v', action='count', dest='verbosity', default=0,
-                     help='increase verbosity (can be repeated)')
-    group.add_option('-q', action='store_true', dest='quiet',
-                     help='no output on stdout, just warnings on stderr')
-    group.add_option('-Q', action='store_true', dest='really_quiet',
-                     help='no output at all, not even warnings')
-    group.add_option('--color', dest='color',
-                     action='store_const', const='yes', default='auto',
-                     help='Do emit colored output (default: auto-detect)')
-    group.add_option('-N', '--no-color', dest='color',
-                     action='store_const', const='no',
-                     help='Do not emit colored output (default: auto-detect)')
-    group.add_option('-w', metavar='FILE', dest='warnfile',
-                     help='write warnings (and errors) to given file')
-    group.add_option('-W', action='store_true', dest='warningiserror',
-                     help='turn warnings into errors')
-    group.add_option('-T', action='store_true', dest='traceback',
-                     help='show full traceback on exception')
-    group.add_option('-P', action='store_true', dest='pdb',
-                     help='run Pdb on exception')
-
-    # parse options
-    try:
-        opts, args = parser.parse_args(argv)
-    except SystemExit as err:
-        return err.code
-
-    # handle basic options
-    if opts.version:
-        print('Sphinx (sphinx-build) %s' % __display_version__)
-        return 0
 
     # get paths (first and second positional argument)
     try:
-        srcdir = abspath(args[0])
+        srcdir = abspath(args.sourcedir)
-        confdir = abspath(opts.confdir or srcdir)
+        confdir = abspath(args.confdir or srcdir)
-        if opts.noconfig:
+        if args.noconfig:
             confdir = None
+
         if not path.isdir(srcdir):
-            print('Error: Cannot find source directory `%s\'.' % srcdir,
+            parser.error('cannot find source directory (%s)' % srcdir)
-                  file=sys.stderr)
+        if not args.noconfig and not path.isfile(path.join(confdir, 'conf.py')):
-            return 1
+            parser.error("config directory doesn't contain a conf.py file "
-        if not opts.noconfig and not path.isfile(path.join(confdir, 'conf.py')):
+                         "(%s)" % confdir)
-            print('Error: Config directory doesn\'t contain a conf.py file.',
+
-                  file=sys.stderr)
+        outdir = abspath(args.outputdir)
-            return 1
-        outdir = abspath(args[1])
         if srcdir == outdir:
-            print('Error: source directory and destination directory are same.',
+            parser.error('source directory and destination directory are same')
-                  file=sys.stderr)
-            return 1
-    except IndexError:
-        parser.print_help()
-        return 1
     except UnicodeError:
-        print(
+        parser.error('multibyte filename not supported on this filesystem '
-            'Error: Multibyte filename not supported on this filesystem '
+                     'encoding (%r)' % fs_encoding)
-            'encoding (%r).' % fs_encoding, file=sys.stderr)
-        return 1
 
     # handle remaining filename arguments
-    filenames = args[2:]
+    filenames = args.filenames
-    errored = False
+    missing_files = []
     for filename in filenames:
         if not path.isfile(filename):
-            print('Error: Cannot find file %r.' % filename, file=sys.stderr)
+            missing_files.append(filename)
-            errored = True
+    if missing_files:
-    if errored:
+        parser.error('cannot find files %r' % missing_files)
-        return 1
 
     # likely encoding used for command-line arguments
     try:
@@ -235,41 +237,39 @@ def main(argv=sys.argv[1:]):  # type: ignore
     except Exception:
         likely_encoding = None
 
-    if opts.force_all and filenames:
+    if args.force_all and filenames:
-        print('Error: Cannot combine -a option and filenames.', file=sys.stderr)
+        parser.error('cannot combine -a option and filenames')
-        return 1
 
-    if opts.color == 'no' or (opts.color == 'auto' and not color_terminal()):
+    if args.color == 'no' or (args.color == 'auto' and not color_terminal()):
         nocolor()
 
-    doctreedir = abspath(opts.doctreedir or path.join(outdir, '.doctrees'))
+    doctreedir = abspath(args.doctreedir or path.join(outdir, '.doctrees'))
 
     status = sys.stdout
     warning = sys.stderr
     error = sys.stderr
 
-    if opts.quiet:
+    if args.quiet:
         status = None
-    if opts.really_quiet:
+
+    if args.really_quiet:
         status = warning = None
-    if warning and opts.warnfile:
+
+    if warning and args.warnfile:
         try:
-            warnfp = open(opts.warnfile, 'w')
+            warnfp = open(args.warnfile, 'w')
         except Exception as exc:
-            print('Error: Cannot open warning file %r: %s' %
+            parser.error('cannot open warning file %r: %s' % (
-                  (opts.warnfile, exc), file=sys.stderr)
+                args.warnfile, exc))
-            sys.exit(1)
         warning = Tee(warning, warnfp)  # type: ignore
         error = warning
 
     confoverrides = {}
-    for val in opts.define:
+    for val in args.define:
         try:
             key, val = val.split('=', 1)
         except ValueError:
-            print('Error: -D option argument must be in the form name=value.',
+            parser.error('-D option argument must be in the form name=value')
-                  file=sys.stderr)
-            return 1
         if likely_encoding and isinstance(val, binary_type):
             try:
                 val = val.decode(likely_encoding)
@@ -277,13 +277,11 @@ def main(argv=sys.argv[1:]):  # type: ignore
                 pass
         confoverrides[key] = val
 
-    for val in opts.htmldefine:
+    for val in args.htmldefine:
         try:
             key, val = val.split('=')
         except ValueError:
-            print('Error: -A option argument must be in the form name=value.',
+            parser.error('-A option argument must be in the form name=value')
-                  file=sys.stderr)
-            return 1
         try:
             val = int(val)
         except ValueError:
@@ -294,17 +292,17 @@ def main(argv=sys.argv[1:]):  # type: ignore
                     pass
         confoverrides['html_context.%s' % key] = val
 
-    if opts.nitpicky:
+    if args.nitpicky:
         confoverrides['nitpicky'] = True
 
     app = None
     try:
         with patch_docutils(), docutils_namespace():
-            app = Sphinx(srcdir, confdir, outdir, doctreedir, opts.builder,
+            app = Sphinx(srcdir, confdir, outdir, doctreedir, args.builder,
-                         confoverrides, status, warning, opts.freshenv,
+                         confoverrides, status, warning, args.freshenv,
-                         opts.warningiserror, opts.tags, opts.verbosity, opts.jobs)
+                         args.warningiserror, args.tags, args.verbosity, args.jobs)
-            app.build(opts.force_all, filenames)
+            app.build(args.force_all, filenames)
             return app.statuscode
     except (Exception, KeyboardInterrupt) as exc:
-        handle_exception(app, opts, exc, error)
+        handle_exception(app, args, exc, error)
-        return 1
+        return 2

sphinx/config.py

@@ -5,15 +5,16 @@
 
     Build configuration file handling.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
+import traceback
 from os import path, getenv
+from typing import TYPE_CHECKING, Any, NamedTuple, Union
 
 from six import PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types
-from typing import Any, NamedTuple, Union
 
 from sphinx.errors import ConfigError
 from sphinx.locale import l_, __
@@ -22,9 +23,8 @@ from sphinx.util.i18n import format_date
 from sphinx.util.osutil import cd
 from sphinx.util.pycompat import execfile_, NoneType
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
+    from typing import Any, Callable, Dict, Iterable, Iterator, List, Tuple, Union  # NOQA
-    from typing import Any, Callable, Dict, Iterable, Iterator, List, Tuple  # NOQA
     from sphinx.util.tags import Tags  # NOQA
 
 logger = logging.getLogger(__name__)
@@ -35,6 +35,7 @@ copyright_year_re = re.compile(r'^((\d{4}-)?)(\d{4})(?=[ ,])')
 CONFIG_SYNTAX_ERROR = "There is a syntax error in your configuration file: %s"
 if PY3:
     CONFIG_SYNTAX_ERROR += "\nDid you change the syntax from 2.x to 3.x?"
+CONFIG_ERROR = "There is a programable error in your configuration file:\n\n%s"
 CONFIG_EXIT_ERROR = "The configuration file (or one of the modules it imports) " \
                     "called sys.exit()"
 CONFIG_ENUM_WARNING = "The config value `{name}` has to be a one of {candidates}, " \
@@ -63,8 +64,11 @@ class ENUM(object):
         self.candidates = candidates
 
     def match(self, value):
-        # type: (unicode) -> bool
+        # type: (Union[unicode,List,Tuple]) -> bool
-        return value in self.candidates
+        if isinstance(value, (list, tuple)):
+            return all(item in self.candidates for item in value)
+        else:
+            return value in self.candidates
 
 
 string_classes = [text_type]  # type: List
@@ -120,6 +124,7 @@ class Config(object):
         primary_domain = ('py', 'env', [NoneType]),
         needs_sphinx = (None, None, string_classes),
         needs_extensions = ({}, None),
+        manpages_url = (None, 'env'),
         nitpicky = (False, None),
         nitpick_ignore = ([], None),
         numfig = (False, 'env'),
@@ -132,6 +137,11 @@ class Config(object):
 
         tls_verify = (True, 'env'),
         tls_cacerts = (None, 'env'),
+        smartquotes = (True, 'env'),
+        smartquotes_action = ('qDe', 'env'),
+        smartquotes_excludes = ({'languages': ['ja'],
+                                 'builders': ['man', 'text']},
+                                'env'),
     )  # type: Dict[unicode, Tuple]
 
     def __init__(self, dirname, filename, overrides, tags):
@@ -152,6 +162,8 @@ class Config(object):
                     raise ConfigError(CONFIG_SYNTAX_ERROR % err)
                 except SystemExit:
                     raise ConfigError(CONFIG_EXIT_ERROR)
+                except Exception:
+                    raise ConfigError(CONFIG_ERROR % traceback.format_exc())
 
         self._raw_config = config
         # these two must be preinitialized because extensions can add their
@@ -288,7 +300,7 @@ class Config(object):
                 logger.warning("%s", exc)
         for name in config:
             if name in self.values:
-                self.__dict__[name] = config[name]
+                self.__dict__[name] = config[name]  # type: ignore
         if isinstance(self.source_suffix, string_types):  # type: ignore
             self.source_suffix = [self.source_suffix]  # type: ignore
 
@@ -329,5 +341,7 @@ class Config(object):
         self.values[name] = (default, rebuild, types)
 
     def filter(self, rebuild):
-        # type: (str) -> Iterator[ConfigValue]
+        # type: (Union[unicode, List[unicode]]) -> Iterator[ConfigValue]
-        return (value for value in self if value.rebuild == rebuild)  # type: ignore
+        if isinstance(rebuild, string_types):
+            rebuild = [rebuild]
+        return (value for value in self if value.rebuild in rebuild)  # type: ignore

sphinx/deprecation.py

@@ -5,7 +5,7 @@
 
     Sphinx deprecation classes and utilities.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

sphinx/directives/__init__.py

@@ -5,11 +5,12 @@
 
     Handlers for additional ReST directives.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives, roles
@@ -29,8 +30,7 @@ from sphinx.directives.patches import (  # noqa
     Figure, Meta
 )
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA

sphinx/directives/code.py

@@ -3,13 +3,14 @@
     sphinx.directives.code
     ~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-import sys
 import codecs
+import sys
 from difflib import unified_diff
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
@@ -21,8 +22,7 @@ from sphinx.util import logging
 from sphinx.util import parselinenos
 from sphinx.util.nodes import set_source_info
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.config import Config  # NOQA

sphinx/directives/other.py

@@ -3,27 +3,27 @@
     sphinx.directives.other
     ~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-from six.moves import range
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
 from docutils.parsers.rst.directives.admonitions import BaseAdmonition
 from docutils.parsers.rst.directives.misc import Class
 from docutils.parsers.rst.directives.misc import Include as BaseInclude
+from six.moves import range
 
 from sphinx import addnodes
 from sphinx.locale import versionlabels, _
 from sphinx.util import url_re, docname_join
+from sphinx.util.matching import patfilter
 from sphinx.util.nodes import explicit_title_re, set_source_info, \
     process_index_entry
-from sphinx.util.matching import patfilter
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
@@ -73,7 +73,9 @@ class TocTree(Directive):
         for entry in self.content:
             if not entry:
                 continue
-            if glob and ('*' in entry or '?' in entry or '[' in entry):
+            # look for explicit titles ("Some Title <document>")
+            explicit = explicit_title_re.match(entry)
+            if glob and ('*' in entry or '?' in entry or '[' in entry) and not explicit:
                 patname = docname_join(env.docname, entry)
                 docnames = sorted(patfilter(all_docnames, patname))
                 for docname in docnames:
@@ -85,11 +87,9 @@ class TocTree(Directive):
                         'toctree glob pattern %r didn\'t match any documents'
                         % entry, line=self.lineno))
             else:
-                # look for explicit titles ("Some Title <document>")
+                if explicit:
-                m = explicit_title_re.match(entry)
+                    ref = explicit.group(2)
-                if m:
+                    title = explicit.group(1)
-                    ref = m.group(2)
-                    title = m.group(1)
                     docname = ref
                 else:
                     ref = docname = entry

sphinx/directives/patches.py

@@ -3,10 +3,12 @@
     sphinx.directives.patches
     ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from typing import TYPE_CHECKING
+
 from docutils import nodes
 from docutils.parsers.rst import directives
 from docutils.parsers.rst.directives import images, html, tables
@@ -14,8 +16,7 @@ from docutils.parsers.rst.directives import images, html, tables
 from sphinx import addnodes
 from sphinx.util.nodes import set_source_info
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Dict, List  # NOQA
     from sphinx.application import Sphinx  # NOQA
 

sphinx/domains/__init__.py

@@ -6,19 +6,19 @@
     Support for domains, which are groupings of description directives
     and roles describing e.g. constructs of one programming language.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import copy
+from typing import TYPE_CHECKING
 
 from six import iteritems
 
 from sphinx.errors import SphinxError
 from sphinx.locale import _
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Callable, Dict, Iterable, List, Tuple, Type, Union  # NOQA
     from docutils import nodes  # NOQA
     from docutils.parsers.rst.states import Inliner  # NOQA
@@ -141,7 +141,7 @@ class Domain(object):
     #: domain label: longer, more descriptive (used in messages)
     label = ''
     #: type (usually directive) name -> ObjType instance
-    object_types = {}       # type: Dict[unicode, Any]
+    object_types = {}       # type: Dict[unicode, ObjType]
     #: directive name -> directive class
     directives = {}         # type: Dict[unicode, Any]
     #: role name -> role callable
@@ -161,6 +161,17 @@ class Domain(object):
     def __init__(self, env):
         # type: (BuildEnvironment) -> None
         self.env = env              # type: BuildEnvironment
+        self._role_cache = {}       # type: Dict[unicode, Callable]
+        self._directive_cache = {}  # type: Dict[unicode, Callable]
+        self._role2type = {}        # type: Dict[unicode, List[unicode]]
+        self._type2role = {}        # type: Dict[unicode, unicode]
+
+        # convert class variables to instance one (to enhance through API)
+        self.object_types = dict(self.object_types)
+        self.directives = dict(self.directives)
+        self.roles = dict(self.roles)
+        self.indices = list(self.indices)
+
         if self.name not in env.domaindata:
             assert isinstance(self.initial_data, dict)
             new_data = copy.deepcopy(self.initial_data)
@@ -170,10 +181,6 @@ class Domain(object):
             self.data = env.domaindata[self.name]
             if self.data['version'] != self.data_version:
                 raise IOError('data of %r domain out of date' % self.label)
-        self._role_cache = {}       # type: Dict[unicode, Callable]
-        self._directive_cache = {}  # type: Dict[unicode, Callable]
-        self._role2type = {}        # type: Dict[unicode, List[unicode]]
-        self._type2role = {}        # type: Dict[unicode, unicode]
         for name, obj in iteritems(self.object_types):
             for rolename in obj.roles:
                 self._role2type.setdefault(rolename, []).append(name)
@@ -181,6 +188,18 @@ class Domain(object):
         self.objtypes_for_role = self._role2type.get    # type: Callable[[unicode], List[unicode]]  # NOQA
         self.role_for_objtype = self._type2role.get     # type: Callable[[unicode], unicode]
 
+    def add_object_type(self, name, objtype):
+        # type: (unicode, ObjType) -> None
+        """Add an object type."""
+        self.object_types[name] = objtype
+        if objtype.roles:
+            self._type2role[name] = objtype.roles[0]
+        else:
+            self._type2role[name] = ''
+
+        for role in objtype.roles:
+            self._role2type.setdefault(role, []).append(name)
+
     def role(self, name):
         # type: (unicode) -> Callable
         """Return a role adapter function that always gives the registered

sphinx/domains/c.py

@@ -5,25 +5,25 @@
 
     The C language domain.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
 import string
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 
 from sphinx import addnodes
-from sphinx.roles import XRefRole
-from sphinx.locale import l_, _
-from sphinx.domains import Domain, ObjType
 from sphinx.directives import ObjectDescription
-from sphinx.util.nodes import make_refnode
+from sphinx.domains import Domain, ObjType
+from sphinx.locale import l_, _
+from sphinx.roles import XRefRole
 from sphinx.util.docfields import Field, TypedField
+from sphinx.util.nodes import make_refnode
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Iterator, List, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.builders import Builder  # NOQA

sphinx/domains/javascript.py

@@ -5,24 +5,25 @@
 
     The JavaScript domain.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from typing import TYPE_CHECKING
+
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
 
 from sphinx import addnodes
-from sphinx.domains import Domain, ObjType
-from sphinx.locale import l_, _
 from sphinx.directives import ObjectDescription
-from sphinx.roles import XRefRole
+from sphinx.domains import Domain, ObjType
 from sphinx.domains.python import _pseudo_parse_arglist
-from sphinx.util.nodes import make_refnode
+from sphinx.locale import l_, _
+from sphinx.roles import XRefRole
 from sphinx.util.docfields import Field, GroupedField, TypedField
+from sphinx.util.nodes import make_refnode
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Iterator, List, Tuple  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.application import Sphinx  # NOQA

sphinx/domains/python.py

@@ -5,28 +5,27 @@
 
     The Python domain.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
-
+from typing import TYPE_CHECKING
-from six import iteritems
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
+from six import iteritems
 
 from sphinx import addnodes
-from sphinx.roles import XRefRole
-from sphinx.locale import l_, _
-from sphinx.domains import Domain, ObjType, Index
 from sphinx.directives import ObjectDescription
+from sphinx.domains import Domain, ObjType, Index
+from sphinx.locale import l_, _
+from sphinx.roles import XRefRole
 from sphinx.util import logging
-from sphinx.util.nodes import make_refnode
 from sphinx.util.docfields import Field, GroupedField, TypedField
+from sphinx.util.nodes import make_refnode
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Iterable, Iterator, List, Tuple, Union  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.builders import Builder  # NOQA
@@ -39,7 +38,7 @@ logger = logging.getLogger(__name__)
 py_sig_re = re.compile(
     r'''^ ([\w.]*\.)?            # class name(s)
           (\w+)  \s*             # thing name
-          (?: \((.*)\)           # optional: arguments
+          (?: \(\s*(.*)\s*\)     # optional: arguments
            (?:\s* -> \s* (.*))?  #           return annotation
           )? $                   # and nothing more
           ''', re.VERBOSE)
@@ -348,6 +347,10 @@ class PyObject(ObjectDescription):
             if self.allow_nesting:
                 classes = self.env.ref_context.setdefault('py:classes', [])
                 classes.append(prefix)
+        if 'module' in self.options:
+            modules = self.env.ref_context.setdefault('py:modules', [])
+            modules.append(self.env.ref_context.get('py:module'))
+            self.env.ref_context['py:module'] = self.options['module']
 
     def after_content(self):
         # type: () -> None
@@ -368,6 +371,12 @@ class PyObject(ObjectDescription):
                 pass
         self.env.ref_context['py:class'] = (classes[-1] if len(classes) > 0
                                             else None)
+        if 'module' in self.options:
+            modules = self.env.ref_context.setdefault('py:modules', [])
+            if modules:
+                self.env.ref_context['py:module'] = modules.pop()
+            else:
+                self.env.ref_context.pop('py:module')
 
 
 class PyModulelevel(PyObject):

sphinx/domains/rst.py

@@ -5,23 +5,23 @@
 
     The reStructuredText domain.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
+from typing import TYPE_CHECKING
 
 from six import iteritems
 
 from sphinx import addnodes
+from sphinx.directives import ObjectDescription
 from sphinx.domains import Domain, ObjType
 from sphinx.locale import l_, _
-from sphinx.directives import ObjectDescription
 from sphinx.roles import XRefRole
 from sphinx.util.nodes import make_refnode
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Iterator, List, Tuple  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.application import Sphinx  # NOQA

sphinx/domains/std.py

@@ -5,29 +5,28 @@
 
     The standard domain.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
 import unicodedata
-
+from typing import TYPE_CHECKING
-from six import iteritems
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
 from docutils.statemachine import ViewList
+from six import iteritems
 
 from sphinx import addnodes
-from sphinx.roles import XRefRole
-from sphinx.locale import l_, _
-from sphinx.domains import Domain, ObjType
 from sphinx.directives import ObjectDescription
+from sphinx.domains import Domain, ObjType
+from sphinx.locale import l_, _
+from sphinx.roles import XRefRole
 from sphinx.util import ws_re, logging, docname_join
 from sphinx.util.nodes import clean_astext, make_refnode
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Callable, Dict, Iterator, List, Tuple, Type, Union  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.builders import Builder  # NOQA
@@ -607,8 +606,9 @@ class StandardDomain(Domain):
             if node.tagname == 'target' and 'refid' in node:  # indirect hyperlink targets
                 node = document.ids.get(node['refid'])
                 labelid = node['names'][0]
-            if name.isdigit() or 'refuri' in node or \
+            if (node.tagname == 'footnote' or
-               node.tagname.startswith('desc_'):
+                    'refuri' in node or
+                    node.tagname.startswith('desc_')):
                 # ignore footnote labels, labels automatically generated from a
                 # link and object descriptions
                 continue
@@ -959,12 +959,18 @@ class StandardDomain(Domain):
 
     def get_full_qualified_name(self, node):
         # type: (nodes.Node) -> unicode
-        progname = node.get('std:program')
+        if node.get('reftype') == 'option':
-        target = node.get('reftarget')
+            progname = node.get('std:program')
-        if progname is None or target is None:
+            command = ws_re.split(node.get('reftarget'))
-            return None
+            if progname:
+                command.insert(0, progname)
+            option = command.pop()
+            if command:
+                return '.'.join(['-'.join(command), option])
+            else:
+                return None
         else:
-            return '.'.join([progname, target])
+            return None
 
 
 def setup(app):

sphinx/environment/__init__.py

@@ -5,56 +5,46 @@
 
     Global creation environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-import re
 import os
+import re
 import sys
 import time
 import types
-import codecs
-import fnmatch
 import warnings
-from os import path
-from copy import copy
 from collections import defaultdict
+from copy import copy
+from os import path
+from typing import TYPE_CHECKING
 
+from docutils.frontend import OptionParser
+from docutils.utils import Reporter, get_source_line
 from six import BytesIO, itervalues, class_types, next
 from six.moves import cPickle as pickle
 
-from docutils.io import NullOutput
+from sphinx import addnodes, versioning
-from docutils.core import Publisher
-from docutils.utils import Reporter, get_source_line, normalize_language_tag
-from docutils.utils.smartquotes import smartchars
-from docutils.parsers.rst import roles
-from docutils.parsers.rst.languages import en as english
-from docutils.frontend import OptionParser
-
-from sphinx import addnodes
-from sphinx.io import SphinxStandaloneReader, SphinxDummyWriter, SphinxFileInput
-from sphinx.util import logging
-from sphinx.util import get_matching_docs, FilenameUniqDict, status_iterator
-from sphinx.util.nodes import is_translatable
-from sphinx.util.osutil import SEP, ensuredir
-from sphinx.util.i18n import find_catalog_files
-from sphinx.util.console import bold  # type: ignore
-from sphinx.util.docutils import sphinx_domains, WarningStream
-from sphinx.util.matching import compile_matchers
-from sphinx.util.parallel import ParallelTasks, parallel_available, make_chunks
-from sphinx.util.websupport import is_commentable
-from sphinx.errors import SphinxError, ExtensionError
-from sphinx.locale import __
-from sphinx.transforms import SphinxTransformer
-from sphinx.versioning import add_uids, merge_doctrees
 from sphinx.deprecation import RemovedInSphinx20Warning
 from sphinx.environment.adapters.indexentries import IndexEntries
 from sphinx.environment.adapters.toctree import TocTree
+from sphinx.errors import SphinxError, ExtensionError
+from sphinx.io import read_doc
+from sphinx.transforms import SphinxTransformer
+from sphinx.util import get_matching_docs, FilenameUniqDict, status_iterator
+from sphinx.util import logging, rst
+from sphinx.util.console import bold  # type: ignore
+from sphinx.util.docutils import sphinx_domains, WarningStream
+from sphinx.util.i18n import find_catalog_files
+from sphinx.util.matching import compile_matchers
+from sphinx.util.nodes import is_translatable
+from sphinx.util.osutil import SEP, ensuredir
+from sphinx.util.parallel import ParallelTasks, parallel_available, make_chunks
+from sphinx.util.websupport import is_commentable
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
+    from typing import Any, Callable, Dict, IO, Iterator, List, Optional, Pattern, Set, Tuple, Type, Union, Generator  # NOQA
-    from typing import Any, Callable, Dict, IO, Iterator, List, Pattern, Set, Tuple, Type, Union  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.builders import Builder  # NOQA
@@ -67,12 +57,15 @@ default_settings = {
     'embed_stylesheet': False,
     'cloak_email_addresses': True,
     'pep_base_url': 'https://www.python.org/dev/peps/',
+    'pep_references': None,
     'rfc_base_url': 'https://tools.ietf.org/html/',
+    'rfc_references': None,
     'input_encoding': 'utf-8-sig',
     'doctitle_xform': False,
     'sectsubtitle_xform': False,
     'halt_level': 5,
     'file_insertion_enabled': True,
+    'smartquotes_locales': [],
 }
 
 # This is increased every time an environment attribute is added
@@ -82,8 +75,6 @@ default_settings = {
 ENV_VERSION = 52 + (sys.version_info[0] - 2)
 
 
-dummy_reporter = Reporter('', 4, 4)
-
 versioning_conditions = {
     'none': False,
     'text': is_translatable,
@@ -110,7 +101,12 @@ class BuildEnvironment(object):
     @staticmethod
     def load(f, app=None):
         # type: (IO, Sphinx) -> BuildEnvironment
-        env = pickle.load(f)
+        try:
+            env = pickle.load(f)
+        except Exception as exc:
+            # This can happen for example when the pickle is from a
+            # different version of Sphinx.
+            raise IOError(exc)
         if env.version != ENV_VERSION:
             raise IOError('build environment version not current')
         if app:
@@ -173,8 +169,8 @@ class BuildEnvironment(object):
         # type: (Sphinx) -> None
         self.app = app
         self.doctreedir = app.doctreedir
-        self.srcdir = app.srcdir
+        self.srcdir = app.srcdir  # type: unicode
-        self.config = app.config
+        self.config = app.config  # type: Config
 
         # the method of doctree versioning; see set_versioning_method
         self.versioning_condition = None  # type: Union[bool, Callable]
@@ -191,7 +187,7 @@ class BuildEnvironment(object):
         self._warnfunc = None  # type: Callable
 
         # this is to invalidate old pickles
-        self.version = ENV_VERSION
+        self.version = ENV_VERSION  # type: int
 
         # All "docnames" here are /-separated and relative and exclude
         # the source suffix.
@@ -256,8 +252,8 @@ class BuildEnvironment(object):
                                     # lineno, module, descname, content)
 
         # these map absolute path -> (docnames, unique filename)
-        self.images = FilenameUniqDict()
+        self.images = FilenameUniqDict()    # type: FilenameUniqDict
-        self.dlfiles = FilenameUniqDict()
+        self.dlfiles = FilenameUniqDict()   # type: FilenameUniqDict
 
         # the original URI for images
         self.original_image_uri = {}  # type: Dict[unicode, unicode]
@@ -344,17 +340,16 @@ class BuildEnvironment(object):
         app.emit('env-merge-info', self, docnames, other)
 
     def path2doc(self, filename):
-        # type: (unicode) -> unicode
+        # type: (unicode) -> Optional[unicode]
         """Return the docname for the filename if the file is document.
 
         *filename* should be absolute or relative to the source directory.
         """
         if filename.startswith(self.srcdir):
-            filename = filename[len(self.srcdir) + 1:]
+            filename = os.path.relpath(filename, self.srcdir)
         for suffix in self.config.source_suffix:
-            if fnmatch.fnmatch(filename, '*' + suffix):
+            if filename.endswith(suffix):
                 return filename[:-len(suffix)]
-        # the file does not have docname
         return None
 
     def doc2path(self, docname, base=True, suffix=None):
@@ -368,15 +363,13 @@ class BuildEnvironment(object):
         """
         docname = docname.replace(SEP, path.sep)
         if suffix is None:
-            candidate_suffix = None  # type: unicode
+            # Use first candidate if there is not a file for any suffix
+            suffix = next(iter(self.config.source_suffix))
             for candidate_suffix in self.config.source_suffix:
                 if path.isfile(path.join(self.srcdir, docname) +
                                candidate_suffix):
                     suffix = candidate_suffix
                     break
-            else:
-                # document does not exist
-                suffix = self.config.source_suffix[0]
         if base is True:
             return path.join(self.srcdir, docname) + suffix
         elif base is None:
@@ -561,21 +554,10 @@ class BuildEnvironment(object):
         self.app.emit('env-before-read-docs', self, docnames)
 
         # check if we should do parallel or serial read
-        par_ok = False
         if parallel_available and len(docnames) > 5 and self.app.parallel > 1:
-            for ext in itervalues(self.app.extensions):
+            par_ok = self.app.is_parallel_allowed('read')
-                if ext.parallel_read_safe is None:
+        else:
-                    logger.warning(__('the %s extension does not declare if it is safe '
+            par_ok = False
-                                      'for parallel reading, assuming it isn\'t - please '
-                                      'ask the extension author to check and make it '
-                                      'explicit'), ext.name)
-                    logger.warning('doing serial read')
-                    break
-                elif ext.parallel_read_safe is False:
-                    break
-            else:
-                # all extensions support parallel-read
-                par_ok = True
 
         if par_ok:
             self._read_parallel(docnames, self.app, nproc=self.app.parallel)
@@ -643,25 +625,9 @@ class BuildEnvironment(object):
 
     # --------- SINGLE FILE READING --------------------------------------------
 
-    def warn_and_replace(self, error):
+    def prepare_settings(self, docname):
-        # type: (Any) -> Tuple
+        # type: (unicode) -> None
-        """Custom decoding error handler that warns and replaces."""
+        """Prepare to set up environment for reading."""
-        linestart = error.object.rfind(b'\n', 0, error.start)
-        lineend = error.object.find(b'\n', error.start)
-        if lineend == -1:
-            lineend = len(error.object)
-        lineno = error.object.count(b'\n', 0, error.start) + 1
-        logger.warning('undecodable source characters, replacing with "?": %r',
-                       (error.object[linestart + 1:error.start] + b'>>>' +
-                        error.object[error.start:error.end] + b'<<<' +
-                        error.object[error.end:lineend]),
-                       location=(self.docname, lineno))
-        return (u'?', error.end)
-
-    def read_doc(self, docname, app=None):
-        # type: (unicode, Sphinx) -> None
-        """Parse a file and add/update inventory entries for the doctree."""
-
         self.temp_data['docname'] = docname
         # defaults to the global default, but can be re-set in a document
         self.temp_data['default_role'] = self.config.default_role
@@ -673,15 +639,15 @@ class BuildEnvironment(object):
             self.config.trim_footnote_reference_space
         self.settings['gettext_compact'] = self.config.gettext_compact
 
-        language = self.config.language or 'en'
+        self.settings['language_code'] = self.config.language or 'en'
-        self.settings['language_code'] = language
+
-        if 'smart_quotes' not in self.settings:
+        # Allow to disable by 3rd party extension (workaround)
-            self.settings['smart_quotes'] = True
+        self.settings.setdefault('smart_quotes', True)
-            for tag in normalize_language_tag(language):
+
-                if tag in smartchars.quotes:
+    def read_doc(self, docname, app=None):
-                    break
+        # type: (unicode, Sphinx) -> None
-            else:
+        """Parse a file and add/update inventory entries for the doctree."""
-                self.settings['smart_quotes'] = False
+        self.prepare_settings(docname)
 
         docutilsconf = path.join(self.srcdir, 'docutils.conf')
         # read docutils.conf from source dir, not from current dir
@@ -689,34 +655,8 @@ class BuildEnvironment(object):
         if path.isfile(docutilsconf):
             self.note_dependency(docutilsconf)
 
-        with sphinx_domains(self):
+        with sphinx_domains(self), rst.default_role(docname, self.config.default_role):
-            if self.config.default_role:
+            doctree = read_doc(self.app, self, self.doc2path(docname))
-                role_fn, messages = roles.role(self.config.default_role, english,
-                                               0, dummy_reporter)
-                if role_fn:
-                    roles._roles[''] = role_fn
-                else:
-                    logger.warning('default role %s not found', self.config.default_role,
-                                   location=docname)
-
-            codecs.register_error('sphinx', self.warn_and_replace)  # type: ignore
-
-            # publish manually
-            reader = SphinxStandaloneReader(self.app,
-                                            parsers=self.app.registry.get_source_parsers())
-            pub = Publisher(reader=reader,
-                            writer=SphinxDummyWriter(),
-                            destination_class=NullOutput)
-            pub.set_components(None, 'restructuredtext', None)
-            pub.process_programmatic_settings(None, self.settings, None)
-            src_path = self.doc2path(docname)
-            source = SphinxFileInput(app, self, source=None, source_path=src_path,
-                                     encoding=self.config.source_encoding)
-            pub.source = source
-            pub.settings._source = src_path
-            pub.set_destination(None, None)
-            pub.publish()
-            doctree = pub.document
 
         # post-processing
         for domain in itervalues(self.domains):
@@ -734,41 +674,14 @@ class BuildEnvironment(object):
             time.time(), path.getmtime(self.doc2path(docname)))
 
         if self.versioning_condition:
-            old_doctree = None
-            if self.versioning_compare:
-                # get old doctree
-                try:
-                    with open(self.doc2path(docname,
-                                            self.doctreedir, '.doctree'), 'rb') as f:
-                        old_doctree = pickle.load(f)
-                except EnvironmentError:
-                    pass
-
             # add uids for versioning
-            if not self.versioning_compare or old_doctree is None:
+            versioning.prepare(doctree)
-                list(add_uids(doctree, self.versioning_condition))
-            else:
-                list(merge_doctrees(
-                    old_doctree, doctree, self.versioning_condition))
-
-        # make it picklable
-        doctree.reporter = None
-        doctree.transformer = None
-        doctree.settings.warning_stream = None
-        doctree.settings.env = None
-        doctree.settings.record_dependencies = None
 
         # cleanup
         self.temp_data.clear()
         self.ref_context.clear()
-        roles._roles.pop('', None)  # if a document has set a local default role
 
-        # save the parsed doctree
+        self.write_doctree(docname, doctree)
-        doctree_filename = self.doc2path(docname, self.doctreedir,
-                                         '.doctree')
-        ensuredir(path.dirname(doctree_filename))
-        with open(doctree_filename, 'wb') as f:
-            pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
 
     # utilities to use while reading a document
 
@@ -872,6 +785,21 @@ class BuildEnvironment(object):
         doctree.reporter = Reporter(self.doc2path(docname), 2, 5, stream=WarningStream())
         return doctree
 
+    def write_doctree(self, docname, doctree):
+        # type: (unicode, nodes.Node) -> None
+        """Write the doctree to a file."""
+        # make it picklable
+        doctree.reporter = None
+        doctree.transformer = None
+        doctree.settings.warning_stream = None
+        doctree.settings.env = None
+        doctree.settings.record_dependencies = None
+
+        doctree_filename = self.doc2path(docname, self.doctreedir, '.doctree')
+        ensuredir(path.dirname(doctree_filename))
+        with open(doctree_filename, 'wb') as f:
+            pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
+
     def get_and_resolve_doctree(self, docname, builder, doctree=None,
                                 prune_toctrees=True, includehidden=False):
         # type: (unicode, Builder, nodes.Node, bool, bool) -> nodes.Node
@@ -928,7 +856,7 @@ class BuildEnvironment(object):
 
             transformer = SphinxTransformer(doctree)
             transformer.set_environment(self)
-            transformer.add_transforms(self.app.post_transforms)
+            transformer.add_transforms(self.app.registry.get_post_transforms())
             transformer.apply_transforms()
         finally:
             self.temp_data = backup

sphinx/environment/adapters/__init__.py

@@ -5,6 +5,6 @@
 
     Sphinx environment adapters
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """

sphinx/environment/adapters/asset.py

@@ -5,12 +5,13 @@
 
     Assets adapter for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-if False:
+from typing import TYPE_CHECKING
-    # For type annotation
+
+if TYPE_CHECKING:
     from sphinx.environment import BuildEnvironment  # NOQA
 
 

sphinx/environment/adapters/indexentries.py

@@ -5,21 +5,21 @@
 
     Index entries adapters for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-import re
 import bisect
+import re
 import unicodedata
 from itertools import groupby
+from typing import TYPE_CHECKING
 
 from six import text_type, iteritems
 
 from sphinx.locale import _
 from sphinx.util import split_into, logging
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, Pattern, List, Tuple  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA

sphinx/environment/adapters/toctree.py

@@ -5,20 +5,20 @@
 
     Toctree adapter for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-from six import iteritems
+from typing import TYPE_CHECKING
 
 from docutils import nodes
+from six import iteritems
 
 from sphinx import addnodes
 from sphinx.util import url_re, logging
-from sphinx.util.nodes import clean_astext
+from sphinx.util.nodes import clean_astext, process_only_nodes
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA
@@ -158,7 +158,7 @@ class TocTree(object):
                         maxdepth = self.env.metadata[ref].get('tocdepth', 0)
                         if ref not in toctree_ancestors or (prune and maxdepth > 0):
                             self._toctree_prune(toc, 2, maxdepth, collapse)
-                        self.process_only_nodes(toc)
+                        process_only_nodes(toc, builder.tags)
                         if title and toc.children and len(toc.children) == 1:
                             child = toc.children[0]
                             for refnode in child.traverse(nodes.reference):
@@ -298,7 +298,7 @@ class TocTree(object):
             # the document does not exist anymore: return a dummy node that
             # renders to nothing
             return nodes.paragraph()
-        self.process_only_nodes(toc)
+        process_only_nodes(toc, builder.tags)
         for node in toc.traverse(nodes.reference):
             node['refuri'] = node['anchorname'] or '#'
         return toc
@@ -323,14 +323,3 @@ class TocTree(object):
         for toctree in toctrees[1:]:
             result.extend(toctree.children)
         return result
-
-    def process_only_nodes(self, doctree):
-        # type: (nodes.Node) -> None
-        # Lazy loading
-        from sphinx.transforms import SphinxTransformer
-        from sphinx.transforms.post_transforms import OnlyNodeTransform
-
-        transformer = SphinxTransformer(doctree)
-        transformer.set_environment(self.env)
-        transformer.add_transform(OnlyNodeTransform)
-        transformer.apply_transforms()

sphinx/environment/collectors/__init__.py

@@ -5,14 +5,15 @@
 
     The data collector components for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from typing import TYPE_CHECKING
+
 from six import itervalues
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Dict, List, Set  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.sphinx import Sphinx  # NOQA

sphinx/environment/collectors/asset.py

@@ -5,18 +5,18 @@
 
     The image collector for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import os
-from os import path
 from glob import glob
-
+from os import path
-from six import iteritems, itervalues
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 from docutils.utils import relative_path
+from six import iteritems, itervalues
 
 from sphinx import addnodes
 from sphinx.environment.collectors import EnvironmentCollector
@@ -24,8 +24,7 @@ from sphinx.util import logging
 from sphinx.util.i18n import get_image_filename_for_language, search_image_for_language
 from sphinx.util.images import guess_mimetype
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Dict, List, Set, Tuple  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.sphinx import Sphinx  # NOQA

sphinx/environment/collectors/dependencies.py

@@ -5,19 +5,19 @@
 
     The dependencies collector components for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from os import path
+from typing import TYPE_CHECKING
 
 from docutils.utils import relative_path
 
-from sphinx.util.osutil import getcwd, fs_encoding
 from sphinx.environment.collectors import EnvironmentCollector
+from sphinx.util.osutil import getcwd, fs_encoding
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Dict, Set  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.sphinx import Sphinx  # NOQA

sphinx/environment/collectors/indexentries.py

@@ -5,16 +5,17 @@
 
     Index entries collector for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-from sphinx import addnodes
+from typing import TYPE_CHECKING
-from sphinx.util import split_index_msg, logging
-from sphinx.environment.collectors import EnvironmentCollector
 
-if False:
+from sphinx import addnodes
-    # For type annotation
+from sphinx.environment.collectors import EnvironmentCollector
+from sphinx.util import split_index_msg, logging
+
+if TYPE_CHECKING:
     from typing import Dict, Set  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.applicatin import Sphinx  # NOQA

sphinx/environment/collectors/metadata.py

@@ -5,16 +5,17 @@
 
     The metadata collector components for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from typing import TYPE_CHECKING
+
 from docutils import nodes
 
 from sphinx.environment.collectors import EnvironmentCollector
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Dict, Set  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.sphinx import Sphinx  # NOQA

sphinx/environment/collectors/title.py

@@ -5,17 +5,18 @@
 
     The title collector components for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from typing import TYPE_CHECKING
+
 from docutils import nodes
 
 from sphinx.environment.collectors import EnvironmentCollector
 from sphinx.transforms import SphinxContentsFilter
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Dict, Set  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.sphinx import Sphinx  # NOQA

sphinx/environment/collectors/toctree.py

@@ -5,22 +5,22 @@
 
     Toctree collector for sphinx.environment.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-from six import iteritems
+from typing import TYPE_CHECKING
 
 from docutils import nodes
+from six import iteritems
 
 from sphinx import addnodes
-from sphinx.util import url_re, logging
-from sphinx.transforms import SphinxContentsFilter
 from sphinx.environment.adapters.toctree import TocTree
 from sphinx.environment.collectors import EnvironmentCollector
+from sphinx.transforms import SphinxContentsFilter
+from sphinx.util import url_re, logging
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Dict, List, Set, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.builders import Builder  # NOQA
@@ -262,7 +262,7 @@ class TocTreeCollector(EnvironmentCollector):
 
                     continue
 
-                figtype = env.get_domain('std').get_figtype(subnode)  # type: ignore
+                figtype = env.get_domain('std').get_figtype(subnode)
                 if figtype and subnode['ids']:
                     register_fignumber(docname, secnum, figtype, subnode)
 

sphinx/errors.py

@@ -6,12 +6,13 @@
     Contains SphinxError and a few subclasses (in an extra module to avoid
     circular import problems).
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-if False:
+from typing import TYPE_CHECKING
-    # For type annotation
+
+if TYPE_CHECKING:
     from typing import Any  # NOQA
 
 

sphinx/events.py

@@ -7,20 +7,20 @@
 
     Gracefully adapted from the TextPress system by Armin.
 
-    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 from __future__ import print_function
 
 from collections import OrderedDict, defaultdict
+from typing import TYPE_CHECKING
 
 from six import itervalues
 
 from sphinx.errors import ExtensionError
 from sphinx.locale import __
 
-if False:
+if TYPE_CHECKING:
-    # For type annotation
     from typing import Any, Callable, Dict, List  # NOQA