Merge branch 'master' into register_author_as_confval

CHANGES

@@ -10,6 +10,8 @@ Incompatible changes
 * #4460: extensions which stores any data to environment should return the
   version of its env data structure as metadata.  In detail, please see
   :ref:`ext-metadata`.
+* Sphinx expects source parser modules to have supported file formats as
+  ``Parser.supported`` attribute
 * The default value of :confval:`epub_author` and :confval:`epub_publisher` are
   changed from ``'unknown'`` to the value of :confval:`author`.  This is same as
   a ``conf.py`` file sphinx-build generates.
@@ -17,12 +19,16 @@ Incompatible changes
 Deprecated
 ----------
 
+* :confval:`source_parsers` is deprecated. Please use ``add_source_parser()``
+  instead.
+
 Features added
 --------------
 
 * Add :event:`config-inited` event
 * Add ``sphinx.config.Any`` to represent the config value accepts any type of
   value
+* :confval:`source_suffix` allows a mapping fileext to file types
 * Add :confval:`author` as a configuration value
 
 Bugs fixed

doc/config.rst

@@ -90,12 +90,32 @@ General configuration
 
 .. confval:: source_suffix
 
-   The file name extension, or list of extensions, of source files.  Only files
+   The file extensions of source files.  Sphinx considers the files with this
-   with this suffix will be read as sources.  Default is ``'.rst'``.
+   suffix as sources.  This value can be a dictionary mapping file extensions
+   to file types.  For example::
+
+      source_suffix = {
+          '.rst': 'restructuredtext',
+          '.txt': 'restructuredtext',
+          '.md': 'markdown',
+      }
+
+   By default, Sphinx only supports ``'restrcturedtext'`` file type.  You can
+   add a new file type using source parser extensions.  Please read a document
+   of the extension to know what file type the extension supports.
+
+   This also allows a list of file extensions.  In that case, Sphinx conciders
+   that all they are ``'restructuredtext'``.  Default is
+   ``{'.rst': 'restructuredtext'}``.
+
+   .. note:: file extensions have to start with dot (like ``.rst``).
 
    .. versionchanged:: 1.3
       Can now be a list of extensions.
 
+   .. versionchanged:: 1.8
+      Support file type mapping
+
 .. confval:: source_encoding
 
    The encoding of all reST source files.  The recommended encoding, and the
@@ -123,6 +143,10 @@ General configuration
 
    .. versionadded:: 1.3
 
+   .. deprecated:: 1.8
+      Now Sphinx provides an API :meth:`Sphinx.add_source_parser` to register
+      a source parser.  Please use it instead.
+
 .. confval:: master_doc
 
    The document name of the "master" document, that is, the document that

doc/extdev/builderapi.rst

@@ -17,6 +17,9 @@ Builder API
    .. autoattribute:: format
    .. autoattribute:: epilog
    .. autoattribute:: supported_image_types
+   .. autoattribute:: supported_remote_images
+   .. autoattribute:: supported_data_uri_images
+   .. autoattribute:: default_translator_class
 
    These methods are predefined and will be called from the application:
 

doc/theming.rst

@@ -422,3 +422,11 @@ Third Party Themes
 
   .. versionchanged:: 1.4
      **sphinx_rtd_theme** has become optional.
+
+
+Besides this, there are a lot of third party themes.  You can find them on
+PyPI__, GitHub__, sphinx-themes.org__ and so on.
+
+.. __: https://pypi.python.org/pypi?:action=browse&c=599
+.. __: https://github.com/search?utf8=%E2%9C%93&q=sphinx+theme&type=
+.. __: https://sphinx-themes.org/

sphinx/application.py

@@ -89,9 +89,11 @@ builtin_extensions = (
     'sphinx.directives.patches',
     'sphinx.io',
     'sphinx.parsers',
+    'sphinx.registry',
     'sphinx.roles',
     'sphinx.transforms.post_transforms',
     'sphinx.transforms.post_transforms.images',
+    'sphinx.util.compat',
     # collectors should be loaded by specific order
     'sphinx.environment.collectors.dependencies',
     'sphinx.environment.collectors.asset',
@@ -249,8 +251,6 @@ class Sphinx(object):
         self.builder = self.create_builder(buildername)
         # check all configuration values for permissible types
         self.config.check_types()
-        # set up source_parsers
-        self._init_source_parsers()
         # set up the build environment
         self._init_env(freshenv)
         # set up the builder
@@ -284,14 +284,6 @@ class Sphinx(object):
             else:
                 logger.info('not available for built-in messages')
 
-    def _init_source_parsers(self):
-        # type: () -> None
-        for suffix, parser in iteritems(self.config.source_parsers):
-            self.add_source_parser(suffix, parser)
-        for suffix, parser in iteritems(self.registry.get_source_parsers()):
-            if suffix not in self.config.source_suffix and suffix != '*':
-                self.config.source_suffix.append(suffix)
-
     def _init_env(self, freshenv):
         # type: (bool) -> None
         if freshenv:

sphinx/builders/__init__.py

@@ -59,8 +59,8 @@ class Builder(object):
     #: ``project``
     epilog = ''  # type: unicode
 
-    # default translator class for the builder.  This will be overrided by
+    #: default translator class for the builder.  This can be overrided by
-    # ``app.set_translator()``.
+    #: :py:meth:`app.set_translator()`.
     default_translator_class = None  # type: nodes.NodeVisitor
     # doctree versioning method
     versioning_method = 'none'  # type: unicode
@@ -73,7 +73,9 @@ class Builder(object):
     #: The list of MIME types of image formats supported by the builder.
     #: Image files are searched in the order in which they appear here.
     supported_image_types = []  # type: List[unicode]
+    #: The builder supports remote images or not.
     supported_remote_images = False
+    #: The builder supports data URIs or not.
     supported_data_uri_images = False
 
     def __init__(self, app):

sphinx/config.py

@@ -12,6 +12,7 @@
 import re
 import traceback
 from os import path, getenv
+from collections import OrderedDict
 
 from six import PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types
 from typing import Any, NamedTuple, Union
@@ -115,7 +116,7 @@ class Config(object):
         figure_language_filename = (u'{root}.{language}{ext}', 'env', [str]),
 
         master_doc = ('contents', 'env'),
-        source_suffix = (['.rst'], 'env', Any),
+        source_suffix = ({'.rst': 'restructuredtext'}, 'env', Any),
         source_encoding = ('utf-8-sig', 'env'),
         source_parsers = ({}, 'env'),
         exclude_patterns = ([], 'env'),
@@ -261,7 +262,9 @@ class Config(object):
             return value
         else:
             defvalue = self.values[name][0]
-            if isinstance(defvalue, dict):
+            if self.values[name][-1] == Any:
+                return value
+            elif isinstance(defvalue, dict):
                 raise ValueError(__('cannot override dictionary config setting %r, '
                                     'ignoring (use %r to set individual elements)') %
                                  (name, name + '.key=value'))
@@ -362,9 +365,28 @@ class Config(object):
 
 def convert_source_suffix(app, config):
     # type: (Sphinx, Config) -> None
-    """This converts source_suffix to string-list."""
+    """This converts old styled source_suffix to new styled one.
-    if isinstance(config.source_suffix, string_types):
+
-        config.source_suffix = [config.source_suffix]  # type: ignore
+    * old style: str or list
+    * new style: a dict which maps from fileext to filetype
+    """
+    source_suffix = config.source_suffix
+    if isinstance(source_suffix, string_types):
+        # if str, considers as default filetype (None)
+        #
+        # The default filetype is determined on later step.
+        # By default, it is considered as restructuredtext.
+        config.source_suffix = OrderedDict({source_suffix: None})  # type: ignore
+    elif isinstance(source_suffix, (list, tuple)):
+        # if list, considers as all of them are default filetype
+        config.source_suffix = OrderedDict([(s, None) for s in source_suffix])  # type: ignore  # NOQA
+    elif isinstance(source_suffix, dict):
+        # if dict, convert it to OrderedDict
+        config.source_suffix = OrderedDict(config.source_suffix)  # type: ignore
+    else:
+        logger.warning(__("The config value `source_suffix' expected to "
+                          "a string, list of strings or dictionary. "
+                          "But `%r' is given." % source_suffix))
 
 
 def setup(app):

sphinx/deprecation.py

@@ -22,4 +22,8 @@ class RemovedInSphinx20Warning(PendingDeprecationWarning):
     pass
 
 
+class RemovedInSphinx30Warning(PendingDeprecationWarning):
+    pass
+
+
 RemovedInNextVersionWarning = RemovedInSphinx18Warning

sphinx/environment/__init__.py

@@ -381,7 +381,7 @@ class BuildEnvironment(object):
                     break
             else:
                 # document does not exist
-                suffix = self.config.source_suffix[0]
+                suffix = list(self.config.source_suffix)[0]
         if base is True:
             return path.join(self.srcdir, docname) + suffix
         elif base is None:

sphinx/ext/autosummary/__init__.py

@@ -603,7 +603,7 @@ def process_generate_options(app):
 
     from sphinx.ext.autosummary.generate import generate_autosummary_docs
 
-    ext = app.config.source_suffix
+    ext = list(app.config.source_suffix)
     genfiles = [genfile + (not genfile.endswith(tuple(ext)) and ext[0] or '')
                 for genfile in genfiles]
 

sphinx/io.py

@@ -16,7 +16,7 @@ from docutils.core import Publisher
 from docutils.readers import standalone
 from docutils.statemachine import StringList, string2lines
 from docutils.writers import UnfilteredWriter
-from six import text_type
+from six import text_type, iteritems
 from typing import Any, Union  # NOQA
 
 from sphinx.transforms import SphinxTransformer
@@ -273,14 +273,29 @@ class SphinxRSTFileInput(SphinxBaseFileInput):
             return lineno
 
 
+class FiletypeNotFoundError(Exception):
+    pass
+
+
+def get_filetype(source_suffix, filename):
+    # type: (Dict[unicode, unicode], unicode) -> unicode
+    for suffix, filetype in iteritems(source_suffix):
+        if filename.endswith(suffix):
+            # If default filetype (None), considered as restructuredtext.
+            return filetype or 'restructuredtext'
+    else:
+        raise FiletypeNotFoundError
+
+
 def read_doc(app, env, filename):
     # type: (Sphinx, BuildEnvironment, unicode) -> nodes.document
     """Parse a document and convert to doctree."""
-    input_class = app.registry.get_source_input(filename)
+    filetype = get_filetype(app.config.source_suffix, filename)
+    input_class = app.registry.get_source_input(filetype)
     reader = SphinxStandaloneReader(app)
     source = input_class(app, env, source=None, source_path=filename,
                          encoding=env.config.source_encoding)
-    parser = app.registry.create_source_parser(app, filename)
+    parser = app.registry.create_source_parser(app, filetype)
 
     pub = Publisher(reader=reader,
                     parser=parser,

sphinx/parsers.py

@@ -91,7 +91,7 @@ class RSTParser(docutils.parsers.rst.Parser):
 
 def setup(app):
     # type: (Sphinx) -> Dict[unicode, Any]
-    app.add_source_parser('*', RSTParser)  # register as a special parser
+    app.add_source_parser('.rst', RSTParser)
 
     return {
         'version': 'builtin',

sphinx/registry.py

@@ -11,19 +11,20 @@
 from __future__ import print_function
 
 import traceback
+import warnings
 
 from pkg_resources import iter_entry_points
-from six import iteritems, itervalues, string_types
+from six import iteritems, itervalues
 
 from sphinx.errors import ExtensionError, SphinxError, VersionRequirementError
 from sphinx.extension import Extension
+from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.domains import ObjType
 from sphinx.domains.std import GenericObject, Target
 from sphinx.locale import __
 from sphinx.parsers import Parser as SphinxParser
 from sphinx.roles import XRefRole
 from sphinx.util import logging
-from sphinx.util import import_object
 from sphinx.util.console import bold  # type: ignore
 from sphinx.util.docutils import directive_helper
 
@@ -61,8 +62,9 @@ class SphinxComponentRegistry(object):
         self.domain_object_types = {}   # type: Dict[unicode, Dict[unicode, ObjType]]
         self.domain_roles = {}          # type: Dict[unicode, Dict[unicode, Union[RoleFunction, XRefRole]]]  # NOQA
         self.post_transforms = []       # type: List[Type[Transform]]
-        self.source_parsers = {}        # type: Dict[unicode, Parser]
+        self.source_parsers = {}        # type: Dict[unicode, Type[Parser]]
         self.source_inputs = {}         # type: Dict[unicode, Input]
+        self.source_suffix = {}         # type: Dict[unicode, unicode]
         self.translators = {}           # type: Dict[unicode, nodes.NodeVisitor]
         self.transforms = []            # type: List[Type[Transform]]
 
@@ -197,28 +199,43 @@ class SphinxComponentRegistry(object):
         object_types = self.domain_object_types.setdefault('std', {})
         object_types[directivename] = ObjType(objname or directivename, rolename)
 
+    def add_source_suffix(self, suffix, filetype):
+        # type: (unicode, unicode) -> None
+        logger.debug('[app] adding source_suffix: %r, %r', suffix, filetype)
+        if suffix in self.source_suffix:
+            raise ExtensionError(__('source_parser for %r is already registered') % suffix)
+        else:
+            self.source_suffix[suffix] = filetype
+
     def add_source_parser(self, suffix, parser):
         # type: (unicode, Type[Parser]) -> None
         logger.debug('[app] adding search source_parser: %r, %r', suffix, parser)
-        if suffix in self.source_parsers:
+        self.add_source_suffix(suffix, suffix)
-            raise ExtensionError(__('source_parser for %r is already registered') % suffix)
+
+        if len(parser.supported) == 0:
+            warnings.warn('Old source_parser has been detected. Please fill Parser.supported '
+                          'attribute: %s' % parser.__name__,
+                          RemovedInSphinx30Warning)
+
+        # create a map from filetype to parser
+        for filetype in parser.supported:
+            if filetype in self.source_parsers:
+                raise ExtensionError(__('source_parser for %r is already registered') %
+                                     filetype)
+            else:
+                self.source_parsers[filetype] = parser
+
+        # also maps suffix to parser
+        #
+        # This allows parsers not having ``supported`` filetypes.
         self.source_parsers[suffix] = parser
 
-    def get_source_parser(self, filename):
+    def get_source_parser(self, filetype):
         # type: (unicode) -> Type[Parser]
-        for suffix, parser_class in iteritems(self.source_parsers):
+        try:
-            if filename.endswith(suffix):
+            return self.source_parsers[filetype]
-                break
+        except KeyError:
-        else:
+            raise SphinxError(__('Source parser for %s not registered') % filetype)
-            # use special parser for unknown file-extension '*' (if exists)
-            parser_class = self.source_parsers.get('*')
-
-        if parser_class is None:
-            raise SphinxError(__('Source parser for %s not registered') % filename)
-        else:
-            if isinstance(parser_class, string_types):
-                parser_class = import_object(parser_class, 'source parser')  # type: ignore
-            return parser_class
 
     def get_source_parsers(self):
         # type: () -> Dict[unicode, Parser]
@@ -238,21 +255,16 @@ class SphinxComponentRegistry(object):
             raise ExtensionError(__('source_input for %r is already registered') % filetype)
         self.source_inputs[filetype] = input_class
 
-    def get_source_input(self, filename):
+    def get_source_input(self, filetype):
         # type: (unicode) -> Type[Input]
-        parser = self.get_source_parser(filename)
+        try:
-        for filetype in parser.supported:
+            return self.source_inputs[filetype]
-            if filetype in self.source_inputs:
+        except KeyError:
-                input_class = self.source_inputs[filetype]
+            try:
-                break
+                # use special source_input for unknown filetype
-        else:
+                return self.source_inputs['*']
-            # use special source_input for unknown file-type '*' (if exists)
+            except KeyError:
-            input_class = self.source_inputs.get('*')
+                raise SphinxError(__('source_input for %s not registered') % filetype)
-
-        if input_class is None:
-            raise SphinxError(__('source_input for %s not registered') % filename)
-        else:
-            return input_class
 
     def add_translator(self, name, translator):
         # type: (unicode, Type[nodes.NodeVisitor]) -> None
@@ -347,3 +359,29 @@ class SphinxComponentRegistry(object):
                       if ext.metadata.get('env_version')}
         envversion['sphinx'] = ENV_VERSION
         return envversion
+
+
+def merge_source_suffix(app):
+    # type: (Sphinx) -> None
+    """Merge source_suffix which specified by user and added by extensions."""
+    for suffix in app.registry.source_suffix:
+        if suffix not in app.config.source_suffix:
+            app.config.source_suffix[suffix] = suffix
+        elif app.config.source_suffix[suffix] is None:
+            # filetype is not specified (default filetype).
+            # So it overrides default filetype by extensions setting.
+            app.config.source_suffix[suffix] = suffix
+
+    # copy config.source_suffix to registry
+    app.registry.source_suffix = app.config.source_suffix
+
+
+def setup(app):
+    # type: (Sphinx) -> Dict[unicode, Any]
+    app.connect('builder-inited', merge_source_suffix)
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

sphinx/transforms/i18n.py

@@ -52,7 +52,7 @@ def publish_msgstr(app, source, source_path, source_line, config, settings):
     from sphinx.io import SphinxI18nReader
     reader = SphinxI18nReader(app)
     reader.set_lineno_for_reporter(source_line)
-    parser = app.registry.create_source_parser(app, '')
+    parser = app.registry.create_source_parser(app, '.rst')
     doc = reader.read(
         source=StringInput(source=source, source_path=source_path),
         parser=parser,

sphinx/util/compat.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.util.compat
+    ~~~~~~~~~~~~~~~~~~
+
+    modules for backward compatibility
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import warnings
+
+from six import string_types, iteritems
+
+from sphinx.deprecation import RemovedInSphinx30Warning
+from sphinx.util import import_object
+
+if False:
+    # For type annotation
+    from typing import Any, Dict  # NOQA
+    from sphinx.application import Sphinx  # NOQA
+    from sphinx.config import Config  # NOQA
+
+
+def deprecate_source_parsers(app, config):
+    # type: (Sphinx, Config) -> None
+    if config.source_parsers:
+        warnings.warn('The config variable "source_parsers" is deprecated. '
+                      'Please use app.add_source_parser() API instead.',
+                      RemovedInSphinx30Warning)
+        for suffix, parser in iteritems(config.source_parsers):
+            if isinstance(parser, string_types):
+                parser = import_object(parser, 'source parser')  # type: ignore
+            app.add_source_parser(suffix, parser)
+
+
+def setup(app):
+    # type: (Sphinx) -> Dict[unicode, Any]
+    app.connect('config-inited', deprecate_source_parsers)
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

tests/roots/test-prolog/prolog_markdown_parser.py

@@ -4,6 +4,8 @@ from docutils.parsers import Parser
 
 
 class DummyMarkdownParser(Parser):
+    supported = ('markdown',)
+
     def parse(self, inputstring, document):
         document.rawsource = inputstring
 

tests/roots/test-root/conf.py

@@ -13,7 +13,6 @@ templates_path = ['_templates']
 
 master_doc = 'contents'
 source_suffix = ['.txt', '.add', '.foo']
-source_parsers = {'.foo': 'parsermod.Parser'}
 
 project = 'Sphinx <Tests>'
 copyright = '2010-2016, Georg Brandl & Team'
@@ -106,9 +105,12 @@ class ClassDirective(Directive):
 
 
 def setup(app):
+    import parsermod
+
     app.add_config_value('value_from_conf_py', 42, False)
     app.add_directive('funcdir', functional_directive, opt=lambda x: x)
     app.add_directive('clsdir', ClassDirective)
     app.add_object_type('userdesc', 'userdescrole', '%s (userdesc)',
                         userdesc_parse, objname='user desc')
     app.add_javascript('file://moo.js')
+    app.add_source_parser('.foo', parsermod.Parser)

tests/roots/test-root/parsermod.py

@@ -3,6 +3,8 @@ from docutils import nodes
 
 
 class Parser(Parser):
+    supported = ('foo',)
+
     def parse(self, input, document):
         section = nodes.section(ids=['id1'])
         section += nodes.title('Generated section', 'Generated section')

tests/test_application.py

@@ -84,7 +84,9 @@ def test_domain_override(app, status, warning):
 @pytest.mark.sphinx(testroot='add_source_parser')
 def test_add_source_parser(app, status, warning):
     assert set(app.config.source_suffix) == set(['.rst', '.md', '.test'])
-    assert set(app.registry.get_source_parsers().keys()) == set(['*', '.md', '.test'])
+    assert '.rst' in app.registry.get_source_parsers()
+    assert '.md' in app.registry.get_source_parsers()
+    assert '.test' in app.registry.get_source_parsers()
     assert app.registry.get_source_parsers()['.md'].__name__ == 'DummyMarkdownParser'
     assert app.registry.get_source_parsers()['.test'].__name__ == 'TestSourceParser'