Integrate source_suffix and source_parsers (refs: #4474)

sphinx/application.py

@@ -89,6 +89,7 @@ builtin_extensions = (
     'sphinx.directives.patches',
     'sphinx.io',
     'sphinx.parsers',
+    'sphinx.registry',
     'sphinx.roles',
     'sphinx.transforms.post_transforms',
     'sphinx.transforms.post_transforms.images',
@@ -251,8 +252,6 @@ class Sphinx(object):
         self.builder = self.create_builder(buildername)
         # check all configuration values for permissible types
         self.config.check_types()
-        # set up source_parsers
-        self._init_source_parsers()
         # set up the build environment
         self._init_env(freshenv)
         # set up the builder
@@ -286,12 +285,6 @@ class Sphinx(object):
             else:
                 logger.info('not available for built-in messages')
 
-    def _init_source_parsers(self):
-        # type: () -> None
-        for suffix, parser in iteritems(self.registry.get_source_parsers()):
-            if suffix not in self.config.source_suffix and suffix != '*':
-                self.config.source_suffix[suffix] = suffix
-
     def _init_env(self, freshenv):
         # type: (bool) -> None
         if freshenv:

sphinx/parsers.py

@@ -91,7 +91,7 @@ class RSTParser(docutils.parsers.rst.Parser):
 
 def setup(app):
     # type: (Sphinx) -> Dict[unicode, Any]
-    app.add_source_parser('*', RSTParser)  # register as a special parser
+    app.add_source_parser('.rst', RSTParser)
 
     return {
         'version': 'builtin',

sphinx/registry.py

@@ -51,6 +51,10 @@ EXTENSION_BLACKLIST = {
 }  # type: Dict[unicode, unicode]
 
 
+class FiletypeNotFoundError(Exception):
+    pass
+
+
 class SphinxComponentRegistry(object):
     def __init__(self):
         self.autodoc_attrgettrs = {}    # type: Dict[Type, Callable[[Any, unicode, Any], Any]]
@@ -62,8 +66,9 @@ class SphinxComponentRegistry(object):
         self.domain_object_types = {}   # type: Dict[unicode, Dict[unicode, ObjType]]
         self.domain_roles = {}          # type: Dict[unicode, Dict[unicode, Union[RoleFunction, XRefRole]]]  # NOQA
         self.post_transforms = []       # type: List[Type[Transform]]
-        self.source_parsers = {}        # type: Dict[unicode, Parser]
+        self.source_parsers = {}        # type: Dict[unicode, Type[Parser]]
         self.source_inputs = {}         # type: Dict[unicode, Input]
+        self.source_suffix = {}         # type: Dict[unicode, unicode]
         self.translators = {}           # type: Dict[unicode, nodes.NodeVisitor]
         self.transforms = []            # type: List[Type[Transform]]
 
@@ -201,27 +206,47 @@ class SphinxComponentRegistry(object):
     def add_source_parser(self, suffix, parser):
         # type: (unicode, Type[Parser]) -> None
         logger.debug('[app] adding search source_parser: %r, %r', suffix, parser)
-        if suffix in self.source_parsers:
+        if suffix in self.source_suffix:
             raise ExtensionError(__('source_parser for %r is already registered') % suffix)
+        else:
+            self.source_suffix[suffix] = suffix
 
         if len(parser.supported) == 0:
             warnings.warn('Old source_parser has been detected. Please fill Parser.supported '
                           'attribute: %s' % parser.__name__,
                           RemovedInSphinx30Warning)
 
+        # create a map from filetype to parser
+        for filetype in parser.supported:
+            if filetype in self.source_parsers:
+                raise ExtensionError(__('source_parser for %r is already registered') %
+                                     filetype)
+            else:
+                self.source_parsers[filetype] = parser
+
+        # also maps suffix to parser
+        #
+        # This allows parsers not having ``supported`` filetypes.
         self.source_parsers[suffix] = parser
 
+    def get_filetype(self, filename):
+        # type: (unicode) -> unicode
+        for suffix, filetype in iteritems(self.source_suffix):
+            if filename.endswith(suffix):
+                return filetype
+        else:
+            raise FiletypeNotFoundError
+
     def get_source_parser(self, filename):
         # type: (unicode) -> Type[Parser]
-        for suffix, parser_class in iteritems(self.source_parsers):
+        try:
-            if filename.endswith(suffix):
+            filetype = self.get_filetype(filename)
-                break
+            parser_class = self.source_parsers[filetype]
-        else:
+        except FiletypeNotFoundError:
-            # use special parser for unknown file-extension '*' (if exists)
+            raise SphinxError(__('Source parser for %s not registered') % filename)
-            parser_class = self.source_parsers.get('*')
 
         if parser_class is None:
-            raise SphinxError(__('Source parser for %s not registered') % filename)
+            raise SphinxError(__('Source parser for %s not registered') % filetype)
         else:
             return parser_class
 
@@ -245,12 +270,10 @@ class SphinxComponentRegistry(object):
 
     def get_source_input(self, filename):
         # type: (unicode) -> Type[Input]
-        parser = self.get_source_parser(filename)
+        try:
-        for filetype in parser.supported:
+            filetype = self.get_filetype(filename)
-            if filetype in self.source_inputs:
+            input_class = self.source_inputs[filetype]
-                input_class = self.source_inputs[filetype]
+        except FiletypeNotFoundError:
-                break
-        else:
             # use special source_input for unknown file-type '*' (if exists)
             input_class = self.source_inputs.get('*')
 
@@ -346,3 +369,25 @@ class SphinxComponentRegistry(object):
 
         app.extensions[extname] = Extension(extname, mod, **metadata)
         app._setting_up_extension.pop()
+
+
+def merge_source_suffix(app):
+    # type: (Sphinx) -> None
+    """Merge source_suffix which specified by user and added by extensions."""
+    for suffix in app.registry.source_suffix:
+        if suffix not in app.config.source_suffix:
+            app.config.source_suffix[suffix] = suffix
+
+    # copy config.source_suffix to registry
+    app.registry.source_suffix = app.config.source_suffix
+
+
+def setup(app):
+    # type: (Sphinx) -> Dict[unicode, Any]
+    app.connect('builder-inited', merge_source_suffix)
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

sphinx/transforms/i18n.py

@@ -52,7 +52,7 @@ def publish_msgstr(app, source, source_path, source_line, config, settings):
     from sphinx.io import SphinxI18nReader
     reader = SphinxI18nReader(app)
     reader.set_lineno_for_reporter(source_line)
-    parser = app.registry.create_source_parser(app, '')
+    parser = app.registry.create_source_parser(app, '.rst')
     doc = reader.read(
         source=StringInput(source=source, source_path=source_path),
         parser=parser,

tests/test_application.py

@@ -84,7 +84,9 @@ def test_domain_override(app, status, warning):
 @pytest.mark.sphinx(testroot='add_source_parser')
 def test_add_source_parser(app, status, warning):
     assert set(app.config.source_suffix) == set(['.rst', '.md', '.test'])
-    assert set(app.registry.get_source_parsers().keys()) == set(['*', '.md', '.test'])
+    assert '.rst' in app.registry.get_source_parsers()
+    assert '.md' in app.registry.get_source_parsers()
+    assert '.test' in app.registry.get_source_parsers()
     assert app.registry.get_source_parsers()['.md'].__name__ == 'DummyMarkdownParser'
     assert app.registry.get_source_parsers()['.test'].__name__ == 'TestSourceParser'