Merge pull request #4594 from tk0miya/4564_rst_prolog_not_working

Change interface of add_source_parser() and add add_source_suffix()

CHANGES

@@ -25,6 +25,8 @@ Deprecated
   ``sphinx.util.import_object()`` instead.
 * Drop function based directive support.  For now, Sphinx only supports class
   based directives.
+* ``Sphinx.add_source_parser()`` has changed;  the *suffix* argument has
+  been deprecated.  Please use ``Sphinx.add_source_suffix()`` instead.
 
 Features added
 --------------

doc/extdev/appapi.rst

@@ -85,7 +85,9 @@ package.
 
 .. automethod:: Sphinx.add_search_language(cls)
 
-.. automethod:: Sphinx.add_source_parser(suffix, parser)
+.. automethod:: Sphinx.add_source_suffix(suffix, filetype)
+
+.. automethod:: Sphinx.add_source_parser(parser)
 
 .. automethod:: Sphinx.add_env_collector(collector)
 

doc/extdev/parserapi.rst

@@ -3,6 +3,35 @@
 Parser API
 ==========
 
+`The docutils documentation describes`__ parsers as follows:
+
+    The Parser analyzes the input document and creates a node tree
+    representation.
+
+__ http://docutils.sourceforge.net/docs/dev/hacking.html#parsing-the-document
+
+In Sphinx, the parser modules works as same as docutils.  The parsers are
+registered to Sphinx by extensions using Application APIs;
+:meth:`Sphinx.add_source_suffix()` and :meth:`Sphinx.add_source_parsers()`.
+
+The *source suffix* is a mapping from file suffix to file type.  For example,
+``.rst`` file is mapped to ``'restructuredtext'`` type.  Sphinx uses the
+file type to looking for parsers from registered list.  On searching,
+Sphinx refers to the ``Parser.supported`` attribute and picks up a parser
+which contains the file type in the attribute.
+
+The users can override the source suffix mappings using
+:confval:`source_suffix` like following::
+
+    # a mapping from file suffix to file types
+    source_suffix = {
+        '.rst': 'restructuredtext',
+        '.md': 'markdown',
+    }
+
+You should indicate file types your parser supports. This will allow users
+to configure their settings appropriately.
+
 .. module:: sphinx.parsers
 
 .. autoclass:: Parser

sphinx/application.py

@@ -1105,13 +1105,25 @@ class Sphinx(object):
         assert issubclass(cls, SearchLanguage)
         languages[cls.lang] = cls
 
-    def add_source_parser(self, suffix, parser):
-        # type: (unicode, Parser) -> None
-        """Register a parser class for specified *suffix*.
+    def add_source_suffix(self, suffix, filetype):
+        # type: (unicode, unicode) -> None
+        """Register a suffix of source files.
+
+        Same as :confval:`source_suffix`.  The users can override this
+        using the setting.
+        """
+        self.registry.add_source_suffix(suffix, filetype)
+
+    def add_source_parser(self, *args):
+        # type: (Any) -> None
+        """Register a parser class.
 
         .. versionadded:: 1.4
+        .. versionchanged:: 1.8
+           *suffix* argument is deprecated.  It only accepts *parser* argument.
+           Use :meth:`add_source_suffix` API to register suffix instead.
         """
-        self.registry.add_source_parser(suffix, parser)
+        self.registry.add_source_parser(*args)
 
     def add_env_collector(self, collector):
         # type: (Type[EnvironmentCollector]) -> None

sphinx/parsers.py

@@ -92,7 +92,7 @@ class RSTParser(docutils.parsers.rst.Parser):
 
 def setup(app):
     # type: (Sphinx) -> Dict[unicode, Any]
-    app.add_source_parser('.rst', RSTParser)
+    app.add_source_parser(RSTParser)
 
     return {
         'version': 'builtin',

sphinx/registry.py

@@ -239,10 +239,23 @@ class SphinxComponentRegistry(object):
         else:
             self.source_suffix[suffix] = filetype
 
-    def add_source_parser(self, suffix, parser):
-        # type: (unicode, Type[Parser]) -> None
-        logger.debug('[app] adding search source_parser: %r, %r', suffix, parser)
-        self.add_source_suffix(suffix, suffix)
+    def add_source_parser(self, *args):
+        # type: (Any) -> None
+        logger.debug('[app] adding search source_parser: %r', args)
+        if len(args) == 1:
+            # new sytle arguments: (source_parser)
+            suffix = None       # type: unicode
+            parser = args[0]    # type: Type[Parser]
+        else:
+            # old style arguments: (suffix, source_parser)
+            warnings.warn('app.add_source_parser() does not support suffix argument. '
+                          'Use app.add_source_suffix() instead.',
+                          RemovedInSphinx30Warning)
+            suffix = args[0]
+            parser = args[1]
+
+        if suffix:
+            self.add_source_suffix(suffix, suffix)
 
         if len(parser.supported) == 0:
             warnings.warn('Old source_parser has been detected. Please fill Parser.supported '
@@ -259,8 +272,9 @@ class SphinxComponentRegistry(object):
 
         # also maps suffix to parser
         #
-        # This allows parsers not having ``supported`` filetypes.
-        self.source_parsers[suffix] = parser
+        # This rescues old styled parsers which does not have ``supported`` filetypes.
+        if suffix:
+            self.source_parsers[suffix] = parser
 
     def get_source_parser(self, filetype):
         # type: (unicode) -> Type[Parser]
@@ -398,13 +412,13 @@ class SphinxComponentRegistry(object):
 def merge_source_suffix(app):
     # type: (Sphinx) -> None
     """Merge source_suffix which specified by user and added by extensions."""
-    for suffix in app.registry.source_suffix:
+    for suffix, filetype in iteritems(app.registry.source_suffix):
         if suffix not in app.config.source_suffix:
-            app.config.source_suffix[suffix] = suffix
+            app.config.source_suffix[suffix] = filetype
         elif app.config.source_suffix[suffix] is None:
             # filetype is not specified (default filetype).
             # So it overrides default filetype by extensions setting.
-            app.config.source_suffix[suffix] = suffix
+            app.config.source_suffix[suffix] = filetype
 
     # copy config.source_suffix to registry
     app.registry.source_suffix = app.config.source_suffix

sphinx/transforms/i18n.py

@@ -52,7 +52,7 @@ def publish_msgstr(app, source, source_path, source_line, config, settings):
     from sphinx.io import SphinxI18nReader
     reader = SphinxI18nReader(app)
     reader.set_lineno_for_reporter(source_line)
-    parser = app.registry.create_source_parser(app, '.rst')
+    parser = app.registry.create_source_parser(app, 'restructuredtext')
     doc = reader.read(
         source=StringInput(source=source, source_path=source_path),
         parser=parser,

tests/roots/test-add_source_parser-conflicts-with-users-setting/conf.py

@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath('.'))
 
 
 class DummyTestParser(Parser):
-    pass
+    supported = ('dummy',)
 
 
 extensions = ['source_parser']

tests/roots/test-add_source_parser-conflicts-with-users-setting/source_parser.py

@@ -4,8 +4,9 @@ from docutils.parsers import Parser
 
 
 class TestSourceParser(Parser):
-    pass
+    supported = ('test',)
 
 
 def setup(app):
-    app.add_source_parser('.test', TestSourceParser)
+    app.add_source_suffix('.test', 'test')
+    app.add_source_parser(TestSourceParser)

tests/roots/test-add_source_parser/conf.py

@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath('.'))
 
 
 class DummyMarkdownParser(Parser):
-    pass
+    supported = ('markdown',)
 
 
 extensions = ['source_parser']

tests/roots/test-add_source_parser/source_parser.py

@@ -4,8 +4,9 @@ from docutils.parsers import Parser
 
 
 class TestSourceParser(Parser):
-    pass
+    supported = ('test',)
 
 
 def setup(app):
-    app.add_source_parser('.test', TestSourceParser)
+    app.add_source_suffix('.test', 'test')
+    app.add_source_parser(TestSourceParser)

tests/roots/test-prolog/prolog_markdown_parser.py

@@ -11,4 +11,5 @@ class DummyMarkdownParser(Parser):
 
 
 def setup(app):
-    app.add_source_parser('.md', DummyMarkdownParser)
+    app.add_source_suffix('.md', 'markdown')
+    app.add_source_parser(DummyMarkdownParser)

tests/roots/test-root/conf.py

@@ -109,4 +109,5 @@ def setup(app):
     app.add_object_type('userdesc', 'userdescrole', '%s (userdesc)',
                         userdesc_parse, objname='user desc')
     app.add_javascript('file://moo.js')
-    app.add_source_parser('.foo', parsermod.Parser)
+    app.add_source_suffix('.foo', 'foo')
+    app.add_source_parser(parsermod.Parser)

tests/test_application.py

@@ -83,11 +83,20 @@ def test_domain_override(app, status, warning):
 @pytest.mark.sphinx(testroot='add_source_parser')
 def test_add_source_parser(app, status, warning):
     assert set(app.config.source_suffix) == set(['.rst', '.md', '.test'])
-    assert '.rst' in app.registry.get_source_parsers()
+
+    # .rst; only in :confval:`source_suffix`
+    assert '.rst' not in app.registry.get_source_parsers()
+    assert app.registry.source_suffix['.rst'] is None
+
+    # .md; configured by :confval:`source_suffix` and :confval:`source_parsers`
     assert '.md' in app.registry.get_source_parsers()
-    assert '.test' in app.registry.get_source_parsers()
+    assert app.registry.source_suffix['.md'] == '.md'
     assert app.registry.get_source_parsers()['.md'].__name__ == 'DummyMarkdownParser'
-    assert app.registry.get_source_parsers()['.test'].__name__ == 'TestSourceParser'
+
+    # .test; configured by API
+    assert app.registry.source_suffix['.test'] == 'test'
+    assert 'test' in app.registry.get_source_parsers()
+    assert app.registry.get_source_parsers()['test'].__name__ == 'TestSourceParser'
 
 
 @pytest.mark.sphinx(testroot='extensions')