Merge pull request #3818 from tk0miya/add_sphinx_rst_parser

Add Sphinx own parser (refs: #3816)

sphinx/application.py

@@ -85,6 +85,7 @@ builtin_extensions = (
     'sphinx.directives.code',
     'sphinx.directives.other',
     'sphinx.directives.patches',
+    'sphinx.parsers',
     'sphinx.roles',
     'sphinx.transforms.post_transforms',
     'sphinx.transforms.post_transforms.images',
@@ -265,7 +266,7 @@ class Sphinx(object):
         for suffix, parser in iteritems(self.config.source_parsers):
             self.add_source_parser(suffix, parser)
         for suffix, parser in iteritems(self.registry.get_source_parsers()):
-            if suffix not in self.config.source_suffix:
+            if suffix not in self.config.source_suffix and suffix != '*':
                 self.config.source_suffix.append(suffix)
 
     def _init_env(self, freshenv):

sphinx/io.py

@@ -63,6 +63,9 @@ class SphinxBaseReader(standalone.Reader):
             if source.source_path.endswith(suffix):
                 self.parser = self.parser_map[suffix]
                 break
+        else:
+            # use special parser for unknown file-extension '*' (if exists)
+            self.parser = self.parser_map.get('*')
 
         if not self.parser:
             self.parser = parser

sphinx/parsers.py

@@ -10,9 +10,15 @@
 """
 
 import docutils.parsers
+import docutils.parsers.rst
+from docutils.transforms.universal import SmartQuotes
+
+from sphinx.transforms import SphinxSmartQuotes
 
 if False:
     # For type annotation
+    from typing import Any, Dict, List, Type  # NOQA
+    from docutils.transforms import Transform  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
 
@@ -47,3 +53,26 @@ class Parser(docutils.parsers.Parser):
         self.env = app.env
         self.warn = app.warn
         self.info = app.info
+
+
+class RSTParser(docutils.parsers.rst.Parser):
+    """A reST parser customized for Sphinx."""
+
+    def get_transforms(self):
+        # type: () -> List[Type[Transform]]
+        """Sphinx's reST parser replaces a transform class for smart-quotes by own's"""
+        transforms = docutils.parsers.rst.Parser.get_transforms(self)
+        transforms.remove(SmartQuotes)
+        transforms.append(SphinxSmartQuotes)
+        return transforms
+
+
+def setup(app):
+    # type: (Sphinx) -> Dict[unicode, Any]
+    app.add_source_parser('*', RSTParser)  # register as a special parser
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

sphinx/transforms/__init__.py

@@ -9,12 +9,11 @@
     :license: BSD, see LICENSE for details.
 """
 
-import docutils
 from docutils import nodes
 from docutils.transforms import Transform, Transformer
 from docutils.transforms.parts import ContentsFilter
 from docutils.utils import new_document
-from docutils.transforms.universal import SmartQuotes as DocutilsSmartQuotes
+from docutils.transforms.universal import SmartQuotes
 
 from sphinx import addnodes
 from sphinx.locale import _
@@ -331,9 +330,11 @@ class SphinxContentsFilter(ContentsFilter):
         raise nodes.SkipNode
 
 
-class SphinxSmartQuotes(DocutilsSmartQuotes):  # NOQA
+class SphinxSmartQuotes(SmartQuotes):
     """
     Customized SmartQuotes to avoid transform for some extra node types.
+
+    refs: sphinx.parsers.RSTParser
     """
     def get_tokens(self, txtnodes):
         # A generator that yields ``(texttype, nodetext)`` tuples for a list
@@ -356,6 +357,3 @@ class SphinxSmartQuotes(DocutilsSmartQuotes):  # NOQA
                                             nodes.raw,
                                             nodes.problematic))]
             yield (nodetype, txtnode.astext())
-
-
-docutils.transforms.universal.SmartQuotes = SphinxSmartQuotes

tests/test_application.py

@@ -83,6 +83,6 @@ def test_domain_override(app, status, warning):
 @pytest.mark.sphinx(testroot='add_source_parser')
 def test_add_source_parser(app, status, warning):
     assert set(app.config.source_suffix) == set(['.rst', '.md', '.test'])
-    assert set(app.registry.get_source_parsers().keys()) == set(['.md', '.test'])
+    assert set(app.registry.get_source_parsers().keys()) == set(['*', '.md', '.test'])
     assert app.registry.get_source_parsers()['.md'].__name__ == 'DummyMarkdownParser'
     assert app.registry.get_source_parsers()['.test'].__name__ == 'TestSourceParser'