Process prolog and epilog on RSTParser (instead Input component)

sphinx/parsers.py

@@ -15,9 +15,11 @@ from docutils.parsers.rst import states
 from docutils.statemachine import StringList
 from docutils.transforms.universal import SmartQuotes
 
+from sphinx.util.rst import append_epilog, prepend_prolog
+
 if False:
     # For type annotation
-    from typing import Any, Dict, List, Type  # NOQA
+    from typing import Any, Dict, List, Type, Union  # NOQA
     from docutils import nodes  # NOQA
     from docutils.transforms import Transform  # NOQA
     from sphinx.application import Sphinx  # NOQA
@@ -65,30 +67,40 @@ class RSTParser(docutils.parsers.rst.Parser, Parser):
         # type: () -> List[Type[Transform]]
         """Sphinx's reST parser replaces a transform class for smart-quotes by own's
 
-        refs: sphinx.io.SphinxStandaloneReader"""
+        refs: sphinx.io.SphinxStandaloneReader
+        """
         transforms = super(RSTParser, self).get_transforms()
         transforms.remove(SmartQuotes)
         return transforms
 
     def parse(self, inputstring, document):
-        # type: (Any, nodes.document) -> None
-        """Parse text and generate a document tree.
+        # type: (Union[str, StringList], nodes.document) -> None
+        """Parse text and generate a document tree."""
+        self.setup_parse(inputstring, document)
+        self.statemachine = states.RSTStateMachine(
+            state_classes=self.state_classes,
+            initial_state=self.initial_state,
+            debug=document.reporter.debug_flag)
 
-        This accepts StringList as an inputstring parameter.
-        It enables to handle mixed contents (cf. :confval:`rst_prolog`) correctly.
-        """
-        if isinstance(inputstring, StringList):
-            self.setup_parse(inputstring, document)
-            self.statemachine = states.RSTStateMachine(
-                state_classes=self.state_classes,
-                initial_state=self.initial_state,
-                debug=document.reporter.debug_flag)
-            # Give inputstring directly to statemachine.
-            self.statemachine.run(inputstring, document, inliner=self.inliner)
-            self.finish_parse()
+        # preprocess inputstring
+        if isinstance(inputstring, str):
+            lines = docutils.statemachine.string2lines(
+                inputstring, tab_width=document.settings.tab_width,
+                convert_whitespace=True)
+
+            inputlines = StringList(lines, document.current_source)
         else:
-            # otherwise, inputstring might be a string. It will be handled by superclass.
-            super(RSTParser, self).parse(inputstring, document)
+            inputlines = inputstring
+
+        self.decorate(inputlines)
+        self.statemachine.run(inputlines, document, inliner=self.inliner)
+        self.finish_parse()
+
+    def decorate(self, content):
+        # type: (StringList) -> None
+        """Preprocess reST content before parsing."""
+        prepend_prolog(content, self.config.rst_prolog)
+        append_epilog(content, self.config.rst_epilog)
 
 
 def setup(app):

tests/test_directive_other.py

@@ -21,9 +21,11 @@ from sphinx.testing.util import assert_node
 
 def parse(app, docname, text):
     app.env.temp_data['docname'] = docname
+    parser = RSTParser()
+    parser.set_application(app)
     return publish_doctree(text, app.srcdir / docname + '.rst',
                            reader=SphinxStandaloneReader(app),
-                           parser=RSTParser(),
+                           parser=parser,
                            settings_overrides={'env': app.env,
                                                'gettext_compact': True})
 

tests/test_parser.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+"""
+    test_sphinx_parsers
+    ~~~~~~~~~~~~~~~~~~~
+
+    Tests parsers module.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from unittest.mock import Mock, patch
+
+import pytest
+
+from sphinx.parsers import RSTParser
+from sphinx.util.docutils import new_document
+
+
+@pytest.mark.sphinx(testroot='basic')
+@patch('docutils.parsers.rst.states.RSTStateMachine')
+def test_RSTParser_prolog_epilog(RSTStateMachine, app):
+    document = new_document('dummy.rst')
+    document.settings = Mock(tab_width=8, language_code='')
+    parser = RSTParser()
+    parser.set_application(app)
+
+    # normal case
+    text = ('hello Sphinx world\n'
+            'Sphinx is a document generator')
+    parser.parse(text, document)
+    (content, _), _ = RSTStateMachine().run.call_args
+
+    assert list(content.xitems()) == [('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator')]
+
+    # with rst_prolog
+    app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!'
+    parser.parse(text, document)
+    (content, _), _ = RSTStateMachine().run.call_args
+    assert list(content.xitems()) == [('<rst_prolog>', 0, 'this is rst_prolog'),
+                                      ('<rst_prolog>', 1, 'hello reST!'),
+                                      ('<generated>', 0, ''),
+                                      ('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator')]
+
+    # with rst_epilog
+    app.env.config.rst_prolog = None
+    app.env.config.rst_epilog = 'this is rst_epilog\ngood-bye reST!'
+    parser.parse(text, document)
+    (content, _), _ = RSTStateMachine().run.call_args
+    assert list(content.xitems()) == [('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator'),
+                                      ('<generated>', 0, ''),
+                                      ('<rst_epilog>', 0, 'this is rst_epilog'),
+                                      ('<rst_epilog>', 1, 'good-bye reST!')]
+
+    # expandtabs / convert whitespaces
+    app.env.config.rst_prolog = None
+    app.env.config.rst_epilog = None
+    text = ('\thello Sphinx world\n'
+            '\v\fSphinx is a document generator')
+    parser.parse(text, document)
+    (content, _), _ = RSTStateMachine().run.call_args
+    assert list(content.xitems()) == [('dummy.rst', 0, '        hello Sphinx world'),
+                                      ('dummy.rst', 1, '  Sphinx is a document generator')]