Merge pull request #5726 from tk0miya/refactor_io2

Process prolog and epilog on RSTParser (instead Input component)

CHANGES

@@ -48,6 +48,7 @@ Deprecated
 * ``sphinx.config.check_unicode()``
 * ``sphinx.ext.autodoc.importer._MockImporter``
 * ``sphinx.ext.doctest.doctest_encode()``
+* ``sphinx.io.SphinxRSTFileInput``
 * ``sphinx.testing.util.remove_unicode_literal()``
 * ``sphinx.util.force_decode()``
 * ``sphinx.util.get_matching_docs()`` is deprecated

doc/extdev/index.rst

@@ -197,6 +197,11 @@ The following is a list of deprecated interfaces.
      - 3.0
      - N/A
 
+   * - ``sphinx.io.SphinxRSTFileInput``
+     - 2.0
+     - 3.0
+     - N/A
+
    * - ``sphinx.writers.latex.LaTeXTranslator._make_visit_admonition()``
      - 2.0
      - 3.0

sphinx/io.py

@@ -9,7 +9,6 @@
     :license: BSD, see LICENSE for details.
 """
 import codecs
-import re
 import warnings
 
 from docutils.core import Publisher
@@ -37,6 +36,7 @@ from sphinx.transforms.i18n import (
 from sphinx.transforms.references import SphinxDomains, SubstitutionDefinitionsRemover
 from sphinx.util import logging
 from sphinx.util.docutils import LoggingReporter
+from sphinx.util.rst import append_epilog, docinfo_re, prepend_prolog
 from sphinx.versioning import UIDTransform
 
 if False:
@@ -51,8 +51,6 @@ if False:
     from sphinx.environment import BuildEnvironment  # NOQA
     from sphinx.util.typing import unicode  # NOQA
 
-docinfo_re = re.compile(':\\w+:.*?')
-
 
 logger = logging.getLogger(__name__)
 
@@ -254,16 +252,17 @@ class SphinxRSTFileInput(SphinxBaseFileInput):
 
     def read(self):
         # type: () -> StringList
+        warnings.warn('SphinxRSTFileInput is deprecated.',
+                      RemovedInSphinx30Warning, stacklevel=2)
+
         inputstring = super(SphinxRSTFileInput, self).read()
         lines = string2lines(inputstring, convert_whitespace=True)
         content = StringList()
         for lineno, line in enumerate(lines):
             content.append(line, self.source_path, lineno)
 
-        if self.env.config.rst_prolog:
-            self.prepend_prolog(content, self.env.config.rst_prolog)
-        if self.env.config.rst_epilog:
-            self.append_epilog(content, self.env.config.rst_epilog)
+        prepend_prolog(content, self.env.config.rst_prolog)
+        append_epilog(content, self.env.config.rst_epilog)
 
         return content
 
@@ -324,7 +323,6 @@ def read_doc(app, env, filename):
 def setup(app):
     # type: (Sphinx) -> Dict[unicode, Any]
     app.registry.add_source_input(SphinxFileInput)
-    app.registry.add_source_input(SphinxRSTFileInput)
 
     return {
         'version': 'builtin',

sphinx/parsers.py

@@ -15,9 +15,11 @@ from docutils.parsers.rst import states
 from docutils.statemachine import StringList
 from docutils.transforms.universal import SmartQuotes
 
+from sphinx.util.rst import append_epilog, prepend_prolog
+
 if False:
     # For type annotation
-    from typing import Any, Dict, List, Type  # NOQA
+    from typing import Any, Dict, List, Type, Union  # NOQA
     from docutils import nodes  # NOQA
     from docutils.transforms import Transform  # NOQA
     from sphinx.application import Sphinx  # NOQA
@@ -58,37 +60,47 @@ class Parser(docutils.parsers.Parser):
         self.env = app.env
 
 
-class RSTParser(docutils.parsers.rst.Parser):
+class RSTParser(docutils.parsers.rst.Parser, Parser):
     """A reST parser for Sphinx."""
 
     def get_transforms(self):
         # type: () -> List[Type[Transform]]
         """Sphinx's reST parser replaces a transform class for smart-quotes by own's
 
-        refs: sphinx.io.SphinxStandaloneReader"""
+        refs: sphinx.io.SphinxStandaloneReader
+        """
         transforms = super(RSTParser, self).get_transforms()
         transforms.remove(SmartQuotes)
         return transforms
 
     def parse(self, inputstring, document):
-        # type: (Any, nodes.document) -> None
-        """Parse text and generate a document tree.
+        # type: (Union[str, StringList], nodes.document) -> None
+        """Parse text and generate a document tree."""
+        self.setup_parse(inputstring, document)
+        self.statemachine = states.RSTStateMachine(
+            state_classes=self.state_classes,
+            initial_state=self.initial_state,
+            debug=document.reporter.debug_flag)
 
-        This accepts StringList as an inputstring parameter.
-        It enables to handle mixed contents (cf. :confval:`rst_prolog`) correctly.
-        """
-        if isinstance(inputstring, StringList):
-            self.setup_parse(inputstring, document)
-            self.statemachine = states.RSTStateMachine(
-                state_classes=self.state_classes,
-                initial_state=self.initial_state,
-                debug=document.reporter.debug_flag)
-            # Give inputstring directly to statemachine.
-            self.statemachine.run(inputstring, document, inliner=self.inliner)
-            self.finish_parse()
+        # preprocess inputstring
+        if isinstance(inputstring, str):
+            lines = docutils.statemachine.string2lines(
+                inputstring, tab_width=document.settings.tab_width,
+                convert_whitespace=True)
+
+            inputlines = StringList(lines, document.current_source)
         else:
-            # otherwise, inputstring might be a string. It will be handled by superclass.
-            super(RSTParser, self).parse(inputstring, document)
+            inputlines = inputstring
+
+        self.decorate(inputlines)
+        self.statemachine.run(inputlines, document, inliner=self.inliner)
+        self.finish_parse()
+
+    def decorate(self, content):
+        # type: (StringList) -> None
+        """Preprocess reST content before parsing."""
+        prepend_prolog(content, self.config.rst_prolog)
+        append_epilog(content, self.config.rst_epilog)
 
 
 def setup(app):

sphinx/util/rst.py

@@ -24,11 +24,14 @@ from sphinx.util import logging
 if False:
     # For type annotation
     from typing import Generator  # NOQA
+    from docutils.statemachine import StringList  # NOQA
     from sphinx.util.typing import unicode  # NOQA
 
-symbols_re = re.compile(r'([!-\-/:-@\[-`{-~])')  # symbols without dot(0x2e)
 logger = logging.getLogger(__name__)
 
+docinfo_re = re.compile(':\\w+:.*?')
+symbols_re = re.compile(r'([!-\-/:-@\[-`{-~])')  # symbols without dot(0x2e)
+
 
 def escape(text):
     # type: (unicode) -> unicode
@@ -51,3 +54,35 @@ def default_role(docname, name):
     yield
 
     docutils.unregister_role('')
+
+
+def prepend_prolog(content, prolog):
+    # type: (StringList, unicode) -> None
+    """Prepend a string to content body as prolog."""
+    if prolog:
+        pos = 0
+        for line in content:
+            if docinfo_re.match(line):
+                pos += 1
+            else:
+                break
+
+        if pos > 0:
+            # insert a blank line after docinfo
+            content.insert(pos, '', '<generated>', 0)
+            pos += 1
+
+        # insert prolog (after docinfo if exists)
+        for lineno, line in enumerate(prolog.splitlines()):
+            content.insert(pos + lineno, line, '<rst_prolog>', lineno)
+
+        content.insert(pos + lineno + 1, '', '<generated>', 0)
+
+
+def append_epilog(content, epilog):
+    # type: (StringList, unicode) -> None
+    """Append a string to content body as epilog."""
+    if epilog:
+        content.append('', '<generated>', 0)
+        for lineno, line in enumerate(epilog.splitlines()):
+            content.append(line, '<rst_epilog>', lineno)

tests/test_directive_other.py

@@ -21,9 +21,11 @@ from sphinx.testing.util import assert_node
 
 def parse(app, docname, text):
     app.env.temp_data['docname'] = docname
+    parser = RSTParser()
+    parser.set_application(app)
     return publish_doctree(text, app.srcdir / docname + '.rst',
                            reader=SphinxStandaloneReader(app),
-                           parser=RSTParser(),
+                           parser=parser,
                            settings_overrides={'env': app.env,
                                                'gettext_compact': True})
 

tests/test_parser.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+"""
+    test_sphinx_parsers
+    ~~~~~~~~~~~~~~~~~~~
+
+    Tests parsers module.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from unittest.mock import Mock, patch
+
+import pytest
+
+from sphinx.parsers import RSTParser
+from sphinx.util.docutils import new_document
+
+
+@pytest.mark.sphinx(testroot='basic')
+@patch('docutils.parsers.rst.states.RSTStateMachine')
+def test_RSTParser_prolog_epilog(RSTStateMachine, app):
+    document = new_document('dummy.rst')
+    document.settings = Mock(tab_width=8, language_code='')
+    parser = RSTParser()
+    parser.set_application(app)
+
+    # normal case
+    text = ('hello Sphinx world\n'
+            'Sphinx is a document generator')
+    parser.parse(text, document)
+    (content, _), _ = RSTStateMachine().run.call_args
+
+    assert list(content.xitems()) == [('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator')]
+
+    # with rst_prolog
+    app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!'
+    parser.parse(text, document)
+    (content, _), _ = RSTStateMachine().run.call_args
+    assert list(content.xitems()) == [('<rst_prolog>', 0, 'this is rst_prolog'),
+                                      ('<rst_prolog>', 1, 'hello reST!'),
+                                      ('<generated>', 0, ''),
+                                      ('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator')]
+
+    # with rst_epilog
+    app.env.config.rst_prolog = None
+    app.env.config.rst_epilog = 'this is rst_epilog\ngood-bye reST!'
+    parser.parse(text, document)
+    (content, _), _ = RSTStateMachine().run.call_args
+    assert list(content.xitems()) == [('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator'),
+                                      ('<generated>', 0, ''),
+                                      ('<rst_epilog>', 0, 'this is rst_epilog'),
+                                      ('<rst_epilog>', 1, 'good-bye reST!')]
+
+    # expandtabs / convert whitespaces
+    app.env.config.rst_prolog = None
+    app.env.config.rst_epilog = None
+    text = ('\thello Sphinx world\n'
+            '\v\fSphinx is a document generator')
+    parser.parse(text, document)
+    (content, _), _ = RSTStateMachine().run.call_args
+    assert list(content.xitems()) == [('dummy.rst', 0, '        hello Sphinx world'),
+                                      ('dummy.rst', 1, '  Sphinx is a document generator')]

tests/test_util_rst.py

@@ -8,7 +8,10 @@
     :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-from sphinx.util.rst import escape
+
+from docutils.statemachine import StringList
+
+from sphinx.util.rst import append_epilog, escape, prepend_prolog
 
 
 def test_escape():
@@ -16,3 +19,68 @@ def test_escape():
     assert escape('footnote [#]_') == r'footnote \[\#\]\_'
     assert escape('sphinx.application') == r'sphinx.application'
     assert escape('.. toctree::') == r'\.. toctree\:\:'
+
+
+def test_append_epilog(app):
+    epilog = 'this is rst_epilog\ngood-bye reST!'
+    content = StringList(['hello Sphinx world',
+                          'Sphinx is a document generator'],
+                         'dummy.rst')
+    append_epilog(content, epilog)
+
+    assert list(content.xitems()) == [('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator'),
+                                      ('<generated>', 0, ''),
+                                      ('<rst_epilog>', 0, 'this is rst_epilog'),
+                                      ('<rst_epilog>', 1, 'good-bye reST!')]
+
+
+def test_prepend_prolog(app):
+    prolog = 'this is rst_prolog\nhello reST!'
+    content = StringList([':title: test of SphinxFileInput',
+                          ':author: Sphinx team',
+                          '',
+                          'hello Sphinx world',
+                          'Sphinx is a document generator'],
+                         'dummy.rst')
+    prepend_prolog(content, prolog)
+
+    assert list(content.xitems()) == [('dummy.rst', 0, ':title: test of SphinxFileInput'),
+                                      ('dummy.rst', 1, ':author: Sphinx team'),
+                                      ('<generated>', 0, ''),
+                                      ('<rst_prolog>', 0, 'this is rst_prolog'),
+                                      ('<rst_prolog>', 1, 'hello reST!'),
+                                      ('<generated>', 0, ''),
+                                      ('dummy.rst', 2, ''),
+                                      ('dummy.rst', 3, 'hello Sphinx world'),
+                                      ('dummy.rst', 4, 'Sphinx is a document generator')]
+
+
+def test_prepend_prolog_with_CR(app):
+    # prolog having CR at tail
+    prolog = 'this is rst_prolog\nhello reST!\n'
+    content = StringList(['hello Sphinx world',
+                          'Sphinx is a document generator'],
+                         'dummy.rst')
+    prepend_prolog(content, prolog)
+
+    assert list(content.xitems()) == [('<rst_prolog>', 0, 'this is rst_prolog'),
+                                      ('<rst_prolog>', 1, 'hello reST!'),
+                                      ('<generated>', 0, ''),
+                                      ('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator')]
+
+
+def test_prepend_prolog_without_CR(app):
+    # prolog not having CR at tail
+    prolog = 'this is rst_prolog\nhello reST!'
+    content = StringList(['hello Sphinx world',
+                          'Sphinx is a document generator'],
+                         'dummy.rst')
+    prepend_prolog(content, prolog)
+
+    assert list(content.xitems()) == [('<rst_prolog>', 0, 'this is rst_prolog'),
+                                      ('<rst_prolog>', 1, 'hello reST!'),
+                                      ('<generated>', 0, ''),
+                                      ('dummy.rst', 0, 'hello Sphinx world'),
+                                      ('dummy.rst', 1, 'Sphinx is a document generator')]