Add test_io.py

sphinx/io.py

@@ -8,13 +8,15 @@
     :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
+import re
 import codecs
 
 from docutils.io import FileInput, NullOutput
 from docutils.core import Publisher
 from docutils.readers import standalone
+from docutils.statemachine import StringList
 from docutils.writers import UnfilteredWriter
-from six import string_types, text_type, iteritems
+from six import text_type
 from typing import Any, Union  # NOQA
 
 from sphinx.transforms import (
@@ -28,7 +30,6 @@ from sphinx.transforms.i18n import (
     PreserveTranslatableMessages, Locale, RemoveTranslatableInline,
 )
 from sphinx.util import logging
-from sphinx.util import import_object, split_docinfo
 from sphinx.util.docutils import LoggingReporter
 
 if False:
@@ -42,6 +43,8 @@ if False:
     from sphinx.builders import Builder  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA
 
+docinfo_re = re.compile(':\\w+:.*?')
+
 
 logger = logging.getLogger(__name__)
 
@@ -59,6 +62,7 @@ class SphinxBaseReader(standalone.Reader):
         document = standalone.Reader.new_document(self)
         reporter = document.reporter
         document.reporter = LoggingReporter.from_reporter(reporter)
+        document.reporter.set_source(self.source)
         return document
 
 
@@ -168,15 +172,50 @@ class SphinxFileInput(SphinxBaseFileInput):
 
 
 class SphinxRSTFileInput(SphinxBaseFileInput):
+    def prepend_prolog(self, text, prolog):
+        # type: (StringList, unicode) -> None
+        docinfo = self.count_docinfo_lines(text)
+        if docinfo:
+            # insert a blank line after docinfo
+            text.insert(docinfo, '', '<generated>', 0)
+            docinfo += 1
+
+        # insert prolog (after docinfo if exists)
+        for lineno, line in enumerate(prolog.splitlines()):
+            text.insert(docinfo + lineno, line, '<rst_prolog>', lineno)
+
+        text.insert(docinfo + lineno + 1, '', '<generated>', 0)
+
+    def append_epilog(self, text, epilog):
+        # type: (StringList, unicode) -> None
+        # append a blank line and rst_epilog
+        text.append('', '<generated>', 0)
+        for lineno, line in enumerate(epilog.splitlines()):
+            text.append(line, '<rst_epilog>', lineno)
+
     def read(self):
-        # type: () -> unicode
+        # type: () -> StringList
         data = SphinxBaseFileInput.read(self)
-        docinfo, data = split_docinfo(data)
+        content = StringList()
-        if self.env.config.rst_epilog:
+        for lineno, line in enumerate(data.splitlines()):
-            data = data + '\n' + self.env.config.rst_epilog + '\n'
+            content.append(line, self.source_path, lineno)
+
         if self.env.config.rst_prolog:
-            data = self.env.config.rst_prolog + '\n' + data
+            self.prepend_prolog(content, self.env.config.rst_prolog)
-        return docinfo + data
+        if self.env.config.rst_epilog:
+            self.append_epilog(content, self.env.config.rst_epilog)
+
+        return content
+
+    def count_docinfo_lines(self, content):
+        # type: (StringList) -> int
+        if len(content) == 0:
+            return 0
+        else:
+            for lineno, line in enumerate(content.data):
+                if not docinfo_re.match(line):
+                    break
+            return lineno
 
 
 def read_doc(app, env, filename):

sphinx/parsers.py

@@ -11,6 +11,8 @@
 
 import docutils.parsers
 import docutils.parsers.rst
+from docutils.parsers.rst import states
+from docutils.statemachine import StringList
 from docutils.transforms.universal import SmartQuotes
 
 from sphinx.transforms import SphinxSmartQuotes
@@ -66,6 +68,26 @@ class RSTParser(docutils.parsers.rst.Parser):
         transforms.append(SphinxSmartQuotes)
         return transforms
 
+    def parse(self, inputstring, document):
+        # type: (Any, nodes.document) -> None
+        """Parse text and generate a document tree.
+
+        This derived method accepts StringList as a inputstring parameter.
+        It enables to handle mixed contents (cf. rst_prolog) correctly.
+        """
+        if isinstance(inputstring, StringList):
+            self.setup_parse(inputstring, document)
+            self.statemachine = states.RSTStateMachine(
+                state_classes=self.state_classes,
+                initial_state=self.initial_state,
+                debug=document.reporter.debug_flag)
+            # Give inputstring directly to statemachine.
+            self.statemachine.run(inputstring, document, inliner=self.inliner)
+            self.finish_parse()
+        else:
+            # otherwise, inputstring might be a string. It will be handled by superclass.
+            docutils.parsers.rst.Parser.parse(self, inputstring, document)
+
 
 def setup(app):
     # type: (Sphinx) -> Dict[unicode, Any]

sphinx/util/__init__.py

@@ -564,16 +564,6 @@ def encode_uri(uri):
     return urlunsplit(split)
 
 
-def split_docinfo(text):
-    # type: (unicode) -> Sequence[unicode]
-    docinfo_re = re.compile('\\A((?:\\s*:\\w+:.*?\n(?:[ \\t]+.*?\n)*)+)', re.M)
-    result = docinfo_re.split(text, 1)  # type: ignore
-    if len(result) == 1:
-        return '', result[0]
-    else:
-        return result[1:]
-
-
 def display_chunk(chunk):
     # type: (Any) -> unicode
     if isinstance(chunk, (list, tuple)):

sphinx/util/docutils.py

@@ -18,8 +18,9 @@ from contextlib import contextmanager
 
 import docutils
 from docutils.languages import get_language
-from docutils.utils import Reporter
+from docutils.statemachine import ViewList
 from docutils.parsers.rst import directives, roles, convert_directive_function
+from docutils.utils import Reporter
 
 from sphinx.errors import ExtensionError
 from sphinx.locale import __
@@ -33,6 +34,7 @@ if False:
     from typing import Any, Callable, Iterator, List, Tuple  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA
+    from sphinx.io import SphinxFileInput  # NOQA
 
 
 __version_info__ = tuple(LooseVersion(docutils.__version__).version)
@@ -180,6 +182,21 @@ class LoggingReporter(Reporter):
         stream = WarningStream()
         Reporter.__init__(self, source, report_level, halt_level,
                           stream, debug, error_handler=error_handler)
+        self.source_and_line = None
+
+    def set_source(self, source):
+        # type: (SphinxFileInput) -> None
+        self.source_and_line = source
+
+    def system_message(self, *args, **kwargs):
+        # type: (Any, Any) -> Any
+        if kwargs.get('line') and isinstance(self.source_and_line, ViewList):
+            # replace source parameter if source is set
+            source, lineno = self.source_and_line.info(kwargs.get('line'))
+            kwargs['source'] = source
+            kwargs['line'] = lineno
+
+        return Reporter.system_message(self, *args, **kwargs)
 
 
 def is_html5_writer_available():

tests/test_io.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+"""
+    test_sphinx_io
+    ~~~~~~~~~~~~~~
+
+    Tests io modules.
+
+    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import pytest
+from six import StringIO
+
+from sphinx.io import SphinxRSTFileInput
+
+
+@pytest.mark.sphinx(testroot='basic')
+def test_SphinxRSTFileInput(app):
+    app.env.temp_data['docname'] = 'index'
+
+    # normal case
+    text = ('hello Sphinx world\n'
+            'Sphinx is a document generator')
+    source = SphinxRSTFileInput(app, app.env, source=StringIO(text),
+                                source_path='dummy.rst', encoding='utf-8')
+    result = source.read()
+    assert result.data == ['hello Sphinx world',
+                           'Sphinx is a document generator']
+    assert result.info(0) == ('dummy.rst', 0)
+    assert result.info(1) == ('dummy.rst', 1)
+    assert result.info(2) == ('dummy.rst', None)  # out of range
+
+    # having rst_prolog ends without CR
+    app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!'
+    source = SphinxRSTFileInput(app, app.env, source=StringIO(text),
+                                source_path='dummy.rst', encoding='utf-8')
+    result = source.read()
+    assert result.data == ['this is rst_prolog',
+                           'hello reST!',
+                           '',
+                           'hello Sphinx world',
+                           'Sphinx is a document generator']
+    assert result.info(0) == ('<rst_prolog>', 0)
+    assert result.info(1) == ('<rst_prolog>', 1)
+    assert result.info(2) == ('<generated>', 0)
+    assert result.info(3) == ('dummy.rst', 0)
+    assert result.info(4) == ('dummy.rst', 1)
+
+    # having rst_prolog ends with CR
+    app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!\n'
+    source = SphinxRSTFileInput(app, app.env, source=StringIO(text),
+                                source_path='dummy.rst', encoding='utf-8')
+    result = source.read()
+    assert result.data == ['this is rst_prolog',
+                           'hello reST!',
+                           '',
+                           'hello Sphinx world',
+                           'Sphinx is a document generator']
+
+    # having docinfo and rst_prolog
+    docinfo_text = (':title: test of SphinxFileInput\n'
+                    ':author: Sphinx team\n'
+                    '\n'
+                    'hello Sphinx world\n'
+                    'Sphinx is a document generator\n')
+    app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!'
+    source = SphinxRSTFileInput(app, app.env, source=StringIO(docinfo_text),
+                                source_path='dummy.rst', encoding='utf-8')
+    result = source.read()
+    assert result.data == [':title: test of SphinxFileInput',
+                           ':author: Sphinx team',
+                           '',
+                           'this is rst_prolog',
+                           'hello reST!',
+                           '',
+                           '',
+                           'hello Sphinx world',
+                           'Sphinx is a document generator']
+    assert result.info(0) == ('dummy.rst', 0)
+    assert result.info(1) == ('dummy.rst', 1)
+    assert result.info(2) == ('<generated>', 0)
+    assert result.info(3) == ('<rst_prolog>', 0)
+    assert result.info(4) == ('<rst_prolog>', 1)
+    assert result.info(5) == ('<generated>', 0)
+    assert result.info(6) == ('dummy.rst', 2)
+    assert result.info(7) == ('dummy.rst', 3)
+    assert result.info(8) == ('dummy.rst', 4)
+    assert result.info(9) == ('dummy.rst', None)  # out of range
+
+    # having rst_epilog
+    app.env.config.rst_prolog = None
+    app.env.config.rst_epilog = 'this is rst_epilog\ngood-bye reST!'
+    source = SphinxRSTFileInput(app, app.env, source=StringIO(text),
+                                source_path='dummy.rst', encoding='utf-8')
+    result = source.read()
+    assert result.data == ['hello Sphinx world',
+                           'Sphinx is a document generator',
+                           '',
+                           'this is rst_epilog',
+                           'good-bye reST!']
+    assert result.info(0) == ('dummy.rst', 0)
+    assert result.info(1) == ('dummy.rst', 1)
+    assert result.info(2) == ('<generated>', 0)
+    assert result.info(3) == ('<rst_epilog>', 0)
+    assert result.info(4) == ('<rst_epilog>', 1)
+    assert result.info(5) == ('<rst_epilog>', None)  # out of range

tests/test_util.py

@@ -14,8 +14,7 @@ from mock import patch
 
 from sphinx.util import logging
 from sphinx.util import (
-    display_chunk, encode_uri, parselinenos, split_docinfo, status_iterator,
+    display_chunk, encode_uri, parselinenos, status_iterator, xmlname_checker
-    xmlname_checker
 )
 
 from sphinx.testing.util import strip_escseq
@@ -36,28 +35,6 @@ def test_encode_uri():
     assert expected, encode_uri(uri)
 
 
-def test_splitdocinfo():
-    source = "Hello world.\n"
-    docinfo, content = split_docinfo(source)
-    assert docinfo == ''
-    assert content == 'Hello world.\n'
-
-    source = ":orphan:\n\nHello world.\n"
-    docinfo, content = split_docinfo(source)
-    assert docinfo == ':orphan:\n'
-    assert content == '\nHello world.\n'
-
-    source = ":author: Georg Brandl\n:title: Manual of Sphinx\n\nHello world.\n"
-    docinfo, content = split_docinfo(source)
-    assert docinfo == ':author: Georg Brandl\n:title: Manual of Sphinx\n'
-    assert content == '\nHello world.\n'
-
-    source = ":multiline: one\n\ttwo\n\tthree\n\nHello world.\n"
-    docinfo, content = split_docinfo(source)
-    assert docinfo == ":multiline: one\n\ttwo\n\tthree\n"
-    assert content == '\nHello world.\n'
-
-
 def test_display_chunk():
     assert display_chunk('hello') == 'hello'
     assert display_chunk(['hello']) == 'hello'