Fix #2287: sphinx.transforms.Locale always uses rst parser. Sphinx i18n feature should support parsers that specified source_parsers.

CHANGES

@@ -9,6 +9,8 @@ Bugs fixed
   with Python 3
 * #2291: Fix pdflatex "Counter too large" error from footnotes inside tables of contents
 * #2292: Fix some footnotes disappear from LaTeX output
+* #2287: ``sphinx.transforms.Locale`` always uses rst parser. Sphinx i18n feature should
+  support parsers that specified source_parsers.
 
 
 Release 1.3.5 (released Jan 24, 2016)

sphinx/application.py

@@ -35,7 +35,8 @@ from sphinx.errors import SphinxError, SphinxWarning, ExtensionError, \
 from sphinx.domains import ObjType, BUILTIN_DOMAINS
 from sphinx.domains.std import GenericObject, Target, StandardDomain
 from sphinx.builders import BUILTIN_BUILDERS
-from sphinx.environment import BuildEnvironment, SphinxStandaloneReader
+from sphinx.environment import BuildEnvironment
+from sphinx.io import SphinxStandaloneReader
 from sphinx.util import pycompat  # noqa: imported for side-effects
 from sphinx.util import import_object
 from sphinx.util.tags import Tags

sphinx/environment.py

@@ -23,22 +23,21 @@ from os import path
 from glob import glob
 from itertools import groupby
 
-from six import iteritems, itervalues, text_type, class_types, string_types, next
+from six import iteritems, itervalues, text_type, class_types, next
 from six.moves import cPickle as pickle
 from docutils import nodes
-from docutils.io import FileInput, NullOutput
+from docutils.io import NullOutput
 from docutils.core import Publisher
 from docutils.utils import Reporter, relative_path, get_source_line
-from docutils.readers import standalone
 from docutils.parsers.rst import roles, directives
 from docutils.parsers.rst.languages import en as english
 from docutils.parsers.rst.directives.html import MetaBody
-from docutils.writers import UnfilteredWriter
 from docutils.frontend import OptionParser
 
 from sphinx import addnodes
+from sphinx.io import SphinxStandaloneReader, SphinxDummyWriter, SphinxFileInput
 from sphinx.util import url_re, get_matching_docs, docname_join, split_into, \
-    FilenameUniqDict, get_figtype, import_object, split_index_msg
+    FilenameUniqDict, get_figtype, split_index_msg
 from sphinx.util.nodes import clean_astext, make_refnode, WarningStream, is_translatable
 from sphinx.util.osutil import SEP, getcwd, fs_encoding, ensuredir
 from sphinx.util.i18n import find_catalog_files
@@ -49,12 +48,7 @@ from sphinx.util.websupport import is_commentable
 from sphinx.errors import SphinxError, ExtensionError
 from sphinx.locale import _
 from sphinx.versioning import add_uids, merge_doctrees
-from sphinx.transforms import (
-    DefaultSubstitutions, MoveModuleTargets, ApplySourceWorkaround,
-    HandleCodeBlocks, AutoNumbering, SortIds, CitationReferences, Locale,
-    RemoveTranslatableInline, SphinxContentsFilter, ExtraTranslatableNodes,
-)
-
+from sphinx.transforms import SphinxContentsFilter
 
 orig_role_function = roles.role
 orig_directive_function = directives.directive
@@ -97,73 +91,6 @@ class NoUri(Exception):
     pass
 
 
-class SphinxStandaloneReader(standalone.Reader):
-    """
-    Add our own transforms.
-    """
-    transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, Locale, CitationReferences,
-                  DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks,
-                  AutoNumbering, SortIds, RemoveTranslatableInline]
-
-    def __init__(self, parsers={}, *args, **kwargs):
-        standalone.Reader.__init__(self, *args, **kwargs)
-        self.parser_map = {}
-        for suffix, parser_class in parsers.items():
-            if isinstance(parser_class, string_types):
-                parser_class = import_object(parser_class, 'source parser')
-            self.parser_map[suffix] = parser_class()
-
-    def read(self, source, parser, settings):
-        self.source = source
-
-        for suffix in self.parser_map:
-            if source.source_path.endswith(suffix):
-                self.parser = self.parser_map[suffix]
-                break
-
-        if not self.parser:
-            self.parser = parser
-        self.settings = settings
-        self.input = self.source.read()
-        self.parse()
-        return self.document
-
-    def get_transforms(self):
-        return standalone.Reader.get_transforms(self) + self.transforms
-
-
-class SphinxDummyWriter(UnfilteredWriter):
-    supported = ('html',)  # needed to keep "meta" nodes
-
-    def translate(self):
-        pass
-
-
-class SphinxFileInput(FileInput):
-    def __init__(self, app, env, *args, **kwds):
-        self.app = app
-        self.env = env
-        kwds['error_handler'] = 'sphinx'  # py3: handle error on open.
-        FileInput.__init__(self, *args, **kwds)
-
-    def decode(self, data):
-        if isinstance(data, text_type):  # py3: `data` already decoded.
-            return data
-        return data.decode(self.encoding, 'sphinx')  # py2: decoding
-
-    def read(self):
-        data = FileInput.read(self)
-        if self.app:
-            arg = [data]
-            self.app.emit('source-read', self.env.docname, arg)
-            data = arg[0]
-        if self.env.config.rst_epilog:
-            data = data + '\n' + self.env.config.rst_epilog + '\n'
-        if self.env.config.rst_prolog:
-            data = self.env.config.rst_prolog + '\n' + data
-        return data
-
-
 class BuildEnvironment:
     """
     The environment in which the ReST files are translated.

sphinx/io.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.io
+    ~~~~~~~~~
+
+    Input/Output files
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+from docutils.io import FileInput
+from docutils.readers import standalone
+from docutils.writers import UnfilteredWriter
+from six import string_types, text_type
+
+from sphinx.transforms import ApplySourceWorkaround, ExtraTranslatableNodes, Locale, \
+    CitationReferences, DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, \
+    AutoNumbering, SortIds, RemoveTranslatableInline
+from sphinx.util import import_object
+
+
+class SphinxBaseReader(standalone.Reader):
+    """
+    Add our source parsers
+    """
+    def __init__(self, parsers={}, *args, **kwargs):
+        standalone.Reader.__init__(self, *args, **kwargs)
+        self.parser_map = {}
+        for suffix, parser_class in parsers.items():
+            if isinstance(parser_class, string_types):
+                parser_class = import_object(parser_class, 'source parser')
+            self.parser_map[suffix] = parser_class()
+
+    def read(self, source, parser, settings):
+        self.source = source
+
+        for suffix in self.parser_map:
+            if source.source_path.endswith(suffix):
+                self.parser = self.parser_map[suffix]
+                break
+
+        if not self.parser:
+            self.parser = parser
+        self.settings = settings
+        self.input = self.source.read()
+        self.parse()
+        return self.document
+
+    def get_transforms(self):
+        return standalone.Reader.get_transforms(self) + self.transforms
+
+
+class SphinxStandaloneReader(SphinxBaseReader):
+    """
+    Add our own transforms.
+    """
+    transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, Locale, CitationReferences,
+                  DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks,
+                  AutoNumbering, SortIds, RemoveTranslatableInline]
+
+
+class SphinxI18nReader(SphinxBaseReader):
+    """
+    Replacer for document.reporter.get_source_and_line method.
+
+    reST text lines for translation do not have the original source line number.
+    This class provides the correct line numbers when reporting.
+    """
+
+    transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, CitationReferences,
+                  DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks,
+                  AutoNumbering, SortIds, RemoveTranslatableInline]
+
+    def __init__(self, *args, **kwargs):
+        SphinxBaseReader.__init__(self, *args, **kwargs)
+        self.lineno = None
+
+    def set_lineno_for_reporter(self, lineno):
+        self.lineno = lineno
+
+    def new_document(self):
+        document = SphinxBaseReader.new_document(self)
+        reporter = document.reporter
+
+        def get_source_and_line(lineno=None):
+            return reporter.source, self.lineno
+
+        reporter.get_source_and_line = get_source_and_line
+        return document
+
+
+class SphinxDummyWriter(UnfilteredWriter):
+    supported = ('html',)  # needed to keep "meta" nodes
+
+    def translate(self):
+        pass
+
+
+class SphinxFileInput(FileInput):
+    def __init__(self, app, env, *args, **kwds):
+        self.app = app
+        self.env = env
+        kwds['error_handler'] = 'sphinx'  # py3: handle error on open.
+        FileInput.__init__(self, *args, **kwds)
+
+    def decode(self, data):
+        if isinstance(data, text_type):  # py3: `data` already decoded.
+            return data
+        return data.decode(self.encoding, 'sphinx')  # py2: decoding
+
+    def read(self):
+        data = FileInput.read(self)
+        if self.app:
+            arg = [data]
+            self.app.emit('source-read', self.env.docname, arg)
+            data = arg[0]
+        if self.env.config.rst_epilog:
+            data = data + '\n' + self.env.config.rst_epilog + '\n'
+        if self.env.config.rst_prolog:
+            data = self.env.config.rst_prolog + '\n' + data
+        return data

sphinx/quickstart.py

@@ -9,6 +9,7 @@
     :license: BSD, see LICENSE for details.
 """
 from __future__ import print_function
+from __future__ import absolute_import
 
 import re
 import os

sphinx/transforms.py

@@ -12,8 +12,8 @@
 from os import path
 
 from docutils import nodes
-from docutils.utils import new_document, relative_path
-from docutils.parsers.rst import Parser as RSTParser
+from docutils.io import StringInput
+from docutils.utils import relative_path
 from docutils.transforms import Transform
 from docutils.transforms.parts import ContentsFilter
 
@@ -202,21 +202,33 @@ class ExtraTranslatableNodes(Transform):
             node['translatable'] = True
 
 
-class CustomLocaleReporter(object):
+def publish_msgstr(source, source_path, source_line, config, settings):
+    """Publish msgstr (single line) into docutils document
+
+    :param unicode source: source text
+    :param unicode source_path: source path for warning indication
+    :param source_line: source line for warning indication
+    :param sphinx.config.Config config: sphinx config
+    :param docutils.frontend.Values settings: docutils settings
+    :return: document
+    :rtype: docutils.nodes.document
     """
-    Replacer for document.reporter.get_source_and_line method.
-
-    reST text lines for translation do not have the original source line number.
-    This class provides the correct line numbers when reporting.
-    """
-    def __init__(self, source, line):
-        self.source, self.line = source, line
-
-    def set_reporter(self, document):
-        document.reporter.get_source_and_line = self.get_source_and_line
-
-    def get_source_and_line(self, lineno=None):
-        return self.source, self.line
+    from sphinx.io import SphinxI18nReader
+    reader = SphinxI18nReader(
+        parsers=config.source_parsers,
+        parser_name='restructuredtext',  # default parser
+    )
+    reader.set_lineno_for_reporter(source_line)
+    doc = reader.read(
+        source=StringInput(source=source, source_path=source_path),
+        parser=reader.parser,
+        settings=settings,
+    )
+    try:
+        doc = doc[0]
+    except IndexError:  # empty node
+        pass
+    return doc
 
 
 class Locale(Transform):
@@ -244,8 +256,6 @@ class Locale(Transform):
         if not has_catalog:
             return
 
-        parser = RSTParser()
-
         # phase1: replace reference ids with translated names
         for node, msg in extract_messages(self.document):
             msgstr = catalog.gettext(msg)
@@ -267,13 +277,7 @@ class Locale(Transform):
             if isinstance(node, LITERAL_TYPE_NODES):
                 msgstr = '::\n\n' + indent(msgstr, ' '*3)
 
-            patch = new_document(source, settings)
-            CustomLocaleReporter(node.source, node.line).set_reporter(patch)
-            parser.parse(msgstr, patch)
-            try:
-                patch = patch[0]
-            except IndexError:  # empty node
-                pass
+            patch = publish_msgstr(msgstr, source, node.line, env.config, settings)
             # XXX doctest and other block markup
             if not isinstance(patch, nodes.paragraph):
                 continue  # skip for now
@@ -390,13 +394,7 @@ class Locale(Transform):
             if isinstance(node, LITERAL_TYPE_NODES):
                 msgstr = '::\n\n' + indent(msgstr, ' '*3)
 
-            patch = new_document(source, settings)
-            CustomLocaleReporter(node.source, node.line).set_reporter(patch)
-            parser.parse(msgstr, patch)
-            try:
-                patch = patch[0]
-            except IndexError:  # empty node
-                pass
+            patch = publish_msgstr(msgstr, source, node.line, env.config, settings)
             # XXX doctest and other block markup
             if not isinstance(
                     patch,