Fix #1994: More supporting non-standard parser (like recommonmark parser) for Translation and WebSupport feature. Now node.rawsource is fall backed to node.astext() during docutils transforming.

CHANGES

@@ -53,6 +53,9 @@ Bugs fixed
 * #1983: i18n translation feature breaks references which uses section name.
 * #1990: Use caption of toctree to title of \tableofcontents in LaTeX
 * #1987: Fix ampersand is ignored in ``:menuselection:`` and ``:guilabel:`` on LaTeX builder
+* #1994: More supporting non-standard parser (like recommonmark parser) for Translation and
+  WebSupport feature. Now node.rawsource is fall backed to node.astext() during docutils
+  transforming.
 
 
 Release 1.3.1 (released Mar 17, 2015)

sphinx/directives/other.py

@@ -142,7 +142,7 @@ class Author(Directive):
         env = self.state.document.settings.env
         if not env.config.show_authors:
             return []
-        para = nodes.paragraph()
+        para = nodes.paragraph(translatable=False)
         emph = nodes.emphasis()
         para += emph
         if self.name == 'sectionauthor':
@@ -205,7 +205,7 @@ class VersionChange(Directive):
         if len(self.arguments) == 2:
             inodes, messages = self.state.inline_text(self.arguments[1],
                                                       self.lineno+1)
-            para = nodes.paragraph(self.arguments[1], '', *inodes)
+            para = nodes.paragraph(self.arguments[1], '', *inodes, translatable=False)
             set_source_info(self, para)
             node.append(para)
         else:
@@ -218,13 +218,14 @@ class VersionChange(Directive):
                 content.source = node[0].source
                 content.line = node[0].line
                 content += node[0].children
-                node[0].replace_self(nodes.paragraph('', '', content))
+                node[0].replace_self(nodes.paragraph('', '', content, translatable=False))
             node[0].insert(0, nodes.inline('', '%s: ' % text,
                                            classes=['versionmodified']))
         else:
             para = nodes.paragraph('', '',
                                    nodes.inline('', '%s.' % text,
-                                                classes=['versionmodified']))
+                                                classes=['versionmodified']),
+                                   translatable=False)
             node.append(para)
         env = self.state.document.settings.env
         # XXX should record node.source as well

sphinx/environment.py

@@ -49,9 +49,11 @@ from sphinx.util.websupport import is_commentable
 from sphinx.errors import SphinxError, ExtensionError
 from sphinx.locale import _
 from sphinx.versioning import add_uids, merge_doctrees
-from sphinx.transforms import DefaultSubstitutions, MoveModuleTargets, \
-    HandleCodeBlocks, AutoNumbering, SortIds, CitationReferences, Locale, \
-    RemoveTranslatableInline, SphinxContentsFilter, ExtraTranslatableNodes
+from sphinx.transforms import (
+    DefaultSubstitutions, MoveModuleTargets, ApplySourceWorkaround,
+    HandleCodeBlocks, AutoNumbering, SortIds, CitationReferences, Locale,
+    RemoveTranslatableInline, SphinxContentsFilter, ExtraTranslatableNodes,
+)
 
 
 orig_role_function = roles.role
@@ -99,7 +101,7 @@ class SphinxStandaloneReader(standalone.Reader):
     """
     Add our own transforms.
     """
-    transforms = [ExtraTranslatableNodes, Locale, CitationReferences,
+    transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, Locale, CitationReferences,
                   DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks,
                   AutoNumbering, SortIds, RemoveTranslatableInline]
 

sphinx/transforms.py

@@ -22,6 +22,7 @@ from sphinx.locale import _, init as init_locale
 from sphinx.util import split_index_msg
 from sphinx.util.nodes import (
     traverse_translatable_index, extract_messages, LITERAL_TYPE_NODES, IMAGE_TYPE_NODES,
+    apply_source_workaround,
 )
 from sphinx.util.osutil import ustrftime
 from sphinx.util.i18n import find_catalog
@@ -169,6 +170,18 @@ TRANSLATABLE_NODES = {
 }
 
 
+class ApplySourceWorkaround(Transform):
+    """
+    update source and rawsource attributes
+    """
+    default_priority = 10
+
+    def apply(self):
+        for n in self.document.traverse():
+            if isinstance(n, nodes.TextElement):
+                apply_source_workaround(n)
+
+
 class ExtraTranslatableNodes(Transform):
     """
     make nodes translatable

sphinx/util/nodes.py

@@ -50,6 +50,10 @@ def apply_source_workaround(node):
         # overwrite: ``term : classifier1 : classifier2`` -> ``term text``
         node.rawsource = node.astext()
 
+    # workaround: recommonmark-0.2.0 doesn't set rawsource attribute
+    if not node.rawsource:
+        node.rawsource = node.astext()
+
     if node.source and node.rawsource:
         return
 
@@ -74,18 +78,20 @@ IGNORED_NODES = (
     nodes.Inline,
     nodes.literal_block,
     nodes.doctest_block,
+    addnodes.versionmodified,
     # XXX there are probably more
 )
 
 
 def is_translatable(node):
     if isinstance(node, nodes.TextElement):
-        apply_source_workaround(node)
-
         if not node.source:
             return False  # built-in message
         if isinstance(node, IGNORED_NODES) and 'translatable' not in node:
             return False
+        if not node.get('translatable', True):
+            # not(node['translatable'] == True or node['translatable'] is None)
+            return False
         # <field_name>orphan</field_name>
         # XXX ignore all metadata (== docinfo)
         if isinstance(node, nodes.field_name) and node.children[0] == 'orphan':

tests/test_util_nodes.py

@@ -16,13 +16,24 @@ from docutils.utils import new_document
 from docutils import frontend
 
 from sphinx.util.nodes import extract_messages
+from sphinx.transforms import ApplySourceWorkaround
 
 
-def _get_doctree(text):
+def _transform(doctree):
+    ApplySourceWorkaround(doctree).apply()
+
+
+def create_new_document():
     settings = frontend.OptionParser(
         components=(rst.Parser,)).get_default_values()
     document = new_document('dummy.txt', settings)
+    return document
+
+
+def _get_doctree(text):
+    document = create_new_document()
     rst.Parser().parse(text, document)
+    _transform(document)
     return document
 
 
@@ -119,3 +130,26 @@ def test_extract_messages():
         extract_messages(_get_doctree(text)),
         nodes.line, 2,
     )
+
+
+def test_extract_messages_without_rawsource():
+    """
+    Check node.rawsource is fall-backed by using node.astext() value.
+
+    `extract_message` which is used from Sphinx i18n feature drop ``not node.rawsource`` nodes.
+    So, all nodes which want to translate must have ``rawsource`` value.
+    However, sometimes node.rawsource is not set.
+
+    For example: recommonmark-0.2.0 doesn't set rawsource to `paragraph` node.
+
+    refs #1994: Fall back to node's astext() during i18n message extraction.
+    """
+    p = nodes.paragraph()
+    p.append(nodes.Text('test'))
+    p.append(nodes.Text('sentence'))
+    assert not p.rawsource  # target node must not have rawsource value
+    document = create_new_document()
+    document.append(p)
+    _transform(document)
+    assert_node_count(extract_messages(document), nodes.TextElement, 1)
+    assert [m for n, m in extract_messages(document)][0], 'text sentence'