Highlighting fallbacks by default

In 423bf7b, I tried to add fallback mechanism to ``python3``. But it breaks
the python3 highlighting on python2 environment.

This adds ``'default'`` to highlighting languages; it works like
``'python3'``, but fallbacks if failed Highlighting. And this removes
try-parse step from ``'python3'`` language. Now, it highlights regardless of
runtime environments.  Thanks to Yoshiki SHIBUKAWA.

CHANGES

@@ -12,9 +12,11 @@ Features added
 Bugs fixed
 ----------
 * Remove ``image/gif`` from supported_image_types of LaTeX writer (#2272)
-* Fix code-block literals raises highlighting warnings by default
 * Fix ValueError is raised if LANGUAGE is empty string
 * Fix unpack warning is shown when the directives generated from ``Sphinx.add_crossref_type`` is used
+* The default highlight language is now ``default``.  This means that source code
+  is highlighted as Python 3 (which is mostly a superset of Python 2) if possible.
+  To get the old behavior back, add ``highlight_language = "python"`` to conf.py.
 
 Documentation
 -------------

doc/config.rst

@@ -333,9 +333,11 @@ Project information
    .. versionadded:: 0.5
 
    .. versionchanged:: 1.4
-      The default is now ``'python3'``, since it is mostly a superset of
+      The default is now ``'default'``. It is similar to ``'python3'``;
-      ``'python'``.  If you prefer Python 2 only highlighting, you can set
+      it is mostly a superset of ``'python'``. but it fallbacks to
-      it back to ``'python'``.
+      ``'none'`` without warning if failed.  ``'python3'`` and other
+      languages will emit warning if failed.  If you prefer Python 2
+      only highlighting, you can set it back to ``'python'``.
 
 .. confval:: highlight_options
 

sphinx/config.py

@@ -65,7 +65,7 @@ class Config(object):
         trim_footnote_reference_space = (False, 'env'),
         show_authors = (False, 'env'),
         pygments_style = (None, 'html', [str]),
-        highlight_language = ('python3', 'env'),
+        highlight_language = ('default', 'env'),
         highlight_options = ({}, 'env'),
         templates_path = ([], 'html'),
         template_bridge = (None, 'html', [str]),

sphinx/highlighting.py

@@ -9,16 +9,7 @@
     :license: BSD, see LICENSE for details.
 """
 
-import re
+from six import text_type
-import textwrap
-
-try:
-    import parser
-except ImportError:
-    # parser is not available on Jython
-    parser = None
-
-from six import PY2, text_type
 
 from sphinx.util.pycompat import htmlescape
 from sphinx.util.texescape import tex_hl_escape_map_new
@@ -100,41 +91,6 @@ class PygmentsBridge(object):
             return '\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n' + \
                    source + '\\end{Verbatim}\n'
 
-    def try_parse(self, src):
-        # Make sure it ends in a newline
-        src += '\n'
-
-        # Ignore consistent indentation.
-        if src.lstrip('\n').startswith(' '):
-            src = textwrap.dedent(src)
-
-        # Replace "..." by a mark which is also a valid python expression
-        # (Note, the highlighter gets the original source, this is only done
-        #  to allow "..." in code and still highlight it as Python code.)
-        mark = "__highlighting__ellipsis__"
-        src = src.replace("...", mark)
-
-        # lines beginning with "..." are probably placeholders for suite
-        src = re.sub(r"(?m)^(\s*)" + mark + "(.)", r"\1" + mark + r"# \2", src)
-
-        if PY2 and isinstance(src, text_type):
-            # Non-ASCII chars will only occur in string literals
-            # and comments.  If we wanted to give them to the parser
-            # correctly, we'd have to find out the correct source
-            # encoding.  Since it may not even be given in a snippet,
-            # just replace all non-ASCII characters.
-            src = src.encode('ascii', 'replace')
-
-        if parser is None:
-            return True
-
-        try:
-            parser.suite(src)
-        except (SyntaxError, UnicodeEncodeError):
-            return False
-        else:
-            return True
-
     def highlight_block(self, source, lang, opts=None, warn=None, force=False, **kwargs):
         if not isinstance(source, text_type):
             source = source.decode()
@@ -146,15 +102,9 @@ class PygmentsBridge(object):
                 lexer = lexers['pycon']
             else:
                 lexer = lexers['python']
-        elif lang in ('py3', 'python3'):
+        elif lang in ('py3', 'python3', 'default'):
             if source.startswith('>>>'):
                 lexer = lexers['pycon3']
-            elif not force:
-                # maybe Python -- try parsing it
-                if self.try_parse(source):
-                    lexer = lexers['python3']
-                else:
-                    lexer = lexers['none']
             else:
                 lexer = lexers['python3']
         elif lang == 'guess':
@@ -189,7 +139,9 @@ class PygmentsBridge(object):
         except ErrorToken as exc:
             # this is most probably not the selected language,
             # so let it pass unhighlighted
-            if warn:
+            if lang == 'default':
+                pass  # automatic highlighting failed.
+            elif warn:
                 warn('Could not lex literal_block as "%s". '
                      'Highlighting skipped.' % lang)
             else:

tests/test_highlighting.py

@@ -11,6 +11,7 @@
 
 from pygments.lexer import RegexLexer
 from pygments.token import Text, Name
+from pygments.filters import ErrorToken
 from pygments.formatters.html import HtmlFormatter
 
 from sphinx.highlighting import PygmentsBridge
@@ -86,3 +87,28 @@ def test_trim_doctest_flags():
         assert ret == '>>> 1+2 \n3\n'
     finally:
         PygmentsBridge.html_formatter = HtmlFormatter
+
+
+def test_default_highlight():
+    bridge = PygmentsBridge('html')
+
+    # default: highlights as python3
+    ret = bridge.highlight_block('print "Hello sphinx world"', 'default')
+    assert ret == ('<div class="highlight"><pre><span></span><span class="nb">print</span> '
+                   '<span class="s2">&quot;Hello sphinx world&quot;</span>\n</pre></div>\n')
+
+    # default: fallbacks to none if highlighting failed
+    ret = bridge.highlight_block('reST ``like`` text', 'default')
+    assert ret == '<div class="highlight"><pre><span></span>reST ``like`` text\n</pre></div>\n'
+
+    # python3: highlights as python3
+    ret = bridge.highlight_block('print "Hello sphinx world"', 'python3')
+    assert ret == ('<div class="highlight"><pre><span></span><span class="nb">print</span> '
+                   '<span class="s2">&quot;Hello sphinx world&quot;</span>\n</pre></div>\n')
+
+    # python3: raises error if highlighting failed
+    try:
+        ret = bridge.highlight_block('reST ``like`` text', 'python3')
+        assert False, "highlight_block() does not raise any exceptions"
+    except ErrorToken:
+        pass  # raise parsing error