Improve support for automatic 2to3 conversion of config files. It now kicks in whenever the original file raises SyntaxErrors on compiling.

sphinx/config.py

@@ -13,14 +13,10 @@ import os
 import re
 import sys
 from os import path
-try:
-    from distutils.util import run_2to3
-except ImportError:
-    run_2to3 = None
 
 from sphinx.errors import ConfigError
 from sphinx.util.osutil import make_filename
-from sphinx.util.pycompat import bytes, b, should_run_2to3, run_2to3
+from sphinx.util.pycompat import bytes, b, convert_with_2to3
 
 nonascii_re = re.compile(b(r'[\x80-\xff]'))
 
@@ -172,17 +168,28 @@ class Config(object):
             config['tags'] = tags
             olddir = os.getcwd()
             try:
+                # we promise to have the config dir as current dir while the
+                # config file is executed
+                os.chdir(dirname)
+                # get config source
+                f = open(config_file, 'rb')
                 try:
-                    os.chdir(dirname)
-                    if should_run_2to3(config_file):
-                        code = run_2to3(config_file)
-                    else:
-                        f = open(config_file, 'rb')
-                        try:
-                            code = f.read()
-                        finally:
-                            f.close()
-                    exec compile(code, config_file, 'exec') in config
+                    source = f.read()
+                finally:
+                    f.close()
+                try:
+                    # compile to a code object, handle syntax errors
+                    try:
+                        code = compile(source, config_file, 'exec')
+                    except SyntaxError:
+                        if convert_with_2to3:
+                            # maybe the file uses 2.x syntax; try to refactor to
+                            # 3.x syntax using 2to3
+                            source = convert_with_2to3(config_file)
+                            code = compile(source, config_file, 'exec')
+                        else:
+                            raise
+                    exec code in config
                 except SyntaxError, err:
                     raise ConfigError(CONFIG_SYNTAX_ERROR % err)
             finally:

sphinx/util/pycompat.py

@@ -30,58 +30,26 @@ else:
     b = str
 
 
-encoding_re = re.compile(b(r'coding[=:]\s*([-\w.]+)'))
-unicode_literal_re = re.compile(ur"""
-(?:
-    "(?:[^"\]]*(?:\\.[^"\\]*)*)"|
-    '(?:[^'\]]*(?:\\.[^'\\]*)*)'
-)
-""", re.VERBOSE)
+# Support for running 2to3 over config files
 
-
-try:
-    from lib2to3.refactor import RefactoringTool, get_fixers_from_package
-except ImportError:
-    _run_2to3 = None
-    def should_run_2to3(filepath):
-        return False
+if sys.version_info < (3, 0):
+    # no need to refactor on 2.x versions
+    convert_with_2to3 = None
 else:
-    def should_run_2to3(filepath):
-        # th default source code encoding for python 2.x
-        encoding = 'ascii'
-        # only the first match of the encoding cookie counts
-        encoding_set = False
-        f = open(filepath, 'rb')
-        try:
-            for i, line in enumerate(f):
-                if line.startswith(b('#')):
-                    if i == 0 and b('python3') in line:
-                        return False
-                    if not encoding_set:
-                        encoding_match = encoding_re.match(line)
-                        if encoding_match:
-                            encoding = encoding_match.group(1)
-                            encodin_set = True
-                elif line.strip():
-                    try:
-                        line = line.decode(encoding)
-                    except UnicodeDecodeError:
-                        # I'm not sure this will work but let's try it anyway
-                        return True
-                    if unicode_literal_re.search(line) is not None:
-                        return True
-        finally:
-            f.close()
-        return False
-
-    def run_2to3(filepath):
-        sys.path.append('..')
+    def convert_with_2to3(filepath):
+        from lib2to3.refactor import RefactoringTool, get_fixers_from_package
+        from lib2to3.pgen2.parse import ParseError
         fixers = get_fixers_from_package('lib2to3.fixes')
-        fixers.extend(get_fixers_from_package('custom_fixers'))
         refactoring_tool = RefactoringTool(fixers)
         source = refactoring_tool._read_python_source(filepath)[0]
-        ast = refactoring_tool.refactor_string(source, 'conf.py')
-        return unicode(ast)
+        try:
+            tree = refactoring_tool.refactor_string(source, 'conf.py')
+        except ParseError, err:
+            # do not propagate lib2to3 exceptions
+            lineno, offset = err.context[1]
+            # try to match ParseError details with SyntaxError details
+            raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
+        return unicode(tree)
 
 
 try:
@@ -93,7 +61,8 @@ except NameError:
 try:
     next = next
 except NameError:
-    # this is on Python 2, where the method is called "next"
+    # this is on Python 2, where the method is called "next" (it is refactored
+    # to __next__ by 2to3, but in that case never executed)
     def next(iterator):
         return iterator.next()
 

tests/test_config.py

@@ -88,6 +88,12 @@ def test_errors_warnings(dir):
     write_file(dir / 'conf.py', u'project = \n', 'ascii')
     raises_msg(ConfigError, 'conf.py', Config, dir, 'conf.py', {}, None)
 
+    # test the automatic conversion of 2.x only code in configs
+    write_file(dir / 'conf.py', u'\n\nproject = u"Jägermeister"\n', 'utf-8')
+    cfg = Config(dir, 'conf.py', {}, None)
+    cfg.init_values()
+    assert cfg.project == u'Jägermeister'
+
     # test the warning for bytestrings with non-ascii content
     # bytestrings with non-ascii content are a syntax error in python3 so we
     # skip the test there