merge with trunk

tests/etree13/ElementTree.py

@@ -1425,12 +1425,16 @@ class XMLParser(object):
         err.position = value.lineno, value.offset
         raise err
 
-    def _fixtext(self, text):
-        # convert text string to ascii, if possible
-        try:
-            return text.encode("ascii")
-        except UnicodeError:
+    if sys.version_info >= (3, 0):
+        def _fixtext(self, text):
             return text
+    else:
+        def _fixtext(self, text):
+            # convert text string to ascii, if possible
+            try:
+                return text.encode("ascii")
+            except UnicodeError:
+                return text
 
     def _fixname(self, key):
         # expand qname, and convert name string to ascii, if possible

tests/root/conf.py

@@ -22,7 +22,7 @@ copyright = '2010, Georg Brandl & Team'
 version = '0.6'
 release = '0.6alpha1'
 today_fmt = '%B %d, %Y'
-#unused_docs = []
+# unused_docs = []
 exclude_patterns = ['_build', '**/excluded.*']
 keep_warnings = True
 pygments_style = 'sphinx'

tests/root/doctest.txt

@@ -30,7 +30,7 @@ Special directives
 
   .. testcode::
 
-     print 1+1
+     print(1+1)
 
   .. testoutput::
 
@@ -50,30 +50,30 @@ Special directives
 
   .. testsetup:: *
 
-     from math import floor
+     from math import factorial
 
   .. doctest::
 
-     >>> floor(1.2)
-     1.0
+     >>> factorial(1)
+     1
 
   .. testcode::
 
-     print floor(1.2)
+     print(factorial(1))
 
   .. testoutput::
 
-     1.0
+     1
 
-  >>> floor(1.2)
-  1.0
+  >>> factorial(1)
+  1
 
 * options for testcode/testoutput blocks
 
   .. testcode::
      :hide:
 
-     print 'Output         text.'
+     print('Output         text.')
 
   .. testoutput::
      :hide:
@@ -85,36 +85,36 @@ Special directives
 
   .. testsetup:: group1
 
-     from math import ceil
+     from math import trunc
 
-  ``ceil`` is now known in "group1", but not in others.
+  ``trunc`` is now known in "group1", but not in others.
 
   .. doctest:: group1
 
-     >>> ceil(0.8)
-     1.0
+     >>> trunc(1.1)
+     1
 
   .. doctest:: group2
 
-     >>> ceil(0.8)
+     >>> trunc(1.1)
      Traceback (most recent call last):
        ...
-     NameError: name 'ceil' is not defined
+     NameError: name 'trunc' is not defined
 
   Interleaving testcode/testoutput:
 
   .. testcode:: group1
 
-     print ceil(0.8)
+     print(factorial(3))
 
   .. testcode:: group2
 
-     print floor(0.8)
+     print(factorial(4))
 
   .. testoutput:: group1
 
-     1.0
+     6
 
   .. testoutput:: group2
 
-     0.0
+     24

tests/root/literal.inc

@@ -1,7 +1,7 @@
 # Literally included file using Python highlighting
 # -*- coding: utf-8 -*-
 
-foo = u"Including Unicode characters: üöä"
+foo = "Including Unicode characters: üöä"
 
 class Foo:
     pass

tests/root/objects.txt

@@ -41,6 +41,10 @@ Testing object descriptions
 
 .. function:: func_without_module2() -> annotation
 
+.. object:: long(parameter, \
+              list)
+	    another one
+
 .. class:: TimeInt
 
    :param moo: |test|
@@ -62,6 +66,8 @@ Testing object descriptions
    :ivar int hour: like *hour*
    :ivar minute: like *minute*
    :vartype minute: int
+   :param hour: Duplicate param.  Should not lead to crashes.
+   :type hour: Duplicate type.
 
 
 C items

tests/run.py

@@ -11,7 +11,17 @@
 """
 
 import sys
-from os import path
+from os import path, chdir, listdir
+
+if sys.version_info >= (3, 0):
+    print('Copying and converting sources to build/lib/tests...')
+    from distutils.util import copydir_run_2to3
+    testroot = path.dirname(__file__) or '.'
+    newroot = path.join(testroot, path.pardir, 'build')
+    newroot = path.join(newroot, listdir(newroot)[0], 'tests')
+    copydir_run_2to3(testroot, newroot)
+    # switch to the converted dir so nose tests the right tests
+    chdir(newroot)
 
 # always test the sphinx package from this directory
 sys.path.insert(0, path.join(path.dirname(__file__), path.pardir))
@@ -19,8 +29,8 @@ sys.path.insert(0, path.join(path.dirname(__file__), path.pardir))
 try:
     import nose
 except ImportError:
-    print "The nose package is needed to run the Sphinx test suite."
+    print("The nose package is needed to run the Sphinx test suite.")
     sys.exit(1)
 
-print "Running Sphinx test suite..."
+print("Running Sphinx test suite...")
 nose.main()

tests/test_application.py

@@ -45,9 +45,11 @@ def test_output():
     app = TestApp(status=status, warning=warnings)
     try:
         status.truncate(0) # __init__ writes to status
+        status.seek(0)
         app.info("Nothing here...")
         assert status.getvalue() == "Nothing here...\n"
         status.truncate(0)
+        status.seek(0)
         app.info("Nothing here...", True)
         assert status.getvalue() == "Nothing here..."
 

tests/test_autosummary.py

@@ -9,8 +9,6 @@
     :license: BSD, see LICENSE for details.
 """
 
-import string
-
 from util import *
 
 from sphinx.ext.autosummary import mangle_signature
@@ -27,7 +25,7 @@ def test_mangle_signature():
     (a, b, c='foobar()', d=123) :: (a, b[, c, d])
     """
 
-    TEST = [map(string.strip, x.split("::")) for x in TEST.split("\n")
+    TEST = [map(lambda x: x.strip(), x.split("::")) for x in TEST.split("\n")
             if '::' in x]
     for inp, outp in TEST:
         res = mangle_signature(inp).strip().replace(u"\u00a0", " ")

tests/test_build_html.py

@@ -12,6 +12,7 @@
 import os
 import re
 import htmlentitydefs
+import sys
 from StringIO import StringIO
 
 try:
@@ -38,7 +39,7 @@ http://www.python.org/logo.png
 reading included file u'wrongenc.inc' seems to be wrong, try giving an \
 :encoding: option\\n?
 %(root)s/includes.txt:4: WARNING: download file not readable: nonexisting.png
-%(root)s/objects.txt:84: WARNING: using old C markup; please migrate to \
+%(root)s/objects.txt:\\d*: WARNING: using old C markup; please migrate to \
 new-style markup \(e.g. c:function instead of cfunction\), see \
 http://sphinx.pocoo.org/domains.html
 """
@@ -50,6 +51,11 @@ HTML_WARNINGS = ENV_WARNINGS + """\
 %(root)s/markup.txt:: WARNING: invalid pair index entry u'keyword; '
 """
 
+if sys.version_info >= (3, 0):
+    ENV_WARNINGS = remove_unicode_literals(ENV_WARNINGS)
+    HTML_WARNINGS = remove_unicode_literals(HTML_WARNINGS)
+
+
 def tail_check(check):
     rex = re.compile(check)
     def checker(nodes):
@@ -161,6 +167,8 @@ HTML_XPATH = {
     'objects.html': [
         (".//dt[@id='mod.Cls.meth1']", ''),
         (".//dt[@id='errmod.Error']", ''),
+        (".//dt/tt", r'long\(parameter,\s* list\)'),
+        (".//dt/tt", 'another one'),
         (".//a[@href='#mod.Cls'][@class='reference internal']", ''),
         (".//dl[@class='userdesc']", ''),
         (".//dt[@id='userdesc-myobj']", ''),
@@ -227,7 +235,7 @@ if pygments:
         (".//div[@class='inc-lines highlight-text']//pre",
             r'^class Foo:\n    pass\nclass Bar:\n$'),
         (".//div[@class='inc-startend highlight-text']//pre",
-            ur'^foo = u"Including Unicode characters: üöä"\n$'),
+            ur'^foo = "Including Unicode characters: üöä"\n$'),
         (".//div[@class='inc-preappend highlight-text']//pre",
             r'(?m)^START CODE$'),
         (".//div[@class='inc-pyobj-dedent highlight-python']//span",

tests/test_build_latex.py

@@ -32,6 +32,9 @@ None:None: WARNING: no matching candidate for image URI u'foo.\\*'
 WARNING: invalid pair index entry u''
 """
 
+if sys.version_info >= (3, 0):
+    LATEX_WARNINGS = remove_unicode_literals(LATEX_WARNINGS)
+
 
 @with_app(buildername='latex', warning=latex_warnfile, cleanenv=True)
 def test_latex(app):

tests/test_config.py

@@ -9,6 +9,7 @@
     :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
+import sys
 
 from util import *
 
@@ -84,11 +85,22 @@ def test_extension_values(app):
 @with_tempdir
 def test_errors_warnings(dir):
     # test the error for syntax errors in the config file
-    write_file(dir / 'conf.py', 'project = \n')
+    write_file(dir / 'conf.py', u'project = \n', 'ascii')
     raises_msg(ConfigError, 'conf.py', Config, dir, 'conf.py', {}, None)
 
+    # test the automatic conversion of 2.x only code in configs
+    write_file(dir / 'conf.py', u'\n\nproject = u"Jägermeister"\n', 'utf-8')
+    cfg = Config(dir, 'conf.py', {}, None)
+    cfg.init_values()
+    assert cfg.project == u'Jägermeister'
+
     # test the warning for bytestrings with non-ascii content
-    write_file(dir / 'conf.py', '# -*- coding: latin-1\nproject = "foo\xe4"\n')
+    # bytestrings with non-ascii content are a syntax error in python3 so we
+    # skip the test there
+    if sys.version_info >= (3, 0):
+        return
+    write_file(dir / 'conf.py', u'# -*- coding: latin-1\nproject = "fooä"\n',
+            'latin-1')
     cfg = Config(dir, 'conf.py', {}, None)
     warned = [False]
     def warn(msg):

tests/test_coverage.py

@@ -33,7 +33,7 @@ def test_build(app):
     assert 'api.h' in c_undoc
     assert ' * Py_SphinxTest' in c_undoc
 
-    undoc_py, undoc_c = pickle.loads((app.outdir / 'undoc.pickle').text())
+    undoc_py, undoc_c = pickle.loads((app.outdir / 'undoc.pickle').bytes())
     assert len(undoc_c) == 1
     # the key is the full path to the header file, which isn't testable
     assert undoc_c.values()[0] == [('function', 'Py_SphinxTest')]

tests/test_env.py

@@ -8,6 +8,7 @@
     :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
+import sys
 
 from util import *
 
@@ -54,8 +55,10 @@ def test_images():
     app._warning.reset()
     htmlbuilder = StandaloneHTMLBuilder(app)
     htmlbuilder.post_process_images(tree)
-    assert "no matching candidate for image URI u'foo.*'" in \
-           app._warning.content[-1]
+    image_uri_message = "no matching candidate for image URI u'foo.*'"
+    if sys.version_info >= (3, 0):
+        image_uri_message = remove_unicode_literals(image_uri_message)
+    assert image_uri_message in app._warning.content[-1]
     assert set(htmlbuilder.images.keys()) == \
         set(['subdir/img.png', 'img.png', 'subdir/simg.png', 'svgimg.svg'])
     assert set(htmlbuilder.images.values()) == \
@@ -64,8 +67,7 @@ def test_images():
     app._warning.reset()
     latexbuilder = LaTeXBuilder(app)
     latexbuilder.post_process_images(tree)
-    assert "no matching candidate for image URI u'foo.*'" in \
-           app._warning.content[-1]
+    assert image_uri_message in app._warning.content[-1]
     assert set(latexbuilder.images.keys()) == \
         set(['subdir/img.png', 'subdir/simg.png', 'img.png', 'img.pdf',
              'svgimg.pdf'])

tests/test_intersphinx.py

@@ -11,7 +11,10 @@
 
 import zlib
 import posixpath
-from cStringIO import StringIO
+try:
+    from io import BytesIO
+except ImportError:
+    from cStringIO import StringIO as BytesIO
 
 from docutils import nodes
 
@@ -28,23 +31,23 @@ inventory_v1 = '''\
 # Version: 1.0
 module mod foo.html
 module.cls class foo.html
-'''
+'''.encode('utf-8')
 
 inventory_v2 = '''\
 # Sphinx inventory version 2
 # Project: foo
 # Version: 2.0
 # The remainder of this file is compressed with zlib.
-''' + zlib.compress('''\
+'''.encode('utf-8') + zlib.compress('''\
 module1 py:module 0 foo.html#module-module1 Long Module desc
 module2 py:module 0 foo.html#module-$ -
 module1.func py:function 1 sub/foo.html#$ -
 CFunc c:function 2 cfunc.html#CFunc -
-''')
+'''.encode('utf-8'))
 
 
 def test_read_inventory_v1():
-    f = StringIO(inventory_v1)
+    f = BytesIO(inventory_v1)
     f.readline()
     invdata = read_inventory_v1(f, '/util', posixpath.join)
     assert invdata['py:module']['module'] == \
@@ -54,12 +57,12 @@ def test_read_inventory_v1():
 
 
 def test_read_inventory_v2():
-    f = StringIO(inventory_v2)
+    f = BytesIO(inventory_v2)
     f.readline()
     invdata1 = read_inventory_v2(f, '/util', posixpath.join)
 
     # try again with a small buffer size to test the chunking algorithm
-    f = StringIO(inventory_v2)
+    f = BytesIO(inventory_v2)
     f.readline()
     invdata2 = read_inventory_v2(f, '/util', posixpath.join, bufsize=5)
 
@@ -94,46 +97,58 @@ def test_missing_reference(tempdir, app):
            ('foo', '2.0', 'http://docs.python.org/foo.html#module-module2', '-')
 
     # create fake nodes and check referencing
-    contnode = nodes.emphasis('foo', 'foo')
-    refnode = addnodes.pending_xref('')
-    refnode['reftarget'] = 'module1.func'
-    refnode['reftype'] = 'func'
-    refnode['refdomain'] = 'py'
 
-    rn = missing_reference(app, app.env, refnode, contnode)
+    def fake_node(domain, type, target, content, **attrs):
+        contnode = nodes.emphasis(content, content)
+        node = addnodes.pending_xref('')
+        node['reftarget'] = target
+        node['reftype'] = type
+        node['refdomain'] = domain
+        node.attributes.update(attrs)
+        node += contnode
+        return node, contnode
+
+    def reference_check(*args, **kwds):
+        node, contnode = fake_node(*args, **kwds)
+        return missing_reference(app, app.env, node, contnode)
+
+    # check resolution when a target is found
+    rn = reference_check('py', 'func', 'module1.func', 'foo')
     assert isinstance(rn, nodes.reference)
     assert rn['refuri'] == 'http://docs.python.org/sub/foo.html#module1.func'
     assert rn['reftitle'] == '(in foo v2.0)'
-    assert rn[0].astext() == 'module1.func'
+    assert rn[0].astext() == 'foo'
 
     # create unresolvable nodes and check None return value
-    refnode['reftype'] = 'foo'
-    assert missing_reference(app, app.env, refnode, contnode) is None
-
-    refnode['reftype'] = 'function'
-    refnode['reftarget'] = 'foo.func'
-    assert missing_reference(app, app.env, refnode, contnode) is None
+    assert reference_check('py', 'foo', 'module1.func', 'foo') is None
+    assert reference_check('py', 'func', 'foo', 'foo') is None
+    assert reference_check('py', 'func', 'foo', 'foo') is None
 
     # check handling of prefixes
 
     # prefix given, target found: prefix is stripped
-    refnode['reftype'] = 'mod'
-    refnode['reftarget'] = 'py3k:module2'
-    rn = missing_reference(app, app.env, refnode, contnode)
+    rn = reference_check('py', 'mod', 'py3k:module2', 'py3k:module2')
     assert rn[0].astext() == 'module2'
 
+    # prefix given, but not in title: nothing stripped
+    rn = reference_check('py', 'mod', 'py3k:module2', 'module2')
+    assert rn[0].astext() == 'module2'
+
+    # prefix given, but explicit: nothing stripped
+    rn = reference_check('py', 'mod', 'py3k:module2', 'py3k:module2',
+                         refexplicit=True)
+    assert rn[0].astext() == 'py3k:module2'
+
     # prefix given, target not found and nonexplicit title: prefix is stripped
-    refnode['reftarget'] = 'py3k:unknown'
-    refnode['refexplicit'] = False
-    contnode[0] = nodes.Text('py3k:unknown')
-    rn = missing_reference(app, app.env, refnode, contnode)
+    node, contnode = fake_node('py', 'mod', 'py3k:unknown', 'py3k:unknown',
+                               refexplicit=False)
+    rn = missing_reference(app, app.env, node, contnode)
     assert rn is None
     assert contnode[0].astext() == 'unknown'
 
     # prefix given, target not found and explicit title: nothing is changed
-    refnode['reftarget'] = 'py3k:unknown'
-    refnode['refexplicit'] = True
-    contnode[0] = nodes.Text('py3k:unknown')
-    rn = missing_reference(app, app.env, refnode, contnode)
+    node, contnode = fake_node('py', 'mod', 'py3k:unknown', 'py3k:unknown',
+                               refexplicit=True)
+    rn = missing_reference(app, app.env, node, contnode)
     assert rn is None
     assert contnode[0].astext() == 'py3k:unknown'

tests/test_markup.py

@@ -17,6 +17,7 @@ from docutils import frontend, utils, nodes
 from docutils.parsers import rst
 
 from sphinx.util import texescape
+from sphinx.util.pycompat import b
 from sphinx.writers.html import HTMLWriter, SmartyPantsHTMLTranslator
 from sphinx.writers.latex import LaTeXWriter, LaTeXTranslator
 
@@ -50,7 +51,7 @@ class ForgivingLaTeXTranslator(LaTeXTranslator, ForgivingTranslator):
 
 
 def verify_re(rst, html_expected, latex_expected):
-    document = utils.new_document('test data', settings)
+    document = utils.new_document(b('test data'), settings)
     document['file'] = 'dummy'
     parser.parse(rst, document)
     for msg in document.traverse(nodes.system_message):

tests/test_quickstart.py

@@ -36,8 +36,13 @@ def mock_raw_input(answers, needanswer=False):
         return ''
     return raw_input
 
+try:
+    real_raw_input = raw_input
+except NameError:
+    real_raw_input = input
+
 def teardown_module():
-    qs.raw_input = __builtin__.raw_input
+    qs.term_input = real_raw_input
     qs.TERM_ENCODING = getattr(sys.stdin, 'encoding', None)
     coloron()
 
@@ -51,7 +56,7 @@ def test_do_prompt():
         'Q5': 'no',
         'Q6': 'foo',
     }
-    qs.raw_input = mock_raw_input(answers)
+    qs.term_input = mock_raw_input(answers)
     try:
         qs.do_prompt(d, 'k1', 'Q1')
     except AssertionError:
@@ -79,13 +84,18 @@ def test_quickstart_defaults(tempdir):
         'Author name': 'Georg Brandl',
         'Project version': '0.1',
     }
-    qs.raw_input = mock_raw_input(answers)
+    qs.term_input = mock_raw_input(answers)
     qs.inner_main([])
 
     conffile = tempdir / 'conf.py'
     assert conffile.isfile()
     ns = {}
-    execfile(conffile, ns)
+    f = open(conffile, 'U')
+    try:
+        code = compile(f.read(), conffile, 'exec')
+    finally:
+        f.close()
+    exec code in ns
     assert ns['extensions'] == []
     assert ns['templates_path'] == ['_templates']
     assert ns['source_suffix'] == '.rst'
@@ -112,8 +122,8 @@ def test_quickstart_all_answers(tempdir):
         'Root path': tempdir,
         'Separate source and build': 'y',
         'Name prefix for templates': '.',
-        'Project name': 'STASI\xe2\x84\xa2',
-        'Author name': 'Wolfgang Sch\xc3\xa4uble & G\'Beckstein',
+        'Project name': u'STASI™'.encode('utf-8'),
+        'Author name': u'Wolfgang Schäuble & G\'Beckstein'.encode('utf-8'),
         'Project version': '2.0',
         'Project release': '2.0.1',
         'Source file suffix': '.txt',
@@ -131,14 +141,19 @@ def test_quickstart_all_answers(tempdir):
         'Create Windows command file': 'no',
         'Do you want to use the epub builder': 'yes',
     }
-    qs.raw_input = mock_raw_input(answers, needanswer=True)
+    qs.term_input = mock_raw_input(answers, needanswer=True)
     qs.TERM_ENCODING = 'utf-8'
     qs.inner_main([])
 
     conffile = tempdir / 'source' / 'conf.py'
     assert conffile.isfile()
     ns = {}
-    execfile(conffile, ns)
+    f = open(conffile, 'U')
+    try:
+        code = compile(f.read(), conffile, 'exec')
+    finally:
+        f.close()
+    exec code in ns
     assert ns['extensions'] == ['sphinx.ext.autodoc', 'sphinx.ext.doctest']
     assert ns['templates_path'] == ['.templates']
     assert ns['source_suffix'] == '.txt'

tests/test_search.py

@@ -13,6 +13,7 @@ from docutils import frontend, utils
 from docutils.parsers import rst
 
 from sphinx.search import IndexBuilder
+from sphinx.util.pycompat import b
 
 
 settings = parser = None
@@ -31,7 +32,7 @@ test that non-comments are indexed: fermion
 '''
 
 def test_wordcollector():
-    doc = utils.new_document('test data', settings)
+    doc = utils.new_document(b('test data'), settings)
     doc['file'] = 'dummy'
     parser.parse(FILE_CONTENTS, doc)
 

tests/util.py

@@ -11,6 +11,8 @@ import sys
 import StringIO
 import tempfile
 import shutil
+import re
+from codecs import open
 
 try:
     from functools import wraps
@@ -31,7 +33,7 @@ __all__ = [
     'raises', 'raises_msg', 'Struct',
     'ListOutput', 'TestApp', 'with_app', 'gen_with_app',
     'path', 'with_tempdir', 'write_file',
-    'sprint',
+    'sprint', 'remove_unicode_literals',
 ]
 
 
@@ -191,11 +193,21 @@ def with_tempdir(func):
     return new_func
 
 
-def write_file(name, contents):
-    f = open(str(name), 'wb')
+def write_file(name, contents, encoding=None):
+    if encoding is None:
+        mode = 'wb'
+        if isinstance(contents, unicode):
+            contents = contents.encode('ascii')
+    else:
+        mode = 'w'
+    f = open(str(name), 'wb', encoding=encoding)
     f.write(contents)
     f.close()
 
 
 def sprint(*args):
     sys.stderr.write(' '.join(map(str, args)) + '\n')
+
+_unicode_literals_re = re.compile(r'u(".*?")|u(\'.*?\')')
+def remove_unicode_literals(s):
+    return _unicode_literals_re.sub(lambda x: x.group(1) or x.group(2), s)