sphinx/tests/test_intl.py

# -*- coding: utf-8 -*-
"""
    test_intl
    ~~~~~~~~~

    Test message patching for internationalization purposes.  Runs the text
    builder in the test root.

    :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
from __future__ import print_function

import os
import re
from subprocess import Popen, PIPE
from xml.etree import ElementTree

from nose.tools import assert_equal
from six import string_types

from util import tempdir, rootdir, path, gen_with_app, SkipTest, \
    assert_re_search, assert_not_re_search, assert_in, assert_not_in, \
    assert_startswith


root = tempdir / 'test-intl'


def gen_with_intl_app(*args, **kw):
    default_kw = {
        'testroot': 'intl',
        'confoverrides': {
            'language': 'xx', 'locale_dirs': ['.'],
            'gettext_compact': False,
        },
    }
    default_kw.update(kw)
    return gen_with_app(*args, **default_kw)


def setup_module():
    if not root.exists():
        (rootdir / 'roots' / 'test-intl').copytree(root)
    # Delete remnants left over after failed build
    # Compile all required catalogs into binary format (*.mo).
    for dirpath, dirs, files in os.walk(root):
        dirpath = path(dirpath)
        for f in [f for f in files if f.endswith('.po')]:
            po = dirpath / f
            mo = root / 'xx' / 'LC_MESSAGES' / (
                os.path.relpath(po[:-3], root) + '.mo')
            if not mo.parent.exists():
                mo.parent.makedirs()
            try:
                p = Popen(['msgfmt', po, '-o', mo],
                          stdout=PIPE, stderr=PIPE)
            except OSError:
                raise SkipTest  # most likely msgfmt was not found
            else:
                stdout, stderr = p.communicate()
                if p.returncode != 0:
                    print(stdout)
                    print(stderr)
                    assert False, \
                        'msgfmt exited with return code %s' % p.returncode
                assert mo.isfile(), 'msgfmt failed'


def elem_gettexts(elem):
    def itertext(self):
        # this function copied from Python-2.7 'ElementTree.itertext'.
        # for compatibility to Python-2.6
        tag = self.tag
        if not isinstance(tag, string_types) and tag is not None:
            return
        if self.text:
            yield self.text
        for e in self:
            for s in itertext(e):
                yield s
            if e.tail:
                yield e.tail
    return [_f for _f in [s.strip() for s in itertext(elem)] if _f]


def elem_getref(elem):
    return elem.attrib.get('refid') or elem.attrib.get('refuri')


def assert_elem(elem, texts=None, refs=None, names=None):
    if texts is not None:
        _texts = elem_gettexts(elem)
        assert _texts == texts
    if refs is not None:
        _refs = [elem_getref(x) for x in elem.findall('reference')]
        assert _refs == refs
    if names is not None:
        _names = elem.attrib.get('names').split()
        assert _names == names


@gen_with_intl_app('text', freshenv=True)
def test_text_builder(app, status, warning):
    app.builder.build_all()

    # --- warnings in translation

    warnings = warning.getvalue().replace(os.sep, '/')
    warning_expr = u'.*/warnings.txt:4: ' \
                   u'WARNING: Inline literal start-string without end-string.\n'
    yield assert_re_search, warning_expr, warnings

    result = (app.outdir / 'warnings.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH REST WARNINGS"
              u"\n***********************\n"
              u"\nLINE OF >>``<<BROKEN LITERAL MARKUP.\n")
    yield assert_equal, result, expect

    # --- simple translation; check title underlines

    result = (app.outdir / 'bom.txt').text(encoding='utf-8')
    expect = (u"\nDatei mit UTF-8"
              u"\n***************\n"  # underline matches new translation
              u"\nThis file has umlauts: äöü.\n")
    yield assert_equal, result, expect

    # --- check translation in subdirs

    result = (app.outdir / 'subdir' / 'contents.txt').text(encoding='utf-8')
    yield assert_startswith, result, u"\nsubdir contents\n***************\n"

    # --- check warnings for inconsistency in number of references

    result = (app.outdir / 'refs_inconsistency.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH REFS INCONSISTENCY"
              u"\n****************************\n"
              u"\n* FOR FOOTNOTE [ref2].\n"
              u"\n* reference FOR reference.\n"
              u"\n* ORPHAN REFERENCE: I18N WITH REFS INCONSISTENCY.\n"
              u"\n[1] THIS IS A AUTO NUMBERED FOOTNOTE.\n"
              u"\n[ref2] THIS IS A NAMED FOOTNOTE.\n"
              u"\n[100] THIS IS A NUMBERED FOOTNOTE.\n")
    yield assert_equal, result, expect

    warnings = warning.getvalue().replace(os.sep, '/')
    warning_fmt = u'.*/refs_inconsistency.txt:\\d+: ' \
                  u'WARNING: inconsistent %s in translated message\n'
    expected_warning_expr = (
        warning_fmt % 'footnote references' +
        warning_fmt % 'references' +
        warning_fmt % 'references')
    yield assert_re_search, expected_warning_expr, warnings

    # --- check warning for literal block

    result = (app.outdir / 'literalblock.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH LITERAL BLOCK"
              u"\n***********************\n"
              u"\nCORRECT LITERAL BLOCK:\n"
              u"\n   this is"
              u"\n   literal block\n"
              u"\nMISSING LITERAL BLOCK:\n"
              u"\n<SYSTEM MESSAGE:")
    yield assert_startswith, result, expect

    warnings = warning.getvalue().replace(os.sep, '/')
    expected_warning_expr = u'.*/literalblock.txt:\\d+: ' \
                            u'WARNING: Literal block expected; none found.'
    yield assert_re_search, expected_warning_expr, warnings

    # --- definition terms: regression test for #975

    result = (app.outdir / 'definition_terms.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH DEFINITION TERMS"
              u"\n**************************\n"
              u"\nSOME TERM"
              u"\n   THE CORRESPONDING DEFINITION\n"
              u"\nSOME OTHER TERM"
              u"\n   THE CORRESPONDING DEFINITION #2\n")
    yield assert_equal, result, expect

    # --- glossary terms: regression test for #1090

    result = (app.outdir / 'glossary_terms.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH GLOSSARY TERMS"
              u"\n************************\n"
              u"\nSOME NEW TERM"
              u"\n   THE CORRESPONDING GLOSSARY\n"
              u"\nSOME OTHER NEW TERM"
              u"\n   THE CORRESPONDING GLOSSARY #2\n"
              u"\nLINK TO *SOME NEW TERM*.\n")
    yield assert_equal, result, expect
    warnings = warning.getvalue().replace(os.sep, '/')
    yield assert_not_in, 'term not in glossary', warnings

    # --- glossary term inconsistencies: regression test for #1090

    result = (app.outdir / 'glossary_terms_inconsistency.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH GLOSSARY TERMS INCONSISTENCY"
              u"\n**************************************\n"
              u"\n1. LINK TO *SOME NEW TERM*.\n")
    yield assert_equal, result, expect

    warnings = warning.getvalue().replace(os.sep, '/')
    expected_warning_expr = (
        u'.*/glossary_terms_inconsistency.txt:\\d+: '
        u'WARNING: inconsistent term references in translated message\n')
    yield assert_re_search, expected_warning_expr, warnings

    # --- seealso

    result = (app.outdir / 'seealso.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH SEEALSO"
              u"\n*****************\n"
              u"\nSee also: SHORT TEXT 1\n"
              u"\nSee also: LONG TEXT 1\n"
              u"\nSee also: SHORT TEXT 2\n"
              u"\n  LONG TEXT 2\n")
    yield assert_equal, result, expect

    # --- figure captions: regression test for #940

    result = (app.outdir / 'figure_caption.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH FIGURE CAPTION"
              u"\n************************\n"
              u"\n   [image]MY CAPTION OF THE FIGURE\n"
              u"\n   MY DESCRIPTION PARAGRAPH1 OF THE FIGURE.\n"
              u"\n   MY DESCRIPTION PARAGRAPH2 OF THE FIGURE.\n"
              u"\n"
              u"\nFIGURE IN THE BLOCK"
              u"\n===================\n"
              u"\nBLOCK\n"
              u"\n      [image]MY CAPTION OF THE FIGURE\n"
              u"\n      MY DESCRIPTION PARAGRAPH1 OF THE FIGURE.\n"
              u"\n      MY DESCRIPTION PARAGRAPH2 OF THE FIGURE.\n")
    yield assert_equal, result, expect

    # --- rubric: regression test for pull request #190

    result = (app.outdir / 'rubric.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH RUBRIC"
              u"\n****************\n"
              u"\n-[ RUBRIC TITLE ]-\n"
              u"\n"
              u"\nRUBRIC IN THE BLOCK"
              u"\n===================\n"
              u"\nBLOCK\n"
              u"\n   -[ RUBRIC TITLE ]-\n")
    yield assert_equal, result, expect

    # --- docfields

    result = (app.outdir / 'docfields.txt').text(encoding='utf-8')
    expect = (u"\nI18N WITH DOCFIELDS"
              u"\n*******************\n"
              u"\nclass class Cls1\n"
              u"\n   Parameters:"
              u"\n      **param** -- DESCRIPTION OF PARAMETER param\n"
              u"\nclass class Cls2\n"
              u"\n   Parameters:"
              u"\n      * **foo** -- DESCRIPTION OF PARAMETER foo\n"
              u"\n      * **bar** -- DESCRIPTION OF PARAMETER bar\n"
              u"\nclass class Cls3(values)\n"
              u"\n   Raises ValueError:"
              u"\n      IF THE VALUES ARE OUT OF RANGE\n"
              u"\nclass class Cls4(values)\n"
              u"\n   Raises:"
              u"\n      * **TypeError** -- IF THE VALUES ARE NOT VALID\n"
              u"\n      * **ValueError** -- IF THE VALUES ARE OUT OF RANGE\n"
              u"\nclass class Cls5\n"
              u"\n   Returns:"
              u'\n      A NEW "Cls3" INSTANCE\n')
    yield assert_equal, result, expect

    # --- admonitions
    # #1206: gettext did not translate admonition directive's title
    # seealso: http://docutils.sourceforge.net/docs/ref/rst/directives.html#admonitions

    result = (app.outdir / 'admonitions.txt').text(encoding='utf-8')
    directives = (
        "attention", "caution", "danger", "error", "hint",
        "important", "note", "tip", "warning", "admonition")
    for d in directives:
        yield assert_in, d.upper() + " TITLE", result
        yield assert_in, d.upper() + " BODY", result


@gen_with_intl_app('html', freshenv=True)
def test_html_builder(app, status, warning):
    app.builder.build_all()

    # --- test for #955 cant-build-html-with-footnotes-when-using

    # expect no error by build
    (app.outdir / 'footnote.html').text(encoding='utf-8')

    # --- links to undefined reference

    result = (app.outdir / 'refs_inconsistency.html').text(encoding='utf-8')

    expected_expr = ('<a class="reference external" '
                     'href="http://www.example.com">reference</a>')
    yield assert_equal, len(re.findall(expected_expr, result)), 2

    expected_expr = ('<a class="reference internal" '
                     'href="#reference">reference</a>')
    yield assert_equal, len(re.findall(expected_expr, result)), 0

    expected_expr = ('<a class="reference internal" '
                     'href="#i18n-with-refs-inconsistency">I18N WITH '
                     'REFS INCONSISTENCY</a>')
    yield assert_equal, len(re.findall(expected_expr, result)), 1

    # --- index entries: regression test for #976

    result = (app.outdir / 'genindex.html').text(encoding='utf-8')

    def wrap(tag, keyword):
        start_tag = "<%s[^>]*>" % tag
        end_tag = "</%s>" % tag
        return r"%s\s*%s\s*%s" % (start_tag, keyword, end_tag)

    expected_exprs = [
        wrap('a', 'NEWSLETTER'),
        wrap('a', 'MAILING LIST'),
        wrap('a', 'RECIPIENTS LIST'),
        wrap('a', 'FIRST SECOND'),
        wrap('a', 'SECOND THIRD'),
        wrap('a', 'THIRD, FIRST'),
        wrap('dt', 'ENTRY'),
        wrap('dt', 'SEE'),
        wrap('a', 'MODULE'),
        wrap('a', 'KEYWORD'),
        wrap('a', 'OPERATOR'),
        wrap('a', 'OBJECT'),
        wrap('a', 'EXCEPTION'),
        wrap('a', 'STATEMENT'),
        wrap('a', 'BUILTIN'),
    ]
    for expr in expected_exprs:
        yield assert_re_search, expr, result, re.M

    # --- versionchanges

    result = (app.outdir / 'versionchange.html').text(encoding='utf-8')

    def get_content(result, name):
        matched = re.search(r'<div class="%s">\n*(.*?)</div>' % name,
                            result, re.DOTALL)
        if matched:
            return matched.group(1)
        else:
            return ''

    expect1 = (
        u"""<p><span class="versionmodified">Deprecated since version 1.0: </span>"""
        u"""THIS IS THE <em>FIRST</em> PARAGRAPH OF DEPRECATED.</p>\n"""
        u"""<p>THIS IS THE <em>SECOND</em> PARAGRAPH OF DEPRECATED.</p>\n""")
    matched_content = get_content(result, "deprecated")
    yield assert_equal, expect1, matched_content

    expect2 = (
        u"""<p><span class="versionmodified">New in version 1.0: </span>"""
        u"""THIS IS THE <em>FIRST</em> PARAGRAPH OF VERSIONADDED.</p>\n""")
    matched_content = get_content(result, "versionadded")
    yield assert_equal, expect2, matched_content

    expect3 = (
        u"""<p><span class="versionmodified">Changed in version 1.0: </span>"""
        u"""THIS IS THE <em>FIRST</em> PARAGRAPH OF VERSIONCHANGED.</p>\n""")
    matched_content = get_content(result, "versionchanged")
    yield assert_equal, expect3, matched_content

    # --- docfields

    # expect no error by build
    (app.outdir / 'docfields.html').text(encoding='utf-8')

    # --- gettext template

    result = (app.outdir / 'index.html').text(encoding='utf-8')
    yield assert_in, "WELCOME", result
    yield assert_in, "SPHINX 2013.120", result

    # --- rebuild by .mo mtime

    app.builder.build_update()
    _, count, _ = app.env.update(app.config, app.srcdir, app.doctreedir, app)
    yield assert_equal, count, 0

    (app.srcdir / 'xx' / 'LC_MESSAGES' / 'bom.mo').utime(None)
    _, count, _ = app.env.update(app.config, app.srcdir, app.doctreedir, app)
    yield assert_equal, count, 1


@gen_with_intl_app('xml', freshenv=True)
def test_xml_builder(app, status, warning):
    app.builder.build_all()

    # --- footnotes: regression test for fix #955, #1176

    et = ElementTree.parse(app.outdir / 'footnote.xml')
    secs = et.findall('section')

    para0 = secs[0].findall('paragraph')
    yield (assert_elem,
           para0[0],
           ['I18N WITH FOOTNOTE', 'INCLUDE THIS CONTENTS',
            '2', '[ref]', '1', '100', '.'],
           ['i18n-with-footnote', 'ref'])

    footnote0 = secs[0].findall('footnote')
    yield (assert_elem,
           footnote0[0],
           ['1', 'THIS IS A AUTO NUMBERED FOOTNOTE.'],
           None,
           ['1'])
    yield (assert_elem,
           footnote0[1],
           ['100', 'THIS IS A NUMBERED FOOTNOTE.'],
           None,
           ['100'])
    yield (assert_elem,
           footnote0[2],
           ['2', 'THIS IS A AUTO NUMBERED NAMED FOOTNOTE.'],
           None,
           ['named'])

    citation0 = secs[0].findall('citation')
    yield (assert_elem,
           citation0[0],
           ['ref', 'THIS IS A NAMED FOOTNOTE.'],
           None,
           ['ref'])

    warnings = warning.getvalue().replace(os.sep, '/')
    warning_expr = u'.*/footnote.xml:\\d*: SEVERE: Duplicate ID: ".*".\n'
    yield assert_not_re_search, warning_expr, warnings

    # --- footnote backlinks: i18n test for #1058

    et = ElementTree.parse(app.outdir / 'footnote.xml')
    secs = et.findall('section')

    para0 = secs[0].findall('paragraph')
    refs0 = para0[0].findall('footnote_reference')
    refid2id = dict([
        (r.attrib.get('refid'), r.attrib.get('ids')) for r in refs0])

    footnote0 = secs[0].findall('footnote')
    for footnote in footnote0:
        ids = footnote.attrib.get('ids')
        backrefs = footnote.attrib.get('backrefs')
        yield assert_equal, refid2id[ids], backrefs

    # --- refs in the Python domain

    et = ElementTree.parse(app.outdir / 'refs_python_domain.xml')
    secs = et.findall('section')

    # regression test for fix #1363
    para0 = secs[0].findall('paragraph')
    yield (assert_elem,
           para0[0],
           ['SEE THIS DECORATOR:', 'sensitive_variables()', '.'],
           ['sensitive.sensitive_variables'])

    # --- keep external links: regression test for #1044

    et = ElementTree.parse(app.outdir / 'external_links.xml')
    secs = et.findall('section')

    para0 = secs[0].findall('paragraph')
    # external link check
    yield (assert_elem,
           para0[0],
           ['EXTERNAL LINK TO', 'Python', '.'],
           ['http://python.org/index.html'])

    # internal link check
    yield (assert_elem,
           para0[1],
           ['EXTERNAL LINKS', 'IS INTERNAL LINK.'],
           ['i18n-with-external-links'])

    # inline link check
    yield (assert_elem,
           para0[2],
           ['INLINE LINK BY', 'THE SPHINX SITE', '.'],
           ['http://sphinx-doc.org'])

    # unnamed link check
    yield (assert_elem,
           para0[3],
           ['UNNAMED', 'LINK', '.'],
           ['http://google.com'])

    # link target swapped translation
    para1 = secs[1].findall('paragraph')
    yield (assert_elem,
           para1[0],
           ['LINK TO', 'external2', 'AND', 'external1', '.'],
           ['http://example.com/external2',
            'http://example.com/external1'])
    yield (assert_elem,
           para1[1],
           ['LINK TO', 'THE PYTHON SITE', 'AND', 'THE SPHINX SITE', '.'],
           ['http://python.org', 'http://sphinx-doc.org'])

    # multiple references in the same line
    para2 = secs[2].findall('paragraph')
    yield (assert_elem,
           para2[0],
           ['LINK TO', 'EXTERNAL LINKS', ',', 'Python', ',',
            'THE SPHINX SITE', ',', 'UNNAMED', 'AND',
            'THE PYTHON SITE', '.'],
           ['i18n-with-external-links', 'http://python.org/index.html',
            'http://sphinx-doc.org', 'http://google.com',
            'http://python.org'])

    # --- role xref: regression test for #1090, #1193

    et = ElementTree.parse(app.outdir / 'role_xref.xml')
    sec1, sec2 = et.findall('section')

    para1, = sec1.findall('paragraph')
    yield (assert_elem,
           para1,
           ['LINK TO', "I18N ROCK'N ROLE XREF", ',', 'CONTENTS', ',',
            'SOME NEW TERM', '.'],
           ['i18n-role-xref', 'contents',
            'glossary_terms#term-some-term'])

    para2 = sec2.findall('paragraph')
    yield (assert_elem,
           para2[0],
           ['LINK TO', 'SOME OTHER NEW TERM', 'AND', 'SOME NEW TERM', '.'],
           ['glossary_terms#term-some-other-term',
            'glossary_terms#term-some-term'])
    yield(assert_elem,
          para2[1],
          ['LINK TO', 'SAME TYPE LINKS', 'AND',
           "I18N ROCK'N ROLE XREF", '.'],
          ['same-type-links', 'i18n-role-xref'])
    yield (assert_elem,
           para2[2],
           ['LINK TO', 'I18N WITH GLOSSARY TERMS', 'AND', 'CONTENTS', '.'],
           ['glossary_terms', 'contents'])
    yield (assert_elem,
           para2[3],
           ['LINK TO', '--module', 'AND', '-m', '.'],
           ['cmdoption--module', 'cmdoption-m'])
    yield (assert_elem,
           para2[4],
           ['LINK TO', 'env2', 'AND', 'env1', '.'],
           ['envvar-env2', 'envvar-env1'])
    yield (assert_elem,
           para2[5],
           ['LINK TO', 'token2', 'AND', 'token1', '.'],
           [])  # TODO: how do I link token role to productionlist?
    yield (assert_elem,
           para2[6],
           ['LINK TO', 'same-type-links', 'AND', "i18n-role-xref", '.'],
           ['same-type-links', 'i18n-role-xref'])

    # warnings
    warnings = warning.getvalue().replace(os.sep, '/')
    yield assert_not_in, 'term not in glossary', warnings
    yield assert_not_in, 'undefined label', warnings
    yield assert_not_in, 'unknown document', warnings

    # --- label targets: regression test for #1193, #1265

    et = ElementTree.parse(app.outdir / 'label_target.xml')
    secs = et.findall('section')

    para0 = secs[0].findall('paragraph')
    yield (assert_elem,
           para0[0],
           ['X SECTION AND LABEL', 'POINT TO', 'implicit-target', 'AND',
            'X SECTION AND LABEL', 'POINT TO', 'section-and-label', '.'],
           ['implicit-target', 'section-and-label'])

    para1 = secs[1].findall('paragraph')
    yield (assert_elem,
           para1[0],
           ['X EXPLICIT-TARGET', 'POINT TO', 'explicit-target', 'AND',
            'X EXPLICIT-TARGET', 'POINT TO DUPLICATED ID LIKE', 'id1',
            '.'],
           ['explicit-target', 'id1'])

    para2 = secs[2].findall('paragraph')
    yield (assert_elem,
           para2[0],
           ['X IMPLICIT SECTION NAME', 'POINT TO',
            'implicit-section-name', '.'],
           ['implicit-section-name'])

    sec2 = secs[2].findall('section')

    para2_0 = sec2[0].findall('paragraph')
    yield (assert_elem,
           para2_0[0],
           ['`X DUPLICATED SUB SECTION`_', 'IS BROKEN LINK.'],
           [])

    para3 = secs[3].findall('paragraph')
    yield (assert_elem,
           para3[0],
           ['X', 'bridge label',
            'IS NOT TRANSLATABLE BUT LINKED TO TRANSLATED ' +
            'SECTION TITLE.'],
           ['label-bridged-target-section'])
    yield (assert_elem,
           para3[1],
           ['X', 'bridge label', 'POINT TO',
            'LABEL BRIDGED TARGET SECTION', 'AND', 'bridge label2',
            'POINT TO', 'SECTION AND LABEL', '. THE SECOND APPEARED',
            'bridge label2', 'POINT TO CORRECT TARGET.'],
           ['label-bridged-target-section',
            'section-and-label',
            'section-and-label'])