Add more internationalisation tests (#12277)

- Add test for basic literal block & literalinclude translation
- Add gettext builder test for litteral blocks additional targets
- Add i18n test for xref roles in titles
- Add i18n tests for strange markup

tests/roots/test-intl/index.txt

@@ -31,6 +31,7 @@ CONTENTS
    section
    translation_progress
    topic
+   markup
 
 .. toctree::
    :maxdepth: 2

tests/roots/test-intl/markup.txt

@@ -0,0 +1,6 @@
+i18n with strange markup
+========================
+
+1. title starting with 1.
+-------------------------
+

tests/roots/test-intl/role_xref.txt

@@ -7,6 +7,9 @@ i18n role xref
 
 link to :term:`Some term`, :ref:`i18n-role-xref`, :doc:`index`.
 
+link to :term:`Some term`, :ref:`i18n-role-xref`, :doc:`index`.
+---------------------------------------------------------------
+
 .. _same-type-links:
 
 same type links

tests/roots/test-intl/xx/LC_MESSAGES/literalblock.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-11-22 08:28+0000\n"
+"POT-Creation-Date: 2024-04-14 15:05+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -22,6 +22,11 @@ msgstr "I18N WITH LITERAL BLOCK"
 msgid "Correct literal block::"
 msgstr "CORRECT LITERAL BLOCK::"
 
+msgid ""
+"this is\n"
+"literal block"
+msgstr "THIS IS\nLITERAL BLOCK"
+
 msgid "Missing literal block::"
 msgstr "MISSING LITERAL BLOCK::"
 
@@ -31,6 +36,25 @@ msgstr "THAT'S ALL."
 msgid "included raw.txt"
 msgstr "INCLUDED RAW.TXT"
 
+msgid ""
+"===\n"
+"Raw\n"
+"===\n"
+"\n"
+".. raw:: html\n"
+"\n"
+"   <iframe src=\"https://sphinx-doc.org\"></iframe>\n"
+"\n"
+msgstr ""
+"===\n"
+"RAW\n"
+"===\n"
+"\n"
+".. raw:: html\n"
+"\n"
+"   <iframe src=\"HTTPS://SPHINX-DOC.ORG\"></iframe>\n"
+"\n"
+
 msgid "code blocks"
 msgstr "CODE-BLOCKS"
 
@@ -43,9 +67,6 @@ msgstr ""
 "   'RESULT'\n"
 "end"
 
-msgid "example of C language"
-msgstr "EXAMPLE OF C LANGUAGE"
-
 msgid ""
 "#include <stdlib.h>\n"
 "int main(int argc, char** argv)\n"
@@ -59,6 +80,9 @@ msgstr ""
 "    return 0;\n"
 "}"
 
+msgid "example of C language"
+msgstr "EXAMPLE OF C LANGUAGE"
+
 msgid ""
 "#include <stdio.h>\n"
 "int main(int argc, char** argv)\n"

tests/roots/test-intl/xx/LC_MESSAGES/markup.po

@@ -0,0 +1,25 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2018, dev
+# This file is distributed under the same license as the sphinx package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2018.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: sphinx 1.0\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2018-05-06 16:44+0900\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.4.0\n"
+
+msgid "i18n with strange markup"
+msgstr "I18N WITH STRANGE MARKUP"
+
+msgid "1. title starting with 1."
+msgstr "1. TITLE STARTING WITH 1."
+

tests/test_builders/test_build_gettext.py

@@ -16,13 +16,12 @@ if sys.version_info[:2] >= (3, 11):
 else:
     from sphinx.util.osutil import _chdir as chdir
 
-_MSGID_PATTERN = re.compile(r'msgid "(.*)"')
+_MSGID_PATTERN = re.compile(r'msgid "((?:\n|.)*?)"\nmsgstr', re.MULTILINE)
 
 
-def msgid_getter(msgid):
+def get_msgids(pot):
-    if m := _MSGID_PATTERN.search(msgid):
+    matches = _MSGID_PATTERN.findall(pot)
-        return m[1]
+    return [m.replace('"\n"', '') for m in matches[1:]]
-    return None
 
 
 def test_Catalog_duplicated_message():
@@ -105,7 +104,7 @@ def test_gettext_index_entries(app):
     app.build(filenames=[app.srcdir / 'index_entries.txt'])
 
     pot = (app.outdir / 'index_entries.pot').read_text(encoding='utf8')
-    msg_ids = list(filter(None, map(msgid_getter, pot.splitlines())))
+    msg_ids = get_msgids(pot)
 
     assert msg_ids == [
         "i18n with index entries",
@@ -134,7 +133,7 @@ def test_gettext_disable_index_entries(app):
     app.build(filenames=[app.srcdir / 'index_entries.txt'])
 
     pot = (app.outdir / 'index_entries.pot').read_text(encoding='utf8')
-    msg_ids = list(filter(None, map(msgid_getter, pot.splitlines())))
+    msg_ids = get_msgids(pot)
 
     assert msg_ids == [
         "i18n with index entries",
@@ -200,7 +199,7 @@ def test_gettext_prolog_epilog_substitution(app):
 
     assert (app.outdir / 'prolog_epilog_substitution.pot').is_file()
     pot = (app.outdir / 'prolog_epilog_substitution.pot').read_text(encoding='utf8')
-    msg_ids = list(filter(None, map(msgid_getter, pot.splitlines())))
+    msg_ids = get_msgids(pot)
 
     assert msg_ids == [
         "i18n with prologue and epilogue substitutions",
@@ -227,9 +226,43 @@ def test_gettext_prolog_epilog_substitution_excluded(app):
 
     assert (app.outdir / 'prolog_epilog_substitution_excluded.pot').is_file()
     pot = (app.outdir / 'prolog_epilog_substitution_excluded.pot').read_text(encoding='utf8')
-    msg_ids = list(filter(None, map(msgid_getter, pot.splitlines())))
+    msg_ids = get_msgids(pot)
 
     assert msg_ids == [
         "i18n without prologue and epilogue substitutions",
         "This is content that does not include prologue and epilogue substitutions.",
     ]
+
+
+@pytest.mark.sphinx(
+    'gettext', srcdir='gettext',
+    confoverrides={'gettext_compact': False,
+                   'gettext_additional_targets': ['literal-block', 'doctest-block']})
+def test_gettext_literalblock_additional(app):
+    app.build(force_all=True)
+
+    assert (app.outdir / 'literalblock.pot').is_file()
+    pot = (app.outdir / 'literalblock.pot').read_text(encoding='utf8')
+    msg_ids = get_msgids(pot)
+
+    assert msg_ids == [
+        'i18n with literal block',
+        'Correct literal block::',
+        'this is\\nliteral block',
+        'Missing literal block::',
+        "That's all.",
+        'included raw.txt',
+        '===\\nRaw\\n===\\n\\n.. raw:: html\\n\\n   <iframe src=\\"https://sphinx-doc.org\\"></iframe>\\n\\n',
+        'code blocks',
+        "def main\\n   'result'\\nend",
+        '#include <stdlib.h>\\nint main(int argc, char** argv)\\n{\\n    return 0;\\n}',
+        'example of C language',
+        '#include <stdio.h>\\nint main(int argc, char** argv)\\n{\\n    return 0;\\n}',
+        'literal-block\\nin list',
+        'test_code_for_noqa()\\ncontinued()',
+        'doctest blocks',
+        '>>> import sys  # sys importing\\n>>> def main():  # define main '
+        "function\\n...     sys.stdout.write('hello')  # call write method of "
+        "stdout object\\n>>>\\n>>> if __name__ == '__main__':  # if run this py "
+        'file as python script\\n...     main()  # call main',
+    ]

tests/test_intl/test_intl.py

@@ -1191,6 +1191,15 @@ def test_xml_role_xref(app):
         ['i18n-role-xref', 'index',
          'glossary_terms#term-Some-term'])
 
+    sec1_1, = sec1.findall('section')
+    title, = sec1_1.findall('title')
+    assert_elem(
+        title,
+        ['LINK TO', "I18N ROCK'N ROLE XREF", ',', 'CONTENTS', ',',
+         'SOME NEW TERM', '.'],
+        ['i18n-role-xref', 'index',
+         'glossary_terms#term-Some-term'])
+
     para2 = sec2.findall('paragraph')
     assert_elem(
         para2[0],
@@ -1293,6 +1302,19 @@ def test_xml_label_targets(app):
          'section-and-label'])
 
 
+@sphinx_intl
+@pytest.mark.sphinx('xml')
+@pytest.mark.test_params(shared_result='test_intl_basic')
+def test_xml_strange_markup(app):
+    app.build()
+    et = etree_parse(app.outdir / 'markup.xml')
+    secs = et.findall('section')
+
+    subsec1, = secs[0].findall('section')
+    title1, = subsec1.findall('title')
+    assert_elem(title1, ['1. TITLE STARTING WITH 1.'])
+
+
 @sphinx_intl
 @pytest.mark.sphinx('html')
 @pytest.mark.test_params(shared_result='test_intl_basic')
@@ -1377,6 +1399,15 @@ def test_additional_targets_should_be_translated(app):
     # [literalblock.txt]
     result = (app.outdir / 'literalblock.html').read_text(encoding='utf8')
 
+    # basic literal bloc should be translated
+    expected_expr = ('<span class="n">THIS</span> <span class="n">IS</span>\n'
+                     '<span class="n">LITERAL</span> <span class="n">BLOCK</span>')
+    assert_count(expected_expr, result, 1)
+
+    # literalinclude should be translated
+    expected_expr = '<span class="s2">&quot;HTTPS://SPHINX-DOC.ORG&quot;</span>'
+    assert_count(expected_expr, result, 1)
+
     # title should be translated
     expected_expr = 'CODE-BLOCKS'
     assert_count(expected_expr, result, 2)