Merge pull request #6750 from tk0miya/6738_new_escape_for_unicode_latex_engine

Fix #6738: Do not replace unicode characters by LaTeX macros on unicode supported LaTeX engine

CHANGES

@@ -42,6 +42,8 @@ Bugs fixed
 
   .. _latex3/latex2e#173: https://github.com/latex3/latex2e/issues/173
 * #6618: LaTeX: Avoid section names at the end of a page
+* #6738: LaTeX: Do not replace unicode characters by LaTeX macros on unicode
+  supported LaTeX engines
 * #6704: linkcheck: Be defensive and handle newly defined HTTP error code
 * #6655: image URLs containing ``data:`` causes gettext builder crashed
 * #6584: i18n: Error when compiling message catalogs on Hindi

sphinx/util/template.py

@@ -63,14 +63,14 @@ class SphinxRenderer(FileRenderer):
 
 
 class LaTeXRenderer(SphinxRenderer):
-    def __init__(self, template_path: str = None) -> None:
+    def __init__(self, template_path: str = None, latex_engine: str = None) -> None:
         if template_path is None:
             template_path = os.path.join(package_dir, 'templates', 'latex')
         super().__init__(template_path)
 
         # use texescape as escape filter
-        self.env.filters['e'] = texescape.escape
-        self.env.filters['escape'] = texescape.escape
+        self.env.filters['e'] = texescape.get_escape_func(latex_engine)
+        self.env.filters['escape'] = texescape.get_escape_func(latex_engine)
         self.env.filters['eabbr'] = texescape.escape_abbr
 
         # use JSP/eRuby like tagging instead because curly bracket; the default

sphinx/util/texescape.py

@@ -9,7 +9,7 @@
 """
 
 import re
-from typing import Dict
+from typing import Callable, Dict
 
 tex_replacements = [
     # map TeX special chars
@@ -46,6 +46,14 @@ tex_replacements = [
     ('|', r'\textbar{}'),
     ('ℯ', r'e'),
     ('ⅈ', r'i'),
+    # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc
+    # OHM SIGN U+2126 is handled by LaTeX textcomp package
+]
+
+# A map Unicode characters to LaTeX representation
+# (for LaTeX engines which don't support unicode)
+unicode_tex_replacements = [
+    # superscript
     ('⁰', r'\(\sp{\text{0}}\)'),
     ('¹', r'\(\sp{\text{1}}\)'),
     ('²', r'\(\sp{\text{2}}\)'),
@@ -56,6 +64,7 @@ tex_replacements = [
     ('⁷', r'\(\sp{\text{7}}\)'),
     ('⁸', r'\(\sp{\text{8}}\)'),
     ('⁹', r'\(\sp{\text{9}}\)'),
+    # subscript
     ('₀', r'\(\sb{\text{0}}\)'),
     ('₁', r'\(\sb{\text{1}}\)'),
     ('₂', r'\(\sb{\text{2}}\)'),
@@ -66,20 +75,32 @@ tex_replacements = [
     ('₇', r'\(\sb{\text{7}}\)'),
     ('₈', r'\(\sb{\text{8}}\)'),
     ('₉', r'\(\sb{\text{9}}\)'),
-    # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc
-    # OHM SIGN U+2126 is handled by LaTeX textcomp package
 ]
 
 tex_escape_map = {}  # type: Dict[int, str]
+tex_escape_map_without_unicode = {}  # type: Dict[int, str]
 tex_replace_map = {}
 tex_hl_escape_map_new = {}
 
 
+def get_escape_func(latex_engine: str) -> Callable[[str], str]:
+    """Get escape() function for given latex_engine."""
+    if latex_engine in ('lualatex', 'xelatex'):
+        return escape_for_unicode_latex_engine
+    else:
+        return escape
+
+
 def escape(s: str) -> str:
     """Escape text for LaTeX output."""
     return s.translate(tex_escape_map)
 
 
+def escape_for_unicode_latex_engine(s: str) -> str:
+    """Escape text for unicode supporting LaTeX engine."""
+    return s.translate(tex_escape_map_without_unicode)
+
+
 def escape_abbr(text: str) -> str:
     """Adjust spacing after abbreviations. Works with @ letter or other."""
     return re.sub(r'\.(?=\s|$)', r'.\@{}', text)
@@ -87,6 +108,11 @@ def escape_abbr(text: str) -> str:
 
 def init() -> None:
     for a, b in tex_replacements:
+        tex_escape_map[ord(a)] = b
+        tex_escape_map_without_unicode[ord(a)] = b
+        tex_replace_map[ord(a)] = '_'
+
+    for a, b in unicode_tex_replacements:
         tex_escape_map[ord(a)] = b
         tex_replace_map[ord(a)] = '_'
 

sphinx/writers/latex.py

@@ -32,7 +32,7 @@ from sphinx.util import split_into, logging
 from sphinx.util.docutils import SphinxTranslator
 from sphinx.util.nodes import clean_astext, get_prev_node
 from sphinx.util.template import LaTeXRenderer
-from sphinx.util.texescape import tex_escape_map, tex_replace_map
+from sphinx.util.texescape import get_escape_func, tex_replace_map
 
 try:
     from docutils.utils.roman import toRoman
@@ -500,6 +500,9 @@ class LaTeXTranslator(SphinxTranslator):
         self.compact_list = 0
         self.first_param = 0
 
+        # escape helper
+        self.escape = get_escape_func(self.config.latex_engine)
+
         # sort out some elements
         self.elements = self.builder.context.copy()
 
@@ -758,8 +761,7 @@ class LaTeXTranslator(SphinxTranslator):
             for i, (letter, entries) in enumerate(content):
                 if i > 0:
                     ret.append('\\indexspace\n')
-                ret.append('\\bigletter{%s}\n' %
-                           str(letter).translate(tex_escape_map))
+                ret.append('\\bigletter{%s}\n' % self.escape(letter))
                 for entry in entries:
                     if not entry[3]:
                         continue
@@ -794,13 +796,14 @@ class LaTeXTranslator(SphinxTranslator):
 
     def render(self, template_name, variables):
         # type: (str, Dict) -> str
+        renderer = LaTeXRenderer(latex_engine=self.config.latex_engine)
         for template_dir in self.builder.config.templates_path:
             template = path.join(self.builder.confdir, template_dir,
                                  template_name)
             if path.exists(template):
-                return LaTeXRenderer().render(template, variables)
+                return renderer.render(template, variables)
 
-        return LaTeXRenderer().render(template_name, variables)
+        return renderer.render(template_name, variables)
 
     def visit_document(self, node):
         # type: (nodes.Element) -> None
@@ -914,14 +917,13 @@ class LaTeXTranslator(SphinxTranslator):
                 if not self.elements['title']:
                     # text needs to be escaped since it is inserted into
                     # the output literally
-                    self.elements['title'] = node.astext().translate(tex_escape_map)
+                    self.elements['title'] = self.escape(node.astext())
                 self.this_is_the_title = 0
                 raise nodes.SkipNode
             else:
                 short = ''
                 if node.traverse(nodes.image):
-                    short = ('[%s]' %
-                             ' '.join(clean_astext(node).split()).translate(tex_escape_map))
+                    short = ('[%s]' % self.escape(' '.join(clean_astext(node).split())))
 
                 try:
                     self.body.append(r'\%s%s{' % (self.sectionnames[self.sectionlevel], short))
@@ -1955,8 +1957,7 @@ class LaTeXTranslator(SphinxTranslator):
         else:
             id = node.get('refuri', '')[1:].replace('#', ':')
 
-        title = node.get('title', '%s')
-        title = str(title).translate(tex_escape_map).replace('\\%s', '%s')
+        title = self.escape(node.get('title', '%s')).replace('\\%s', '%s')
         if '\\{name\\}' in title or '\\{number\\}' in title:
             # new style format (cf. "Fig.%{number}")
             title = title.replace('\\{name\\}', '{name}').replace('\\{number\\}', '{number}')
@@ -2404,7 +2405,7 @@ class LaTeXTranslator(SphinxTranslator):
 
     def encode(self, text):
         # type: (str) -> str
-        text = str(text).translate(tex_escape_map)
+        text = self.escape(text)
         if self.literal_whitespace:
             # Insert a blank before the newline, to avoid
             # ! LaTeX Error: There's no line here to end.
@@ -2615,33 +2616,31 @@ class LaTeXTranslator(SphinxTranslator):
         ret = []  # type: List[str]
         figure = self.builder.config.numfig_format['figure'].split('%s', 1)
         if len(figure) == 1:
-            ret.append('\\def\\fnum@figure{%s}\n' %
-                       str(figure[0]).strip().translate(tex_escape_map))
+            ret.append('\\def\\fnum@figure{%s}\n' % self.escape(figure[0]).strip())
         else:
-            definition = escape_abbr(str(figure[0]).translate(tex_escape_map))
+            definition = escape_abbr(self.escape(figure[0]))
             ret.append(self.babel_renewcommand('\\figurename', definition))
             ret.append('\\makeatletter\n')
             ret.append('\\def\\fnum@figure{\\figurename\\thefigure{}%s}\n' %
-                       str(figure[1]).translate(tex_escape_map))
+                       self.escape(figure[1]))
             ret.append('\\makeatother\n')
 
         table = self.builder.config.numfig_format['table'].split('%s', 1)
         if len(table) == 1:
-            ret.append('\\def\\fnum@table{%s}\n' %
-                       str(table[0]).strip().translate(tex_escape_map))
+            ret.append('\\def\\fnum@table{%s}\n' % self.escape(table[0]).strip())
         else:
-            definition = escape_abbr(str(table[0]).translate(tex_escape_map))
+            definition = escape_abbr(self.escape(table[0]))
             ret.append(self.babel_renewcommand('\\tablename', definition))
             ret.append('\\makeatletter\n')
             ret.append('\\def\\fnum@table{\\tablename\\thetable{}%s}\n' %
-                       str(table[1]).translate(tex_escape_map))
+                       self.escape(table[1]))
             ret.append('\\makeatother\n')
 
         codeblock = self.builder.config.numfig_format['code-block'].split('%s', 1)
         if len(codeblock) == 1:
             pass  # FIXME
         else:
-            definition = str(codeblock[0]).strip().translate(tex_escape_map)
+            definition = self.escape(codeblock[0]).strip()
             ret.append(self.babel_renewcommand('\\literalblockname', definition))
             if codeblock[1]:
                 pass  # FIXME

tests/roots/test-latex-unicode/index.rst

@@ -0,0 +1,7 @@
+test-latex-unicode
+==================
+
+* script small e: ℯ
+* double struck italic small i: ⅈ
+* superscript: ⁰, ¹
+* subscript: ₀, ₁

tests/test_build_latex.py

@@ -1439,6 +1439,30 @@ def test_index_on_title(app, status, warning):
             in result)
 
 
+@pytest.mark.sphinx('latex', testroot='latex-unicode',
+                    confoverrides={'latex_engine': 'pdflatex'})
+def test_texescape_for_non_unicode_supported_engine(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'python.tex').text()
+    print(result)
+    assert 'script small e: e' in result
+    assert 'double struck italic small i: i' in result
+    assert r'superscript: \(\sp{\text{0}}\), \(\sp{\text{1}}\)' in result
+    assert r'subscript: \(\sb{\text{0}}\), \(\sb{\text{1}}\)' in result
+
+
+@pytest.mark.sphinx('latex', testroot='latex-unicode',
+                    confoverrides={'latex_engine': 'xelatex'})
+def test_texescape_for_unicode_supported_engine(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'python.tex').text()
+    print(result)
+    assert 'script small e: e' in result
+    assert 'double struck italic small i: i' in result
+    assert 'superscript: ⁰, ¹' in result
+    assert 'subscript: ₀, ₁' in result
+
+    
 @pytest.mark.sphinx('latex', testroot='basic',
                     confoverrides={'latex_elements': {'extrapackages': r'\usepackage{foo}'}})
 def test_latex_elements_extrapackages(app, status, warning):