Fix #3851 and fix #3706 about box drawing characters for PDF output

Do not tex-escape them, but provide suitable definitions for pdflatex
only (and platex). Define the 4 box drawing characters needed by unix
tree command with charset utf8. Also define the U+2572 as it was
previously escaped for all engines.

No definitions for xelatex/lualatex as it is then only a matter of using
a font providing them (a priori, only the mono font needs definition, as
the characters will be used in literal blocks).

sphinx/texinputs/sphinx.sty

@@ -1478,6 +1478,41 @@
 % figure legend comes after caption and may contain arbitrary body elements
 \newenvironment{sphinxlegend}{\par\small}{\par}
 
+% Declare Unicode characters used by linux tree command to pdflatex utf8/utf8x
+\def\spx@bd#1#2{%
+  \leavevmode
+  \begingroup
+  \ifx\spx@bd@height   \@undefined\def\spx@bd@height{\baselineskip}\fi
+  \ifx\spx@bd@width    \@undefined\setbox0\hbox{0}\def\spx@bd@width{\wd0 }\fi
+  \ifx\spx@bd@thickness\@undefined\def\spx@bd@thickness{.6\p@}\fi
+  \ifx\spx@bd@raise    \@undefined\def\spx@bd@lower{\dp\strutbox}\fi
+  \lower\spx@bd@lower#1{#2}%
+  \endgroup
+}%
+\@namedef{sphinx@u2500}% BOX DRAWINGS LIGHT HORIZONTAL
+  {\spx@bd{\vbox to\spx@bd@height}
+          {\vss\hrule\@height\spx@bd@thickness
+                     \@width\spx@bd@width\vss}}%
+\@namedef{sphinx@u2502}% BOX DRAWINGS LIGHT VERTICAL
+  {\spx@bd{\hb@xt@\spx@bd@width}
+          {\hss\vrule\@height\spx@bd@height
+                     \@width \spx@bd@thickness\hss}}%
+\@namedef{sphinx@u2514}% BOX DRAWINGS LIGHT UP AND RIGHT
+  {\spx@bd{\hb@xt@\spx@bd@width}
+          {\hss\raise.5\spx@bd@height
+           \hb@xt@\z@{\hss\vrule\@height.5\spx@bd@height
+                                \@width \spx@bd@thickness\hss}%
+           \vbox to\spx@bd@height{\vss\hrule\@height\spx@bd@thickness
+                                            \@width.5\spx@bd@width\vss}}}%
+\@namedef{sphinx@u251C}% BOX DRAWINGS LIGHT VERTICAL AND RIGHT
+  {\spx@bd{\hb@xt@\spx@bd@width}
+          {\hss
+           \hb@xt@\z@{\hss\vrule\@height\spx@bd@height
+                                \@width \spx@bd@thickness\hss}%
+           \vbox to\spx@bd@height{\vss\hrule\@height\spx@bd@thickness
+                                            \@width.5\spx@bd@width\vss}}}%
+\protected\def\sphinxunichar#1{\@nameuse{sphinx@u#1}}%
+
 % Tell TeX about pathological hyphenation cases:
 \hyphenation{Base-HTTP-Re-quest-Hand-ler}
 \endinput

sphinx/util/texescape.py

@@ -40,12 +40,9 @@ tex_replacements = [
     # used to separate -- in options
     ('', r'{}'),
     # map some special Unicode characters to similar ASCII ones
-    ('─', r'-'),
     ('⎽', r'\_'),
-    ('╲', r'\textbackslash{}'),
     ('–', r'\textendash{}'),
     ('|', r'\textbar{}'),
-    ('│', r'\textbar{}'),
     ('ℯ', r'e'),
     ('ⅈ', r'i'),
     ('⁰', r'\(\sp{\text{0}}\)'),

sphinx/writers/latex.py

@@ -105,9 +105,22 @@ ADDITIONAL_SETTINGS = {
     'pdflatex': {
         'inputenc':     '\\usepackage[utf8]{inputenc}',
         'utf8extra':   ('\\ifdefined\\DeclareUnicodeCharacter\n'
-                        ' \\ifdefined\\DeclareUnicodeCharacterAsOptional\\else\n'
+                        ' \\ifdefined\\DeclareUnicodeCharacterAsOptional\n'
+                        '  \\DeclareUnicodeCharacter{"00A0}{\\nobreakspace}\n'
+                        '  \\DeclareUnicodeCharacter{"2500}{\\sphinxunichar{2500}}\n'
+                        '  \\DeclareUnicodeCharacter{"2502}{\\sphinxunichar{2502}}\n'
+                        '  \\DeclareUnicodeCharacter{"2514}{\\sphinxunichar{2514}}\n'
+                        '  \\DeclareUnicodeCharacter{"251C}{\\sphinxunichar{251C}}\n'
+                        '  \\DeclareUnicodeCharacter{"2572}{\\textbackslash}\n'
+                        ' \\else\n'
                         '  \\DeclareUnicodeCharacter{00A0}{\\nobreakspace}\n'
-                        '\\fi\\fi'),
+                        '  \\DeclareUnicodeCharacter{2500}{\\sphinxunichar{2500}}\n'
+                        '  \\DeclareUnicodeCharacter{2502}{\\sphinxunichar{2502}}\n'
+                        '  \\DeclareUnicodeCharacter{2514}{\\sphinxunichar{2514}}\n'
+                        '  \\DeclareUnicodeCharacter{251C}{\\sphinxunichar{251C}}\n'
+                        '  \\DeclareUnicodeCharacter{2572}{\\textbackslash}\n'
+                        ' \\fi\n'
+                        '\\fi'),
     },
     'xelatex': {
         'latex_engine': 'xelatex',