Merge pull request #10251 from jbms/fix-inline-syntax-highlight

Make code role highlighting consistent with code-block directive

doc/usage/restructuredtext/directives.rst

@@ -499,8 +499,10 @@ __ https://pygments.org/docs/lexers
    The directive's alias name :rst:dir:`sourcecode` works as well.  This
    directive takes a language name as an argument.  It can be `any lexer alias
    supported by Pygments <https://pygments.org/docs/lexers/>`_.  If it is not
-   given, the setting of :rst:dir:`highlight` directive will be used.
-   If not set, :confval:`highlight_language` will be used.
+   given, the setting of :rst:dir:`highlight` directive will be used.  If not
+   set, :confval:`highlight_language` will be used.  To display a code example
+   *inline* within other text, rather than as a separate block, you can use the
+   :rst:role:`code` role instead.
 
    .. versionchanged:: 2.0
       The ``language`` argument becomes optional.

doc/usage/restructuredtext/roles.rst

@@ -276,6 +276,34 @@ The following role creates a cross-reference to a term in a
    If you use a term that's not explained in a glossary, you'll get a warning
    during build.
 
+Inline code highlighting
+------------------------
+
+.. rst:role:: code
+
+   An *inline* code example.  When used directly, this role just displays the
+   text *without* syntax highlighting, as a literal.
+
+   .. code-block:: rst
+
+      By default, inline code such as :code:`1 + 2` just displays without
+      highlighting.
+
+   Unlike the :rst:dir:`code-block` directive, this role does not respect the
+   default language set by the :rst:dir:`highlight` directive.
+
+   To enable syntax highlighting, you must first use the ``role`` directive to
+   define a custom ``code`` role for a particular language:
+
+   .. code-block:: rst
+
+      .. role:: python(code)
+         :language: python
+
+      In Python, :python:`1 + 2` is equal to :python:`3`.
+
+   To display a multi-line code example, use the :rst:dir:`code-block` directive
+   instead.
 
 Math
 ----

sphinx/roles.py

@@ -3,6 +3,9 @@
 import re
 from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type
 
+import docutils.parsers.rst.directives
+import docutils.parsers.rst.roles
+import docutils.parsers.rst.states
 from docutils import nodes, utils
 from docutils.nodes import Element, Node, TextElement, system_message
 
@@ -333,6 +336,57 @@ class Abbreviation(SphinxRole):
         return [nodes.abbreviation(self.rawtext, text, **options)], []
 
 
+# Sphinx provides the `code-block` directive for highlighting code blocks.
+# Docutils provides the `code` role which in theory can be used similarly by
+# defining a custom role for a given programming language:
+#
+#     .. .. role:: python(code)
+#          :language: python
+#          :class: highlight
+#
+# In practice this does not produce correct highlighting because it uses a
+# separate highlighting mechanism that results in the "long" pygments class
+# names rather than "short" pygments class names produced by the Sphinx
+# `code-block` directive and for which this extension contains CSS rules.
+#
+# In addition, even if that issue is fixed, because the highlighting
+# implementation in docutils, despite being based on pygments, differs from that
+# used by Sphinx, the output does not exactly match that produced by the Sphinx
+# `code-block` directive.
+#
+# This issue is noted here: //github.com/sphinx-doc/sphinx/issues/5157
+#
+# This overrides the docutils `code` role to perform highlighting in the same
+# way as the Sphinx `code-block` directive.
+#
+# TODO: Change to use `SphinxRole` once SphinxRole is fixed to support options.
+def code_role(name: str, rawtext: str, text: str, lineno: int,
+              inliner: docutils.parsers.rst.states.Inliner,
+              options: Dict = {}, content: List[str] = []
+              ) -> Tuple[List[Node], List[system_message]]:
+    options = options.copy()
+    docutils.parsers.rst.roles.set_classes(options)
+    language = options.get('language', '')
+    classes = ['code']
+    if language:
+        classes.append('highlight')
+    if 'classes' in options:
+        classes.extend(options['classes'])
+
+    if language and language not in classes:
+        classes.append(language)
+
+    node = nodes.literal(rawtext, text, classes=classes, language=language)
+
+    return [node], []
+
+
+code_role.options = {  # type: ignore
+    'class': docutils.parsers.rst.directives.class_option,
+    'language': docutils.parsers.rst.directives.unchanged,
+}
+
+
 specific_docroles: Dict[str, RoleFunction] = {
     # links to download references
     'download': XRefRole(nodeclass=addnodes.download_reference),
@@ -360,6 +414,10 @@ def setup(app: "Sphinx") -> Dict[str, Any]:
     for rolename, func in specific_docroles.items():
         roles.register_local_role(rolename, func)
 
+    # Since docutils registers it as a canonical role, override it as a
+    # canonical role as well.
+    roles.register_canonical_role('code', code_role)
+
     return {
         'version': 'builtin',
         'parallel_read_safe': True,

sphinx/writers/html.py

@@ -503,10 +503,25 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator):
         if 'kbd' in node['classes']:
             self.body.append(self.starttag(node, 'kbd', '',
                                            CLASS='docutils literal notranslate'))
-        else:
+            return
+        lang = node.get("language", None)
+        if 'code' not in node['classes'] or not lang:
             self.body.append(self.starttag(node, 'code', '',
                                            CLASS='docutils literal notranslate'))
             self.protect_literal_text += 1
+            return
+
+        opts = self.config.highlight_options.get(lang, {})
+        highlighted = self.highlighter.highlight_block(
+            node.astext(), lang, opts=opts, location=node, nowrap=True)
+        starttag = self.starttag(
+            node,
+            "code",
+            suffix="",
+            CLASS="docutils literal highlight highlight-%s" % lang,
+        )
+        self.body.append(starttag + highlighted.strip() + "</code>")
+        raise nodes.SkipNode
 
     def depart_literal(self, node: Element) -> None:
         if 'kbd' in node['classes']:

sphinx/writers/html5.py

@@ -462,10 +462,25 @@ class HTML5Translator(SphinxTranslator, BaseTranslator):
         if 'kbd' in node['classes']:
             self.body.append(self.starttag(node, 'kbd', '',
                                            CLASS='docutils literal notranslate'))
-        else:
+            return
+        lang = node.get("language", None)
+        if 'code' not in node['classes'] or not lang:
             self.body.append(self.starttag(node, 'code', '',
                                            CLASS='docutils literal notranslate'))
             self.protect_literal_text += 1
+            return
+
+        opts = self.config.highlight_options.get(lang, {})
+        highlighted = self.highlighter.highlight_block(
+            node.astext(), lang, opts=opts, location=node, nowrap=True)
+        starttag = self.starttag(
+            node,
+            "code",
+            suffix="",
+            CLASS="docutils literal highlight highlight-%s" % lang,
+        )
+        self.body.append(starttag + highlighted.strip() + "</code>")
+        raise nodes.SkipNode
 
     def depart_literal(self, node: Element) -> None:
         if 'kbd' in node['classes']:

sphinx/writers/latex.py

@@ -1740,10 +1740,27 @@ class LaTeXTranslator(SphinxTranslator):
     def visit_literal(self, node: Element) -> None:
         if self.in_title:
             self.body.append(r'\sphinxstyleliteralintitle{\sphinxupquote{')
+            return
         elif 'kbd' in node['classes']:
             self.body.append(r'\sphinxkeyboard{\sphinxupquote{')
-        else:
+            return
+        lang = node.get("language", None)
+        if 'code' not in node['classes'] or not lang:
             self.body.append(r'\sphinxcode{\sphinxupquote{')
+            return
+
+        opts = self.config.highlight_options.get(lang, {})
+        hlcode = self.highlighter.highlight_block(
+            node.astext(), lang, opts=opts, location=node)
+        # TODO: Use nowrap option once LaTeX formatter supports it
+        # https://github.com/pygments/pygments/pull/1343
+        hlcode = hlcode.replace(r'\begin{Verbatim}[commandchars=\\\{\}]',
+                                r'\sphinxcode{\sphinxupquote{')
+        # get consistent trailer
+        hlcode = hlcode.rstrip()[:-14]  # strip \end{Verbatim}
+        self.body.append(hlcode)
+        self.body.append('}}')
+        raise nodes.SkipNode
 
     def depart_literal(self, node: Element) -> None:
         self.body.append('}}')

tests/roots/test-reST-code-role/index.rst

@@ -0,0 +1,9 @@
+.. role:: python(code)
+   :language: python
+   :class: highlight
+
+Inline :python:`def foo(1 + 2 + None + "abc"): pass` code block
+
+.. code-block:: python
+
+   def foo(1 + 2 + None + "abc"): pass

tests/test_build_html.py

@@ -1710,3 +1710,27 @@ def test_html_signaturereturn_icon(app):
     content = (app.outdir / 'index.html').read_text()
 
     assert ('<span class="sig-return-icon">&#x2192;</span>' in content)
+
+
+@pytest.mark.sphinx('html', testroot='reST-code-role')
+def test_html_code_role(app):
+    app.build()
+    content = (app.outdir / 'index.html').read_text()
+
+    common_content = (
+        '<span class="k">def</span> <span class="nf">foo</span>'
+        '<span class="p">(</span>'
+        '<span class="mi">1</span> '
+        '<span class="o">+</span> '
+        '<span class="mi">2</span> '
+        '<span class="o">+</span> '
+        '<span class="kc">None</span> '
+        '<span class="o">+</span> '
+        '<span class="s2">&quot;abc&quot;</span>'
+        '<span class="p">):</span> '
+        '<span class="k">pass</span>')
+    assert ('<p>Inline <code class="code highlight python docutils literal highlight-python">' +
+            common_content + '</code> code block</p>') in content
+    assert ('<div class="highlight-python notranslate">' +
+            '<div class="highlight"><pre><span></span>' +
+            common_content) in content

tests/test_build_latex.py

@@ -1620,3 +1620,30 @@ def test_latex_container(app, status, warning):
     result = (app.outdir / 'python.tex').read_text()
     assert r'\begin{sphinxuseclass}{classname}' in result
     assert r'\end{sphinxuseclass}' in result
+
+
+@pytest.mark.sphinx('latex', testroot='reST-code-role')
+def test_latex_code_role(app):
+    app.build()
+    content = (app.outdir / 'python.tex').read_text()
+
+    common_content = (
+        r'\PYG{k}{def} '
+        r'\PYG{n+nf}{foo}'
+        r'\PYG{p}{(}'
+        r'\PYG{l+m+mi}{1} '
+        r'\PYG{o}{+} '
+        r'\PYG{l+m+mi}{2} '
+        r'\PYG{o}{+} '
+        r'\PYG{k+kc}{None} '
+        r'\PYG{o}{+} '
+        r'\PYG{l+s+s2}{\PYGZdq{}}'
+        r'\PYG{l+s+s2}{abc}'
+        r'\PYG{l+s+s2}{\PYGZdq{}}'
+        r'\PYG{p}{)}'
+        r'\PYG{p}{:} '
+        r'\PYG{k}{pass}')
+    assert (r'Inline \sphinxcode{\sphinxupquote{' + '\n' +
+            common_content + '\n}} code block') in content
+    assert (r'\begin{sphinxVerbatim}[commandchars=\\\{\}]' +
+            '\n' + common_content + '\n' + r'\end{sphinxVerbatim}') in content