Refactor: Run highlightlang on resolving phase

CHANGES

@@ -49,6 +49,7 @@ Incompatible changes
   upgrading Sphinx, please clean latex build repertory of existing project
   before new build.
 * #5163: html: hlist items are now aligned to top
+* ``highlightlang`` directive is processed on resolving phase
 
 Deprecated
 ----------
@@ -96,7 +97,16 @@ Deprecated
 * ``sphinx.writers.latex.LaTeXTranslator.push_hyperlink_ids()`` is deprecated
 * ``sphinx.writers.latex.LaTeXTranslator.pop_hyperlink_ids()`` is deprecated
 * ``sphinx.writers.latex.LaTeXTranslator.bibitems`` is deprecated
+* ``sphinx.writers.latex.LaTeXTranslator.hlsettingstack`` is deprecated
 * ``sphinx.writers.latex.ExtBabel.get_shorthandoff()`` is deprecated
+* ``sphinx.writers.html.HTMLTranslator.highlightlang`` is deprecated
+* ``sphinx.writers.html.HTMLTranslator.highlightlang_base`` is deprecated
+* ``sphinx.writers.html.HTMLTranslator.highlightlangopts`` is deprecated
+* ``sphinx.writers.html.HTMLTranslator.highlightlinenothreshold`` is deprecated
+* ``sphinx.writers.html5.HTMLTranslator.highlightlang`` is deprecated
+* ``sphinx.writers.html5.HTMLTranslator.highlightlang_base`` is deprecated
+* ``sphinx.writers.html5.HTMLTranslator.highlightlangopts`` is deprecated
+* ``sphinx.writers.html5.HTMLTranslator.highlightlinenothreshold`` is deprecated
 * ``sphinx.ext.mathbase.math`` node is deprecated
 * ``sphinx.ext.mathbase.displaymath`` node is deprecated
 * ``sphinx.ext.mathbase.eqref`` node is deprecated

doc/extdev/index.rst

@@ -221,11 +221,56 @@ The following is a list of deprecated interface.
      - 3.0
      - N/A
 
+   * - ``sphinx.writers.latex.LaTeXTranslator.hlsettingstack``
+     - 1.8
+     - 3.0
+     - N/A
+
    * - ``sphinx.writers.latex.ExtBabel.get_shorthandoff()``
      - 1.8
      - 3.0
      - N/A
 
+   * - ``sphinx.writers.html.HTMLTranslator.highlightlang()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html.HTMLTranslator.highlightlang_base()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html.HTMLTranslator.highlightlangopts()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html.HTMLTranslator.highlightlinenothreshold()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html5.HTMLTranslator.highlightlang()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html5.HTMLTranslator.highlightlang_base()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html5.HTMLTranslator.highlightlangopts()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html5.HTMLTranslator.highlightlinenothreshold()``
+     - 1.8
+     - 3.0
+     - N/A
+
    * - ``sphinx.application.CONFIG_FILENAME``
      - 1.8
      - 3.0

sphinx/application.py

@@ -96,6 +96,7 @@ builtin_extensions = (
     'sphinx.registry',
     'sphinx.roles',
     'sphinx.transforms.post_transforms',
+    'sphinx.transforms.post_transforms.code',
     'sphinx.transforms.post_transforms.images',
     'sphinx.transforms.post_transforms.compat',
     'sphinx.util.compat',

sphinx/transforms/post_transforms/code.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.transforms.post_transforms.code
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    transforms for code-blocks.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys
+from typing import NamedTuple
+
+from docutils import nodes
+from six import text_type
+
+from sphinx import addnodes
+from sphinx.transforms import SphinxTransform
+
+if False:
+    # For type annotation
+    from typing import Any, Dict, List  # NOQA
+    from sphinx.application import Sphinx  # NOQA
+
+
+HighlightSetting = NamedTuple('HighlightSetting', [('language', text_type),
+                                                   ('lineno_threshold', int)])
+
+
+class HighlightLanguageTransform(SphinxTransform):
+    """
+    Apply highlight_language to all literal_block nodes.
+
+    This refers both :confval:`highlight_language` setting and
+    :rst:dir:`highlightlang` directive.  After processing, this transform
+    removes ``highlightlang`` node from doctree.
+    """
+    default_priority = 400
+
+    def apply(self):
+        visitor = HighlightLanguageVisitor(self.document,
+                                           self.config.highlight_language)
+        self.document.walkabout(visitor)
+
+        for node in self.document.traverse(addnodes.highlightlang):
+            node.parent.remove(node)
+
+
+class HighlightLanguageVisitor(nodes.NodeVisitor):
+    def __init__(self, document, default_language):
+        # type: (nodes.document, unicode) -> None
+        self.default_setting = HighlightSetting(default_language, sys.maxsize)
+        self.settings = []  # type: List[HighlightSetting]
+        nodes.NodeVisitor.__init__(self, document)
+
+    def unknown_visit(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def unknown_departure(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_document(self, node):
+        # type: (nodes.Node) -> None
+        self.settings.append(self.default_setting)
+
+    def depart_document(self, node):
+        # type: (nodes.Node) -> None
+        self.settings.pop()
+
+    def visit_start_of_file(self, node):
+        # type: (nodes.Node) -> None
+        self.settings.append(self.default_setting)
+
+    def depart_start_of_file(self, node):
+        # type: (nodes.Node) -> None
+        self.settings.pop()
+
+    def visit_highlightlang(self, node):
+        # type: (addnodes.highlightlang) -> None
+        self.settings[-1] = HighlightSetting(node['lang'], node['linenothreshold'])
+
+    def visit_literal_block(self, node):
+        # type: (nodes.literal_block) -> None
+        setting = self.settings[-1]
+        if 'language' not in node:
+            node['language'] = setting.language
+            node['force_highlighting'] = False
+        else:
+            node['force_highlighting'] = True
+        if 'linenos' not in node:
+            lines = node.astext().count('\n')
+            node['linenos'] = (lines >= setting.lineno_threshold - 1)
+
+
+def setup(app):
+    # type: (Sphinx) -> Dict[unicode, Any]
+    app.add_post_transform(HighlightLanguageTransform)
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

sphinx/writers/html.py

@@ -13,12 +13,14 @@ import copy
 import os
 import posixpath
 import sys
+import warnings
 
 from docutils import nodes
 from docutils.writers.html4css1 import Writer, HTMLTranslator as BaseTranslator
 from six import string_types
 
 from sphinx import addnodes
+from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.locale import admonitionlabels, _, __
 from sphinx.util import logging
 from sphinx.util.images import get_image_size
@@ -74,10 +76,6 @@ class HTMLTranslator(BaseTranslator):
         BaseTranslator.__init__(self, *args, **kwds)
         self.highlighter = builder.highlighter
         self.builder = builder
-        self.highlightlang = self.highlightlang_base = \
-            builder.config.highlight_language
-        self.highlightopts = builder.config.highlight_options
-        self.highlightlinenothreshold = sys.maxsize
         self.docnames = [builder.current_docname]  # for singlehtml builder
         self.manpages_url = builder.config.manpages_url
         self.protect_literal_text = 0
@@ -423,19 +421,14 @@ class HTMLTranslator(BaseTranslator):
         if node.rawsource != node.astext():
             # most probably a parsed-literal block -- don't highlight
             return BaseTranslator.visit_literal_block(self, node)
-        lang = self.highlightlang
-        linenos = node.rawsource.count('\n') >= \
-            self.highlightlinenothreshold - 1
+
+        lang = node.get('language', 'default')
+        linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        if 'language' in node:
-            # code-block directives
-            lang = node['language']
-            highlight_args['force'] = True
-        if 'linenos' in node:
-            linenos = node['linenos']
-        if lang is self.highlightlang_base:
+        highlight_args['force'] = node.get('force_highlighting', False)
+        if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
-            opts = self.highlightopts
+            opts = self.builder.config.highlight_options
         else:
             opts = {}
 
@@ -569,15 +562,6 @@ class HTMLTranslator(BaseTranslator):
         # type: (nodes.Node) -> None
         pass
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        self.highlightlang = node['lang']
-        self.highlightlinenothreshold = node['linenothreshold']
-
-    def depart_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
     def visit_download_reference(self, node):
         # type: (nodes.Node) -> None
         if self.builder.download_support and node.hasattr('filename'):
@@ -883,3 +867,33 @@ class HTMLTranslator(BaseTranslator):
     def unknown_visit(self, node):
         # type: (nodes.Node) -> None
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
+
+    # --------- METHODS FOR COMPATIBILITY --------------------------------------
+
+    @property
+    def highlightlang(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightlang is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_language
+
+    @property
+    def highlightlang_base(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightlang_base is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_language
+
+    @property
+    def highlightopts(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightopts is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_options
+
+    @property
+    def highlightlinenothreshold(self):
+        # type: () -> int
+        warnings.warn('HTMLTranslator.highlightlinenothreshold is deprecated.',
+                      RemovedInSphinx30Warning)
+        return sys.maxsize

sphinx/writers/html5.py

@@ -12,12 +12,14 @@
 import os
 import posixpath
 import sys
+import warnings
 
 from docutils import nodes
 from docutils.writers.html5_polyglot import HTMLTranslator as BaseTranslator
 from six import string_types
 
 from sphinx import addnodes
+from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.locale import admonitionlabels, _, __
 from sphinx.util import logging
 from sphinx.util.images import get_image_size
@@ -44,10 +46,6 @@ class HTML5Translator(BaseTranslator):
         BaseTranslator.__init__(self, *args, **kwds)
         self.highlighter = builder.highlighter
         self.builder = builder
-        self.highlightlang = self.highlightlang_base = \
-            builder.config.highlight_language
-        self.highlightopts = builder.config.highlight_options
-        self.highlightlinenothreshold = sys.maxsize
         self.docnames = [builder.current_docname]  # for singlehtml builder
         self.manpages_url = builder.config.manpages_url
         self.protect_literal_text = 0
@@ -369,19 +367,14 @@ class HTML5Translator(BaseTranslator):
         if node.rawsource != node.astext():
             # most probably a parsed-literal block -- don't highlight
             return BaseTranslator.visit_literal_block(self, node)
-        lang = self.highlightlang
-        linenos = node.rawsource.count('\n') >= \
-            self.highlightlinenothreshold - 1
+
+        lang = node.get('language', 'default')
+        linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        if 'language' in node:
-            # code-block directives
-            lang = node['language']
-            highlight_args['force'] = True
-        if 'linenos' in node:
-            linenos = node['linenos']
-        if lang is self.highlightlang_base:
+        highlight_args['force'] = node.get('force_highlighting', False)
+        if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
-            opts = self.highlightopts
+            opts = self.builder.config.highlight_options
         else:
             opts = {}
 
@@ -515,15 +508,6 @@ class HTML5Translator(BaseTranslator):
         # type: (nodes.Node) -> None
         pass
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        self.highlightlang = node['lang']
-        self.highlightlinenothreshold = node['linenothreshold']
-
-    def depart_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
     def visit_download_reference(self, node):
         # type: (nodes.Node) -> None
         if self.builder.download_support and node.hasattr('filename'):
@@ -834,3 +818,33 @@ class HTML5Translator(BaseTranslator):
     def unknown_visit(self, node):
         # type: (nodes.Node) -> None
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
+
+    # --------- METHODS FOR COMPATIBILITY --------------------------------------
+
+    @property
+    def highlightlang(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightlang is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_language
+
+    @property
+    def highlightlang_base(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightlang_base is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_language
+
+    @property
+    def highlightopts(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightopts is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_options
+
+    @property
+    def highlightlinenothreshold(self):
+        # type: () -> int
+        warnings.warn('HTMLTranslator.highlightlinenothreshold is deprecated.',
+                      RemovedInSphinx30Warning)
+        return sys.maxsize

sphinx/writers/latex.py

@@ -704,12 +704,6 @@ class LaTeXTranslator(nodes.NodeVisitor):
         self.descstack = []             # type: List[unicode]
         self.table = None               # type: Table
         self.next_table_colspec = None  # type: unicode
-        # stack of [language, linenothreshold] settings per file
-        # the first item here is the default and must not be changed
-        # the second item is the default for the master file and can be changed
-        # by .. highlight:: directive in the master file
-        self.hlsettingstack = 2 * [[builder.config.highlight_language,
-                                    sys.maxsize]]
         self.bodystack = []             # type: List[List[unicode]]
         self.footnote_restricted = False
         self.pending_footnotes = []     # type: List[nodes.footnote_reference]
@@ -945,8 +939,6 @@ class LaTeXTranslator(nodes.NodeVisitor):
     def visit_start_of_file(self, node):
         # type: (nodes.Node) -> None
         self.curfilestack.append(node['docname'])
-        # use default highlight settings for new file
-        self.hlsettingstack.append(self.hlsettingstack[0])
 
     def collect_footnotes(self, node):
         # type: (nodes.Node) -> Dict[unicode, List[Union[collected_footnote, bool]]]
@@ -971,12 +963,6 @@ class LaTeXTranslator(nodes.NodeVisitor):
     def depart_start_of_file(self, node):
         # type: (nodes.Node) -> None
         self.curfilestack.pop()
-        self.hlsettingstack.pop()
-
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        self.hlsettingstack[-1] = [node['lang'], node['linenothreshold']]
-        raise nodes.SkipNode
 
     def visit_section(self, node):
         # type: (nodes.Node) -> None
@@ -2246,26 +2232,18 @@ class LaTeXTranslator(nodes.NodeVisitor):
             if labels and not self.in_footnote:
                 self.body.append('\n\\def\\sphinxLiteralBlockLabel{' + labels + '}')
 
-            code = node.astext()
-            lang = self.hlsettingstack[-1][0]
-            linenos = code.count('\n') >= self.hlsettingstack[-1][1] - 1
+            lang = node.get('language', 'default')
+            linenos = node.get('linenos', False)
             highlight_args = node.get('highlight_args', {})
-            hllines = '\\fvset{hllines={, %s,}}%%' %\
-                      str(highlight_args.get('hl_lines', []))[1:-1]
-            if 'language' in node:
-                # code-block directives
-                lang = node['language']
-                highlight_args['force'] = True
-            if 'linenos' in node:
-                linenos = node['linenos']
-            if lang is self.hlsettingstack[0][0]:
+            highlight_args['force'] = node.get('force_highlighting', False)
+            if lang is self.builder.config.highlight_language:
                 # only pass highlighter options for original language
                 opts = self.builder.config.highlight_options
             else:
                 opts = {}
 
             hlcode = self.highlighter.highlight_block(
-                code, lang, opts=opts, linenos=linenos,
+                node.rawsource, lang, opts=opts, linenos=linenos,
                 location=(self.curfilestack[-1], node.line), **highlight_args
             )
             # workaround for Unicode issue
@@ -2290,6 +2268,9 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 hlcode += '\\end{sphinxVerbatimintable}'
             else:
                 hlcode += '\\end{sphinxVerbatim}'
+
+            hllines = '\\fvset{hllines={, %s,}}%%' %\
+                      str(highlight_args.get('hl_lines', []))[1:-1]
             self.body.append('\n' + hllines + '\n' + hlcode + '\n')
             raise nodes.SkipNode
 
@@ -2639,6 +2620,13 @@ class LaTeXTranslator(nodes.NodeVisitor):
                       RemovedInSphinx30Warning)
         return set()
 
+    @property
+    def hlsettingstack(self):
+        # type: () -> List[List[Union[unicode, int]]]
+        warnings.warn('LaTeXTranslator.hlsettingstack is deprecated.',
+                      RemovedInSphinx30Warning)
+        return [[self.builder.config.highlight_language, sys.maxsize]]
+
 
 # Import old modules here for compatibility
 # They should be imported after `LaTeXTranslator` to avoid recursive import.

sphinx/writers/manpage.py

@@ -378,14 +378,6 @@ class ManualPageTranslator(BaseTranslator):
         # type: (nodes.Node) -> None
         pass
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
-    def depart_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
     def visit_download_reference(self, node):
         # type: (nodes.Node) -> None
         pass

sphinx/writers/texinfo.py

@@ -1533,14 +1533,6 @@ class TexinfoTranslator(nodes.NodeVisitor):
         self.body.append('\n\n')
         raise nodes.SkipNode
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
-    def depart_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
     # -- Desc
 
     def visit_desc(self, node):

sphinx/writers/text.py

@@ -251,10 +251,6 @@ class TextTranslator(nodes.NodeVisitor):
                                  for line in lines)
         # XXX header/footer?
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        raise nodes.SkipNode
-
     def visit_section(self, node):
         # type: (nodes.Node) -> None
         self._title_char = self.sectionchars[self.sectionlevel]