Merge pull request #5199 from tk0miya/refactor_highlighter

Apply trim_doctest_flags to all builders (cf. text, manpages)

CHANGES

@@ -49,6 +49,7 @@ Incompatible changes
   upgrading Sphinx, please clean latex build repertory of existing project
   before new build.
 * #5163: html: hlist items are now aligned to top
+* ``highlightlang`` directive is processed on resolving phase
 
 Deprecated
 ----------
@@ -96,13 +97,25 @@ Deprecated
 * ``sphinx.writers.latex.LaTeXTranslator.push_hyperlink_ids()`` is deprecated
 * ``sphinx.writers.latex.LaTeXTranslator.pop_hyperlink_ids()`` is deprecated
 * ``sphinx.writers.latex.LaTeXTranslator.bibitems`` is deprecated
+* ``sphinx.writers.latex.LaTeXTranslator.hlsettingstack`` is deprecated
 * ``sphinx.writers.latex.ExtBabel.get_shorthandoff()`` is deprecated
+* ``sphinx.writers.html.HTMLTranslator.highlightlang`` is deprecated
+* ``sphinx.writers.html.HTMLTranslator.highlightlang_base`` is deprecated
+* ``sphinx.writers.html.HTMLTranslator.highlightlangopts`` is deprecated
+* ``sphinx.writers.html.HTMLTranslator.highlightlinenothreshold`` is deprecated
+* ``sphinx.writers.html5.HTMLTranslator.highlightlang`` is deprecated
+* ``sphinx.writers.html5.HTMLTranslator.highlightlang_base`` is deprecated
+* ``sphinx.writers.html5.HTMLTranslator.highlightlangopts`` is deprecated
+* ``sphinx.writers.html5.HTMLTranslator.highlightlinenothreshold`` is deprecated
 * ``sphinx.ext.mathbase.math`` node is deprecated
 * ``sphinx.ext.mathbase.displaymath`` node is deprecated
 * ``sphinx.ext.mathbase.eqref`` node is deprecated
 * ``sphinx.ext.mathbase.is_in_section_title()`` is deprecated
 * ``sphinx.ext.mathbase.MathDomain`` is deprecated
 * ``sphinx.ext.mathbase.setup_math()`` is deprecated
+* ``sphinx.highlighting.PygmentsBridge.unhighlight()`` is deprecated
+* The ``trim_doctest_flags`` argument of ``sphinx.highlighting.PygmentsBridge``
+  is deprecated
 
 For more details, see `deprecation APIs list
 <http://www.sphinx-doc.org/en/master/extdev/index.html#deprecated-apis>`_
@@ -165,6 +178,7 @@ Features added
 * #5122: setuptools: support nitpicky option
 * #2820: autoclass directive supports nested class
 * Add ``app.add_html_math_renderer()`` to register a math renderer for HTML
+* Apply :confval:`trim_doctest_flags` to all builders (cf. text, manpages)
 
 Bugs fixed
 ----------

doc/extdev/index.rst

@@ -131,6 +131,17 @@ The following is a list of deprecated interface.
      - 4.0
      - :meth:`~sphinx.application.Sphinx.add_js_file()`
 
+   * - ``sphinx.highlighting.PygmentsBridge.unhighlight()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``trim_doctest_flags`` arguments of
+       ``sphinx.highlighting.PygmentsBridge``
+     - 1.8
+     - 3.0
+     - N/A
+
    * - ``sphinx.ext.mathbase.MathDomain``
      - 1.8
      - 3.0
@@ -221,11 +232,56 @@ The following is a list of deprecated interface.
      - 3.0
      - N/A
 
+   * - ``sphinx.writers.latex.LaTeXTranslator.hlsettingstack``
+     - 1.8
+     - 3.0
+     - N/A
+
    * - ``sphinx.writers.latex.ExtBabel.get_shorthandoff()``
      - 1.8
      - 3.0
      - N/A
 
+   * - ``sphinx.writers.html.HTMLTranslator.highlightlang()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html.HTMLTranslator.highlightlang_base()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html.HTMLTranslator.highlightlangopts()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html.HTMLTranslator.highlightlinenothreshold()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html5.HTMLTranslator.highlightlang()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html5.HTMLTranslator.highlightlang_base()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html5.HTMLTranslator.highlightlangopts()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.writers.html5.HTMLTranslator.highlightlinenothreshold()``
+     - 1.8
+     - 3.0
+     - N/A
+
    * - ``sphinx.application.CONFIG_FILENAME``
      - 1.8
      - 3.0

sphinx/application.py

@@ -96,6 +96,7 @@ builtin_extensions = (
     'sphinx.registry',
     'sphinx.roles',
     'sphinx.transforms.post_transforms',
+    'sphinx.transforms.post_transforms.code',
     'sphinx.transforms.post_transforms.images',
     'sphinx.transforms.post_transforms.compat',
     'sphinx.util.compat',

sphinx/builders/html.py

@@ -386,8 +386,7 @@ class StandaloneHTMLBuilder(Builder):
             style = self.theme.get_config('theme', 'pygments_style', 'none')
         else:
             style = 'sphinx'
-        self.highlighter = PygmentsBridge('html', style,
-                                          self.config.trim_doctest_flags)
+        self.highlighter = PygmentsBridge('html', style)
 
     def init_css_files(self):
         # type: () -> None

sphinx/builders/latex/__init__.py

@@ -169,8 +169,7 @@ class LaTeXBuilder(Builder):
 
     def write_stylesheet(self):
         # type: () -> None
-        highlighter = highlighting.PygmentsBridge(
-            'latex', self.config.pygments_style, self.config.trim_doctest_flags)
+        highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style)
         stylesheet = path.join(self.outdir, 'sphinxhighlight.sty')
         with open(stylesheet, 'w') as f:
             f.write('\\NeedsTeXFormat{LaTeX2e}[1995/12/01]\n')

sphinx/highlighting.py

@@ -9,6 +9,8 @@
     :license: BSD, see LICENSE for details.
 """
 
+import warnings
+
 from pygments import highlight
 from pygments.filters import ErrorToken
 from pygments.formatters import HtmlFormatter, LatexFormatter
@@ -20,6 +22,7 @@ from pygments.styles import get_style_by_name
 from pygments.util import ClassNotFound
 from six import text_type
 
+from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.ext import doctest
 from sphinx.locale import __
 from sphinx.pygments_styles import SphinxStyle, NoneStyle
@@ -65,7 +68,7 @@ class PygmentsBridge(object):
     html_formatter = HtmlFormatter
     latex_formatter = LatexFormatter
 
-    def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=False):
+    def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=None):
         # type: (unicode, unicode, bool) -> None
         self.dest = dest
         if stylename is None or stylename == 'sphinx':
@@ -78,7 +81,6 @@ class PygmentsBridge(object):
                             stylename)
         else:
             style = get_style_by_name(stylename)
-        self.trim_doctest_flags = trim_doctest_flags
         self.formatter_args = {'style': style}  # type: Dict[unicode, Any]
         if dest == 'html':
             self.formatter = self.html_formatter
@@ -86,6 +88,11 @@ class PygmentsBridge(object):
             self.formatter = self.latex_formatter
             self.formatter_args['commandprefix'] = 'PYG'
 
+        self.trim_doctest_flags = trim_doctest_flags
+        if trim_doctest_flags is not None:
+            warnings.warn('trim_doctest_flags option for PygmentsBridge is now deprecated.',
+                          RemovedInSphinx30Warning)
+
     def get_formatter(self, **kwargs):
         # type: (Any) -> Formatter
         kwargs.update(self.formatter_args)  # type: ignore
@@ -93,6 +100,8 @@ class PygmentsBridge(object):
 
     def unhighlighted(self, source):
         # type: (unicode) -> unicode
+        warnings.warn('PygmentsBridge.unhighlighted() is now deprecated.',
+                      RemovedInSphinx30Warning)
         if self.dest == 'html':
             return '<pre>' + htmlescape(source) + '</pre>\n'
         else:
@@ -161,8 +170,6 @@ class PygmentsBridge(object):
         if self.dest == 'html':
             return hlsource
         else:
-            if not isinstance(hlsource, text_type):  # Py2 / Pygments < 1.6
-                hlsource = hlsource.decode()
             return hlsource.translate(tex_hl_escape_map_new)
 
     def get_stylesheet(self):

sphinx/transforms/post_transforms/code.py

@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.transforms.post_transforms.code
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    transforms for code-blocks.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys
+from typing import NamedTuple
+
+from docutils import nodes
+from pygments.lexers import PythonConsoleLexer, guess_lexer
+from six import text_type
+
+from sphinx import addnodes
+from sphinx.ext import doctest
+from sphinx.transforms import SphinxTransform
+
+if False:
+    # For type annotation
+    from typing import Any, Dict, List  # NOQA
+    from sphinx.application import Sphinx  # NOQA
+
+
+HighlightSetting = NamedTuple('HighlightSetting', [('language', text_type),
+                                                   ('lineno_threshold', int)])
+
+
+class HighlightLanguageTransform(SphinxTransform):
+    """
+    Apply highlight_language to all literal_block nodes.
+
+    This refers both :confval:`highlight_language` setting and
+    :rst:dir:`highlightlang` directive.  After processing, this transform
+    removes ``highlightlang`` node from doctree.
+    """
+    default_priority = 400
+
+    def apply(self):
+        visitor = HighlightLanguageVisitor(self.document,
+                                           self.config.highlight_language)
+        self.document.walkabout(visitor)
+
+        for node in self.document.traverse(addnodes.highlightlang):
+            node.parent.remove(node)
+
+
+class HighlightLanguageVisitor(nodes.NodeVisitor):
+    def __init__(self, document, default_language):
+        # type: (nodes.document, unicode) -> None
+        self.default_setting = HighlightSetting(default_language, sys.maxsize)
+        self.settings = []  # type: List[HighlightSetting]
+        nodes.NodeVisitor.__init__(self, document)
+
+    def unknown_visit(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def unknown_departure(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_document(self, node):
+        # type: (nodes.Node) -> None
+        self.settings.append(self.default_setting)
+
+    def depart_document(self, node):
+        # type: (nodes.Node) -> None
+        self.settings.pop()
+
+    def visit_start_of_file(self, node):
+        # type: (nodes.Node) -> None
+        self.settings.append(self.default_setting)
+
+    def depart_start_of_file(self, node):
+        # type: (nodes.Node) -> None
+        self.settings.pop()
+
+    def visit_highlightlang(self, node):
+        # type: (addnodes.highlightlang) -> None
+        self.settings[-1] = HighlightSetting(node['lang'], node['linenothreshold'])
+
+    def visit_literal_block(self, node):
+        # type: (nodes.literal_block) -> None
+        setting = self.settings[-1]
+        if 'language' not in node:
+            node['language'] = setting.language
+            node['force_highlighting'] = False
+        else:
+            node['force_highlighting'] = True
+        if 'linenos' not in node:
+            lines = node.astext().count('\n')
+            node['linenos'] = (lines >= setting.lineno_threshold - 1)
+
+
+class TrimDoctestFlagsTransform(SphinxTransform):
+    """
+    Trim doctest flags like ``# doctest: +FLAG`` from python code-blocks.
+
+    see :confval:`trim_doctest_flags` for more information.
+    """
+    default_priority = HighlightLanguageTransform.default_priority + 1
+
+    def apply(self):
+        if not self.config.trim_doctest_flags:
+            return
+
+        for node in self.document.traverse(nodes.literal_block):
+            if self.is_pyconsole(node):
+                source = node.rawsource
+                source = doctest.blankline_re.sub('', source)
+                source = doctest.doctestopt_re.sub('', source)
+                node.rawsource = source
+                node[:] = [nodes.Text(source)]
+
+    @staticmethod
+    def is_pyconsole(node):
+        # type: (nodes.literal_block) -> bool
+        if node.rawsource != node.astext():
+            return False  # skip parsed-literal node
+
+        language = node.get('language')
+        if language in ('pycon', 'pycon3'):
+            return True
+        elif language in ('py', 'py3', 'python', 'python3', 'default'):
+            return node.rawsource.startswith('>>>')
+        elif language == 'guess':
+            try:
+                lexer = guess_lexer(node.rawsource)
+                return isinstance(lexer, PythonConsoleLexer)
+            except Exception:
+                pass
+
+        return False
+
+
+def setup(app):
+    # type: (Sphinx) -> Dict[unicode, Any]
+    app.add_post_transform(HighlightLanguageTransform)
+    app.add_post_transform(TrimDoctestFlagsTransform)
+
+    return {
+        'version': 'builtin',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

sphinx/writers/html.py

@@ -13,12 +13,14 @@ import copy
 import os
 import posixpath
 import sys
+import warnings
 
 from docutils import nodes
 from docutils.writers.html4css1 import Writer, HTMLTranslator as BaseTranslator
 from six import string_types
 
 from sphinx import addnodes
+from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.locale import admonitionlabels, _, __
 from sphinx.util import logging
 from sphinx.util.images import get_image_size
@@ -74,10 +76,6 @@ class HTMLTranslator(BaseTranslator):
         BaseTranslator.__init__(self, *args, **kwds)
         self.highlighter = builder.highlighter
         self.builder = builder
-        self.highlightlang = self.highlightlang_base = \
-            builder.config.highlight_language
-        self.highlightopts = builder.config.highlight_options
-        self.highlightlinenothreshold = sys.maxsize
         self.docnames = [builder.current_docname]  # for singlehtml builder
         self.manpages_url = builder.config.manpages_url
         self.protect_literal_text = 0
@@ -423,19 +421,14 @@ class HTMLTranslator(BaseTranslator):
         if node.rawsource != node.astext():
             # most probably a parsed-literal block -- don't highlight
             return BaseTranslator.visit_literal_block(self, node)
-        lang = self.highlightlang
-        linenos = node.rawsource.count('\n') >= \
-            self.highlightlinenothreshold - 1
+
+        lang = node.get('language', 'default')
+        linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        if 'language' in node:
-            # code-block directives
-            lang = node['language']
-            highlight_args['force'] = True
-        if 'linenos' in node:
-            linenos = node['linenos']
-        if lang is self.highlightlang_base:
+        highlight_args['force'] = node.get('force_highlighting', False)
+        if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
-            opts = self.highlightopts
+            opts = self.builder.config.highlight_options
         else:
             opts = {}
 
@@ -569,15 +562,6 @@ class HTMLTranslator(BaseTranslator):
         # type: (nodes.Node) -> None
         pass
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        self.highlightlang = node['lang']
-        self.highlightlinenothreshold = node['linenothreshold']
-
-    def depart_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
     def visit_download_reference(self, node):
         # type: (nodes.Node) -> None
         if self.builder.download_support and node.hasattr('filename'):
@@ -901,3 +885,33 @@ class HTMLTranslator(BaseTranslator):
     def unknown_visit(self, node):
         # type: (nodes.Node) -> None
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
+
+    # --------- METHODS FOR COMPATIBILITY --------------------------------------
+
+    @property
+    def highlightlang(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightlang is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_language
+
+    @property
+    def highlightlang_base(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightlang_base is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_language
+
+    @property
+    def highlightopts(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightopts is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_options
+
+    @property
+    def highlightlinenothreshold(self):
+        # type: () -> int
+        warnings.warn('HTMLTranslator.highlightlinenothreshold is deprecated.',
+                      RemovedInSphinx30Warning)
+        return sys.maxsize

sphinx/writers/html5.py

@@ -12,12 +12,14 @@
 import os
 import posixpath
 import sys
+import warnings
 
 from docutils import nodes
 from docutils.writers.html5_polyglot import HTMLTranslator as BaseTranslator
 from six import string_types
 
 from sphinx import addnodes
+from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.locale import admonitionlabels, _, __
 from sphinx.util import logging
 from sphinx.util.images import get_image_size
@@ -44,10 +46,6 @@ class HTML5Translator(BaseTranslator):
         BaseTranslator.__init__(self, *args, **kwds)
         self.highlighter = builder.highlighter
         self.builder = builder
-        self.highlightlang = self.highlightlang_base = \
-            builder.config.highlight_language
-        self.highlightopts = builder.config.highlight_options
-        self.highlightlinenothreshold = sys.maxsize
         self.docnames = [builder.current_docname]  # for singlehtml builder
         self.manpages_url = builder.config.manpages_url
         self.protect_literal_text = 0
@@ -369,19 +367,14 @@ class HTML5Translator(BaseTranslator):
         if node.rawsource != node.astext():
             # most probably a parsed-literal block -- don't highlight
             return BaseTranslator.visit_literal_block(self, node)
-        lang = self.highlightlang
-        linenos = node.rawsource.count('\n') >= \
-            self.highlightlinenothreshold - 1
+
+        lang = node.get('language', 'default')
+        linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        if 'language' in node:
-            # code-block directives
-            lang = node['language']
-            highlight_args['force'] = True
-        if 'linenos' in node:
-            linenos = node['linenos']
-        if lang is self.highlightlang_base:
+        highlight_args['force'] = node.get('force_highlighting', False)
+        if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
-            opts = self.highlightopts
+            opts = self.builder.config.highlight_options
         else:
             opts = {}
 
@@ -515,15 +508,6 @@ class HTML5Translator(BaseTranslator):
         # type: (nodes.Node) -> None
         pass
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        self.highlightlang = node['lang']
-        self.highlightlinenothreshold = node['linenothreshold']
-
-    def depart_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
     def visit_download_reference(self, node):
         # type: (nodes.Node) -> None
         if self.builder.download_support and node.hasattr('filename'):
@@ -852,3 +836,33 @@ class HTML5Translator(BaseTranslator):
     def unknown_visit(self, node):
         # type: (nodes.Node) -> None
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
+
+    # --------- METHODS FOR COMPATIBILITY --------------------------------------
+
+    @property
+    def highlightlang(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightlang is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_language
+
+    @property
+    def highlightlang_base(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightlang_base is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_language
+
+    @property
+    def highlightopts(self):
+        # type: () -> unicode
+        warnings.warn('HTMLTranslator.highlightopts is deprecated.',
+                      RemovedInSphinx30Warning)
+        return self.builder.config.highlight_options
+
+    @property
+    def highlightlinenothreshold(self):
+        # type: () -> int
+        warnings.warn('HTMLTranslator.highlightlinenothreshold is deprecated.',
+                      RemovedInSphinx30Warning)
+        return sys.maxsize

sphinx/writers/latex.py

@@ -697,19 +697,11 @@ class LaTeXTranslator(nodes.NodeVisitor):
             self.babel_defmacro('\\pageautorefname', self.encode(_('page')))
         self.elements['numfig_format'] = self.generate_numfig_format(builder)
 
-        self.highlighter = highlighting.PygmentsBridge(
-            'latex',
-            builder.config.pygments_style, builder.config.trim_doctest_flags)
+        self.highlighter = highlighting.PygmentsBridge('latex', builder.config.pygments_style)
         self.context = []               # type: List[Any]
         self.descstack = []             # type: List[unicode]
         self.table = None               # type: Table
         self.next_table_colspec = None  # type: unicode
-        # stack of [language, linenothreshold] settings per file
-        # the first item here is the default and must not be changed
-        # the second item is the default for the master file and can be changed
-        # by .. highlight:: directive in the master file
-        self.hlsettingstack = 2 * [[builder.config.highlight_language,
-                                    sys.maxsize]]
         self.bodystack = []             # type: List[List[unicode]]
         self.footnote_restricted = False
         self.pending_footnotes = []     # type: List[nodes.footnote_reference]
@@ -945,8 +937,6 @@ class LaTeXTranslator(nodes.NodeVisitor):
     def visit_start_of_file(self, node):
         # type: (nodes.Node) -> None
         self.curfilestack.append(node['docname'])
-        # use default highlight settings for new file
-        self.hlsettingstack.append(self.hlsettingstack[0])
 
     def collect_footnotes(self, node):
         # type: (nodes.Node) -> Dict[unicode, List[Union[collected_footnote, bool]]]
@@ -971,12 +961,6 @@ class LaTeXTranslator(nodes.NodeVisitor):
     def depart_start_of_file(self, node):
         # type: (nodes.Node) -> None
         self.curfilestack.pop()
-        self.hlsettingstack.pop()
-
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        self.hlsettingstack[-1] = [node['lang'], node['linenothreshold']]
-        raise nodes.SkipNode
 
     def visit_section(self, node):
         # type: (nodes.Node) -> None
@@ -2246,26 +2230,18 @@ class LaTeXTranslator(nodes.NodeVisitor):
             if labels and not self.in_footnote:
                 self.body.append('\n\\def\\sphinxLiteralBlockLabel{' + labels + '}')
 
-            code = node.astext()
-            lang = self.hlsettingstack[-1][0]
-            linenos = code.count('\n') >= self.hlsettingstack[-1][1] - 1
+            lang = node.get('language', 'default')
+            linenos = node.get('linenos', False)
             highlight_args = node.get('highlight_args', {})
-            hllines = '\\fvset{hllines={, %s,}}%%' %\
-                      str(highlight_args.get('hl_lines', []))[1:-1]
-            if 'language' in node:
-                # code-block directives
-                lang = node['language']
-                highlight_args['force'] = True
-            if 'linenos' in node:
-                linenos = node['linenos']
-            if lang is self.hlsettingstack[0][0]:
+            highlight_args['force'] = node.get('force_highlighting', False)
+            if lang is self.builder.config.highlight_language:
                 # only pass highlighter options for original language
                 opts = self.builder.config.highlight_options
             else:
                 opts = {}
 
             hlcode = self.highlighter.highlight_block(
-                code, lang, opts=opts, linenos=linenos,
+                node.rawsource, lang, opts=opts, linenos=linenos,
                 location=(self.curfilestack[-1], node.line), **highlight_args
             )
             # workaround for Unicode issue
@@ -2290,6 +2266,9 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 hlcode += '\\end{sphinxVerbatimintable}'
             else:
                 hlcode += '\\end{sphinxVerbatim}'
+
+            hllines = '\\fvset{hllines={, %s,}}%%' %\
+                      str(highlight_args.get('hl_lines', []))[1:-1]
             self.body.append('\n' + hllines + '\n' + hlcode + '\n')
             raise nodes.SkipNode
 
@@ -2639,6 +2618,13 @@ class LaTeXTranslator(nodes.NodeVisitor):
                       RemovedInSphinx30Warning)
         return set()
 
+    @property
+    def hlsettingstack(self):
+        # type: () -> List[List[Union[unicode, int]]]
+        warnings.warn('LaTeXTranslator.hlsettingstack is deprecated.',
+                      RemovedInSphinx30Warning)
+        return [[self.builder.config.highlight_language, sys.maxsize]]
+
 
 # Import old modules here for compatibility
 # They should be imported after `LaTeXTranslator` to avoid recursive import.

sphinx/writers/manpage.py

@@ -378,14 +378,6 @@ class ManualPageTranslator(BaseTranslator):
         # type: (nodes.Node) -> None
         pass
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
-    def depart_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
     def visit_download_reference(self, node):
         # type: (nodes.Node) -> None
         pass

sphinx/writers/texinfo.py

@@ -1533,14 +1533,6 @@ class TexinfoTranslator(nodes.NodeVisitor):
         self.body.append('\n\n')
         raise nodes.SkipNode
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
-    def depart_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        pass
-
     # -- Desc
 
     def visit_desc(self, node):

sphinx/writers/text.py

@@ -251,10 +251,6 @@ class TextTranslator(nodes.NodeVisitor):
                                  for line in lines)
         # XXX header/footer?
 
-    def visit_highlightlang(self, node):
-        # type: (nodes.Node) -> None
-        raise nodes.SkipNode
-
     def visit_section(self, node):
         # type: (nodes.Node) -> None
         self._title_char = self.sectionchars[self.sectionlevel]

tests/roots/test-trim_doctest_flags/conf.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+master_doc = 'index'
+
+extensions = ['sphinx.ext.doctest']
+
+latex_documents = [
+    (master_doc, 'test.tex', 'The basic Sphinx documentation for testing', 'Sphinx', 'report')
+]

tests/roots/test-trim_doctest_flags/index.rst

@@ -0,0 +1,23 @@
+test-trim_doctest_flags
+=======================
+
+.. code-block:: pycon
+
+   >>> datetime.date.now()   # doctest: +FOO
+   datetime.date(2008, 1, 1)
+
+.. code-block:: none
+
+   >>> datetime.date.now()   # doctest: +BAR
+   datetime.date(2008, 1, 1)
+
+.. code-block:: guess
+
+   # vim: set filetype=pycon
+   >>> datetime.date.now()   # doctest: +BAZ
+   datetime.date(2008, 1, 1)
+
+.. testcode::
+
+   >>> datetime.date.now()   # doctest: +QUX
+   datetime.date(2008, 1, 1)

tests/test_highlighting.py

@@ -76,16 +76,6 @@ def test_set_formatter():
         PygmentsBridge.html_formatter = HtmlFormatter
 
 
-def test_trim_doctest_flags():
-    PygmentsBridge.html_formatter = MyFormatter
-    try:
-        bridge = PygmentsBridge('html', trim_doctest_flags=True)
-        ret = bridge.highlight_block('>>> 1+2 # doctest: SKIP\n3\n', 'pycon')
-        assert ret == '>>> 1+2 \n3\n'
-    finally:
-        PygmentsBridge.html_formatter = HtmlFormatter
-
-
 @mock.patch('sphinx.highlighting.logger')
 def test_default_highlight(logger):
     bridge = PygmentsBridge('html')

tests/test_transforms_post_transforms_code.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+"""
+    test_transforms_post_transforms_code
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import pytest
+
+
+@pytest.mark.sphinx('html', testroot='trim_doctest_flags')
+def test_trim_doctest_flags_html(app, status, warning):
+    app.build()
+
+    result = (app.outdir / 'index.html').text(encoding='utf8')
+    assert 'FOO' not in result
+    assert 'BAR' in result
+    assert 'BAZ' not in result
+    assert 'QUX' not in result
+
+
+@pytest.mark.sphinx('latex', testroot='trim_doctest_flags')
+def test_trim_doctest_flags_latex(app, status, warning):
+    app.build()
+
+    result = (app.outdir / 'test.tex').text(encoding='utf8')
+    assert 'FOO' not in result
+    assert 'BAR' in result
+    assert 'BAZ' not in result
+    assert 'QUX' not in result