Merge commit '54d5fcfaebc3364044761d30c0fed6bd4d3052c3'

CHANGES

@@ -36,6 +36,8 @@ Incompatible changes
   by non-ASCII characters
 * #4550: html: Centering tables by default using CSS
 * #6239: latex: xelatex and xeCJK are used for Chinese documents by default
+* ``Sphinx.add_lexer()`` now takes a Lexer class instead of instance.  An
+  instance of lexers are still supported until Sphinx-3.x.
 
 Deprecated
 ----------

doc/usage/restructuredtext/directives.rst

@@ -36,8 +36,8 @@ tables of contents.  The ``toctree`` directive is the central element.
 
 .. note::
 
-   For local tables of contents, use the standard reST :dudir:`contents
+   To create table of contents for current document (.rst file), use the
-   directive <table-of-contents>`.
+   standard reST :dudir:`contents directive <table-of-contents>`.
 
 .. rst:directive:: toctree
 
@@ -463,6 +463,12 @@ __ http://pygments.org/docs/lexers/
 
    This will produce line numbers for all code blocks longer than five lines.
 
+   To ignore minor errors on highlighting, you can specifiy ``:force:`` option.
+
+   .. versionchanged:: 2.1
+
+      ``:force:`` option.
+
 .. rst:directive:: .. code-block:: [language]
 
    Example::
@@ -525,6 +531,8 @@ __ http://pygments.org/docs/lexers/
 
              some ruby code
 
+   A ``force`` option can ignore minor errors on highlighting.
+
    .. versionchanged:: 1.1
       The ``emphasize-lines`` option has been added.
 
@@ -538,6 +546,10 @@ __ http://pygments.org/docs/lexers/
    .. versionchanged:: 2.0
       The ``language`` argument becomes optional.
 
+   .. versionchanged:: 2.1
+
+      ``:force:`` option has been added.
+
 .. rst:directive:: .. literalinclude:: filename
 
    Longer displays of verbatim text may be included by storing the example text
@@ -630,6 +642,8 @@ __ http://pygments.org/docs/lexers/
    This shows the diff between ``example.py`` and ``example.py.orig`` with
    unified diff format.
 
+   A ``force`` option can ignore minor errors on highlighting.
+
    .. versionchanged:: 0.4.3
       Added the ``encoding`` option.
 
@@ -651,6 +665,9 @@ __ http://pygments.org/docs/lexers/
       With both ``start-after`` and ``lines`` in use, the first line as per
       ``start-after`` is considered to be with line number ``1`` for ``lines``.
 
+   .. versionchanged:: 2.1
+      Added the ``force`` option.
+
 .. _glossary-directive:
 
 Glossary

sphinx/application.py

@@ -19,6 +19,7 @@ from io import StringIO
 from os import path
 
 from docutils.parsers.rst import Directive, roles
+from pygments.lexer import Lexer
 
 import sphinx
 from sphinx import package_dir, locale
@@ -27,6 +28,7 @@ from sphinx.deprecation import RemovedInSphinx40Warning
 from sphinx.environment import BuildEnvironment
 from sphinx.errors import ApplicationError, ConfigError, VersionRequirementError
 from sphinx.events import EventManager
+from sphinx.highlighting import lexer_classes, lexers
 from sphinx.locale import __
 from sphinx.project import Project
 from sphinx.registry import SphinxComponentRegistry
@@ -969,19 +971,24 @@ class Sphinx:
         self.registry.add_latex_package(packagename, options)
 
     def add_lexer(self, alias, lexer):
-        # type: (str, Any) -> None
+        # type: (str, Union[Lexer, Type[Lexer]]) -> None
         """Register a new lexer for source code.
 
-        Use *lexer*, which must be an instance of a Pygments lexer class, to
+        Use *lexer* to highlight code blocks with the given language *alias*.
-        highlight code blocks with the given language *alias*.
 
         .. versionadded:: 0.6
+        .. versionchanged:: 2.1
+           Take a lexer class as an argument.  An instance of lexers are
+           still supported until Sphinx-3.x.
         """
         logger.debug('[app] adding lexer: %r', (alias, lexer))
-        from sphinx.highlighting import lexers
+        if isinstance(lexer, Lexer):
-        if lexers is None:
+            warnings.warn('app.add_lexer() API changed; '
-            return
+                          'Please give lexer class instead instance',
-        lexers[alias] = lexer
+                          RemovedInSphinx40Warning)
+            lexers[alias] = lexer
+        else:
+            lexer_classes[alias] = lexer
 
     def add_autodocumenter(self, cls):
         # type: (Any) -> None

sphinx/directives/code.py

@@ -41,6 +41,7 @@ class Highlight(SphinxDirective):
     optional_arguments = 0
     final_argument_whitespace = False
     option_spec = {
+        'force': directives.flag,
         'linenothreshold': directives.positive_int,
     }
 
@@ -48,9 +49,12 @@ class Highlight(SphinxDirective):
         # type: () -> List[nodes.Node]
         language = self.arguments[0].strip()
         linenothreshold = self.options.get('linenothreshold', sys.maxsize)
+        force = 'force' in self.options
 
         self.env.temp_data['highlight_language'] = language
-        return [addnodes.highlightlang(lang=language, linenothreshold=linenothreshold)]
+        return [addnodes.highlightlang(lang=language,
+                                       force=force,
+                                       linenothreshold=linenothreshold)]
 
 
 class HighlightLang(Highlight):
@@ -115,6 +119,7 @@ class CodeBlock(SphinxDirective):
     optional_arguments = 1
     final_argument_whitespace = False
     option_spec = {
+        'force': directives.flag,
         'linenos': directives.flag,
         'dedent': int,
         'lineno-start': int,
@@ -156,17 +161,16 @@ class CodeBlock(SphinxDirective):
         if 'linenos' in self.options or 'lineno-start' in self.options:
             literal['linenos'] = True
         literal['classes'] += self.options.get('class', [])
+        literal['force'] = 'force' in self.options
         if self.arguments:
             # highlight language specified
             literal['language'] = self.arguments[0]
-            literal['force_highlighting'] = True
         else:
             # no highlight language specified.  Then this directive refers the current
             # highlight setting via ``highlight`` directive or ``highlight_language``
             # configuration.
             literal['language'] = self.env.temp_data.get('highlight_language',
                                                          self.config.highlight_language)
-            literal['force_highlighting'] = False
         extra_args = literal['highlight_args'] = {}
         if hl_lines is not None:
             extra_args['hl_lines'] = hl_lines
@@ -409,6 +413,7 @@ class LiteralInclude(SphinxDirective):
         'lineno-match': directives.flag,
         'tab-width': int,
         'language': directives.unchanged_required,
+        'force': directives.flag,
         'encoding': directives.encoding,
         'pyobject': directives.unchanged_required,
         'lines': directives.unchanged_required,
@@ -445,6 +450,7 @@ class LiteralInclude(SphinxDirective):
             text, lines = reader.read(location=location)
 
             retnode = nodes.literal_block(text, text, source=filename)  # type: nodes.Element
+            retnode['force'] = 'force' in self.options
             self.set_source_info(retnode)
             if self.options.get('diff'):  # if diff is set, set udiff
                 retnode['language'] = 'udiff'

sphinx/directives/patches.py

@@ -119,6 +119,7 @@ class Code(SphinxDirective):
     optional_arguments = 1
     option_spec = {
         'class': directives.class_option,
+        'force': directives.flag,
         'name': directives.unchanged,
         'number-lines': optional_int,
     }
@@ -131,6 +132,7 @@ class Code(SphinxDirective):
         code = '\n'.join(self.content)
         node = nodes.literal_block(code, code,
                                    classes=self.options.get('classes', []),
+                                   force='force' in self.options,
                                    highlight_args={})
         self.add_name(node)
         set_source_info(self, node)
@@ -138,14 +140,12 @@ class Code(SphinxDirective):
         if self.arguments:
             # highlight language specified
             node['language'] = self.arguments[0]
-            node['force_highlighting'] = True
         else:
             # no highlight language specified.  Then this directive refers the current
             # highlight setting via ``highlight`` directive or ``highlight_language``
             # configuration.
             node['language'] = self.env.temp_data.get('highlight_language',
                                                       self.config.highlight_language)
-            node['force_highlighting'] = False
 
         if 'number-lines' in self.options:
             node['linenos'] = True

sphinx/highlighting.py

@@ -8,6 +8,8 @@
     :license: BSD, see LICENSE for details.
 """
 
+from functools import partial
+
 from pygments import highlight
 from pygments.filters import ErrorToken
 from pygments.formatters import HtmlFormatter, LatexFormatter
@@ -32,17 +34,16 @@ if False:
 
 logger = logging.getLogger(__name__)
 
-lexers = {
+lexers = {}  # type: Dict[str, Lexer]
-    'none': TextLexer(stripnl=False),
+lexer_classes = {
-    'python': PythonLexer(stripnl=False),
+    'none': partial(TextLexer, stripnl=False),
-    'python3': Python3Lexer(stripnl=False),
+    'python': partial(PythonLexer, stripnl=False),
-    'pycon': PythonConsoleLexer(stripnl=False),
+    'python3': partial(Python3Lexer, stripnl=False),
-    'pycon3': PythonConsoleLexer(python3=True, stripnl=False),
+    'pycon': partial(PythonConsoleLexer, stripnl=False),
-    'rest': RstLexer(stripnl=False),
+    'pycon3': partial(PythonConsoleLexer, python3=True, stripnl=False),
-    'c': CLexer(stripnl=False),
+    'rest': partial(RstLexer, stripnl=False),
+    'c': partial(CLexer, stripnl=False),
 }  # type: Dict[str, Lexer]
-for _lexer in lexers.values():
-    _lexer.add_filter('raiseonerror')
 
 
 escape_hl_chars = {ord('\\'): '\\PYGZbs{}',
@@ -91,46 +92,55 @@ class PygmentsBridge:
         kwargs.update(self.formatter_args)
         return self.formatter(**kwargs)
 
-    def get_lexer(self, source, lang, opts=None, location=None):
+    def get_lexer(self, source, lang, opts=None, force=False, location=None):
-        # type: (str, str, Any, Any) -> Lexer
+        # type: (str, str, Dict, bool, Any) -> Lexer
+        if not opts:
+            opts = {}
+
         # find out which lexer to use
         if lang in ('py', 'python'):
             if source.startswith('>>>'):
                 # interactive session
-                lexer = lexers['pycon']
+                lang = 'pycon'
             else:
-                lexer = lexers['python']
+                lang = 'python'
         elif lang in ('py3', 'python3', 'default'):
             if source.startswith('>>>'):
-                lexer = lexers['pycon3']
+                lang = 'pycon3'
             else:
-                lexer = lexers['python3']
+                lang = 'python3'
         elif lang == 'guess':
             try:
                 lexer = guess_lexer(source)
             except Exception:
                 lexer = lexers['none']
+
+        if lang in lexers:
+            lexer = lexers[lang]
+        elif lang in lexer_classes:
+            lexer = lexer_classes[lang](**opts)
         else:
-            if lang in lexers:
+            try:
-                lexer = lexers[lang]
+                if lang == 'guess':
-            else:
+                    lexer = guess_lexer(lang, **opts)
-                try:
-                    lexer = lexers[lang] = get_lexer_by_name(lang, **(opts or {}))
-                except ClassNotFound:
-                    logger.warning(__('Pygments lexer name %r is not known'), lang,
-                                   location=location)
-                    lexer = lexers['none']
                 else:
-                    lexer.add_filter('raiseonerror')
+                    lexer = get_lexer_by_name(lang, **opts)
+            except ClassNotFound:
+                logger.warning(__('Pygments lexer name %r is not known'), lang,
+                               location=location)
+                lexer = lexer_classes['none'](**opts)
+
+        if not force:
+            lexer.add_filter('raiseonerror')
 
         return lexer
 
-    def highlight_block(self, source, lang, opts=None, location=None, force=False, **kwargs):
+    def highlight_block(self, source, lang, opts=None, force=False, location=None, **kwargs):
-        # type: (str, str, Any, Any, bool, Any) -> str
+        # type: (str, str, Dict, bool, Any, Any) -> str
         if not isinstance(source, str):
             source = source.decode()
 
-        lexer = self.get_lexer(source, lang, opts, location)
+        lexer = self.get_lexer(source, lang, opts, force, location)
 
         # highlight via Pygments
         formatter = self.get_formatter(**kwargs)
@@ -146,7 +156,8 @@ class PygmentsBridge:
                                   'Highlighting skipped.'), lang,
                                type='misc', subtype='highlighting_failure',
                                location=location)
-            hlsource = highlight(source, lexers['none'], formatter)
+            lexer = self.get_lexer(source, 'none', opts, force, location)
+            hlsource = highlight(source, lexer, formatter)
 
         if self.dest == 'html':
             return hlsource

sphinx/transforms/post_transforms/code.py

@@ -25,6 +25,7 @@ if False:
 
 
 HighlightSetting = NamedTuple('HighlightSetting', [('language', str),
+                                                   ('force', bool),
                                                    ('lineno_threshold', int)])
 
 
@@ -51,7 +52,7 @@ class HighlightLanguageTransform(SphinxTransform):
 class HighlightLanguageVisitor(nodes.NodeVisitor):
     def __init__(self, document, default_language):
         # type: (nodes.document, str) -> None
-        self.default_setting = HighlightSetting(default_language, sys.maxsize)
+        self.default_setting = HighlightSetting(default_language, False, sys.maxsize)
         self.settings = []  # type: List[HighlightSetting]
         super().__init__(document)
 
@@ -81,16 +82,16 @@ class HighlightLanguageVisitor(nodes.NodeVisitor):
 
     def visit_highlightlang(self, node):
         # type: (addnodes.highlightlang) -> None
-        self.settings[-1] = HighlightSetting(node['lang'], node['linenothreshold'])
+        self.settings[-1] = HighlightSetting(node['lang'],
+                                             node['force'],
+                                             node['linenothreshold'])
 
     def visit_literal_block(self, node):
         # type: (nodes.literal_block) -> None
         setting = self.settings[-1]
         if 'language' not in node:
             node['language'] = setting.language
-            node['force_highlighting'] = False
+            node['force'] = setting.force
-        elif 'force_highlighting' not in node:
-            node['force_highlighting'] = True
         if 'linenos' not in node:
             lines = node.astext().count('\n')
             node['linenos'] = (lines >= setting.lineno_threshold - 1)

sphinx/writers/html.py

@@ -475,7 +475,7 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator):
         lang = node.get('language', 'default')
         linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        highlight_args['force'] = node.get('force_highlighting', False)
+        highlight_args['force'] = node.get('force', False)
         if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
             opts = self.builder.config.highlight_options

sphinx/writers/html5.py

@@ -422,7 +422,7 @@ class HTML5Translator(SphinxTranslator, BaseTranslator):
         lang = node.get('language', 'default')
         linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        highlight_args['force'] = node.get('force_highlighting', False)
+        highlight_args['force'] = node.get('force', False)
         if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
             opts = self.builder.config.highlight_options

sphinx/writers/latex.py

@@ -2097,7 +2097,7 @@ class LaTeXTranslator(SphinxTranslator):
             lang = node.get('language', 'default')
             linenos = node.get('linenos', False)
             highlight_args = node.get('highlight_args', {})
-            highlight_args['force'] = node.get('force_highlighting', False)
+            highlight_args['force'] = node.get('force', False)
             if lang is self.builder.config.highlight_language:
                 # only pass highlighter options for original language
                 opts = self.builder.config.highlight_options

tests/roots/test-directive-code/error.inc

@@ -0,0 +1 @@
+not a python script!

tests/roots/test-directive-code/force.rst

@@ -0,0 +1,16 @@
+force option
+============
+
+.. code:: python
+   :force:
+
+   not a python script!
+
+.. code-block:: python
+   :force:
+
+   not a python script!
+
+.. literalinclude:: error.inc
+   :language: python
+   :force:

tests/test_directive_code.py

@@ -307,6 +307,12 @@ def test_code_block(app, status, warning):
     assert actual == expect
 
 
+@pytest.mark.sphinx('html', testroot='directive-code')
+def test_force_option(app, status, warning):
+    app.builder.build(['force'])
+    assert 'force.rst' not in warning.getvalue()
+
+
 @pytest.mark.sphinx('html', testroot='directive-code')
 def test_code_block_caption_html(app, status, warning):
     app.builder.build(['caption'])