From c6d840d95a8001bebcea23b8f1a2a3fb10745b15 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 1 Jun 2019 11:27:35 +0900 Subject: [PATCH 1/5] doc: Change wording for contents directive (refs: #6265) --- doc/usage/restructuredtext/directives.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/usage/restructuredtext/directives.rst b/doc/usage/restructuredtext/directives.rst index d00979255..21d01b6c0 100644 --- a/doc/usage/restructuredtext/directives.rst +++ b/doc/usage/restructuredtext/directives.rst @@ -36,8 +36,8 @@ tables of contents. The ``toctree`` directive is the central element. .. note:: - For local tables of contents, use the standard reST :dudir:`contents - directive `. + To create table of contents for current document (.rst file), use the + standard reST :dudir:`contents directive `. .. rst:directive:: toctree From e9c5e3656fc74c66eb31f897c9f485a10912a2c9 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 29 May 2019 01:26:18 +0900 Subject: [PATCH 2/5] highlight: Enable raiseonerror only if force=False --- sphinx/highlighting.py | 67 ++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py index dd602708a..fba7226c2 100644 --- a/sphinx/highlighting.py +++ b/sphinx/highlighting.py @@ -10,6 +10,7 @@ import html import warnings +from functools import partial from pygments import highlight from pygments.filters import ErrorToken @@ -37,17 +38,16 @@ if False: logger = logging.getLogger(__name__) -lexers = { - 'none': TextLexer(stripnl=False), - 'python': PythonLexer(stripnl=False), - 'python3': Python3Lexer(stripnl=False), - 'pycon': PythonConsoleLexer(stripnl=False), - 'pycon3': PythonConsoleLexer(python3=True, stripnl=False), - 'rest': RstLexer(stripnl=False), - 'c': CLexer(stripnl=False), +lexers = {} # type: Dict[str, Lexer] +lexer_classes = { + 'none': TextLexer, + 'python': PythonLexer, + 'python3': Python3Lexer, + 'pycon': PythonConsoleLexer, + 'pycon3': partial(PythonConsoleLexer, python3=True), + 'rest': RstLexer, + 'c': CLexer, } # type: Dict[str, Lexer] -for _lexer in lexers.values(): - _lexer.add_filter('raiseonerror') escape_hl_chars = {ord('\\'): '\\PYGZbs{}', @@ -115,46 +115,55 @@ class PygmentsBridge: return '\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n' + \ source + '\\end{Verbatim}\n' - def get_lexer(self, source, lang, opts=None, location=None): - # type: (str, str, Any, Any) -> Lexer + def get_lexer(self, source, lang, opts=None, force=False, location=None): + # type: (str, str, Dict, bool, Any) -> Lexer + lexer_options = {'strip': True} + lexer_options.update(opts) + # find out which lexer to use if lang in ('py', 'python'): if source.startswith('>>>'): # interactive session - lexer = lexers['pycon'] + lang = 'pycon' else: - lexer = lexers['python'] + lang = 'python' elif lang in ('py3', 'python3', 'default'): if source.startswith('>>>'): - lexer = lexers['pycon3'] + lang = 'pycon3' else: - lexer = lexers['python3'] + lang = 'python3' elif lang == 'guess': try: lexer = guess_lexer(source) except Exception: lexer = lexers['none'] + + if lang in lexers: + lexer = lexers[lang] + elif lang in lexer_classes: + lexer = lexer_classes[lang](**lexer_options) else: - if lang in lexers: - lexer = lexers[lang] - else: - try: - lexer = lexers[lang] = get_lexer_by_name(lang, **(opts or {})) - except ClassNotFound: - logger.warning(__('Pygments lexer name %r is not known'), lang, - location=location) - lexer = lexers['none'] + try: + if lang == 'guess': + lexer = guess_lexer(lang, **lexer_options) else: - lexer.add_filter('raiseonerror') + lexer = get_lexer_by_name(lang, **lexer_options) + except ClassNotFound: + logger.warning(__('Pygments lexer name %r is not known'), lang, + location=location) + lexer = lexer_classes['none'](**lexer_options) + + if not force: + lexer.add_filter('raiseonerror') return lexer - def highlight_block(self, source, lang, opts=None, location=None, force=False, **kwargs): - # type: (str, str, Any, Any, bool, Any) -> str + def highlight_block(self, source, lang, opts=None, force=False, location=None, **kwargs): + # type: (str, str, Dict, bool, Any, Any) -> str if not isinstance(source, str): source = source.decode() - lexer = self.get_lexer(source, lang, opts, location) + lexer = self.get_lexer(source, lang, opts, force, location) # trim doctest options if wanted if isinstance(lexer, PythonConsoleLexer) and self.trim_doctest_flags: From e001d358d0dd83f260ec696e2c14a65bd63cde13 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 30 May 2019 10:44:48 +0900 Subject: [PATCH 3/5] Add :force: option to code directives --- doc/usage/restructuredtext/directives.rst | 17 +++++++++++++ sphinx/directives/code.py | 12 +++++++--- sphinx/directives/patches.py | 4 ++-- sphinx/highlighting.py | 29 ++++++++++++----------- sphinx/transforms/post_transforms/code.py | 11 +++++---- sphinx/writers/html.py | 2 +- sphinx/writers/html5.py | 2 +- sphinx/writers/latex.py | 2 +- tests/roots/test-directive-code/error.inc | 1 + tests/roots/test-directive-code/force.rst | 16 +++++++++++++ tests/test_directive_code.py | 6 +++++ 11 files changed, 75 insertions(+), 27 deletions(-) create mode 100644 tests/roots/test-directive-code/error.inc create mode 100644 tests/roots/test-directive-code/force.rst diff --git a/doc/usage/restructuredtext/directives.rst b/doc/usage/restructuredtext/directives.rst index d00979255..1dd9078e8 100644 --- a/doc/usage/restructuredtext/directives.rst +++ b/doc/usage/restructuredtext/directives.rst @@ -463,6 +463,12 @@ __ http://pygments.org/docs/lexers/ This will produce line numbers for all code blocks longer than five lines. + To ignore minor errors on highlighting, you can specifiy ``:force:`` option. + + .. versionchanged:: 2.1 + + ``:force:`` option. + .. rst:directive:: .. code-block:: [language] Example:: @@ -525,6 +531,8 @@ __ http://pygments.org/docs/lexers/ some ruby code + A ``force`` option can ignore minor errors on highlighting. + .. versionchanged:: 1.1 The ``emphasize-lines`` option has been added. @@ -538,6 +546,10 @@ __ http://pygments.org/docs/lexers/ .. versionchanged:: 2.0 The ``language`` argument becomes optional. + .. versionchanged:: 2.1 + + ``:force:`` option has been added. + .. rst:directive:: .. literalinclude:: filename Longer displays of verbatim text may be included by storing the example text @@ -630,6 +642,8 @@ __ http://pygments.org/docs/lexers/ This shows the diff between ``example.py`` and ``example.py.orig`` with unified diff format. + A ``force`` option can ignore minor errors on highlighting. + .. versionchanged:: 0.4.3 Added the ``encoding`` option. @@ -651,6 +665,9 @@ __ http://pygments.org/docs/lexers/ With both ``start-after`` and ``lines`` in use, the first line as per ``start-after`` is considered to be with line number ``1`` for ``lines``. + .. versionchanged:: 2.1 + Added the ``force`` option. + .. _glossary-directive: Glossary diff --git a/sphinx/directives/code.py b/sphinx/directives/code.py index 75ab61796..ff21bcdab 100644 --- a/sphinx/directives/code.py +++ b/sphinx/directives/code.py @@ -41,6 +41,7 @@ class Highlight(SphinxDirective): optional_arguments = 0 final_argument_whitespace = False option_spec = { + 'force': directives.flag, 'linenothreshold': directives.positive_int, } @@ -48,9 +49,12 @@ class Highlight(SphinxDirective): # type: () -> List[nodes.Node] language = self.arguments[0].strip() linenothreshold = self.options.get('linenothreshold', sys.maxsize) + force = 'force' in self.options self.env.temp_data['highlight_language'] = language - return [addnodes.highlightlang(lang=language, linenothreshold=linenothreshold)] + return [addnodes.highlightlang(lang=language, + force=force, + linenothreshold=linenothreshold)] class HighlightLang(Highlight): @@ -115,6 +119,7 @@ class CodeBlock(SphinxDirective): optional_arguments = 1 final_argument_whitespace = False option_spec = { + 'force': directives.flag, 'linenos': directives.flag, 'dedent': int, 'lineno-start': int, @@ -156,17 +161,16 @@ class CodeBlock(SphinxDirective): if 'linenos' in self.options or 'lineno-start' in self.options: literal['linenos'] = True literal['classes'] += self.options.get('class', []) + literal['force'] = 'force' in self.options if self.arguments: # highlight language specified literal['language'] = self.arguments[0] - literal['force_highlighting'] = True else: # no highlight language specified. Then this directive refers the current # highlight setting via ``highlight`` directive or ``highlight_language`` # configuration. literal['language'] = self.env.temp_data.get('highlight_language', self.config.highlight_language) - literal['force_highlighting'] = False extra_args = literal['highlight_args'] = {} if hl_lines is not None: extra_args['hl_lines'] = hl_lines @@ -409,6 +413,7 @@ class LiteralInclude(SphinxDirective): 'lineno-match': directives.flag, 'tab-width': int, 'language': directives.unchanged_required, + 'force': directives.flag, 'encoding': directives.encoding, 'pyobject': directives.unchanged_required, 'lines': directives.unchanged_required, @@ -445,6 +450,7 @@ class LiteralInclude(SphinxDirective): text, lines = reader.read(location=location) retnode = nodes.literal_block(text, text, source=filename) # type: nodes.Element + retnode['force'] = 'force' in self.options self.set_source_info(retnode) if self.options.get('diff'): # if diff is set, set udiff retnode['language'] = 'udiff' diff --git a/sphinx/directives/patches.py b/sphinx/directives/patches.py index c102f3461..189e1f7e3 100644 --- a/sphinx/directives/patches.py +++ b/sphinx/directives/patches.py @@ -119,6 +119,7 @@ class Code(SphinxDirective): optional_arguments = 1 option_spec = { 'class': directives.class_option, + 'force': directives.flag, 'name': directives.unchanged, 'number-lines': optional_int, } @@ -131,6 +132,7 @@ class Code(SphinxDirective): code = '\n'.join(self.content) node = nodes.literal_block(code, code, classes=self.options.get('classes', []), + force='force' in self.options, highlight_args={}) self.add_name(node) set_source_info(self, node) @@ -138,14 +140,12 @@ class Code(SphinxDirective): if self.arguments: # highlight language specified node['language'] = self.arguments[0] - node['force_highlighting'] = True else: # no highlight language specified. Then this directive refers the current # highlight setting via ``highlight`` directive or ``highlight_language`` # configuration. node['language'] = self.env.temp_data.get('highlight_language', self.config.highlight_language) - node['force_highlighting'] = False if 'number-lines' in self.options: node['linenos'] = True diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py index fba7226c2..2d825e9f1 100644 --- a/sphinx/highlighting.py +++ b/sphinx/highlighting.py @@ -40,13 +40,13 @@ logger = logging.getLogger(__name__) lexers = {} # type: Dict[str, Lexer] lexer_classes = { - 'none': TextLexer, - 'python': PythonLexer, - 'python3': Python3Lexer, - 'pycon': PythonConsoleLexer, - 'pycon3': partial(PythonConsoleLexer, python3=True), - 'rest': RstLexer, - 'c': CLexer, + 'none': partial(TextLexer, stripnl=False), + 'python': partial(PythonLexer, stripnl=False), + 'python3': partial(Python3Lexer, stripnl=False), + 'pycon': partial(PythonConsoleLexer, stripnl=False), + 'pycon3': partial(PythonConsoleLexer, python3=True, stripnl=False), + 'rest': partial(RstLexer, stripnl=False), + 'c': partial(CLexer, stripnl=False), } # type: Dict[str, Lexer] @@ -117,8 +117,8 @@ class PygmentsBridge: def get_lexer(self, source, lang, opts=None, force=False, location=None): # type: (str, str, Dict, bool, Any) -> Lexer - lexer_options = {'strip': True} - lexer_options.update(opts) + if not opts: + opts = {} # find out which lexer to use if lang in ('py', 'python'): @@ -141,17 +141,17 @@ class PygmentsBridge: if lang in lexers: lexer = lexers[lang] elif lang in lexer_classes: - lexer = lexer_classes[lang](**lexer_options) + lexer = lexer_classes[lang](**opts) else: try: if lang == 'guess': - lexer = guess_lexer(lang, **lexer_options) + lexer = guess_lexer(lang, **opts) else: - lexer = get_lexer_by_name(lang, **lexer_options) + lexer = get_lexer_by_name(lang, **opts) except ClassNotFound: logger.warning(__('Pygments lexer name %r is not known'), lang, location=location) - lexer = lexer_classes['none'](**lexer_options) + lexer = lexer_classes['none'](**opts) if not force: lexer.add_filter('raiseonerror') @@ -184,7 +184,8 @@ class PygmentsBridge: 'Highlighting skipped.'), lang, type='misc', subtype='highlighting_failure', location=location) - hlsource = highlight(source, lexers['none'], formatter) + lexer = self.get_lexer(source, 'none', opts, force, location) + hlsource = highlight(source, lexer, formatter) if self.dest == 'html': return hlsource diff --git a/sphinx/transforms/post_transforms/code.py b/sphinx/transforms/post_transforms/code.py index fb8533e9c..a42de4bf1 100644 --- a/sphinx/transforms/post_transforms/code.py +++ b/sphinx/transforms/post_transforms/code.py @@ -25,6 +25,7 @@ if False: HighlightSetting = NamedTuple('HighlightSetting', [('language', str), + ('force', bool), ('lineno_threshold', int)]) @@ -51,7 +52,7 @@ class HighlightLanguageTransform(SphinxTransform): class HighlightLanguageVisitor(nodes.NodeVisitor): def __init__(self, document, default_language): # type: (nodes.document, str) -> None - self.default_setting = HighlightSetting(default_language, sys.maxsize) + self.default_setting = HighlightSetting(default_language, False, sys.maxsize) self.settings = [] # type: List[HighlightSetting] super().__init__(document) @@ -81,16 +82,16 @@ class HighlightLanguageVisitor(nodes.NodeVisitor): def visit_highlightlang(self, node): # type: (addnodes.highlightlang) -> None - self.settings[-1] = HighlightSetting(node['lang'], node['linenothreshold']) + self.settings[-1] = HighlightSetting(node['lang'], + node['force'], + node['linenothreshold']) def visit_literal_block(self, node): # type: (nodes.literal_block) -> None setting = self.settings[-1] if 'language' not in node: node['language'] = setting.language - node['force_highlighting'] = False - elif 'force_highlighting' not in node: - node['force_highlighting'] = True + node['force'] = setting.force if 'linenos' not in node: lines = node.astext().count('\n') node['linenos'] = (lines >= setting.lineno_threshold - 1) diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py index 3eb9a0911..381457926 100644 --- a/sphinx/writers/html.py +++ b/sphinx/writers/html.py @@ -476,7 +476,7 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator): lang = node.get('language', 'default') linenos = node.get('linenos', False) highlight_args = node.get('highlight_args', {}) - highlight_args['force'] = node.get('force_highlighting', False) + highlight_args['force'] = node.get('force', False) if lang is self.builder.config.highlight_language: # only pass highlighter options for original language opts = self.builder.config.highlight_options diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py index dc0895f8a..e412ea167 100644 --- a/sphinx/writers/html5.py +++ b/sphinx/writers/html5.py @@ -423,7 +423,7 @@ class HTML5Translator(SphinxTranslator, BaseTranslator): lang = node.get('language', 'default') linenos = node.get('linenos', False) highlight_args = node.get('highlight_args', {}) - highlight_args['force'] = node.get('force_highlighting', False) + highlight_args['force'] = node.get('force', False) if lang is self.builder.config.highlight_language: # only pass highlighter options for original language opts = self.builder.config.highlight_options diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 899df04c9..9afb1f0c7 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -2139,7 +2139,7 @@ class LaTeXTranslator(SphinxTranslator): lang = node.get('language', 'default') linenos = node.get('linenos', False) highlight_args = node.get('highlight_args', {}) - highlight_args['force'] = node.get('force_highlighting', False) + highlight_args['force'] = node.get('force', False) if lang is self.builder.config.highlight_language: # only pass highlighter options for original language opts = self.builder.config.highlight_options diff --git a/tests/roots/test-directive-code/error.inc b/tests/roots/test-directive-code/error.inc new file mode 100644 index 000000000..472828080 --- /dev/null +++ b/tests/roots/test-directive-code/error.inc @@ -0,0 +1 @@ +not a python script! diff --git a/tests/roots/test-directive-code/force.rst b/tests/roots/test-directive-code/force.rst new file mode 100644 index 000000000..1834b3ac6 --- /dev/null +++ b/tests/roots/test-directive-code/force.rst @@ -0,0 +1,16 @@ +force option +============ + +.. code:: python + :force: + + not a python script! + +.. code-block:: python + :force: + + not a python script! + +.. literalinclude:: error.inc + :language: python + :force: diff --git a/tests/test_directive_code.py b/tests/test_directive_code.py index 2e1a1fde2..d1f7c03dc 100644 --- a/tests/test_directive_code.py +++ b/tests/test_directive_code.py @@ -307,6 +307,12 @@ def test_code_block(app, status, warning): assert actual == expect +@pytest.mark.sphinx('html', testroot='directive-code') +def test_force_option(app, status, warning): + app.builder.build(['force']) + assert 'force.rst' not in warning.getvalue() + + @pytest.mark.sphinx('html', testroot='directive-code') def test_code_block_caption_html(app, status, warning): app.builder.build(['caption']) From 5d8c25b60507056e0a9227c3249f937e7d8e4098 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 1 Jun 2019 12:03:38 +0900 Subject: [PATCH 4/5] refactor: app.add_lexer() --- sphinx/application.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sphinx/application.py b/sphinx/application.py index 96dd3f936..8a4c6c35d 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -30,6 +30,7 @@ from sphinx.deprecation import ( from sphinx.environment import BuildEnvironment from sphinx.errors import ApplicationError, ConfigError, VersionRequirementError from sphinx.events import EventManager +from sphinx.highlighting import lexers from sphinx.locale import __ from sphinx.project import Project from sphinx.registry import SphinxComponentRegistry @@ -50,6 +51,7 @@ if False: from docutils import nodes # NOQA from docutils.parsers import Parser # NOQA from docutils.transforms import Transform # NOQA + from pygments.lexer import Lexer # NOQA from sphinx.builders import Builder # NOQA from sphinx.domains import Domain, Index # NOQA from sphinx.environment.collectors import EnvironmentCollector # NOQA @@ -1033,7 +1035,7 @@ class Sphinx: self.registry.add_latex_package(packagename, options) def add_lexer(self, alias, lexer): - # type: (str, Any) -> None + # type: (str, Lexer) -> None """Register a new lexer for source code. Use *lexer*, which must be an instance of a Pygments lexer class, to @@ -1042,9 +1044,6 @@ class Sphinx: .. versionadded:: 0.6 """ logger.debug('[app] adding lexer: %r', (alias, lexer)) - from sphinx.highlighting import lexers - if lexers is None: - return lexers[alias] = lexer def add_autodocumenter(self, cls): From 71842264b238b31dedab77200eeaab72140d3433 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 1 Jun 2019 12:45:08 +0900 Subject: [PATCH 5/5] Sphinx.add_lexer() now takes a Lexer class instead of instance --- CHANGES | 2 ++ sphinx/application.py | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index fc3c7eb66..2363ea759 100644 --- a/CHANGES +++ b/CHANGES @@ -13,6 +13,8 @@ Incompatible changes by non-ASCII characters * #4550: html: Centering tables by default using CSS * #6239: latex: xelatex and xeCJK are used for Chinese documents by default +* ``Sphinx.add_lexer()`` now takes a Lexer class instead of instance. An + instance of lexers are still supported until Sphinx-3.x. Deprecated ---------- diff --git a/sphinx/application.py b/sphinx/application.py index 8a4c6c35d..96471be0c 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -20,6 +20,7 @@ from io import StringIO from os import path from docutils.parsers.rst import Directive, roles +from pygments.lexer import Lexer import sphinx from sphinx import package_dir, locale @@ -30,7 +31,7 @@ from sphinx.deprecation import ( from sphinx.environment import BuildEnvironment from sphinx.errors import ApplicationError, ConfigError, VersionRequirementError from sphinx.events import EventManager -from sphinx.highlighting import lexers +from sphinx.highlighting import lexer_classes, lexers from sphinx.locale import __ from sphinx.project import Project from sphinx.registry import SphinxComponentRegistry @@ -51,7 +52,6 @@ if False: from docutils import nodes # NOQA from docutils.parsers import Parser # NOQA from docutils.transforms import Transform # NOQA - from pygments.lexer import Lexer # NOQA from sphinx.builders import Builder # NOQA from sphinx.domains import Domain, Index # NOQA from sphinx.environment.collectors import EnvironmentCollector # NOQA @@ -1035,16 +1035,24 @@ class Sphinx: self.registry.add_latex_package(packagename, options) def add_lexer(self, alias, lexer): - # type: (str, Lexer) -> None + # type: (str, Union[Lexer, Type[Lexer]]) -> None """Register a new lexer for source code. - Use *lexer*, which must be an instance of a Pygments lexer class, to - highlight code blocks with the given language *alias*. + Use *lexer* to highlight code blocks with the given language *alias*. .. versionadded:: 0.6 + .. versionchanged:: 2.1 + Take a lexer class as an argument. An instance of lexers are + still supported until Sphinx-3.x. """ logger.debug('[app] adding lexer: %r', (alias, lexer)) - lexers[alias] = lexer + if isinstance(lexer, Lexer): + warnings.warn('app.add_lexer() API changed; ' + 'Please give lexer class instead instance', + RemovedInSphinx40Warning) + lexers[alias] = lexer + else: + lexer_classes[alias] = lexer def add_autodocumenter(self, cls): # type: (Any) -> None