From dc45877d3cb9f67348d3e2857bc489e16b71f61a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 30 Oct 2017 13:46:12 +0900 Subject: [PATCH 01/10] Deprecate config value: source_parsers --- CHANGES | 3 +++ doc/config.rst | 4 ++++ sphinx/application.py | 3 +-- sphinx/deprecation.py | 4 ++++ sphinx/registry.py | 5 +---- sphinx/util/compat.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 sphinx/util/compat.py diff --git a/CHANGES b/CHANGES index 3a5131e7b..695430673 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,9 @@ Incompatible changes Deprecated ---------- +* :confval:`source_parsers` is deprecated. Please use ``add_source_parser()`` + instead. + Features added -------------- diff --git a/doc/config.rst b/doc/config.rst index 830fb69a0..5af846ee7 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -123,6 +123,10 @@ General configuration .. versionadded:: 1.3 + .. deprecated:: 1.8 + Now Sphinx provides an API :meth:`Sphinx.add_source_parser` to register + a source parser. Please use it instead. + .. confval:: master_doc The document name of the "master" document, that is, the document that diff --git a/sphinx/application.py b/sphinx/application.py index db4122b16..8a5d2f60d 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -92,6 +92,7 @@ builtin_extensions = ( 'sphinx.roles', 'sphinx.transforms.post_transforms', 'sphinx.transforms.post_transforms.images', + 'sphinx.util.compat', # collectors should be loaded by specific order 'sphinx.environment.collectors.dependencies', 'sphinx.environment.collectors.asset', @@ -287,8 +288,6 @@ class Sphinx(object): def _init_source_parsers(self): # type: () -> None - for suffix, parser in iteritems(self.config.source_parsers): - self.add_source_parser(suffix, parser) for suffix, parser in iteritems(self.registry.get_source_parsers()): if suffix not in self.config.source_suffix and suffix != '*': self.config.source_suffix.append(suffix) diff --git a/sphinx/deprecation.py b/sphinx/deprecation.py index e28e0f916..cd6e1ae2a 100644 --- a/sphinx/deprecation.py +++ b/sphinx/deprecation.py @@ -22,4 +22,8 @@ class RemovedInSphinx20Warning(PendingDeprecationWarning): pass +class RemovedInSphinx30Warning(PendingDeprecationWarning): + pass + + RemovedInNextVersionWarning = RemovedInSphinx18Warning diff --git a/sphinx/registry.py b/sphinx/registry.py index e48c12f96..3d7784d47 100644 --- a/sphinx/registry.py +++ b/sphinx/registry.py @@ -13,7 +13,7 @@ from __future__ import print_function import traceback from pkg_resources import iter_entry_points -from six import iteritems, itervalues, string_types +from six import iteritems, itervalues from sphinx.errors import ExtensionError, SphinxError, VersionRequirementError from sphinx.extension import Extension @@ -23,7 +23,6 @@ from sphinx.locale import __ from sphinx.parsers import Parser as SphinxParser from sphinx.roles import XRefRole from sphinx.util import logging -from sphinx.util import import_object from sphinx.util.console import bold # type: ignore from sphinx.util.docutils import directive_helper @@ -216,8 +215,6 @@ class SphinxComponentRegistry(object): if parser_class is None: raise SphinxError(__('Source parser for %s not registered') % filename) else: - if isinstance(parser_class, string_types): - parser_class = import_object(parser_class, 'source parser') # type: ignore return parser_class def get_source_parsers(self): diff --git a/sphinx/util/compat.py b/sphinx/util/compat.py new file mode 100644 index 000000000..edd8cac61 --- /dev/null +++ b/sphinx/util/compat.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +""" + sphinx.util.compat + ~~~~~~~~~~~~~~~~~~ + + modules for backward compatibility + + :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import warnings + +from six import string_types, iteritems + +from sphinx.deprecation import RemovedInSphinx30Warning +from sphinx.util import import_object + +if False: + # For type annotation + from typing import Any, Dict # NOQA + from sphinx.application import Sphinx # NOQA + from sphinx.config import Config # NOQA + + +def deprecate_source_parsers(app, config): + # type: (Sphinx, Config) -> None + if config.source_parsers: + warnings.warn('The config variable "source_parsers" is deprecated. ' + 'Please use app.add_source_parser() API instead.', + RemovedInSphinx30Warning) + for suffix, parser in iteritems(config.source_parsers): + if isinstance(parser, string_types): + parser = import_object(parser, 'source parser') # type: ignore + app.add_source_parser(suffix, parser) + + +def setup(app): + # type: (Sphinx) -> Dict[unicode, Any] + app.connect('config-inited', deprecate_source_parsers) + + return { + 'version': 'builtin', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } From bece0484e5e21e9488f248eb24d17fa38c4c6cc1 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 22 Jan 2018 11:38:24 +0900 Subject: [PATCH 02/10] source_parsers should fill Parser.supported attribute to let Sphinx know supported formats --- CHANGES | 3 +++ sphinx/registry.py | 8 ++++++++ tests/roots/test-prolog/prolog_markdown_parser.py | 2 ++ tests/roots/test-root/conf.py | 4 +++- tests/roots/test-root/parsermod.py | 2 ++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 695430673..9ef1102b2 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,9 @@ Dependencies Incompatible changes -------------------- +* Sphinx expects source parser modules to have supported file formats as + ``Parser.supported`` attribute + Deprecated ---------- diff --git a/sphinx/registry.py b/sphinx/registry.py index 3d7784d47..550df3360 100644 --- a/sphinx/registry.py +++ b/sphinx/registry.py @@ -11,12 +11,14 @@ from __future__ import print_function import traceback +import warnings from pkg_resources import iter_entry_points from six import iteritems, itervalues from sphinx.errors import ExtensionError, SphinxError, VersionRequirementError from sphinx.extension import Extension +from sphinx.deprecation import RemovedInSphinx30Warning from sphinx.domains import ObjType from sphinx.domains.std import GenericObject, Target from sphinx.locale import __ @@ -201,6 +203,12 @@ class SphinxComponentRegistry(object): logger.debug('[app] adding search source_parser: %r, %r', suffix, parser) if suffix in self.source_parsers: raise ExtensionError(__('source_parser for %r is already registered') % suffix) + + if len(parser.supported) == 0: + warnings.warn('Old source_parser has been detected. Please fill Parser.supported ' + 'attribute: %s' % parser.__name__, + RemovedInSphinx30Warning) + self.source_parsers[suffix] = parser def get_source_parser(self, filename): diff --git a/tests/roots/test-prolog/prolog_markdown_parser.py b/tests/roots/test-prolog/prolog_markdown_parser.py index f28c37b4e..6053da7be 100644 --- a/tests/roots/test-prolog/prolog_markdown_parser.py +++ b/tests/roots/test-prolog/prolog_markdown_parser.py @@ -4,6 +4,8 @@ from docutils.parsers import Parser class DummyMarkdownParser(Parser): + supported = ('markdown',) + def parse(self, inputstring, document): document.rawsource = inputstring diff --git a/tests/roots/test-root/conf.py b/tests/roots/test-root/conf.py index 04cd87d7b..0f5c20c8e 100644 --- a/tests/roots/test-root/conf.py +++ b/tests/roots/test-root/conf.py @@ -13,7 +13,6 @@ templates_path = ['_templates'] master_doc = 'contents' source_suffix = ['.txt', '.add', '.foo'] -source_parsers = {'.foo': 'parsermod.Parser'} project = 'Sphinx ' copyright = '2010-2016, Georg Brandl & Team' @@ -106,9 +105,12 @@ class ClassDirective(Directive): def setup(app): + import parsermod + app.add_config_value('value_from_conf_py', 42, False) app.add_directive('funcdir', functional_directive, opt=lambda x: x) app.add_directive('clsdir', ClassDirective) app.add_object_type('userdesc', 'userdescrole', '%s (userdesc)', userdesc_parse, objname='user desc') app.add_javascript('file://moo.js') + app.add_source_parser('.foo', parsermod.Parser) diff --git a/tests/roots/test-root/parsermod.py b/tests/roots/test-root/parsermod.py index 3e5330ac8..f98d82f3e 100644 --- a/tests/roots/test-root/parsermod.py +++ b/tests/roots/test-root/parsermod.py @@ -3,6 +3,8 @@ from docutils import nodes class Parser(Parser): + supported = ('foo',) + def parse(self, input, document): section = nodes.section(ids=['id1']) section += nodes.title('Generated section', 'Generated section') From 69f69628ed8e97f9e3caed3704aea2106477a2d8 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 22 Jan 2018 12:33:55 +0900 Subject: [PATCH 03/10] :confval:`source_suffix` allows a mapping fileext to file types --- CHANGES | 1 + doc/config.rst | 24 ++++++++++++++++++++++-- sphinx/application.py | 2 +- sphinx/config.py | 29 ++++++++++++++++++++++++----- sphinx/environment/__init__.py | 2 +- sphinx/ext/autosummary/__init__.py | 2 +- 6 files changed, 50 insertions(+), 10 deletions(-) diff --git a/CHANGES b/CHANGES index 9ef1102b2..73375f6dc 100644 --- a/CHANGES +++ b/CHANGES @@ -22,6 +22,7 @@ Features added * Add :event:`config-inited` event * Add ``sphinx.config.Any`` to represent the config value accepts any type of value +* :confval:`source_suffix` allows a mapping fileext to file types Bugs fixed ---------- diff --git a/doc/config.rst b/doc/config.rst index 5af846ee7..f1ec78dcd 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -90,12 +90,32 @@ General configuration .. confval:: source_suffix - The file name extension, or list of extensions, of source files. Only files - with this suffix will be read as sources. Default is ``'.rst'``. + The file extensions of source files. Sphinx considers the files with this + suffix as sources. This value can be a dictionary mapping file extensions + to file types. For example:: + + source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'restructuredtext', + '.md': 'markdown', + } + + By default, Sphinx only supports ``'restrcturedtext'`` file type. You can + add a new file type using source parser extensions. Please read a document + of the extension to know what file type the extension supports. + + This also allows a list of file extensions. In that case, Sphinx conciders + that all they are ``'restructuredtext'``. Default is + ``{'.rst': 'restructuredtext'}``. + + .. note:: file extensions have to start with dot (like ``.rst``). .. versionchanged:: 1.3 Can now be a list of extensions. + .. vesionchanged:: 1.8 + Support file type mapping + .. confval:: source_encoding The encoding of all reST source files. The recommended encoding, and the diff --git a/sphinx/application.py b/sphinx/application.py index 8a5d2f60d..3dd98165c 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -290,7 +290,7 @@ class Sphinx(object): # type: () -> None for suffix, parser in iteritems(self.registry.get_source_parsers()): if suffix not in self.config.source_suffix and suffix != '*': - self.config.source_suffix.append(suffix) + self.config.source_suffix[suffix] = suffix def _init_env(self, freshenv): # type: (bool) -> None diff --git a/sphinx/config.py b/sphinx/config.py index 29883b0cd..88453b891 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -12,6 +12,7 @@ import re import traceback from os import path, getenv +from collections import OrderedDict from six import PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types from typing import Any, NamedTuple, Union @@ -114,7 +115,7 @@ class Config(object): figure_language_filename = (u'{root}.{language}{ext}', 'env', [str]), master_doc = ('contents', 'env'), - source_suffix = (['.rst'], 'env', Any), + source_suffix = ({'.rst': 'restructuredtext'}, 'env', Any), source_encoding = ('utf-8-sig', 'env'), source_parsers = ({}, 'env'), exclude_patterns = ([], 'env'), @@ -260,7 +261,9 @@ class Config(object): return value else: defvalue = self.values[name][0] - if isinstance(defvalue, dict): + if self.values[name][-1] == Any: + return value + elif isinstance(defvalue, dict): raise ValueError(__('cannot override dictionary config setting %r, ' 'ignoring (use %r to set individual elements)') % (name, name + '.key=value')) @@ -361,9 +364,25 @@ class Config(object): def convert_source_suffix(app, config): # type: (Sphinx, Config) -> None - """This converts source_suffix to string-list.""" - if isinstance(config.source_suffix, string_types): - config.source_suffix = [config.source_suffix] # type: ignore + """This converts old styled source_suffix to new styled one. + + * old style: str or list + * new style: a dict which maps from fileext to filetype + """ + source_suffix = config.source_suffix + if isinstance(source_suffix, string_types): + # if str, considers as reST + config.source_suffix = OrderedDict({source_suffix: 'restructuredtext'}) # type: ignore + elif isinstance(source_suffix, (list, tuple)): + # if list, considers as all of them are reST + config.source_suffix = OrderedDict([(s, 'restructuredtext') for s in source_suffix]) # type: ignore # NOQA + elif isinstance(source_suffix, dict): + # if dict, convert it to OrderedDict + config.source_suffix = OrderedDict(config.source_suffix) # type: ignore + else: + logger.warning(__("The config value `source_suffix' expected to " + "a string, list of strings or dictionary. " + "But `%r' is given." % source_suffix)) def setup(app): diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index 17f9667a1..cc88f7f6d 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -372,7 +372,7 @@ class BuildEnvironment(object): break else: # document does not exist - suffix = self.config.source_suffix[0] + suffix = self.config.source_suffix.keys()[0] if base is True: return path.join(self.srcdir, docname) + suffix elif base is None: diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index 7a9e59c73..c95296077 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -603,7 +603,7 @@ def process_generate_options(app): from sphinx.ext.autosummary.generate import generate_autosummary_docs - ext = app.config.source_suffix + ext = app.config.source_suffix.keys() genfiles = [genfile + (not genfile.endswith(tuple(ext)) and ext[0] or '') for genfile in genfiles] From 0cdd9ee72a5def1528d3f6a80622b26ae3cdacbc Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 24 Jan 2018 00:48:02 +0900 Subject: [PATCH 04/10] Integrate source_suffix and source_parsers (refs: #4474) --- sphinx/application.py | 9 +---- sphinx/parsers.py | 2 +- sphinx/registry.py | 75 +++++++++++++++++++++++++++++++-------- sphinx/transforms/i18n.py | 2 +- tests/test_application.py | 4 ++- 5 files changed, 66 insertions(+), 26 deletions(-) diff --git a/sphinx/application.py b/sphinx/application.py index 3dd98165c..df64874bd 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -89,6 +89,7 @@ builtin_extensions = ( 'sphinx.directives.patches', 'sphinx.io', 'sphinx.parsers', + 'sphinx.registry', 'sphinx.roles', 'sphinx.transforms.post_transforms', 'sphinx.transforms.post_transforms.images', @@ -251,8 +252,6 @@ class Sphinx(object): self.builder = self.create_builder(buildername) # check all configuration values for permissible types self.config.check_types() - # set up source_parsers - self._init_source_parsers() # set up the build environment self._init_env(freshenv) # set up the builder @@ -286,12 +285,6 @@ class Sphinx(object): else: logger.info('not available for built-in messages') - def _init_source_parsers(self): - # type: () -> None - for suffix, parser in iteritems(self.registry.get_source_parsers()): - if suffix not in self.config.source_suffix and suffix != '*': - self.config.source_suffix[suffix] = suffix - def _init_env(self, freshenv): # type: (bool) -> None if freshenv: diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 34822898f..3a009321e 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -91,7 +91,7 @@ class RSTParser(docutils.parsers.rst.Parser): def setup(app): # type: (Sphinx) -> Dict[unicode, Any] - app.add_source_parser('*', RSTParser) # register as a special parser + app.add_source_parser('.rst', RSTParser) return { 'version': 'builtin', diff --git a/sphinx/registry.py b/sphinx/registry.py index 550df3360..ba01978cc 100644 --- a/sphinx/registry.py +++ b/sphinx/registry.py @@ -51,6 +51,10 @@ EXTENSION_BLACKLIST = { } # type: Dict[unicode, unicode] +class FiletypeNotFoundError(Exception): + pass + + class SphinxComponentRegistry(object): def __init__(self): self.autodoc_attrgettrs = {} # type: Dict[Type, Callable[[Any, unicode, Any], Any]] @@ -62,8 +66,9 @@ class SphinxComponentRegistry(object): self.domain_object_types = {} # type: Dict[unicode, Dict[unicode, ObjType]] self.domain_roles = {} # type: Dict[unicode, Dict[unicode, Union[RoleFunction, XRefRole]]] # NOQA self.post_transforms = [] # type: List[Type[Transform]] - self.source_parsers = {} # type: Dict[unicode, Parser] + self.source_parsers = {} # type: Dict[unicode, Type[Parser]] self.source_inputs = {} # type: Dict[unicode, Input] + self.source_suffix = {} # type: Dict[unicode, unicode] self.translators = {} # type: Dict[unicode, nodes.NodeVisitor] self.transforms = [] # type: List[Type[Transform]] @@ -201,27 +206,47 @@ class SphinxComponentRegistry(object): def add_source_parser(self, suffix, parser): # type: (unicode, Type[Parser]) -> None logger.debug('[app] adding search source_parser: %r, %r', suffix, parser) - if suffix in self.source_parsers: + if suffix in self.source_suffix: raise ExtensionError(__('source_parser for %r is already registered') % suffix) + else: + self.source_suffix[suffix] = suffix if len(parser.supported) == 0: warnings.warn('Old source_parser has been detected. Please fill Parser.supported ' 'attribute: %s' % parser.__name__, RemovedInSphinx30Warning) + # create a map from filetype to parser + for filetype in parser.supported: + if filetype in self.source_parsers: + raise ExtensionError(__('source_parser for %r is already registered') % + filetype) + else: + self.source_parsers[filetype] = parser + + # also maps suffix to parser + # + # This allows parsers not having ``supported`` filetypes. self.source_parsers[suffix] = parser + def get_filetype(self, filename): + # type: (unicode) -> unicode + for suffix, filetype in iteritems(self.source_suffix): + if filename.endswith(suffix): + return filetype + else: + raise FiletypeNotFoundError + def get_source_parser(self, filename): # type: (unicode) -> Type[Parser] - for suffix, parser_class in iteritems(self.source_parsers): - if filename.endswith(suffix): - break - else: - # use special parser for unknown file-extension '*' (if exists) - parser_class = self.source_parsers.get('*') + try: + filetype = self.get_filetype(filename) + parser_class = self.source_parsers[filetype] + except FiletypeNotFoundError: + raise SphinxError(__('Source parser for %s not registered') % filename) if parser_class is None: - raise SphinxError(__('Source parser for %s not registered') % filename) + raise SphinxError(__('Source parser for %s not registered') % filetype) else: return parser_class @@ -245,12 +270,10 @@ class SphinxComponentRegistry(object): def get_source_input(self, filename): # type: (unicode) -> Type[Input] - parser = self.get_source_parser(filename) - for filetype in parser.supported: - if filetype in self.source_inputs: - input_class = self.source_inputs[filetype] - break - else: + try: + filetype = self.get_filetype(filename) + input_class = self.source_inputs[filetype] + except FiletypeNotFoundError: # use special source_input for unknown file-type '*' (if exists) input_class = self.source_inputs.get('*') @@ -346,3 +369,25 @@ class SphinxComponentRegistry(object): app.extensions[extname] = Extension(extname, mod, **metadata) app._setting_up_extension.pop() + + +def merge_source_suffix(app): + # type: (Sphinx) -> None + """Merge source_suffix which specified by user and added by extensions.""" + for suffix in app.registry.source_suffix: + if suffix not in app.config.source_suffix: + app.config.source_suffix[suffix] = suffix + + # copy config.source_suffix to registry + app.registry.source_suffix = app.config.source_suffix + + +def setup(app): + # type: (Sphinx) -> Dict[unicode, Any] + app.connect('builder-inited', merge_source_suffix) + + return { + 'version': 'builtin', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py index db67aae97..f5ff97e5b 100644 --- a/sphinx/transforms/i18n.py +++ b/sphinx/transforms/i18n.py @@ -52,7 +52,7 @@ def publish_msgstr(app, source, source_path, source_line, config, settings): from sphinx.io import SphinxI18nReader reader = SphinxI18nReader(app) reader.set_lineno_for_reporter(source_line) - parser = app.registry.create_source_parser(app, '') + parser = app.registry.create_source_parser(app, '.rst') doc = reader.read( source=StringInput(source=source, source_path=source_path), parser=parser, diff --git a/tests/test_application.py b/tests/test_application.py index 12b6bbe60..c993f47bb 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -84,7 +84,9 @@ def test_domain_override(app, status, warning): @pytest.mark.sphinx(testroot='add_source_parser') def test_add_source_parser(app, status, warning): assert set(app.config.source_suffix) == set(['.rst', '.md', '.test']) - assert set(app.registry.get_source_parsers().keys()) == set(['*', '.md', '.test']) + assert '.rst' in app.registry.get_source_parsers() + assert '.md' in app.registry.get_source_parsers() + assert '.test' in app.registry.get_source_parsers() assert app.registry.get_source_parsers()['.md'].__name__ == 'DummyMarkdownParser' assert app.registry.get_source_parsers()['.test'].__name__ == 'TestSourceParser' From 2b6c8b33be0e66160c6f5d14fd78c7a47e3389b0 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 24 Jan 2018 21:33:33 +0900 Subject: [PATCH 05/10] Fix all files are treated as reST if old styled source_suffix used --- sphinx/config.py | 11 +++++++---- sphinx/registry.py | 11 ++++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/sphinx/config.py b/sphinx/config.py index 88453b891..34cbf0883 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -371,11 +371,14 @@ def convert_source_suffix(app, config): """ source_suffix = config.source_suffix if isinstance(source_suffix, string_types): - # if str, considers as reST - config.source_suffix = OrderedDict({source_suffix: 'restructuredtext'}) # type: ignore + # if str, considers as default filetype (None) + # + # The default filetype is determined on later step. + # By default, it is considered as restructuredtext. + config.source_suffix = OrderedDict({source_suffix: None}) # type: ignore elif isinstance(source_suffix, (list, tuple)): - # if list, considers as all of them are reST - config.source_suffix = OrderedDict([(s, 'restructuredtext') for s in source_suffix]) # type: ignore # NOQA + # if list, considers as all of them are default filetype + config.source_suffix = OrderedDict([(s, None) for s in source_suffix]) # type: ignore # NOQA elif isinstance(source_suffix, dict): # if dict, convert it to OrderedDict config.source_suffix = OrderedDict(config.source_suffix) # type: ignore diff --git a/sphinx/registry.py b/sphinx/registry.py index ba01978cc..953fcf211 100644 --- a/sphinx/registry.py +++ b/sphinx/registry.py @@ -233,7 +233,8 @@ class SphinxComponentRegistry(object): # type: (unicode) -> unicode for suffix, filetype in iteritems(self.source_suffix): if filename.endswith(suffix): - return filetype + # If default filetype (None), considered as restructuredtext. + return filetype or 'restructuredtext' else: raise FiletypeNotFoundError @@ -273,8 +274,8 @@ class SphinxComponentRegistry(object): try: filetype = self.get_filetype(filename) input_class = self.source_inputs[filetype] - except FiletypeNotFoundError: - # use special source_input for unknown file-type '*' (if exists) + except (FiletypeNotFoundError, KeyError): + # use special source_input for unknown filetype input_class = self.source_inputs.get('*') if input_class is None: @@ -377,6 +378,10 @@ def merge_source_suffix(app): for suffix in app.registry.source_suffix: if suffix not in app.config.source_suffix: app.config.source_suffix[suffix] = suffix + elif app.config.source_suffix[suffix] is None: + # filetype is not specified (default filetype). + # So it overrides default filetype by extensions setting. + app.config.source_suffix[suffix] = suffix # copy config.source_suffix to registry app.registry.source_suffix = app.config.source_suffix From 3047fdbbf427ae86d4202ece005c03363834e162 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 24 Jan 2018 21:33:33 +0900 Subject: [PATCH 06/10] Move get_filetype() to sphinx.io --- sphinx/io.py | 21 ++++++++++++++++++--- sphinx/registry.py | 44 +++++++++++--------------------------------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/sphinx/io.py b/sphinx/io.py index 9a68146ac..cd0a07c26 100644 --- a/sphinx/io.py +++ b/sphinx/io.py @@ -16,7 +16,7 @@ from docutils.core import Publisher from docutils.readers import standalone from docutils.statemachine import StringList, string2lines from docutils.writers import UnfilteredWriter -from six import text_type +from six import text_type, iteritems from typing import Any, Union # NOQA from sphinx.transforms import SphinxTransformer @@ -273,14 +273,29 @@ class SphinxRSTFileInput(SphinxBaseFileInput): return lineno +class FiletypeNotFoundError(Exception): + pass + + +def get_filetype(source_suffix, filename): + # type: (Dict[unicode, unicode]) -> unicode + for suffix, filetype in iteritems(source_suffix): + if filename.endswith(suffix): + # If default filetype (None), considered as restructuredtext. + return filetype or 'restructuredtext' + else: + raise FiletypeNotFoundError + + def read_doc(app, env, filename): # type: (Sphinx, BuildEnvironment, unicode) -> nodes.document """Parse a document and convert to doctree.""" - input_class = app.registry.get_source_input(filename) + filetype = get_filetype(app.config.source_suffix, filename) + input_class = app.registry.get_source_input(filetype) reader = SphinxStandaloneReader(app) source = input_class(app, env, source=None, source_path=filename, encoding=env.config.source_encoding) - parser = app.registry.create_source_parser(app, filename) + parser = app.registry.create_source_parser(app, filetype) pub = Publisher(reader=reader, parser=parser, diff --git a/sphinx/registry.py b/sphinx/registry.py index 953fcf211..ec39878b6 100644 --- a/sphinx/registry.py +++ b/sphinx/registry.py @@ -51,10 +51,6 @@ EXTENSION_BLACKLIST = { } # type: Dict[unicode, unicode] -class FiletypeNotFoundError(Exception): - pass - - class SphinxComponentRegistry(object): def __init__(self): self.autodoc_attrgettrs = {} # type: Dict[Type, Callable[[Any, unicode, Any], Any]] @@ -229,27 +225,12 @@ class SphinxComponentRegistry(object): # This allows parsers not having ``supported`` filetypes. self.source_parsers[suffix] = parser - def get_filetype(self, filename): - # type: (unicode) -> unicode - for suffix, filetype in iteritems(self.source_suffix): - if filename.endswith(suffix): - # If default filetype (None), considered as restructuredtext. - return filetype or 'restructuredtext' - else: - raise FiletypeNotFoundError - - def get_source_parser(self, filename): + def get_source_parser(self, filetype): # type: (unicode) -> Type[Parser] try: - filetype = self.get_filetype(filename) - parser_class = self.source_parsers[filetype] - except FiletypeNotFoundError: - raise SphinxError(__('Source parser for %s not registered') % filename) - - if parser_class is None: + return self.source_parsers[filetype] + except KeyError: raise SphinxError(__('Source parser for %s not registered') % filetype) - else: - return parser_class def get_source_parsers(self): # type: () -> Dict[unicode, Parser] @@ -269,19 +250,16 @@ class SphinxComponentRegistry(object): raise ExtensionError(__('source_input for %r is already registered') % filetype) self.source_inputs[filetype] = input_class - def get_source_input(self, filename): + def get_source_input(self, filetype): # type: (unicode) -> Type[Input] try: - filetype = self.get_filetype(filename) - input_class = self.source_inputs[filetype] - except (FiletypeNotFoundError, KeyError): - # use special source_input for unknown filetype - input_class = self.source_inputs.get('*') - - if input_class is None: - raise SphinxError(__('source_input for %s not registered') % filename) - else: - return input_class + return self.source_inputs[filetype] + except KeyError: + try: + # use special source_input for unknown filetype + return self.source_inputs['*'] + except KeyError: + raise SphinxError(__('source_input for %s not registered') % filetype) def add_translator(self, name, translator): # type: (unicode, Type[nodes.NodeVisitor]) -> None From ae294d39d1649567fece3cc7fb232c4a417663e9 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 24 Jan 2018 21:33:34 +0900 Subject: [PATCH 07/10] Add register.add_source_suffix() --- sphinx/registry.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sphinx/registry.py b/sphinx/registry.py index ec39878b6..175bceff5 100644 --- a/sphinx/registry.py +++ b/sphinx/registry.py @@ -199,13 +199,18 @@ class SphinxComponentRegistry(object): object_types = self.domain_object_types.setdefault('std', {}) object_types[directivename] = ObjType(objname or directivename, rolename) - def add_source_parser(self, suffix, parser): - # type: (unicode, Type[Parser]) -> None - logger.debug('[app] adding search source_parser: %r, %r', suffix, parser) + def add_source_suffix(self, suffix, filetype): + # type: (unicode, unicode) -> None + logger.debug('[app] adding source_suffix: %r, %r', suffix, filetype) if suffix in self.source_suffix: raise ExtensionError(__('source_parser for %r is already registered') % suffix) else: - self.source_suffix[suffix] = suffix + self.source_suffix[suffix] = filetype + + def add_source_parser(self, suffix, parser): + # type: (unicode, Type[Parser]) -> None + logger.debug('[app] adding search source_parser: %r, %r', suffix, parser) + self.add_source_suffix(suffix, suffix) if len(parser.supported) == 0: warnings.warn('Old source_parser has been detected. Please fill Parser.supported ' From 2be87e485157a79a8fb13dc59440f0c5b63e3076 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 25 Jan 2018 10:51:04 +0900 Subject: [PATCH 08/10] Fix mypy violation --- sphinx/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/io.py b/sphinx/io.py index cd0a07c26..d4413f357 100644 --- a/sphinx/io.py +++ b/sphinx/io.py @@ -278,7 +278,7 @@ class FiletypeNotFoundError(Exception): def get_filetype(source_suffix, filename): - # type: (Dict[unicode, unicode]) -> unicode + # type: (Dict[unicode, unicode], unicode) -> unicode for suffix, filetype in iteritems(source_suffix): if filename.endswith(suffix): # If default filetype (None), considered as restructuredtext. From 5cdec60c2614a61ce3f5780a58934b0a7db60a99 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 25 Jan 2018 10:53:44 +0900 Subject: [PATCH 09/10] Fix KeysView object does not support indexing in py3 --- sphinx/environment/__init__.py | 2 +- sphinx/ext/autosummary/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index cc88f7f6d..c2f94e459 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -372,7 +372,7 @@ class BuildEnvironment(object): break else: # document does not exist - suffix = self.config.source_suffix.keys()[0] + suffix = list(self.config.source_suffix)[0] if base is True: return path.join(self.srcdir, docname) + suffix elif base is None: diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index c95296077..8a939c8bc 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -603,7 +603,7 @@ def process_generate_options(app): from sphinx.ext.autosummary.generate import generate_autosummary_docs - ext = app.config.source_suffix.keys() + ext = list(app.config.source_suffix) genfiles = [genfile + (not genfile.endswith(tuple(ext)) and ext[0] or '') for genfile in genfiles] From 53096a35d891d5803c84decfc88f585e051c2d28 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 26 Jan 2018 01:29:52 +0900 Subject: [PATCH 10/10] doc: Fix typo --- doc/config.rst | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/config.rst b/doc/config.rst index f1ec78dcd..e85a4027d 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -113,7 +113,7 @@ General configuration .. versionchanged:: 1.3 Can now be a list of extensions. - .. vesionchanged:: 1.8 + .. versionchanged:: 1.8 Support file type mapping .. confval:: source_encoding diff --git a/setup.py b/setup.py index f35e5f88d..be0c89728 100644 --- a/setup.py +++ b/setup.py @@ -214,7 +214,7 @@ setup( 'Topic :: Utilities', ], platforms='any', - packages=find_packages(exclude=['tests']), + packages=find_packages(exclude=['tests', 'utils']), include_package_data=True, entry_points={ 'console_scripts': [