diff --git a/CHANGES b/CHANGES index cad3b9b0a..75383c554 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,8 @@ Incompatible changes * #4460: extensions which stores any data to environment should return the version of its env data structure as metadata. In detail, please see :ref:`ext-metadata`. +* Sphinx expects source parser modules to have supported file formats as + ``Parser.supported`` attribute * The default value of :confval:`epub_author` and :confval:`epub_publisher` are changed from ``'unknown'`` to the value of :confval:`author`. This is same as a ``conf.py`` file sphinx-build generates. @@ -17,12 +19,16 @@ Incompatible changes Deprecated ---------- +* :confval:`source_parsers` is deprecated. Please use ``add_source_parser()`` + instead. + Features added -------------- * Add :event:`config-inited` event * Add ``sphinx.config.Any`` to represent the config value accepts any type of value +* :confval:`source_suffix` allows a mapping fileext to file types * Add :confval:`author` as a configuration value Bugs fixed diff --git a/doc/config.rst b/doc/config.rst index 2e9ddf07c..a0ec91cf0 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -90,12 +90,32 @@ General configuration .. confval:: source_suffix - The file name extension, or list of extensions, of source files. Only files - with this suffix will be read as sources. Default is ``'.rst'``. + The file extensions of source files. Sphinx considers the files with this + suffix as sources. This value can be a dictionary mapping file extensions + to file types. For example:: + + source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'restructuredtext', + '.md': 'markdown', + } + + By default, Sphinx only supports ``'restrcturedtext'`` file type. You can + add a new file type using source parser extensions. Please read a document + of the extension to know what file type the extension supports. + + This also allows a list of file extensions. In that case, Sphinx conciders + that all they are ``'restructuredtext'``. Default is + ``{'.rst': 'restructuredtext'}``. + + .. note:: file extensions have to start with dot (like ``.rst``). .. versionchanged:: 1.3 Can now be a list of extensions. + .. versionchanged:: 1.8 + Support file type mapping + .. confval:: source_encoding The encoding of all reST source files. The recommended encoding, and the @@ -123,6 +143,10 @@ General configuration .. versionadded:: 1.3 + .. deprecated:: 1.8 + Now Sphinx provides an API :meth:`Sphinx.add_source_parser` to register + a source parser. Please use it instead. + .. confval:: master_doc The document name of the "master" document, that is, the document that diff --git a/doc/extdev/builderapi.rst b/doc/extdev/builderapi.rst index b8ff0595b..2c2cf12e3 100644 --- a/doc/extdev/builderapi.rst +++ b/doc/extdev/builderapi.rst @@ -17,6 +17,9 @@ Builder API .. autoattribute:: format .. autoattribute:: epilog .. autoattribute:: supported_image_types + .. autoattribute:: supported_remote_images + .. autoattribute:: supported_data_uri_images + .. autoattribute:: default_translator_class These methods are predefined and will be called from the application: diff --git a/doc/theming.rst b/doc/theming.rst index 80fc6e685..67dd6adc2 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -422,3 +422,11 @@ Third Party Themes .. versionchanged:: 1.4 **sphinx_rtd_theme** has become optional. + + +Besides this, there are a lot of third party themes. You can find them on +PyPI__, GitHub__, sphinx-themes.org__ and so on. + +.. __: https://pypi.python.org/pypi?:action=browse&c=599 +.. __: https://github.com/search?utf8=%E2%9C%93&q=sphinx+theme&type= +.. __: https://sphinx-themes.org/ diff --git a/sphinx/application.py b/sphinx/application.py index d36925f8e..3699bdfeb 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -89,9 +89,11 @@ builtin_extensions = ( 'sphinx.directives.patches', 'sphinx.io', 'sphinx.parsers', + 'sphinx.registry', 'sphinx.roles', 'sphinx.transforms.post_transforms', 'sphinx.transforms.post_transforms.images', + 'sphinx.util.compat', # collectors should be loaded by specific order 'sphinx.environment.collectors.dependencies', 'sphinx.environment.collectors.asset', @@ -249,8 +251,6 @@ class Sphinx(object): self.builder = self.create_builder(buildername) # check all configuration values for permissible types self.config.check_types() - # set up source_parsers - self._init_source_parsers() # set up the build environment self._init_env(freshenv) # set up the builder @@ -284,14 +284,6 @@ class Sphinx(object): else: logger.info('not available for built-in messages') - def _init_source_parsers(self): - # type: () -> None - for suffix, parser in iteritems(self.config.source_parsers): - self.add_source_parser(suffix, parser) - for suffix, parser in iteritems(self.registry.get_source_parsers()): - if suffix not in self.config.source_suffix and suffix != '*': - self.config.source_suffix.append(suffix) - def _init_env(self, freshenv): # type: (bool) -> None if freshenv: diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 51578a1d6..662b94227 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -59,8 +59,8 @@ class Builder(object): #: ``project`` epilog = '' # type: unicode - # default translator class for the builder. This will be overrided by - # ``app.set_translator()``. + #: default translator class for the builder. This can be overrided by + #: :py:meth:`app.set_translator()`. default_translator_class = None # type: nodes.NodeVisitor # doctree versioning method versioning_method = 'none' # type: unicode @@ -73,7 +73,9 @@ class Builder(object): #: The list of MIME types of image formats supported by the builder. #: Image files are searched in the order in which they appear here. supported_image_types = [] # type: List[unicode] + #: The builder supports remote images or not. supported_remote_images = False + #: The builder supports data URIs or not. supported_data_uri_images = False def __init__(self, app): diff --git a/sphinx/config.py b/sphinx/config.py index 5d9a9258c..ad08b4ca4 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -12,6 +12,7 @@ import re import traceback from os import path, getenv +from collections import OrderedDict from six import PY2, PY3, iteritems, string_types, binary_type, text_type, integer_types from typing import Any, NamedTuple, Union @@ -115,7 +116,7 @@ class Config(object): figure_language_filename = (u'{root}.{language}{ext}', 'env', [str]), master_doc = ('contents', 'env'), - source_suffix = (['.rst'], 'env', Any), + source_suffix = ({'.rst': 'restructuredtext'}, 'env', Any), source_encoding = ('utf-8-sig', 'env'), source_parsers = ({}, 'env'), exclude_patterns = ([], 'env'), @@ -261,7 +262,9 @@ class Config(object): return value else: defvalue = self.values[name][0] - if isinstance(defvalue, dict): + if self.values[name][-1] == Any: + return value + elif isinstance(defvalue, dict): raise ValueError(__('cannot override dictionary config setting %r, ' 'ignoring (use %r to set individual elements)') % (name, name + '.key=value')) @@ -362,9 +365,28 @@ class Config(object): def convert_source_suffix(app, config): # type: (Sphinx, Config) -> None - """This converts source_suffix to string-list.""" - if isinstance(config.source_suffix, string_types): - config.source_suffix = [config.source_suffix] # type: ignore + """This converts old styled source_suffix to new styled one. + + * old style: str or list + * new style: a dict which maps from fileext to filetype + """ + source_suffix = config.source_suffix + if isinstance(source_suffix, string_types): + # if str, considers as default filetype (None) + # + # The default filetype is determined on later step. + # By default, it is considered as restructuredtext. + config.source_suffix = OrderedDict({source_suffix: None}) # type: ignore + elif isinstance(source_suffix, (list, tuple)): + # if list, considers as all of them are default filetype + config.source_suffix = OrderedDict([(s, None) for s in source_suffix]) # type: ignore # NOQA + elif isinstance(source_suffix, dict): + # if dict, convert it to OrderedDict + config.source_suffix = OrderedDict(config.source_suffix) # type: ignore + else: + logger.warning(__("The config value `source_suffix' expected to " + "a string, list of strings or dictionary. " + "But `%r' is given." % source_suffix)) def setup(app): diff --git a/sphinx/deprecation.py b/sphinx/deprecation.py index e28e0f916..cd6e1ae2a 100644 --- a/sphinx/deprecation.py +++ b/sphinx/deprecation.py @@ -22,4 +22,8 @@ class RemovedInSphinx20Warning(PendingDeprecationWarning): pass +class RemovedInSphinx30Warning(PendingDeprecationWarning): + pass + + RemovedInNextVersionWarning = RemovedInSphinx18Warning diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index 330dda284..86b2e5fba 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -381,7 +381,7 @@ class BuildEnvironment(object): break else: # document does not exist - suffix = self.config.source_suffix[0] + suffix = list(self.config.source_suffix)[0] if base is True: return path.join(self.srcdir, docname) + suffix elif base is None: diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py index 7a9e59c73..8a939c8bc 100644 --- a/sphinx/ext/autosummary/__init__.py +++ b/sphinx/ext/autosummary/__init__.py @@ -603,7 +603,7 @@ def process_generate_options(app): from sphinx.ext.autosummary.generate import generate_autosummary_docs - ext = app.config.source_suffix + ext = list(app.config.source_suffix) genfiles = [genfile + (not genfile.endswith(tuple(ext)) and ext[0] or '') for genfile in genfiles] diff --git a/sphinx/io.py b/sphinx/io.py index 9a68146ac..d4413f357 100644 --- a/sphinx/io.py +++ b/sphinx/io.py @@ -16,7 +16,7 @@ from docutils.core import Publisher from docutils.readers import standalone from docutils.statemachine import StringList, string2lines from docutils.writers import UnfilteredWriter -from six import text_type +from six import text_type, iteritems from typing import Any, Union # NOQA from sphinx.transforms import SphinxTransformer @@ -273,14 +273,29 @@ class SphinxRSTFileInput(SphinxBaseFileInput): return lineno +class FiletypeNotFoundError(Exception): + pass + + +def get_filetype(source_suffix, filename): + # type: (Dict[unicode, unicode], unicode) -> unicode + for suffix, filetype in iteritems(source_suffix): + if filename.endswith(suffix): + # If default filetype (None), considered as restructuredtext. + return filetype or 'restructuredtext' + else: + raise FiletypeNotFoundError + + def read_doc(app, env, filename): # type: (Sphinx, BuildEnvironment, unicode) -> nodes.document """Parse a document and convert to doctree.""" - input_class = app.registry.get_source_input(filename) + filetype = get_filetype(app.config.source_suffix, filename) + input_class = app.registry.get_source_input(filetype) reader = SphinxStandaloneReader(app) source = input_class(app, env, source=None, source_path=filename, encoding=env.config.source_encoding) - parser = app.registry.create_source_parser(app, filename) + parser = app.registry.create_source_parser(app, filetype) pub = Publisher(reader=reader, parser=parser, diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 34822898f..3a009321e 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -91,7 +91,7 @@ class RSTParser(docutils.parsers.rst.Parser): def setup(app): # type: (Sphinx) -> Dict[unicode, Any] - app.add_source_parser('*', RSTParser) # register as a special parser + app.add_source_parser('.rst', RSTParser) return { 'version': 'builtin', diff --git a/sphinx/registry.py b/sphinx/registry.py index 455ca4fb5..042781dfb 100644 --- a/sphinx/registry.py +++ b/sphinx/registry.py @@ -11,19 +11,20 @@ from __future__ import print_function import traceback +import warnings from pkg_resources import iter_entry_points -from six import iteritems, itervalues, string_types +from six import iteritems, itervalues from sphinx.errors import ExtensionError, SphinxError, VersionRequirementError from sphinx.extension import Extension +from sphinx.deprecation import RemovedInSphinx30Warning from sphinx.domains import ObjType from sphinx.domains.std import GenericObject, Target from sphinx.locale import __ from sphinx.parsers import Parser as SphinxParser from sphinx.roles import XRefRole from sphinx.util import logging -from sphinx.util import import_object from sphinx.util.console import bold # type: ignore from sphinx.util.docutils import directive_helper @@ -61,8 +62,9 @@ class SphinxComponentRegistry(object): self.domain_object_types = {} # type: Dict[unicode, Dict[unicode, ObjType]] self.domain_roles = {} # type: Dict[unicode, Dict[unicode, Union[RoleFunction, XRefRole]]] # NOQA self.post_transforms = [] # type: List[Type[Transform]] - self.source_parsers = {} # type: Dict[unicode, Parser] + self.source_parsers = {} # type: Dict[unicode, Type[Parser]] self.source_inputs = {} # type: Dict[unicode, Input] + self.source_suffix = {} # type: Dict[unicode, unicode] self.translators = {} # type: Dict[unicode, nodes.NodeVisitor] self.transforms = [] # type: List[Type[Transform]] @@ -197,28 +199,43 @@ class SphinxComponentRegistry(object): object_types = self.domain_object_types.setdefault('std', {}) object_types[directivename] = ObjType(objname or directivename, rolename) + def add_source_suffix(self, suffix, filetype): + # type: (unicode, unicode) -> None + logger.debug('[app] adding source_suffix: %r, %r', suffix, filetype) + if suffix in self.source_suffix: + raise ExtensionError(__('source_parser for %r is already registered') % suffix) + else: + self.source_suffix[suffix] = filetype + def add_source_parser(self, suffix, parser): # type: (unicode, Type[Parser]) -> None logger.debug('[app] adding search source_parser: %r, %r', suffix, parser) - if suffix in self.source_parsers: - raise ExtensionError(__('source_parser for %r is already registered') % suffix) + self.add_source_suffix(suffix, suffix) + + if len(parser.supported) == 0: + warnings.warn('Old source_parser has been detected. Please fill Parser.supported ' + 'attribute: %s' % parser.__name__, + RemovedInSphinx30Warning) + + # create a map from filetype to parser + for filetype in parser.supported: + if filetype in self.source_parsers: + raise ExtensionError(__('source_parser for %r is already registered') % + filetype) + else: + self.source_parsers[filetype] = parser + + # also maps suffix to parser + # + # This allows parsers not having ``supported`` filetypes. self.source_parsers[suffix] = parser - def get_source_parser(self, filename): + def get_source_parser(self, filetype): # type: (unicode) -> Type[Parser] - for suffix, parser_class in iteritems(self.source_parsers): - if filename.endswith(suffix): - break - else: - # use special parser for unknown file-extension '*' (if exists) - parser_class = self.source_parsers.get('*') - - if parser_class is None: - raise SphinxError(__('Source parser for %s not registered') % filename) - else: - if isinstance(parser_class, string_types): - parser_class = import_object(parser_class, 'source parser') # type: ignore - return parser_class + try: + return self.source_parsers[filetype] + except KeyError: + raise SphinxError(__('Source parser for %s not registered') % filetype) def get_source_parsers(self): # type: () -> Dict[unicode, Parser] @@ -238,21 +255,16 @@ class SphinxComponentRegistry(object): raise ExtensionError(__('source_input for %r is already registered') % filetype) self.source_inputs[filetype] = input_class - def get_source_input(self, filename): + def get_source_input(self, filetype): # type: (unicode) -> Type[Input] - parser = self.get_source_parser(filename) - for filetype in parser.supported: - if filetype in self.source_inputs: - input_class = self.source_inputs[filetype] - break - else: - # use special source_input for unknown file-type '*' (if exists) - input_class = self.source_inputs.get('*') - - if input_class is None: - raise SphinxError(__('source_input for %s not registered') % filename) - else: - return input_class + try: + return self.source_inputs[filetype] + except KeyError: + try: + # use special source_input for unknown filetype + return self.source_inputs['*'] + except KeyError: + raise SphinxError(__('source_input for %s not registered') % filetype) def add_translator(self, name, translator): # type: (unicode, Type[nodes.NodeVisitor]) -> None @@ -347,3 +359,29 @@ class SphinxComponentRegistry(object): if ext.metadata.get('env_version')} envversion['sphinx'] = ENV_VERSION return envversion + + +def merge_source_suffix(app): + # type: (Sphinx) -> None + """Merge source_suffix which specified by user and added by extensions.""" + for suffix in app.registry.source_suffix: + if suffix not in app.config.source_suffix: + app.config.source_suffix[suffix] = suffix + elif app.config.source_suffix[suffix] is None: + # filetype is not specified (default filetype). + # So it overrides default filetype by extensions setting. + app.config.source_suffix[suffix] = suffix + + # copy config.source_suffix to registry + app.registry.source_suffix = app.config.source_suffix + + +def setup(app): + # type: (Sphinx) -> Dict[unicode, Any] + app.connect('builder-inited', merge_source_suffix) + + return { + 'version': 'builtin', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/sphinx/transforms/i18n.py b/sphinx/transforms/i18n.py index db67aae97..f5ff97e5b 100644 --- a/sphinx/transforms/i18n.py +++ b/sphinx/transforms/i18n.py @@ -52,7 +52,7 @@ def publish_msgstr(app, source, source_path, source_line, config, settings): from sphinx.io import SphinxI18nReader reader = SphinxI18nReader(app) reader.set_lineno_for_reporter(source_line) - parser = app.registry.create_source_parser(app, '') + parser = app.registry.create_source_parser(app, '.rst') doc = reader.read( source=StringInput(source=source, source_path=source_path), parser=parser, diff --git a/sphinx/util/compat.py b/sphinx/util/compat.py new file mode 100644 index 000000000..edd8cac61 --- /dev/null +++ b/sphinx/util/compat.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +""" + sphinx.util.compat + ~~~~~~~~~~~~~~~~~~ + + modules for backward compatibility + + :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import warnings + +from six import string_types, iteritems + +from sphinx.deprecation import RemovedInSphinx30Warning +from sphinx.util import import_object + +if False: + # For type annotation + from typing import Any, Dict # NOQA + from sphinx.application import Sphinx # NOQA + from sphinx.config import Config # NOQA + + +def deprecate_source_parsers(app, config): + # type: (Sphinx, Config) -> None + if config.source_parsers: + warnings.warn('The config variable "source_parsers" is deprecated. ' + 'Please use app.add_source_parser() API instead.', + RemovedInSphinx30Warning) + for suffix, parser in iteritems(config.source_parsers): + if isinstance(parser, string_types): + parser = import_object(parser, 'source parser') # type: ignore + app.add_source_parser(suffix, parser) + + +def setup(app): + # type: (Sphinx) -> Dict[unicode, Any] + app.connect('config-inited', deprecate_source_parsers) + + return { + 'version': 'builtin', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/tests/roots/test-prolog/prolog_markdown_parser.py b/tests/roots/test-prolog/prolog_markdown_parser.py index f28c37b4e..6053da7be 100644 --- a/tests/roots/test-prolog/prolog_markdown_parser.py +++ b/tests/roots/test-prolog/prolog_markdown_parser.py @@ -4,6 +4,8 @@ from docutils.parsers import Parser class DummyMarkdownParser(Parser): + supported = ('markdown',) + def parse(self, inputstring, document): document.rawsource = inputstring diff --git a/tests/roots/test-root/conf.py b/tests/roots/test-root/conf.py index 04cd87d7b..0f5c20c8e 100644 --- a/tests/roots/test-root/conf.py +++ b/tests/roots/test-root/conf.py @@ -13,7 +13,6 @@ templates_path = ['_templates'] master_doc = 'contents' source_suffix = ['.txt', '.add', '.foo'] -source_parsers = {'.foo': 'parsermod.Parser'} project = 'Sphinx ' copyright = '2010-2016, Georg Brandl & Team' @@ -106,9 +105,12 @@ class ClassDirective(Directive): def setup(app): + import parsermod + app.add_config_value('value_from_conf_py', 42, False) app.add_directive('funcdir', functional_directive, opt=lambda x: x) app.add_directive('clsdir', ClassDirective) app.add_object_type('userdesc', 'userdescrole', '%s (userdesc)', userdesc_parse, objname='user desc') app.add_javascript('file://moo.js') + app.add_source_parser('.foo', parsermod.Parser) diff --git a/tests/roots/test-root/parsermod.py b/tests/roots/test-root/parsermod.py index 3e5330ac8..f98d82f3e 100644 --- a/tests/roots/test-root/parsermod.py +++ b/tests/roots/test-root/parsermod.py @@ -3,6 +3,8 @@ from docutils import nodes class Parser(Parser): + supported = ('foo',) + def parse(self, input, document): section = nodes.section(ids=['id1']) section += nodes.title('Generated section', 'Generated section') diff --git a/tests/test_application.py b/tests/test_application.py index 12b6bbe60..c993f47bb 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -84,7 +84,9 @@ def test_domain_override(app, status, warning): @pytest.mark.sphinx(testroot='add_source_parser') def test_add_source_parser(app, status, warning): assert set(app.config.source_suffix) == set(['.rst', '.md', '.test']) - assert set(app.registry.get_source_parsers().keys()) == set(['*', '.md', '.test']) + assert '.rst' in app.registry.get_source_parsers() + assert '.md' in app.registry.get_source_parsers() + assert '.test' in app.registry.get_source_parsers() assert app.registry.get_source_parsers()['.md'].__name__ == 'DummyMarkdownParser' assert app.registry.get_source_parsers()['.test'].__name__ == 'TestSourceParser'