IntenseWebs/sphinx
3
0
Fork 0
mirror of https://github.com/sphinx-doc/sphinx.git synced 2025-02-25 18:55:22 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3313 from tk0miya/collectors

Browse Source 
Refactor sphinx.environment; Add EnvironmentCollectors
This commit is contained in:
Takeshi KOMIYA 2017-01-31 01:50:28 +09:00 committed by GitHub
commit 238d92beb3
25 changed files with 940 additions and 655 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGES
View File

@ -62,6 +62,14 @@ Deprecated
* ``Sphinx.status_iterator()` and ``Sphinx.old_status_iterator()`` is now
deprecated. Please use ``sphinx.util:status_iterator()`` intead.
* ``BuildEnvironment.set_warnfunc()`` is now deprecated
* Following methods of ``BuildEnvironment`` is now deprecated.
- ``BuildEnvironment.note_toctree()``
- ``BuildEnvironment.get_toc_for()``
- ``BuildEnvironment.get_toctree_for()``
- ``BuildEnvironment.create_index()``
Please use ``sphinx.environment.adapters`` modules instead.
Release 1.5.3 (in development)
==============================

6
doc/extdev/appapi.rst
View File

@ -359,6 +359,12 @@ package.
.. versionadded:: 1.4
.. method:: Sphinx.add_env_collector(collector)
Register a environment collector class (refs: :ref:`collector-api`)
.. versionadded:: 1.6
.. method:: Sphinx.require_sphinx(version)
Compare *version* (which must be a ``major.minor`` version string,

9
doc/extdev/collectorapi.rst Normal file
View File

@ -0,0 +1,9 @@
.. _collector-api:
Environment Collector API
-------------------------
.. module:: sphinx.environment.collectors
.. autoclass:: EnvironmentCollector
:members:

1
doc/extdev/index.rst
View File

@ -50,6 +50,7 @@ APIs used for writing extensions
appapi
envapi
builderapi
collectorapi
markupapi
domainapi
parserapi

15
sphinx/application.py
View File

@ -55,11 +55,13 @@ if False:
from docutils.transform import Transform # NOQA
from sphinx.builders import Builder # NOQA
from sphinx.domains import Domain, Index # NOQA
from sphinx.environment.collectors import EnvironmentCollector # NOQA
# List of all known core events. Maps name to arguments description.
events = {
'builder-inited': '',
'env-get-outdated': 'env, added, changed, removed',
'env-get-updated': 'env',
'env-purge-doc': 'env, docname',
'env-before-read-docs': 'env, docnames',
'source-read': 'docname, source text',
@ -101,6 +103,13 @@ builtin_extensions = (
'sphinx.directives.other',
'sphinx.directives.patches',
'sphinx.roles',
# collectors should be loaded by specific order
'sphinx.environment.collectors.dependencies',
'sphinx.environment.collectors.asset',
'sphinx.environment.collectors.metadata',
'sphinx.environment.collectors.title',
'sphinx.environment.collectors.toctree',
'sphinx.environment.collectors.indexentries',
) # type: Tuple[unicode, ...]
CONFIG_FILENAME = 'conf.py'
@ -300,7 +309,6 @@ class Sphinx(object):
logger.info(bold('loading pickled environment... '), nonl=True)
self.env = BuildEnvironment.frompickle(
self.srcdir, self.config, path.join(self.doctreedir, ENV_PICKLE_FILENAME))
self.env.init_managers()
self.env.domains = {}
for domain in self.domains.keys():
# this can raise if the data version doesn't fit
@ -833,6 +841,11 @@ class Sphinx(object):
type='app', subtype='add_source_parser')
self._additional_source_parsers[suffix] = parser
def add_env_collector(self, collector):
# type: (Type[EnvironmentCollector]) -> None
logger.debug('[app] adding environment collector: %r', collector)
collector().enable(self)
class TemplateBridge(object):
"""

2
sphinx/builders/__init__.py
View File

@ -292,7 +292,7 @@ class Builder(object):
doccount = len(updated_docnames)
logger.info(bold('looking for now-outdated files... '), nonl=1)
for docname in self.env.check_dependents(updated_docnames):
for docname in self.env.check_dependents(self.app, updated_docnames):
updated_docnames.add(docname)
outdated = len(updated_docnames) - doccount
if outdated:

3
sphinx/builders/devhelp.py
View File

@ -22,6 +22,7 @@ from sphinx import addnodes
from sphinx.util import logging
from sphinx.util.osutil import make_filename
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.environment.adapters.indexentries import IndexEntries
try:
import xml.etree.ElementTree as etree
@ -104,7 +105,7 @@ class DevhelpBuilder(StandaloneHTMLBuilder):
# Index
functions = etree.SubElement(root, 'functions')
index = self.env.create_index(self)
index = IndexEntries(self.env).create_index(self)
def write_index(title, refs, subitems):
# type: (unicode, List[Any], Any) -> None

13
sphinx/builders/html.py
View File

@ -45,6 +45,8 @@ from sphinx.highlighting import PygmentsBridge
from sphinx.util.console import bold, darkgreen # type: ignore
from sphinx.writers.html import HTMLWriter, HTMLTranslator, \
SmartyPantsHTMLTranslator
from sphinx.environment.adapters.toctree import TocTree
from sphinx.environment.adapters.indexentries import IndexEntries
if False:
# For type annotation
@ -439,7 +441,7 @@ class StandaloneHTMLBuilder(Builder):
meta = self.env.metadata.get(docname)
# local TOC and global TOC tree
self_toc = self.env.get_toc_for(docname, self)
self_toc = TocTree(self.env).get_toc_for(docname, self)
toc = self.render_partial(self_toc)['fragment']
return dict(
@ -541,7 +543,7 @@ class StandaloneHTMLBuilder(Builder):
# type: () -> None
# the total count of lines for each index letter, used to distribute
# the entries into two columns
genindex = self.env.create_index(self)
genindex = IndexEntries(self.env).create_index(self)
indexcounts = []
for _k, entries in genindex:
indexcounts.append(sum(1 + len(subitems)
@ -763,7 +765,7 @@ class StandaloneHTMLBuilder(Builder):
# type: (unicode, bool, Any) -> unicode
if 'includehidden' not in kwds:
kwds['includehidden'] = False
return self.render_partial(self.env.get_toctree_for(
return self.render_partial(TocTree(self.env).get_toctree_for(
docname, self, collapse, **kwds))['fragment']
def get_outfilename(self, pagename):
@ -1010,7 +1012,7 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
# type: (unicode, bool, Any) -> unicode
if 'includehidden' not in kwds:
kwds['includehidden'] = False
toctree = self.env.get_toctree_for(docname, self, collapse, **kwds)
toctree = TocTree(self.env).get_toctree_for(docname, self, collapse, **kwds)
self.fix_refuris(toctree)
return self.render_partial(toctree)['fragment']
@ -1066,7 +1068,8 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
def get_doc_context(self, docname, body, metatags):
# type: (unicode, unicode, Dict) -> Dict
# no relation links...
toc = self.env.get_toctree_for(self.config.master_doc, self, False) # type: Any
toc = TocTree(self.env).get_toctree_for(self.config.master_doc,
self, False)
# if there is no toctree, toc is None
if toc:
self.fix_refuris(toc)

3
sphinx/builders/htmlhelp.py
View File

@ -19,6 +19,7 @@ from docutils import nodes
from sphinx import addnodes
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.environment.adapters.indexentries import IndexEntries
from sphinx.util import logging
from sphinx.util.osutil import make_filename
from sphinx.util.pycompat import htmlescape
@ -281,7 +282,7 @@ class HTMLHelpBuilder(StandaloneHTMLBuilder):
f.write(contents_footer)
logger.info('writing index file...')
index = self.env.create_index(self)
index = IndexEntries(self.env).create_index(self)
with self.open_file(outdir, outname + '.hhk') as f:
f.write('<UL>\n')

3
sphinx/builders/qthelp.py
View File

@ -21,6 +21,7 @@ from docutils import nodes
from sphinx import addnodes
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.environment.adapters.indexentries import IndexEntries
from sphinx.util import force_decode, logging
from sphinx.util.osutil import make_filename
from sphinx.util.pycompat import htmlescape
@ -170,7 +171,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
# keywords
keywords = []
index = self.env.create_index(self, group_entries=False)
index = IndexEntries(self.env).create_index(self, group_entries=False)
for (key, group) in index:
for title, (refs, subitems, key_) in group:
keywords.extend(self.build_keywords(title, refs, subitems))

269
sphinx/environment/__init__.py
View File

@ -18,16 +18,15 @@ import codecs
import fnmatch
import warnings
from os import path
from glob import glob
from collections import defaultdict
from six import iteritems, itervalues, class_types, next
from six import itervalues, class_types, next
from six.moves import cPickle as pickle
from docutils import nodes
from docutils.io import NullOutput
from docutils.core import Publisher
from docutils.utils import Reporter, relative_path, get_source_line
from docutils.utils import Reporter, get_source_line
from docutils.parsers.rst import roles
from docutils.parsers.rst.languages import en as english
from docutils.frontend import OptionParser
@ -38,10 +37,8 @@ from sphinx.util import logging
from sphinx.util import get_matching_docs, docname_join, FilenameUniqDict, status_iterator
from sphinx.util.nodes import clean_astext, WarningStream, is_translatable, \
process_only_nodes
from sphinx.util.osutil import SEP, getcwd, fs_encoding, ensuredir
from sphinx.util.images import guess_mimetype
from sphinx.util.i18n import find_catalog_files, get_image_filename_for_language, \
search_image_for_language
from sphinx.util.osutil import SEP, ensuredir
from sphinx.util.i18n import find_catalog_files
from sphinx.util.console import bold # type: ignore
from sphinx.util.docutils import sphinx_domains
from sphinx.util.matching import compile_matchers
@ -49,10 +46,9 @@ from sphinx.util.parallel import ParallelTasks, parallel_available, make_chunks
from sphinx.util.websupport import is_commentable
from sphinx.errors import SphinxError, ExtensionError
from sphinx.versioning import add_uids, merge_doctrees
from sphinx.transforms import SphinxContentsFilter
from sphinx.deprecation import RemovedInSphinx20Warning
from sphinx.environment.managers.indexentries import IndexEntries
from sphinx.environment.managers.toctree import Toctree
from sphinx.environment.adapters.indexentries import IndexEntries
from sphinx.environment.adapters.toctree import TocTree
if False:
# For type annotation
@ -61,7 +57,6 @@ if False:
from sphinx.builders import Builder # NOQA
from sphinx.config import Config # NOQA
from sphinx.domains import Domain # NOQA
from sphinx.environment.managers import EnvironmentManager # NOQA
logger = logging.getLogger(__name__)
@ -128,7 +123,6 @@ class BuildEnvironment(object):
del self.config.values
domains = self.domains
del self.domains
managers = self.detach_managers()
# remove potentially pickling-problematic values from config
for key, val in list(vars(self.config).items()):
if key.startswith('_') or \
@ -139,7 +133,6 @@ class BuildEnvironment(object):
with open(filename, 'wb') as picklefile:
pickle.dump(self, picklefile, pickle.HIGHEST_PROTOCOL)
# reset attributes
self.attach_managers(managers)
self.domains = domains
self.config.values = values
@ -189,7 +182,7 @@ class BuildEnvironment(object):
# next build
# File metadata
self.metadata = {} # type: Dict[unicode, Dict[unicode, Any]]
self.metadata = defaultdict(dict) # type: Dict[unicode, Dict[unicode, Any]]
# docname -> dict of metadata items
# TOC inventory
@ -244,31 +237,6 @@ class BuildEnvironment(object):
# attributes of "any" cross references
self.ref_context = {} # type: Dict[unicode, Any]
self.managers = {} # type: Dict[unicode, EnvironmentManager]
self.init_managers()
def init_managers(self):
# type: () -> None
managers = {}
manager_class = None # type: Type[EnvironmentManager]
for manager_class in [IndexEntries, Toctree]: # type: ignore
managers[manager_class.name] = manager_class(self)
self.attach_managers(managers)
def attach_managers(self, managers):
# type: (Dict[unicode, EnvironmentManager]) -> None
for name, manager in iteritems(managers):
self.managers[name] = manager
manager.attach(self)
def detach_managers(self):
# type: () -> Dict[unicode, EnvironmentManager]
managers = self.managers
self.managers = {}
for _, manager in iteritems(managers):
manager.detach(self)
return managers
def set_warnfunc(self, func):
# type: (Callable) -> None
warnings.warn('env.set_warnfunc() is now deprecated. Use sphinx.util.logging instead.',
@ -315,20 +283,11 @@ class BuildEnvironment(object):
if docname in self.all_docs:
self.all_docs.pop(docname, None)
self.reread_always.discard(docname)
self.metadata.pop(docname, None)
self.dependencies.pop(docname, None)
self.titles.pop(docname, None)
self.longtitles.pop(docname, None)
self.images.purge_doc(docname)
self.dlfiles.purge_doc(docname)
for version, changes in self.versionchanges.items():
new = [change for change in changes if change[1] != docname]
changes[:] = new
for manager in itervalues(self.managers):
manager.clear_doc(docname)
for domain in self.domains.values():
domain.clear_doc(docname)
@ -344,21 +303,11 @@ class BuildEnvironment(object):
self.all_docs[docname] = other.all_docs[docname]
if docname in other.reread_always:
self.reread_always.add(docname)
self.metadata[docname] = other.metadata[docname]
if docname in other.dependencies:
self.dependencies[docname] = other.dependencies[docname]
self.titles[docname] = other.titles[docname]
self.longtitles[docname] = other.longtitles[docname]
self.images.merge_other(docnames, other.images)
self.dlfiles.merge_other(docnames, other.dlfiles)
for version, changes in other.versionchanges.items():
self.versionchanges.setdefault(version, []).extend(
change for change in changes if change[1] in docnames)
for manager in itervalues(self.managers):
manager.merge_other(docnames, other)
for domainname, domain in self.domains.items():
domain.merge_domaindata(docnames, other.domaindata[domainname])
app.emit('env-merge-info', self, docnames, other)
@ -662,10 +611,11 @@ class BuildEnvironment(object):
logger.info(bold('waiting for workers...'))
tasks.join()
def check_dependents(self, already):
# type: (Set[unicode]) -> Iterator[unicode]
to_rewrite = (self.toctree.assign_section_numbers() + # type: ignore
self.toctree.assign_figure_numbers()) # type: ignore
def check_dependents(self, app, already):
# type: (Sphinx, Set[unicode]) -> Iterator[unicode]
to_rewrite = [] # type: List[unicode]
for docnames in app.emit('env-get-updated', self):
to_rewrite.extend(docnames)
for docname in set(to_rewrite):
if docname not in already:
yield docname
@ -736,13 +686,6 @@ class BuildEnvironment(object):
doctree = pub.document
# post-processing
self.process_dependencies(docname, doctree)
self.process_images(docname, doctree)
self.process_downloads(docname, doctree)
self.process_metadata(docname, doctree)
self.create_title_from(docname, doctree)
for manager in itervalues(self.managers):
manager.process_doc(docname, doctree)
for domain in itervalues(self.domains):
domain.process_doc(self, docname, doctree)
@ -865,180 +808,31 @@ class BuildEnvironment(object):
self.ref_context.get('py:module'),
self.temp_data.get('object'), node.astext()))
# post-processing of read doctrees
def process_dependencies(self, docname, doctree):
# type: (unicode, nodes.Node) -> None
"""Process docutils-generated dependency info."""
cwd = getcwd()
frompath = path.join(path.normpath(self.srcdir), 'dummy')
deps = doctree.settings.record_dependencies
if not deps:
return
for dep in deps.list:
# the dependency path is relative to the working dir, so get
# one relative to the srcdir
if isinstance(dep, bytes):
dep = dep.decode(fs_encoding)
relpath = relative_path(frompath,
path.normpath(path.join(cwd, dep)))
self.dependencies[docname].add(relpath)
def process_downloads(self, docname, doctree):
# type: (unicode, nodes.Node) -> None
"""Process downloadable file paths. """
for node in doctree.traverse(addnodes.download_reference):
targetname = node['reftarget']
rel_filename, filename = self.relfn2path(targetname, docname)
self.dependencies[docname].add(rel_filename)
if not os.access(filename, os.R_OK):
logger.warning('download file not readable: %s', filename,
location=node)
continue
uniquename = self.dlfiles.add_file(docname, filename)
node['filename'] = uniquename
def process_images(self, docname, doctree):
# type: (unicode, nodes.Node) -> None
"""Process and rewrite image URIs."""
def collect_candidates(imgpath, candidates):
globbed = {} # type: Dict[unicode, List[unicode]]
for filename in glob(imgpath):
new_imgpath = relative_path(path.join(self.srcdir, 'dummy'),
filename)
try:
mimetype = guess_mimetype(filename)
if mimetype not in candidates:
globbed.setdefault(mimetype, []).append(new_imgpath)
except (OSError, IOError) as err:
logger.warning('image file %s not readable: %s', filename, err,
location=node)
for key, files in iteritems(globbed):
candidates[key] = sorted(files, key=len)[0] # select by similarity
for node in doctree.traverse(nodes.image):
# Map the mimetype to the corresponding image. The writer may
# choose the best image from these candidates. The special key * is
# set if there is only single candidate to be used by a writer.
# The special key ? is set for nonlocal URIs.
node['candidates'] = candidates = {}
imguri = node['uri']
if imguri.startswith('data:'):
logger.warning('image data URI found. some builders might not support',
location=node, type='image', subtype='data_uri')
candidates['?'] = imguri
continue
elif imguri.find('://') != -1:
logger.warning('nonlocal image URI found: %s', imguri,
location=node, type='image', subtype='nonlocal_uri')
candidates['?'] = imguri
continue
rel_imgpath, full_imgpath = self.relfn2path(imguri, docname)
if self.config.language:
# substitute figures (ex. foo.png -> foo.en.png)
i18n_full_imgpath = search_image_for_language(full_imgpath, self)
if i18n_full_imgpath != full_imgpath:
full_imgpath = i18n_full_imgpath
rel_imgpath = relative_path(path.join(self.srcdir, 'dummy'),
i18n_full_imgpath)
# set imgpath as default URI
node['uri'] = rel_imgpath
if rel_imgpath.endswith(os.extsep + '*'):
if self.config.language:
# Search language-specific figures at first
i18n_imguri = get_image_filename_for_language(imguri, self)
_, full_i18n_imgpath = self.relfn2path(i18n_imguri, docname)
collect_candidates(full_i18n_imgpath, candidates)
collect_candidates(full_imgpath, candidates)
else:
candidates['*'] = rel_imgpath
# map image paths to unique image names (so that they can be put
# into a single directory)
for imgpath in itervalues(candidates):
self.dependencies[docname].add(imgpath)
if not os.access(path.join(self.srcdir, imgpath), os.R_OK):
logger.warning('image file not readable: %s', imgpath,
location=node)
continue
self.images.add_file(docname, imgpath)
def process_metadata(self, docname, doctree):
# type: (unicode, nodes.Node) -> None
"""Process the docinfo part of the doctree as metadata.
Keep processing minimal -- just return what docutils says.
"""
self.metadata[docname] = {}
md = self.metadata[docname]
try:
docinfo = doctree[0]
except IndexError:
# probably an empty document
return
if docinfo.__class__ is not nodes.docinfo:
# nothing to see here
return
for node in docinfo:
# nodes are multiply inherited...
if isinstance(node, nodes.authors):
md['authors'] = [author.astext() for author in node]
elif isinstance(node, nodes.TextElement): # e.g. author
md[node.__class__.__name__] = node.astext()
else:
name, body = node
md[name.astext()] = body.astext()
for name, value in md.items():
if name in ('tocdepth',):
try:
value = int(value)
except ValueError:
value = 0
md[name] = value
del doctree[0]
def create_title_from(self, docname, document):
# type: (unicode, nodes.Node) -> None
"""Add a title node to the document (just copy the first section title),
and store that title in the environment.
"""
titlenode = nodes.title()
longtitlenode = titlenode
# explicit title set with title directive; use this only for
# the <title> tag in HTML output
if 'title' in document:
longtitlenode = nodes.title()
longtitlenode += nodes.Text(document['title'])
# look for first section title and use that as the title
for node in document.traverse(nodes.section):
visitor = SphinxContentsFilter(document)
node[0].walkabout(visitor)
titlenode += visitor.get_entry_text()
break
else:
# document has no title
titlenode += nodes.Text('<no title>')
self.titles[docname] = titlenode
self.longtitles[docname] = longtitlenode
def note_toctree(self, docname, toctreenode):
# type: (unicode, addnodes.toctree) -> None
"""Note a TOC tree directive in a document and gather information about
file relations from it.
"""
self.toctree.note_toctree(docname, toctreenode) # type: ignore
warnings.warn('env.note_toctree() is deprecated. '
'Use sphinx.environment.adapters.toctre.TocTree instead.',
RemovedInSphinx20Warning)
TocTree(self).note(docname, toctreenode)
def get_toc_for(self, docname, builder):
# type: (unicode, Builder) -> addnodes.toctree
# type: (unicode, Builder) -> Dict[unicode, nodes.Node]
"""Return a TOC nodetree -- for use on the same page only!"""
return self.toctree.get_toc_for(docname, builder) # type: ignore
warnings.warn('env.get_toc_for() is deprecated. '
'Use sphinx.environment.adapters.toctre.TocTree instead.',
RemovedInSphinx20Warning)
return TocTree(self).get_toc_for(docname, builder)
def get_toctree_for(self, docname, builder, collapse, **kwds):
# type: (unicode, Builder, bool, Any) -> addnodes.toctree
"""Return the global TOC nodetree."""
return self.toctree.get_toctree_for(docname, builder, collapse, **kwds) # type: ignore
warnings.warn('env.get_toctree_for() is deprecated. '
'Use sphinx.environment.adapters.toctre.TocTree instead.',
RemovedInSphinx20Warning)
return TocTree(self).get_toctree_for(docname, builder, collapse, **kwds)
def get_domain(self, domainname):
# type: (unicode) -> Domain
@ -1078,7 +872,7 @@ class BuildEnvironment(object):
# now, resolve all toctree nodes
for toctreenode in doctree.traverse(addnodes.toctree):
result = self.resolve_toctree(docname, builder, toctreenode,
result = TocTree(self).resolve(docname, builder, toctreenode,
prune=prune_toctrees,
includehidden=includehidden)
if result is None:
@ -1102,7 +896,7 @@ class BuildEnvironment(object):
If *collapse* is True, all branches not containing docname will
be collapsed.
"""
return self.toctree.resolve_toctree(docname, builder, toctree, prune, # type: ignore
return TocTree(self).resolve(docname, builder, toctree, prune,
maxdepth, titles_only, collapse,
includehidden)
@ -1239,8 +1033,13 @@ class BuildEnvironment(object):
def create_index(self, builder, group_entries=True,
_fixre=re.compile(r'(.*) ([(][^()]*[)])')):
# type: (Builder, bool, Pattern) -> Any
return self.indices.create_index(builder, group_entries=group_entries, _fixre=_fixre) # type: ignore # NOQA
# type: (Builder, bool, Pattern) -> List[Tuple[unicode, List[Tuple[unicode, List[unicode]]]]] # NOQA
warnings.warn('env.create_index() is deprecated. '
'Use sphinx.environment.adapters.indexentreis.IndexEntries instead.',
RemovedInSphinx20Warning)
return IndexEntries(self).create_index(builder,
group_entries=group_entries,
_fixre=_fixre)
def collect_relations(self):
# type: () -> Dict[unicode, List[unicode]]

10
sphinx/environment/adapters/__init__.py Normal file
View File

@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.adapters
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Sphinx environment adapters
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""

50
sphinx/environment/managers/indexentries.py → sphinx/environment/adapters/indexentries.py
View File

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.managers.indexentries
sphinx.environment.adapters.indexentries
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Index entries manager for sphinx.environment.
Index entries adapters for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
@ -15,59 +15,27 @@ import string
from itertools import groupby
from six import text_type
from sphinx import addnodes
from sphinx.util import iteritems, split_index_msg, split_into, logging
from sphinx.locale import _
from sphinx.environment.managers import EnvironmentManager
from sphinx.util import iteritems, split_into, logging
if False:
# For type annotation
from typing import Pattern, Tuple # NOQA
from docutils import nodes # NOQA
from typing import Any, Pattern, Tuple # NOQA
from sphinx.builders import Builder # NOQA
from sphinx.environment import BuildEnvironment # NOQA
logger = logging.getLogger(__name__)
class IndexEntries(EnvironmentManager):
name = 'indices'
class IndexEntries(object):
def __init__(self, env):
# type: (BuildEnvironment) -> None
super(IndexEntries, self).__init__(env)
self.data = env.indexentries
def clear_doc(self, docname):
# type: (unicode) -> None
self.data.pop(docname, None)
def merge_other(self, docnames, other):
# type: (List[unicode], BuildEnvironment) -> None
for docname in docnames:
self.data[docname] = other.indexentries[docname]
def process_doc(self, docname, doctree):
# type: (unicode, nodes.Node) -> None
entries = self.data[docname] = []
for node in doctree.traverse(addnodes.index):
try:
for entry in node['entries']:
split_index_msg(entry[0], entry[1])
except ValueError as exc:
logger.warning(str(exc), location=node)
node.parent.remove(node)
else:
for entry in node['entries']:
if len(entry) == 5:
# Since 1.4: new index structure including index_key (5th column)
entries.append(entry)
else:
entries.append(entry + (None,))
self.env = env
def create_index(self, builder, group_entries=True,
_fixre=re.compile(r'(.*) ([(][^()]*[)])')):
# type: (Builder, bool, Pattern) -> List[Tuple[unicode, List[Tuple[unicode, List[unicode]]]]] # NOQA
# type: (Builder, bool, Pattern) -> List[Tuple[unicode, List[Tuple[unicode, Any]]]] # NOQA
"""Create the real index from the collected index entries."""
from sphinx.environment import NoUri
@ -92,7 +60,7 @@ class IndexEntries(EnvironmentManager):
# maintain links in sorted/deterministic order
bisect.insort(entry[0], (main, uri))
for fn, entries in iteritems(self.data):
for fn, entries in iteritems(self.env.indexentries):
# new entry types must be listed in directives/other.py!
for type, value, tid, main, index_key in entries:
try:

355
sphinx/environment/managers/toctree.py → sphinx/environment/adapters/toctree.py
View File

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.managers.toctree
sphinx.environment.adapters.toctree
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Toctree manager for sphinx.environment.
Toctree adapter for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
@ -16,189 +16,38 @@ from docutils import nodes
from sphinx import addnodes
from sphinx.util import url_re, logging
from sphinx.util.nodes import clean_astext, process_only_nodes
from sphinx.transforms import SphinxContentsFilter
from sphinx.environment.managers import EnvironmentManager
if False:
# For type annotation
from typing import Any, Tuple # NOQA
from typing import Any # NOQA
from sphinx.builders import Builder # NOQA
from sphinx.environment import BuildEnvironment # NOQA
logger = logging.getLogger(__name__)
class Toctree(EnvironmentManager):
name = 'toctree'
class TocTree(object):
def __init__(self, env):
# type: (BuildEnvironment) -> None
super(Toctree, self).__init__(env)
self.env = env
self.tocs = env.tocs
self.toc_num_entries = env.toc_num_entries
self.toc_secnumbers = env.toc_secnumbers
self.toc_fignumbers = env.toc_fignumbers
self.toctree_includes = env.toctree_includes
self.files_to_rebuild = env.files_to_rebuild
self.glob_toctrees = env.glob_toctrees
self.numbered_toctrees = env.numbered_toctrees
def clear_doc(self, docname):
# type: (unicode) -> None
self.tocs.pop(docname, None)
self.toc_secnumbers.pop(docname, None)
self.toc_fignumbers.pop(docname, None)
self.toc_num_entries.pop(docname, None)
self.toctree_includes.pop(docname, None)
self.glob_toctrees.discard(docname)
self.numbered_toctrees.discard(docname)
for subfn, fnset in list(self.files_to_rebuild.items()):
fnset.discard(docname)
if not fnset:
del self.files_to_rebuild[subfn]
def merge_other(self, docnames, other):
# type: (List[unicode], BuildEnvironment) -> None
for docname in docnames:
self.tocs[docname] = other.tocs[docname]
self.toc_num_entries[docname] = other.toc_num_entries[docname]
if docname in other.toctree_includes:
self.toctree_includes[docname] = other.toctree_includes[docname]
if docname in other.glob_toctrees:
self.glob_toctrees.add(docname)
if docname in other.numbered_toctrees:
self.numbered_toctrees.add(docname)
for subfn, fnset in other.files_to_rebuild.items():
self.files_to_rebuild.setdefault(subfn, set()).update(fnset & set(docnames))
def process_doc(self, docname, doctree):
# type: (unicode, nodes.Node) -> None
"""Build a TOC from the doctree and store it in the inventory."""
numentries = [0] # nonlocal again...
def traverse_in_section(node, cls):
"""Like traverse(), but stay within the same section."""
result = []
if isinstance(node, cls):
result.append(node)
for child in node.children:
if isinstance(child, nodes.section):
continue
result.extend(traverse_in_section(child, cls))
return result
def build_toc(node, depth=1):
entries = []
for sectionnode in node:
# find all toctree nodes in this section and add them
# to the toc (just copying the toctree node which is then
# resolved in self.get_and_resolve_doctree)
if isinstance(sectionnode, addnodes.only):
onlynode = addnodes.only(expr=sectionnode['expr'])
blist = build_toc(sectionnode, depth)
if blist:
onlynode += blist.children
entries.append(onlynode)
continue
if not isinstance(sectionnode, nodes.section):
for toctreenode in traverse_in_section(sectionnode,
addnodes.toctree):
item = toctreenode.copy()
entries.append(item)
# important: do the inventory stuff
self.note_toctree(docname, toctreenode)
continue
title = sectionnode[0]
# copy the contents of the section title, but without references
# and unnecessary stuff
visitor = SphinxContentsFilter(doctree)
title.walkabout(visitor)
nodetext = visitor.get_entry_text()
if not numentries[0]:
# for the very first toc entry, don't add an anchor
# as it is the file's title anyway
anchorname = ''
else:
anchorname = '#' + sectionnode['ids'][0]
numentries[0] += 1
# make these nodes:
# list_item -> compact_paragraph -> reference
reference = nodes.reference(
'', '', internal=True, refuri=docname,
anchorname=anchorname, *nodetext)
para = addnodes.compact_paragraph('', '', reference)
item = nodes.list_item('', para)
sub_item = build_toc(sectionnode, depth + 1)
item += sub_item
entries.append(item)
if entries:
return nodes.bullet_list('', *entries)
return []
toc = build_toc(doctree)
if toc:
self.tocs[docname] = toc
else:
self.tocs[docname] = nodes.bullet_list('')
self.toc_num_entries[docname] = numentries[0]
def note_toctree(self, docname, toctreenode):
def note(self, docname, toctreenode):
# type: (unicode, addnodes.toctree) -> None
"""Note a TOC tree directive in a document and gather information about
file relations from it.
"""
if toctreenode['glob']:
self.glob_toctrees.add(docname)
self.env.glob_toctrees.add(docname)
if toctreenode.get('numbered'):
self.numbered_toctrees.add(docname)
self.env.numbered_toctrees.add(docname)
includefiles = toctreenode['includefiles']
for includefile in includefiles:
# note that if the included file is rebuilt, this one must be
# too (since the TOC of the included file could have changed)
self.files_to_rebuild.setdefault(includefile, set()).add(docname)
self.toctree_includes.setdefault(docname, []).extend(includefiles)
self.env.files_to_rebuild.setdefault(includefile, set()).add(docname)
self.env.toctree_includes.setdefault(docname, []).extend(includefiles)
def get_toc_for(self, docname, builder):
# type: (unicode, Builder) -> None
"""Return a TOC nodetree -- for use on the same page only!"""
tocdepth = self.env.metadata[docname].get('tocdepth', 0)
try:
toc = self.tocs[docname].deepcopy()
self._toctree_prune(toc, 2, tocdepth)
except KeyError:
# the document does not exist anymore: return a dummy node that
# renders to nothing
return nodes.paragraph()
process_only_nodes(toc, builder.tags)
for node in toc.traverse(nodes.reference):
node['refuri'] = node['anchorname'] or '#'
return toc
def get_toctree_for(self, docname, builder, collapse, **kwds):
# type: (unicode, Builder, bool, Any) -> nodes.Node
"""Return the global TOC nodetree."""
doctree = self.env.get_doctree(self.env.config.master_doc)
toctrees = []
if 'includehidden' not in kwds:
kwds['includehidden'] = True
if 'maxdepth' not in kwds:
kwds['maxdepth'] = 0
kwds['collapse'] = collapse
for toctreenode in doctree.traverse(addnodes.toctree):
toctree = self.env.resolve_toctree(docname, builder, toctreenode,
prune=True, **kwds)
if toctree:
toctrees.append(toctree)
if not toctrees:
return None
result = toctrees[0]
for toctree in toctrees[1:]:
result.extend(toctree.children)
return result
def resolve_toctree(self, docname, builder, toctree, prune=True, maxdepth=0,
def resolve(self, docname, builder, toctree, prune=True, maxdepth=0,
titles_only=False, collapse=False, includehidden=False):
# type: (unicode, Builder, addnodes.toctree, bool, int, bool, bool, bool) -> nodes.Node
"""Resolve a *toctree* node into individual bullet lists with titles
@ -304,7 +153,7 @@ class Toctree(EnvironmentManager):
location=ref)
continue
refdoc = ref
toc = self.tocs[ref].deepcopy()
toc = self.env.tocs[ref].deepcopy()
maxdepth = self.env.metadata[ref].get('tocdepth', 0)
if ref not in toctree_ancestors or (prune and maxdepth > 0):
self._toctree_prune(toc, 2, maxdepth, collapse)
@ -405,7 +254,7 @@ class Toctree(EnvironmentManager):
def get_toctree_ancestors(self, docname):
# type: (unicode) -> List[unicode]
parent = {}
for p, children in iteritems(self.toctree_includes):
for p, children in iteritems(self.env.toctree_includes):
for child in children:
parent[child] = p
ancestors = [] # type: List[unicode]
@ -437,149 +286,39 @@ class Toctree(EnvironmentManager):
# recurse on visible children
self._toctree_prune(subnode, depth + 1, maxdepth, collapse)
def assign_section_numbers(self):
# type: () -> List[unicode]
"""Assign a section number to each heading under a numbered toctree."""
# a list of all docnames whose section numbers changed
rewrite_needed = []
def get_toc_for(self, docname, builder):
# type: (unicode, Builder) -> Dict[unicode, nodes.Node]
"""Return a TOC nodetree -- for use on the same page only!"""
tocdepth = self.env.metadata[docname].get('tocdepth', 0)
try:
toc = self.env.tocs[docname].deepcopy()
self._toctree_prune(toc, 2, tocdepth)
except KeyError:
# the document does not exist anymore: return a dummy node that
# renders to nothing
return nodes.paragraph()
process_only_nodes(toc, builder.tags)
for node in toc.traverse(nodes.reference):
node['refuri'] = node['anchorname'] or '#'
return toc
assigned = set() # type: Set[unicode]
old_secnumbers = self.toc_secnumbers
self.toc_secnumbers = self.env.toc_secnumbers = {}
def _walk_toc(node, secnums, depth, titlenode=None):
# titlenode is the title of the document, it will get assigned a
# secnumber too, so that it shows up in next/prev/parent rellinks
for subnode in node.children:
if isinstance(subnode, nodes.bullet_list):
numstack.append(0)
_walk_toc(subnode, secnums, depth - 1, titlenode)
numstack.pop()
titlenode = None
elif isinstance(subnode, nodes.list_item):
_walk_toc(subnode, secnums, depth, titlenode)
titlenode = None
elif isinstance(subnode, addnodes.only):
# at this stage we don't know yet which sections are going
# to be included; just include all of them, even if it leads
# to gaps in the numbering
_walk_toc(subnode, secnums, depth, titlenode)
titlenode = None
elif isinstance(subnode, addnodes.compact_paragraph):
numstack[-1] += 1
if depth > 0:
number = tuple(numstack)
else:
number = None
secnums[subnode[0]['anchorname']] = \
subnode[0]['secnumber'] = number
if titlenode:
titlenode['secnumber'] = number
titlenode = None
elif isinstance(subnode, addnodes.toctree):
_walk_toctree(subnode, depth)
def _walk_toctree(toctreenode, depth):
if depth == 0:
return
for (title, ref) in toctreenode['entries']:
if url_re.match(ref) or ref == 'self':
# don't mess with those
continue
elif ref in assigned:
logger.warning('%s is already assigned section numbers '
'(nested numbered toctree?)', ref,
location=toctreenode, type='toc', subtype='secnum')
elif ref in self.tocs:
secnums = self.toc_secnumbers[ref] = {}
assigned.add(ref)
_walk_toc(self.tocs[ref], secnums, depth,
self.env.titles.get(ref))
if secnums != old_secnumbers.get(ref):
rewrite_needed.append(ref)
for docname in self.numbered_toctrees:
assigned.add(docname)
doctree = self.env.get_doctree(docname)
def get_toctree_for(self, docname, builder, collapse, **kwds):
# type: (unicode, Builder, bool, Any) -> nodes.Node
"""Return the global TOC nodetree."""
doctree = self.env.get_doctree(self.env.config.master_doc)
toctrees = []
if 'includehidden' not in kwds:
kwds['includehidden'] = True
if 'maxdepth' not in kwds:
kwds['maxdepth'] = 0
kwds['collapse'] = collapse
for toctreenode in doctree.traverse(addnodes.toctree):
depth = toctreenode.get('numbered', 0)
if depth:
# every numbered toctree gets new numbering
numstack = [0]
_walk_toctree(toctreenode, depth)
return rewrite_needed
def assign_figure_numbers(self):
# type: () -> List[unicode]
"""Assign a figure number to each figure under a numbered toctree."""
rewrite_needed = []
assigned = set() # type: Set[unicode]
old_fignumbers = self.toc_fignumbers
self.toc_fignumbers = self.env.toc_fignumbers = {}
fignum_counter = {} # type: Dict[unicode, Dict[Tuple[int], int]]
def get_section_number(docname, section):
anchorname = '#' + section['ids'][0]
secnumbers = self.toc_secnumbers.get(docname, {})
if anchorname in secnumbers:
secnum = secnumbers.get(anchorname)
else:
secnum = secnumbers.get('')
return secnum or tuple()
def get_next_fignumber(figtype, secnum):
counter = fignum_counter.setdefault(figtype, {})
secnum = secnum[:self.env.config.numfig_secnum_depth]
counter[secnum] = counter.get(secnum, 0) + 1
return secnum + (counter[secnum],)
def register_fignumber(docname, secnum, figtype, fignode):
self.toc_fignumbers.setdefault(docname, {})
fignumbers = self.toc_fignumbers[docname].setdefault(figtype, {})
figure_id = fignode['ids'][0]
fignumbers[figure_id] = get_next_fignumber(figtype, secnum)
def _walk_doctree(docname, doctree, secnum):
for subnode in doctree.children:
if isinstance(subnode, nodes.section):
next_secnum = get_section_number(docname, subnode)
if next_secnum:
_walk_doctree(docname, subnode, next_secnum)
else:
_walk_doctree(docname, subnode, secnum)
continue
elif isinstance(subnode, addnodes.toctree):
for title, subdocname in subnode['entries']:
if url_re.match(subdocname) or subdocname == 'self':
# don't mess with those
continue
_walk_doc(subdocname, secnum)
continue
figtype = self.env.get_domain('std').get_figtype(subnode) # type: ignore
if figtype and subnode['ids']:
register_fignumber(docname, secnum, figtype, subnode)
_walk_doctree(docname, subnode, secnum)
def _walk_doc(docname, secnum):
if docname not in assigned:
assigned.add(docname)
doctree = self.env.get_doctree(docname)
_walk_doctree(docname, doctree, secnum)
if self.env.config.numfig:
_walk_doc(self.env.config.master_doc, tuple())
for docname, fignums in iteritems(self.toc_fignumbers):
if fignums != old_fignumbers.get(docname):
rewrite_needed.append(docname)
return rewrite_needed
toctree = self.resolve(docname, builder, toctreenode, prune=True, **kwds)
if toctree:
toctrees.append(toctree)
if not toctrees:
return None
result = toctrees[0]
for toctree in toctrees[1:]:
result.extend(toctree.children)
return result

84
sphinx/environment/collectors/__init__.py Normal file
View File

@ -0,0 +1,84 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.collectors
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The data collector components for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from six import itervalues
if False:
# For type annotation
from docutils import nodes # NOQA
from sphinx.sphinx import Sphinx # NOQA
from sphinx.environment import BuildEnvironment # NOQA
class EnvironmentCollector(object):
"""An EnvironmentCollector is a specific data collector from each document.
It gathers data and stores :py:class:`BuildEnvironment
<sphinx.environment.BuildEnvironment>` as a database. Examples of specific
data would be images, download files, section titles, metadatas, index
entries and toctrees, etc.
"""
listener_ids = None # type: Dict[unicode, int]
def enable(self, app):
# type: (Sphinx) -> None
assert self.listener_ids is None
self.listener_ids = {
'doctree-read': app.connect('doctree-read', self.process_doc),
'env-merge-info': app.connect('env-merge-info', self.merge_other),
'env-purge-doc': app.connect('env-purge-doc', self.clear_doc),
'env-get-updated': app.connect('env-get-updated', self.get_updated_docs),
'env-get-outdated': app.connect('env-get-outdated', self.get_outdated_docs),
}
def disable(self, app):
# type: (Sphinx) -> None
assert self.listener_ids is not None
for listener_id in itervalues(self.listener_ids):
app.disconnect(listener_id)
self.listener_ids = None
def clear_doc(self, app, env, docname):
# type: (Sphinx, BuildEnvironment, unicode) -> None
"""Remove specified data of a document.
This method is called on the removal of the document."""
raise NotImplementedError
def merge_other(self, app, env, docnames, other):
# type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
"""Merge in specified data regarding docnames from a different `BuildEnvironment`
object which coming from a subprocess in parallel builds."""
raise NotImplementedError
def process_doc(self, app, doctree):
# type: (Sphinx, nodes.Node) -> None
"""Process a document and gather specific data from it.
This method is called after the document is read."""
raise NotImplementedError
def get_updated_docs(self, app, env):
# type: (Sphinx, BuildEnvironment) -> List[unicode]
"""Return a list of docnames to re-read.
This methods is called after reading the whole of documents (experimental).
"""
return []
def get_outdated_docs(self, app, env, added, changed, removed):
# type: (Sphinx, BuildEnvironment, unicode, Set[unicode], Set[unicode], Set[unicode]) -> List[unicode] # NOQA
"""Return a list of docnames to re-read.
This methods is called before reading the documents.
"""
return []

148
sphinx/environment/collectors/asset.py Normal file
View File

@ -0,0 +1,148 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.collectors.asset
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The image collector for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import os
from os import path
from glob import glob
from six import iteritems, itervalues
from docutils import nodes
from docutils.utils import relative_path
from sphinx import addnodes
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.util import logging
from sphinx.util.i18n import get_image_filename_for_language, search_image_for_language
from sphinx.util.images import guess_mimetype
if False:
# For type annotation
from typing import Tuple # NOQA
from docutils import nodes # NOQA
from sphinx.sphinx import Sphinx # NOQA
from sphinx.environment import BuildEnvironment # NOQA
logger = logging.getLogger(__name__)
class ImageCollector(EnvironmentCollector):
"""Image files collector for sphinx.environment."""
def clear_doc(self, app, env, docname):
# type: (Sphinx, BuildEnvironment, unicode) -> None
env.images.purge_doc(docname)
def merge_other(self, app, env, docnames, other):
# type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
env.images.merge_other(docnames, other.images)
def process_doc(self, app, doctree):
# type: (Sphinx, nodes.Node) -> None
"""Process and rewrite image URIs."""
docname = app.env.docname
for node in doctree.traverse(nodes.image):
# Map the mimetype to the corresponding image. The writer may
# choose the best image from these candidates. The special key * is
# set if there is only single candidate to be used by a writer.
# The special key ? is set for nonlocal URIs.
candidates = {} # type: Dict[unicode, unicode]
node['candidates'] = candidates
imguri = node['uri']
if imguri.startswith('data:'):
logger.warning('image data URI found. some builders might not support',
location=node, type='image', subtype='data_uri')
candidates['?'] = imguri
continue
elif imguri.find('://') != -1:
logger.warning('nonlocal image URI found: %s' % imguri,
location=node,
type='image', subtype='nonlocal_uri')
candidates['?'] = imguri
continue
rel_imgpath, full_imgpath = app.env.relfn2path(imguri, docname)
if app.config.language:
# substitute figures (ex. foo.png -> foo.en.png)
i18n_full_imgpath = search_image_for_language(full_imgpath, app.env)
if i18n_full_imgpath != full_imgpath:
full_imgpath = i18n_full_imgpath
rel_imgpath = relative_path(path.join(app.srcdir, 'dummy'),
i18n_full_imgpath)
# set imgpath as default URI
node['uri'] = rel_imgpath
if rel_imgpath.endswith(os.extsep + '*'):
if app.config.language:
# Search language-specific figures at first
i18n_imguri = get_image_filename_for_language(imguri, app.env)
_, full_i18n_imgpath = app.env.relfn2path(i18n_imguri, docname)
self.collect_candidates(app.env, full_i18n_imgpath, candidates, node)
self.collect_candidates(app.env, full_imgpath, candidates, node)
else:
candidates['*'] = rel_imgpath
# map image paths to unique image names (so that they can be put
# into a single directory)
for imgpath in itervalues(candidates):
app.env.dependencies[docname].add(imgpath)
if not os.access(path.join(app.srcdir, imgpath), os.R_OK):
logger.warning('image file not readable: %s' % imgpath,
location=node)
continue
app.env.images.add_file(docname, imgpath)
def collect_candidates(self, env, imgpath, candidates, node):
# type: (BuildEnvironment, unicode, Dict[unicode, unicode], nodes.Node) -> None
globbed = {} # type: Dict[unicode, List[unicode]]
for filename in glob(imgpath):
new_imgpath = relative_path(path.join(env.srcdir, 'dummy'),
filename)
try:
mimetype = guess_mimetype(filename)
if mimetype not in candidates:
globbed.setdefault(mimetype, []).append(new_imgpath)
except (OSError, IOError) as err:
logger.warning('image file %s not readable: %s' % (filename, err),
location=node)
for key, files in iteritems(globbed):
candidates[key] = sorted(files, key=len)[0] # select by similarity
class DownloadFileCollector(EnvironmentCollector):
"""Download files collector for sphinx.environment."""
def clear_doc(self, app, env, docname):
# type: (Sphinx, BuildEnvironment, unicode) -> None
env.dlfiles.purge_doc(docname)
def merge_other(self, app, env, docnames, other):
# type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
env.dlfiles.merge_other(docnames, other.dlfiles)
def process_doc(self, app, doctree):
# type: (Sphinx, nodes.Node) -> None
"""Process downloadable file paths. """
for node in doctree.traverse(addnodes.download_reference):
targetname = node['reftarget']
rel_filename, filename = app.env.relfn2path(targetname, app.env.docname)
app.env.dependencies[app.env.docname].add(rel_filename)
if not os.access(filename, os.R_OK):
logger.warning('download file not readable: %s' % filename,
location=node)
continue
node['filename'] = app.env.dlfiles.add_file(app.env.docname, filename)
def setup(app):
# type: (Sphinx) -> None
app.add_env_collector(ImageCollector)
app.add_env_collector(DownloadFileCollector)

59
sphinx/environment/collectors/dependencies.py Normal file
View File

@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.collectors.dependencies
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The dependencies collector components for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from os import path
from docutils.utils import relative_path
from sphinx.util.osutil import getcwd, fs_encoding
from sphinx.environment.collectors import EnvironmentCollector
if False:
# For type annotation
from docutils import nodes # NOQA
from sphinx.sphinx import Sphinx # NOQA
from sphinx.environment import BuildEnvironment # NOQA
class DependenciesCollector(EnvironmentCollector):
"""dependencies collector for sphinx.environment."""
def clear_doc(self, app, env, docname):
# type: (Sphinx, BuildEnvironment, unicode) -> None
env.dependencies.pop(docname, None)
def merge_other(self, app, env, docnames, other):
# type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
for docname in docnames:
if docname in other.dependencies:
env.dependencies[docname] = other.dependencies[docname]
def process_doc(self, app, doctree):
# type: (Sphinx, nodes.Node) -> None
"""Process docutils-generated dependency info."""
cwd = getcwd()
frompath = path.join(path.normpath(app.srcdir), 'dummy')
deps = doctree.settings.record_dependencies
if not deps:
return
for dep in deps.list:
# the dependency path is relative to the working dir, so get
# one relative to the srcdir
if isinstance(dep, bytes):
dep = dep.decode(fs_encoding)
relpath = relative_path(frompath,
path.normpath(path.join(cwd, dep)))
app.env.dependencies[app.env.docname].add(relpath)
def setup(app):
# type: (Sphinx) -> None
app.add_env_collector(DependenciesCollector)

59
sphinx/environment/collectors/indexentries.py Normal file
View File

@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.collectors.indexentries
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Index entries collector for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from sphinx import addnodes
from sphinx.util import split_index_msg, logging
from sphinx.environment.collectors import EnvironmentCollector
if False:
# For type annotation
from docutils import nodes # NOQA
from sphinx.applicatin import Sphinx # NOQA
from sphinx.environment import BuildEnvironment # NOQA
logger = logging.getLogger(__name__)
class IndexEntriesCollector(EnvironmentCollector):
name = 'indices'
def clear_doc(self, app, env, docname):
# type: (Sphinx, BuildEnvironment, unicode) -> None
env.indexentries.pop(docname, None)
def merge_other(self, app, env, docnames, other):
# type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
for docname in docnames:
env.indexentries[docname] = other.indexentries[docname]
def process_doc(self, app, doctree):
# type: (Sphinx, nodes.Node) -> None
docname = app.env.docname
entries = app.env.indexentries[docname] = []
for node in doctree.traverse(addnodes.index):
try:
for entry in node['entries']:
split_index_msg(entry[0], entry[1])
except ValueError as exc:
logger.warning(str(exc), location=node)
node.parent.remove(node)
else:
for entry in node['entries']:
if len(entry) == 5:
# Since 1.4: new index structure including index_key (5th column)
entries.append(entry)
else:
entries.append(entry + (None,))
def setup(app):
# type: (Sphinx) -> None
app.add_env_collector(IndexEntriesCollector)

72
sphinx/environment/collectors/metadata.py Normal file
View File

@ -0,0 +1,72 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.collectors.metadata
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The metadata collector components for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from docutils import nodes
from sphinx.environment.collectors import EnvironmentCollector
if False:
# For type annotation
from docutils import nodes # NOQA
from sphinx.sphinx import Sphinx # NOQA
from sphinx.environment import BuildEnvironment # NOQA
class MetadataCollector(EnvironmentCollector):
"""metadata collector for sphinx.environment."""
def clear_doc(self, app, env, docname):
# type: (Sphinx, BuildEnvironment, unicode) -> None
env.metadata.pop(docname, None)
def merge_other(self, app, env, docnames, other):
# type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
for docname in docnames:
env.metadata[docname] = other.metadata[docname]
def process_doc(self, app, doctree):
# type: (Sphinx, nodes.Node) -> None
"""Process the docinfo part of the doctree as metadata.
Keep processing minimal -- just return what docutils says.
"""
md = app.env.metadata[app.env.docname]
try:
docinfo = doctree[0]
except IndexError:
# probably an empty document
return
if docinfo.__class__ is not nodes.docinfo:
# nothing to see here
return
for node in docinfo:
# nodes are multiply inherited...
if isinstance(node, nodes.authors):
md['authors'] = [author.astext() for author in node]
elif isinstance(node, nodes.TextElement): # e.g. author
md[node.__class__.__name__] = node.astext()
else:
name, body = node
md[name.astext()] = body.astext()
for name, value in md.items():
if name in ('tocdepth',):
try:
value = int(value)
except ValueError:
value = 0
md[name] = value
del doctree[0]
def setup(app):
# type: (Sphinx) -> None
app.add_env_collector(MetadataCollector)

65
sphinx/environment/collectors/title.py Normal file
View File

@ -0,0 +1,65 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.collectors.title
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The title collector components for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from docutils import nodes
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.transforms import SphinxContentsFilter
if False:
# For type annotation
from docutils import nodes # NOQA
from sphinx.sphinx import Sphinx # NOQA
from sphinx.environment import BuildEnvironment # NOQA
class TitleCollector(EnvironmentCollector):
"""title collector for sphinx.environment."""
def clear_doc(self, app, env, docname):
# type: (Sphinx, BuildEnvironment, unicode) -> None
env.titles.pop(docname, None)
env.longtitles.pop(docname, None)
def merge_other(self, app, env, docnames, other):
# type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
for docname in docnames:
env.titles[docname] = other.titles[docname]
env.longtitles[docname] = other.longtitles[docname]
def process_doc(self, app, doctree):
# type: (Sphinx, nodes.Node) -> None
"""Add a title node to the document (just copy the first section title),
and store that title in the environment.
"""
titlenode = nodes.title()
longtitlenode = titlenode
# explicit title set with title directive; use this only for
# the <title> tag in HTML output
if 'title' in doctree:
longtitlenode = nodes.title()
longtitlenode += nodes.Text(doctree['title'])
# look for first section title and use that as the title
for node in doctree.traverse(nodes.section):
visitor = SphinxContentsFilter(doctree)
node[0].walkabout(visitor)
titlenode += visitor.get_entry_text()
break
else:
# document has no title
titlenode += nodes.Text('<no title>')
app.env.titles[app.env.docname] = titlenode
app.env.longtitles[app.env.docname] = longtitlenode
def setup(app):
# type: (Sphinx) -> None
app.add_env_collector(TitleCollector)

288
sphinx/environment/collectors/toctree.py Normal file
View File

@ -0,0 +1,288 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.collectors.toctree
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Toctree collector for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from six import iteritems
from docutils import nodes
from sphinx import addnodes
from sphinx.util import url_re, logging
from sphinx.transforms import SphinxContentsFilter
from sphinx.environment.adapters.toctree import TocTree
from sphinx.environment.collectors import EnvironmentCollector
if False:
# For type annotation
from typing import Any, Tuple # NOQA
from sphinx.application import Sphinx # NOQA
from sphinx.builders import Builder # NOQA
from sphinx.environment import BuildEnvironment # NOQA
logger = logging.getLogger(__name__)
class TocTreeCollector(EnvironmentCollector):
def clear_doc(self, app, env, docname):
# type: (Sphinx, BuildEnvironment, unicode) -> None
env.tocs.pop(docname, None)
env.toc_secnumbers.pop(docname, None)
env.toc_fignumbers.pop(docname, None)
env.toc_num_entries.pop(docname, None)
env.toctree_includes.pop(docname, None)
env.glob_toctrees.discard(docname)
env.numbered_toctrees.discard(docname)
for subfn, fnset in list(env.files_to_rebuild.items()):
fnset.discard(docname)
if not fnset:
del env.files_to_rebuild[subfn]
def merge_other(self, app, env, docnames, other):
# type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
for docname in docnames:
env.tocs[docname] = other.tocs[docname]
env.toc_num_entries[docname] = other.toc_num_entries[docname]
if docname in other.toctree_includes:
env.toctree_includes[docname] = other.toctree_includes[docname]
if docname in other.glob_toctrees:
env.glob_toctrees.add(docname)
if docname in other.numbered_toctrees:
env.numbered_toctrees.add(docname)
for subfn, fnset in other.files_to_rebuild.items():
env.files_to_rebuild.setdefault(subfn, set()).update(fnset & set(docnames))
def process_doc(self, app, doctree):
# type: (Sphinx, nodes.Node) -> None
"""Build a TOC from the doctree and store it in the inventory."""
docname = app.env.docname
numentries = [0] # nonlocal again...
def traverse_in_section(node, cls):
"""Like traverse(), but stay within the same section."""
result = []
if isinstance(node, cls):
result.append(node)
for child in node.children:
if isinstance(child, nodes.section):
continue
result.extend(traverse_in_section(child, cls))
return result
def build_toc(node, depth=1):
entries = []
for sectionnode in node:
# find all toctree nodes in this section and add them
# to the toc (just copying the toctree node which is then
# resolved in self.get_and_resolve_doctree)
if isinstance(sectionnode, addnodes.only):
onlynode = addnodes.only(expr=sectionnode['expr'])
blist = build_toc(sectionnode, depth)
if blist:
onlynode += blist.children
entries.append(onlynode)
continue
if not isinstance(sectionnode, nodes.section):
for toctreenode in traverse_in_section(sectionnode,
addnodes.toctree):
item = toctreenode.copy()
entries.append(item)
# important: do the inventory stuff
TocTree(app.env).note(docname, toctreenode)
continue
title = sectionnode[0]
# copy the contents of the section title, but without references
# and unnecessary stuff
visitor = SphinxContentsFilter(doctree)
title.walkabout(visitor)
nodetext = visitor.get_entry_text()
if not numentries[0]:
# for the very first toc entry, don't add an anchor
# as it is the file's title anyway
anchorname = ''
else:
anchorname = '#' + sectionnode['ids'][0]
numentries[0] += 1
# make these nodes:
# list_item -> compact_paragraph -> reference
reference = nodes.reference(
'', '', internal=True, refuri=docname,
anchorname=anchorname, *nodetext)
para = addnodes.compact_paragraph('', '', reference)
item = nodes.list_item('', para)
sub_item = build_toc(sectionnode, depth + 1)
item += sub_item
entries.append(item)
if entries:
return nodes.bullet_list('', *entries)
return []
toc = build_toc(doctree)
if toc:
app.env.tocs[docname] = toc
else:
app.env.tocs[docname] = nodes.bullet_list('')
app.env.toc_num_entries[docname] = numentries[0]
def get_updated_docs(self, app, env):
# type: (Sphinx, BuildEnvironment) -> List[unicode]
return self.assign_section_numbers(env) + self.assign_figure_numbers(env)
def assign_section_numbers(self, env):
# type: (BuildEnvironment) -> List[unicode]
"""Assign a section number to each heading under a numbered toctree."""
# a list of all docnames whose section numbers changed
rewrite_needed = []
assigned = set() # type: Set[unicode]
old_secnumbers = env.toc_secnumbers
env.toc_secnumbers = {}
def _walk_toc(node, secnums, depth, titlenode=None):
# titlenode is the title of the document, it will get assigned a
# secnumber too, so that it shows up in next/prev/parent rellinks
for subnode in node.children:
if isinstance(subnode, nodes.bullet_list):
numstack.append(0)
_walk_toc(subnode, secnums, depth - 1, titlenode)
numstack.pop()
titlenode = None
elif isinstance(subnode, nodes.list_item):
_walk_toc(subnode, secnums, depth, titlenode)
titlenode = None
elif isinstance(subnode, addnodes.only):
# at this stage we don't know yet which sections are going
# to be included; just include all of them, even if it leads
# to gaps in the numbering
_walk_toc(subnode, secnums, depth, titlenode)
titlenode = None
elif isinstance(subnode, addnodes.compact_paragraph):
numstack[-1] += 1
if depth > 0:
number = tuple(numstack)
else:
number = None
secnums[subnode[0]['anchorname']] = \
subnode[0]['secnumber'] = number
if titlenode:
titlenode['secnumber'] = number
titlenode = None
elif isinstance(subnode, addnodes.toctree):
_walk_toctree(subnode, depth)
def _walk_toctree(toctreenode, depth):
if depth == 0:
return
for (title, ref) in toctreenode['entries']:
if url_re.match(ref) or ref == 'self':
# don't mess with those
continue
elif ref in assigned:
logger.warning('%s is already assigned section numbers '
'(nested numbered toctree?)', ref,
location=toctreenode, type='toc', subtype='secnum')
elif ref in env.tocs:
secnums = env.toc_secnumbers[ref] = {}
assigned.add(ref)
_walk_toc(env.tocs[ref], secnums, depth,
env.titles.get(ref))
if secnums != old_secnumbers.get(ref):
rewrite_needed.append(ref)
for docname in env.numbered_toctrees:
assigned.add(docname)
doctree = env.get_doctree(docname)
for toctreenode in doctree.traverse(addnodes.toctree):
depth = toctreenode.get('numbered', 0)
if depth:
# every numbered toctree gets new numbering
numstack = [0]
_walk_toctree(toctreenode, depth)
return rewrite_needed
def assign_figure_numbers(self, env):
# type: (BuildEnvironment) -> List[unicode]
"""Assign a figure number to each figure under a numbered toctree."""
rewrite_needed = []
assigned = set() # type: Set[unicode]
old_fignumbers = env.toc_fignumbers
env.toc_fignumbers = {}
fignum_counter = {} # type: Dict[unicode, Dict[Tuple[int], int]]
def get_section_number(docname, section):
anchorname = '#' + section['ids'][0]
secnumbers = env.toc_secnumbers.get(docname, {})
if anchorname in secnumbers:
secnum = secnumbers.get(anchorname)
else:
secnum = secnumbers.get('')
return secnum or tuple()
def get_next_fignumber(figtype, secnum):
counter = fignum_counter.setdefault(figtype, {})
secnum = secnum[:env.config.numfig_secnum_depth]
counter[secnum] = counter.get(secnum, 0) + 1
return secnum + (counter[secnum],)
def register_fignumber(docname, secnum, figtype, fignode):
env.toc_fignumbers.setdefault(docname, {})
fignumbers = env.toc_fignumbers[docname].setdefault(figtype, {})
figure_id = fignode['ids'][0]
fignumbers[figure_id] = get_next_fignumber(figtype, secnum)
def _walk_doctree(docname, doctree, secnum):
for subnode in doctree.children:
if isinstance(subnode, nodes.section):
next_secnum = get_section_number(docname, subnode)
if next_secnum:
_walk_doctree(docname, subnode, next_secnum)
else:
_walk_doctree(docname, subnode, secnum)
continue
elif isinstance(subnode, addnodes.toctree):
for title, subdocname in subnode['entries']:
if url_re.match(subdocname) or subdocname == 'self':
# don't mess with those
continue
_walk_doc(subdocname, secnum)
continue
figtype = env.get_domain('std').get_figtype(subnode) # type: ignore
if figtype and subnode['ids']:
register_fignumber(docname, secnum, figtype, subnode)
_walk_doctree(docname, subnode, secnum)
def _walk_doc(docname, secnum):
if docname not in assigned:
assigned.add(docname)
doctree = env.get_doctree(docname)
_walk_doctree(docname, doctree, secnum)
if env.config.numfig:
_walk_doc(env.config.master_doc, tuple())
for docname, fignums in iteritems(env.toc_fignumbers):
if fignums != old_fignumbers.get(docname):
rewrite_needed.append(docname)
return rewrite_needed
def setup(app):
# type: (Sphinx) -> None
app.add_env_collector(TocTreeCollector)

50
sphinx/environment/managers/__init__.py
View File

@ -1,50 +0,0 @@
# -*- coding: utf-8 -*-
"""
sphinx.environment.managers
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Manager components for sphinx.environment.
:copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
if False:
# For type annotation
from typing import Any # NOQA
from docutils import nodes # NOQA
from sphinx.environment import BuildEnvironment # NOQA
class EnvironmentManager(object):
"""Base class for sphinx.environment managers."""
name = None # type: unicode
env = None # type: BuildEnvironment
def __init__(self, env):
# type: (BuildEnvironment) -> None
self.env = env
def attach(self, env):
# type: (BuildEnvironment) -> None
self.env = env
if self.name:
setattr(env, self.name, self)
def detach(self, env):
# type: (BuildEnvironment) -> None
self.env = None
if self.name:
delattr(env, self.name)
def clear_doc(self, docname):
# type: (unicode) -> None
raise NotImplementedError
def merge_other(self, docnames, other):
# type: (List[unicode], Any) -> None
raise NotImplementedError
def process_doc(self, docname, doctree):
# type: (unicode, nodes.Node) -> None
raise NotImplementedError

3
sphinx/ext/autosummary/__init__.py
View File

@ -69,6 +69,7 @@ from docutils import nodes
import sphinx
from sphinx import addnodes
from sphinx.environment.adapters.toctree import TocTree
from sphinx.util import import_object, rst, logging
from sphinx.pycode import ModuleAnalyzer, PycodeError
from sphinx.ext.autodoc import Options
@ -104,7 +105,7 @@ def process_autosummary_toc(app, doctree):
try:
if (isinstance(subnode, autosummary_toc) and
isinstance(subnode[0], addnodes.toctree)):
env.note_toctree(env.docname, subnode[0])
TocTree(env).note(env.docname, subnode[0])
continue
except IndexError:
continue

2
sphinx/util/__init__.py
View File

@ -147,7 +147,7 @@ class FilenameUniqDict(dict):
self._existing.discard(unique)
def merge_other(self, docnames, other):
# type: (List[unicode], Dict[unicode, Tuple[Set[unicode], Any]]) -> None
# type: (Set[unicode], Dict[unicode, Tuple[Set[unicode], Any]]) -> None
for filename, (docs, unique) in other.items():
for doc in docs & set(docnames):
self.add_file(doc, filename)

2
tests/test_environment_indexentries.py
View File

@ -11,7 +11,7 @@
from collections import namedtuple
from sphinx import locale
from sphinx.environment.managers.indexentries import IndexEntries
from sphinx.environment.adapters.indexentries import IndexEntries
import mock