diff --git a/CHANGES b/CHANGES
index 392a2a25e..e53aabdb2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -62,6 +62,14 @@ Deprecated
 * ``Sphinx.status_iterator()` and ``Sphinx.old_status_iterator()`` is now
   deprecated.  Please use ``sphinx.util:status_iterator()`` intead.
 * ``BuildEnvironment.set_warnfunc()`` is now deprecated
+* Following methods of ``BuildEnvironment`` is now deprecated.
+
+  - ``BuildEnvironment.note_toctree()``
+  - ``BuildEnvironment.get_toc_for()``
+  - ``BuildEnvironment.get_toctree_for()``
+  - ``BuildEnvironment.create_index()``
+
+  Please use ``sphinx.environment.adapters`` modules instead.
 
 Release 1.5.3 (in development)
 ==============================
diff --git a/doc/extdev/appapi.rst b/doc/extdev/appapi.rst
index f6d21c057..c02e85933 100644
--- a/doc/extdev/appapi.rst
+++ b/doc/extdev/appapi.rst
@@ -359,6 +359,12 @@ package.
 
    .. versionadded:: 1.4
 
+.. method:: Sphinx.add_env_collector(collector)
+
+   Register a environment collector class (refs: :ref:`collector-api`)
+
+   .. versionadded:: 1.6
+
 .. method:: Sphinx.require_sphinx(version)
 
    Compare *version* (which must be a ``major.minor`` version string,
diff --git a/doc/extdev/collectorapi.rst b/doc/extdev/collectorapi.rst
new file mode 100644
index 000000000..cb4c30bf3
--- /dev/null
+++ b/doc/extdev/collectorapi.rst
@@ -0,0 +1,9 @@
+.. _collector-api:
+
+Environment Collector API
+-------------------------
+
+.. module:: sphinx.environment.collectors
+
+.. autoclass:: EnvironmentCollector
+   :members:
diff --git a/doc/extdev/index.rst b/doc/extdev/index.rst
index 1f3871c21..85172abb6 100644
--- a/doc/extdev/index.rst
+++ b/doc/extdev/index.rst
@@ -50,6 +50,7 @@ APIs used for writing extensions
    appapi
    envapi
    builderapi
+   collectorapi
    markupapi
    domainapi
    parserapi
diff --git a/sphinx/application.py b/sphinx/application.py
index 839d0101e..d10b89b99 100644
--- a/sphinx/application.py
+++ b/sphinx/application.py
@@ -55,11 +55,13 @@ if False:
     from docutils.transform import Transform  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.domains import Domain, Index  # NOQA
+    from sphinx.environment.collectors import EnvironmentCollector  # NOQA
 
 # List of all known core events. Maps name to arguments description.
 events = {
     'builder-inited': '',
     'env-get-outdated': 'env, added, changed, removed',
+    'env-get-updated': 'env',
     'env-purge-doc': 'env, docname',
     'env-before-read-docs': 'env, docnames',
     'source-read': 'docname, source text',
@@ -101,6 +103,13 @@ builtin_extensions = (
     'sphinx.directives.other',
     'sphinx.directives.patches',
     'sphinx.roles',
+    # collectors should be loaded by specific order
+    'sphinx.environment.collectors.dependencies',
+    'sphinx.environment.collectors.asset',
+    'sphinx.environment.collectors.metadata',
+    'sphinx.environment.collectors.title',
+    'sphinx.environment.collectors.toctree',
+    'sphinx.environment.collectors.indexentries',
 )  # type: Tuple[unicode, ...]
 
 CONFIG_FILENAME = 'conf.py'
@@ -300,7 +309,6 @@ class Sphinx(object):
                 logger.info(bold('loading pickled environment... '), nonl=True)
                 self.env = BuildEnvironment.frompickle(
                     self.srcdir, self.config, path.join(self.doctreedir, ENV_PICKLE_FILENAME))
-                self.env.init_managers()
                 self.env.domains = {}
                 for domain in self.domains.keys():
                     # this can raise if the data version doesn't fit
@@ -833,6 +841,11 @@ class Sphinx(object):
                            type='app', subtype='add_source_parser')
         self._additional_source_parsers[suffix] = parser
 
+    def add_env_collector(self, collector):
+        # type: (Type[EnvironmentCollector]) -> None
+        logger.debug('[app] adding environment collector: %r', collector)
+        collector().enable(self)
+
 
 class TemplateBridge(object):
     """
diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py
index 1461c3e68..dbeca1a34 100644
--- a/sphinx/builders/__init__.py
+++ b/sphinx/builders/__init__.py
@@ -292,7 +292,7 @@ class Builder(object):
 
         doccount = len(updated_docnames)
         logger.info(bold('looking for now-outdated files... '), nonl=1)
-        for docname in self.env.check_dependents(updated_docnames):
+        for docname in self.env.check_dependents(self.app, updated_docnames):
             updated_docnames.add(docname)
         outdated = len(updated_docnames) - doccount
         if outdated:
diff --git a/sphinx/builders/devhelp.py b/sphinx/builders/devhelp.py
index af8bcfeed..031c55184 100644
--- a/sphinx/builders/devhelp.py
+++ b/sphinx/builders/devhelp.py
@@ -22,6 +22,7 @@ from sphinx import addnodes
 from sphinx.util import logging
 from sphinx.util.osutil import make_filename
 from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.environment.adapters.indexentries import IndexEntries
 
 try:
     import xml.etree.ElementTree as etree
@@ -104,7 +105,7 @@ class DevhelpBuilder(StandaloneHTMLBuilder):
 
         # Index
         functions = etree.SubElement(root, 'functions')
-        index = self.env.create_index(self)
+        index = IndexEntries(self.env).create_index(self)
 
         def write_index(title, refs, subitems):
             # type: (unicode, List[Any], Any) -> None
diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py
index 3a074a1cc..062c56669 100644
--- a/sphinx/builders/html.py
+++ b/sphinx/builders/html.py
@@ -45,6 +45,8 @@ from sphinx.highlighting import PygmentsBridge
 from sphinx.util.console import bold, darkgreen  # type: ignore
 from sphinx.writers.html import HTMLWriter, HTMLTranslator, \
     SmartyPantsHTMLTranslator
+from sphinx.environment.adapters.toctree import TocTree
+from sphinx.environment.adapters.indexentries import IndexEntries
 
 if False:
     # For type annotation
@@ -439,7 +441,7 @@ class StandaloneHTMLBuilder(Builder):
         meta = self.env.metadata.get(docname)
 
         # local TOC and global TOC tree
-        self_toc = self.env.get_toc_for(docname, self)
+        self_toc = TocTree(self.env).get_toc_for(docname, self)
         toc = self.render_partial(self_toc)['fragment']
 
         return dict(
@@ -541,7 +543,7 @@ class StandaloneHTMLBuilder(Builder):
         # type: () -> None
         # the total count of lines for each index letter, used to distribute
         # the entries into two columns
-        genindex = self.env.create_index(self)
+        genindex = IndexEntries(self.env).create_index(self)
         indexcounts = []
         for _k, entries in genindex:
             indexcounts.append(sum(1 + len(subitems)
@@ -763,7 +765,7 @@ class StandaloneHTMLBuilder(Builder):
         # type: (unicode, bool, Any) -> unicode
         if 'includehidden' not in kwds:
             kwds['includehidden'] = False
-        return self.render_partial(self.env.get_toctree_for(
+        return self.render_partial(TocTree(self.env).get_toctree_for(
             docname, self, collapse, **kwds))['fragment']
 
     def get_outfilename(self, pagename):
@@ -1010,7 +1012,7 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
         # type: (unicode, bool, Any) -> unicode
         if 'includehidden' not in kwds:
             kwds['includehidden'] = False
-        toctree = self.env.get_toctree_for(docname, self, collapse, **kwds)
+        toctree = TocTree(self.env).get_toctree_for(docname, self, collapse, **kwds)
         self.fix_refuris(toctree)
         return self.render_partial(toctree)['fragment']
 
@@ -1066,7 +1068,8 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
     def get_doc_context(self, docname, body, metatags):
         # type: (unicode, unicode, Dict) -> Dict
         # no relation links...
-        toc = self.env.get_toctree_for(self.config.master_doc, self, False)  # type: Any
+        toc = TocTree(self.env).get_toctree_for(self.config.master_doc,
+                                                self, False)
         # if there is no toctree, toc is None
         if toc:
             self.fix_refuris(toc)
diff --git a/sphinx/builders/htmlhelp.py b/sphinx/builders/htmlhelp.py
index 08e6f9df4..68fd3b1db 100644
--- a/sphinx/builders/htmlhelp.py
+++ b/sphinx/builders/htmlhelp.py
@@ -19,6 +19,7 @@ from docutils import nodes
 
 from sphinx import addnodes
 from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.environment.adapters.indexentries import IndexEntries
 from sphinx.util import logging
 from sphinx.util.osutil import make_filename
 from sphinx.util.pycompat import htmlescape
@@ -281,7 +282,7 @@ class HTMLHelpBuilder(StandaloneHTMLBuilder):
             f.write(contents_footer)
 
         logger.info('writing index file...')
-        index = self.env.create_index(self)
+        index = IndexEntries(self.env).create_index(self)
         with self.open_file(outdir, outname + '.hhk') as f:
             f.write('<UL>\n')
 
diff --git a/sphinx/builders/qthelp.py b/sphinx/builders/qthelp.py
index 27178676f..25dec7586 100644
--- a/sphinx/builders/qthelp.py
+++ b/sphinx/builders/qthelp.py
@@ -21,6 +21,7 @@ from docutils import nodes
 
 from sphinx import addnodes
 from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.environment.adapters.indexentries import IndexEntries
 from sphinx.util import force_decode, logging
 from sphinx.util.osutil import make_filename
 from sphinx.util.pycompat import htmlescape
@@ -170,7 +171,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
 
         # keywords
         keywords = []
-        index = self.env.create_index(self, group_entries=False)
+        index = IndexEntries(self.env).create_index(self, group_entries=False)
         for (key, group) in index:
             for title, (refs, subitems, key_) in group:
                 keywords.extend(self.build_keywords(title, refs, subitems))
diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py
index c8a05a22b..17fafe8d2 100644
--- a/sphinx/environment/__init__.py
+++ b/sphinx/environment/__init__.py
@@ -18,16 +18,15 @@ import codecs
 import fnmatch
 import warnings
 from os import path
-from glob import glob
 from collections import defaultdict
 
-from six import iteritems, itervalues, class_types, next
+from six import itervalues, class_types, next
 from six.moves import cPickle as pickle
 
 from docutils import nodes
 from docutils.io import NullOutput
 from docutils.core import Publisher
-from docutils.utils import Reporter, relative_path, get_source_line
+from docutils.utils import Reporter, get_source_line
 from docutils.parsers.rst import roles
 from docutils.parsers.rst.languages import en as english
 from docutils.frontend import OptionParser
@@ -38,10 +37,8 @@ from sphinx.util import logging
 from sphinx.util import get_matching_docs, docname_join, FilenameUniqDict, status_iterator
 from sphinx.util.nodes import clean_astext, WarningStream, is_translatable, \
     process_only_nodes
-from sphinx.util.osutil import SEP, getcwd, fs_encoding, ensuredir
-from sphinx.util.images import guess_mimetype
-from sphinx.util.i18n import find_catalog_files, get_image_filename_for_language, \
-    search_image_for_language
+from sphinx.util.osutil import SEP, ensuredir
+from sphinx.util.i18n import find_catalog_files
 from sphinx.util.console import bold  # type: ignore
 from sphinx.util.docutils import sphinx_domains
 from sphinx.util.matching import compile_matchers
@@ -49,10 +46,9 @@ from sphinx.util.parallel import ParallelTasks, parallel_available, make_chunks
 from sphinx.util.websupport import is_commentable
 from sphinx.errors import SphinxError, ExtensionError
 from sphinx.versioning import add_uids, merge_doctrees
-from sphinx.transforms import SphinxContentsFilter
 from sphinx.deprecation import RemovedInSphinx20Warning
-from sphinx.environment.managers.indexentries import IndexEntries
-from sphinx.environment.managers.toctree import Toctree
+from sphinx.environment.adapters.indexentries import IndexEntries
+from sphinx.environment.adapters.toctree import TocTree
 
 if False:
     # For type annotation
@@ -61,7 +57,6 @@ if False:
     from sphinx.builders import Builder  # NOQA
     from sphinx.config import Config  # NOQA
     from sphinx.domains import Domain  # NOQA
-    from sphinx.environment.managers import EnvironmentManager  # NOQA
 
 logger = logging.getLogger(__name__)
 
@@ -128,7 +123,6 @@ class BuildEnvironment(object):
         del self.config.values
         domains = self.domains
         del self.domains
-        managers = self.detach_managers()
         # remove potentially pickling-problematic values from config
         for key, val in list(vars(self.config).items()):
             if key.startswith('_') or \
@@ -139,7 +133,6 @@ class BuildEnvironment(object):
         with open(filename, 'wb') as picklefile:
             pickle.dump(self, picklefile, pickle.HIGHEST_PROTOCOL)
         # reset attributes
-        self.attach_managers(managers)
         self.domains = domains
         self.config.values = values
 
@@ -189,8 +182,8 @@ class BuildEnvironment(object):
                                     # next build
 
         # File metadata
-        self.metadata = {}          # type: Dict[unicode, Dict[unicode, Any]]
-                                    # docname -> dict of metadata items
+        self.metadata = defaultdict(dict)       # type: Dict[unicode, Dict[unicode, Any]]
+                                                # docname -> dict of metadata items
 
         # TOC inventory
         self.titles = {}            # type: Dict[unicode, nodes.Node]
@@ -244,31 +237,6 @@ class BuildEnvironment(object):
         # attributes of "any" cross references
         self.ref_context = {}       # type: Dict[unicode, Any]
 
-        self.managers = {}          # type: Dict[unicode, EnvironmentManager]
-        self.init_managers()
-
-    def init_managers(self):
-        # type: () -> None
-        managers = {}
-        manager_class = None  # type: Type[EnvironmentManager]
-        for manager_class in [IndexEntries, Toctree]:  # type: ignore
-            managers[manager_class.name] = manager_class(self)
-        self.attach_managers(managers)
-
-    def attach_managers(self, managers):
-        # type: (Dict[unicode, EnvironmentManager]) -> None
-        for name, manager in iteritems(managers):
-            self.managers[name] = manager
-            manager.attach(self)
-
-    def detach_managers(self):
-        # type: () -> Dict[unicode, EnvironmentManager]
-        managers = self.managers
-        self.managers = {}
-        for _, manager in iteritems(managers):
-            manager.detach(self)
-        return managers
-
     def set_warnfunc(self, func):
         # type: (Callable) -> None
         warnings.warn('env.set_warnfunc() is now deprecated. Use sphinx.util.logging instead.',
@@ -315,20 +283,11 @@ class BuildEnvironment(object):
         if docname in self.all_docs:
             self.all_docs.pop(docname, None)
             self.reread_always.discard(docname)
-            self.metadata.pop(docname, None)
-            self.dependencies.pop(docname, None)
-            self.titles.pop(docname, None)
-            self.longtitles.pop(docname, None)
-            self.images.purge_doc(docname)
-            self.dlfiles.purge_doc(docname)
 
             for version, changes in self.versionchanges.items():
                 new = [change for change in changes if change[1] != docname]
                 changes[:] = new
 
-        for manager in itervalues(self.managers):
-            manager.clear_doc(docname)
-
         for domain in self.domains.values():
             domain.clear_doc(docname)
 
@@ -344,21 +303,11 @@ class BuildEnvironment(object):
             self.all_docs[docname] = other.all_docs[docname]
             if docname in other.reread_always:
                 self.reread_always.add(docname)
-            self.metadata[docname] = other.metadata[docname]
-            if docname in other.dependencies:
-                self.dependencies[docname] = other.dependencies[docname]
-            self.titles[docname] = other.titles[docname]
-            self.longtitles[docname] = other.longtitles[docname]
-
-        self.images.merge_other(docnames, other.images)
-        self.dlfiles.merge_other(docnames, other.dlfiles)
 
         for version, changes in other.versionchanges.items():
             self.versionchanges.setdefault(version, []).extend(
                 change for change in changes if change[1] in docnames)
 
-        for manager in itervalues(self.managers):
-            manager.merge_other(docnames, other)
         for domainname, domain in self.domains.items():
             domain.merge_domaindata(docnames, other.domaindata[domainname])
         app.emit('env-merge-info', self, docnames, other)
@@ -662,10 +611,11 @@ class BuildEnvironment(object):
         logger.info(bold('waiting for workers...'))
         tasks.join()
 
-    def check_dependents(self, already):
-        # type: (Set[unicode]) -> Iterator[unicode]
-        to_rewrite = (self.toctree.assign_section_numbers() +  # type: ignore
-                      self.toctree.assign_figure_numbers())  # type: ignore
+    def check_dependents(self, app, already):
+        # type: (Sphinx, Set[unicode]) -> Iterator[unicode]
+        to_rewrite = []  # type: List[unicode]
+        for docnames in app.emit('env-get-updated', self):
+            to_rewrite.extend(docnames)
         for docname in set(to_rewrite):
             if docname not in already:
                 yield docname
@@ -736,13 +686,6 @@ class BuildEnvironment(object):
             doctree = pub.document
 
         # post-processing
-        self.process_dependencies(docname, doctree)
-        self.process_images(docname, doctree)
-        self.process_downloads(docname, doctree)
-        self.process_metadata(docname, doctree)
-        self.create_title_from(docname, doctree)
-        for manager in itervalues(self.managers):
-            manager.process_doc(docname, doctree)
         for domain in itervalues(self.domains):
             domain.process_doc(self, docname, doctree)
 
@@ -865,180 +808,31 @@ class BuildEnvironment(object):
              self.ref_context.get('py:module'),
              self.temp_data.get('object'), node.astext()))
 
-    # post-processing of read doctrees
-
-    def process_dependencies(self, docname, doctree):
-        # type: (unicode, nodes.Node) -> None
-        """Process docutils-generated dependency info."""
-        cwd = getcwd()
-        frompath = path.join(path.normpath(self.srcdir), 'dummy')
-        deps = doctree.settings.record_dependencies
-        if not deps:
-            return
-        for dep in deps.list:
-            # the dependency path is relative to the working dir, so get
-            # one relative to the srcdir
-            if isinstance(dep, bytes):
-                dep = dep.decode(fs_encoding)
-            relpath = relative_path(frompath,
-                                    path.normpath(path.join(cwd, dep)))
-            self.dependencies[docname].add(relpath)
-
-    def process_downloads(self, docname, doctree):
-        # type: (unicode, nodes.Node) -> None
-        """Process downloadable file paths. """
-        for node in doctree.traverse(addnodes.download_reference):
-            targetname = node['reftarget']
-            rel_filename, filename = self.relfn2path(targetname, docname)
-            self.dependencies[docname].add(rel_filename)
-            if not os.access(filename, os.R_OK):
-                logger.warning('download file not readable: %s', filename,
-                               location=node)
-                continue
-            uniquename = self.dlfiles.add_file(docname, filename)
-            node['filename'] = uniquename
-
-    def process_images(self, docname, doctree):
-        # type: (unicode, nodes.Node) -> None
-        """Process and rewrite image URIs."""
-        def collect_candidates(imgpath, candidates):
-            globbed = {}  # type: Dict[unicode, List[unicode]]
-            for filename in glob(imgpath):
-                new_imgpath = relative_path(path.join(self.srcdir, 'dummy'),
-                                            filename)
-                try:
-                    mimetype = guess_mimetype(filename)
-                    if mimetype not in candidates:
-                        globbed.setdefault(mimetype, []).append(new_imgpath)
-                except (OSError, IOError) as err:
-                    logger.warning('image file %s not readable: %s', filename, err,
-                                   location=node)
-            for key, files in iteritems(globbed):
-                candidates[key] = sorted(files, key=len)[0]  # select by similarity
-
-        for node in doctree.traverse(nodes.image):
-            # Map the mimetype to the corresponding image.  The writer may
-            # choose the best image from these candidates.  The special key * is
-            # set if there is only single candidate to be used by a writer.
-            # The special key ? is set for nonlocal URIs.
-            node['candidates'] = candidates = {}
-            imguri = node['uri']
-            if imguri.startswith('data:'):
-                logger.warning('image data URI found. some builders might not support',
-                               location=node, type='image', subtype='data_uri')
-                candidates['?'] = imguri
-                continue
-            elif imguri.find('://') != -1:
-                logger.warning('nonlocal image URI found: %s', imguri,
-                               location=node, type='image', subtype='nonlocal_uri')
-                candidates['?'] = imguri
-                continue
-            rel_imgpath, full_imgpath = self.relfn2path(imguri, docname)
-            if self.config.language:
-                # substitute figures (ex. foo.png -> foo.en.png)
-                i18n_full_imgpath = search_image_for_language(full_imgpath, self)
-                if i18n_full_imgpath != full_imgpath:
-                    full_imgpath = i18n_full_imgpath
-                    rel_imgpath = relative_path(path.join(self.srcdir, 'dummy'),
-                                                i18n_full_imgpath)
-            # set imgpath as default URI
-            node['uri'] = rel_imgpath
-            if rel_imgpath.endswith(os.extsep + '*'):
-                if self.config.language:
-                    # Search language-specific figures at first
-                    i18n_imguri = get_image_filename_for_language(imguri, self)
-                    _, full_i18n_imgpath = self.relfn2path(i18n_imguri, docname)
-                    collect_candidates(full_i18n_imgpath, candidates)
-
-                collect_candidates(full_imgpath, candidates)
-            else:
-                candidates['*'] = rel_imgpath
-
-            # map image paths to unique image names (so that they can be put
-            # into a single directory)
-            for imgpath in itervalues(candidates):
-                self.dependencies[docname].add(imgpath)
-                if not os.access(path.join(self.srcdir, imgpath), os.R_OK):
-                    logger.warning('image file not readable: %s', imgpath,
-                                   location=node)
-                    continue
-                self.images.add_file(docname, imgpath)
-
-    def process_metadata(self, docname, doctree):
-        # type: (unicode, nodes.Node) -> None
-        """Process the docinfo part of the doctree as metadata.
-
-        Keep processing minimal -- just return what docutils says.
-        """
-        self.metadata[docname] = {}
-        md = self.metadata[docname]
-        try:
-            docinfo = doctree[0]
-        except IndexError:
-            # probably an empty document
-            return
-        if docinfo.__class__ is not nodes.docinfo:
-            # nothing to see here
-            return
-        for node in docinfo:
-            # nodes are multiply inherited...
-            if isinstance(node, nodes.authors):
-                md['authors'] = [author.astext() for author in node]
-            elif isinstance(node, nodes.TextElement):  # e.g. author
-                md[node.__class__.__name__] = node.astext()
-            else:
-                name, body = node
-                md[name.astext()] = body.astext()
-        for name, value in md.items():
-            if name in ('tocdepth',):
-                try:
-                    value = int(value)
-                except ValueError:
-                    value = 0
-                md[name] = value
-
-        del doctree[0]
-
-    def create_title_from(self, docname, document):
-        # type: (unicode, nodes.Node) -> None
-        """Add a title node to the document (just copy the first section title),
-        and store that title in the environment.
-        """
-        titlenode = nodes.title()
-        longtitlenode = titlenode
-        # explicit title set with title directive; use this only for
-        # the <title> tag in HTML output
-        if 'title' in document:
-            longtitlenode = nodes.title()
-            longtitlenode += nodes.Text(document['title'])
-        # look for first section title and use that as the title
-        for node in document.traverse(nodes.section):
-            visitor = SphinxContentsFilter(document)
-            node[0].walkabout(visitor)
-            titlenode += visitor.get_entry_text()
-            break
-        else:
-            # document has no title
-            titlenode += nodes.Text('<no title>')
-        self.titles[docname] = titlenode
-        self.longtitles[docname] = longtitlenode
-
     def note_toctree(self, docname, toctreenode):
         # type: (unicode, addnodes.toctree) -> None
         """Note a TOC tree directive in a document and gather information about
         file relations from it.
         """
-        self.toctree.note_toctree(docname, toctreenode)  # type: ignore
+        warnings.warn('env.note_toctree() is deprecated. '
+                      'Use sphinx.environment.adapters.toctre.TocTree instead.',
+                      RemovedInSphinx20Warning)
+        TocTree(self).note(docname, toctreenode)
 
     def get_toc_for(self, docname, builder):
-        # type: (unicode, Builder) -> addnodes.toctree
+        # type: (unicode, Builder) -> Dict[unicode, nodes.Node]
         """Return a TOC nodetree -- for use on the same page only!"""
-        return self.toctree.get_toc_for(docname, builder)  # type: ignore
+        warnings.warn('env.get_toc_for() is deprecated. '
+                      'Use sphinx.environment.adapters.toctre.TocTree instead.',
+                      RemovedInSphinx20Warning)
+        return TocTree(self).get_toc_for(docname, builder)
 
     def get_toctree_for(self, docname, builder, collapse, **kwds):
         # type: (unicode, Builder, bool, Any) -> addnodes.toctree
         """Return the global TOC nodetree."""
-        return self.toctree.get_toctree_for(docname, builder, collapse, **kwds)  # type: ignore
+        warnings.warn('env.get_toctree_for() is deprecated. '
+                      'Use sphinx.environment.adapters.toctre.TocTree instead.',
+                      RemovedInSphinx20Warning)
+        return TocTree(self).get_toctree_for(docname, builder, collapse, **kwds)
 
     def get_domain(self, domainname):
         # type: (unicode) -> Domain
@@ -1078,9 +872,9 @@ class BuildEnvironment(object):
 
         # now, resolve all toctree nodes
         for toctreenode in doctree.traverse(addnodes.toctree):
-            result = self.resolve_toctree(docname, builder, toctreenode,
-                                          prune=prune_toctrees,
-                                          includehidden=includehidden)
+            result = TocTree(self).resolve(docname, builder, toctreenode,
+                                           prune=prune_toctrees,
+                                           includehidden=includehidden)
             if result is None:
                 toctreenode.replace_self([])
             else:
@@ -1102,9 +896,9 @@ class BuildEnvironment(object):
         If *collapse* is True, all branches not containing docname will
         be collapsed.
         """
-        return self.toctree.resolve_toctree(docname, builder, toctree, prune,  # type: ignore
-                                            maxdepth, titles_only, collapse,
-                                            includehidden)
+        return TocTree(self).resolve(docname, builder, toctree, prune,
+                                     maxdepth, titles_only, collapse,
+                                     includehidden)
 
     def resolve_references(self, doctree, fromdocname, builder):
         # type: (nodes.Node, unicode, Builder) -> None
@@ -1239,8 +1033,13 @@ class BuildEnvironment(object):
 
     def create_index(self, builder, group_entries=True,
                      _fixre=re.compile(r'(.*) ([(][^()]*[)])')):
-        # type: (Builder, bool, Pattern) -> Any
-        return self.indices.create_index(builder, group_entries=group_entries, _fixre=_fixre)  # type: ignore  # NOQA
+        # type: (Builder, bool, Pattern) -> List[Tuple[unicode, List[Tuple[unicode, List[unicode]]]]]  # NOQA
+        warnings.warn('env.create_index() is deprecated. '
+                      'Use sphinx.environment.adapters.indexentreis.IndexEntries instead.',
+                      RemovedInSphinx20Warning)
+        return IndexEntries(self).create_index(builder,
+                                               group_entries=group_entries,
+                                               _fixre=_fixre)
 
     def collect_relations(self):
         # type: () -> Dict[unicode, List[unicode]]
diff --git a/sphinx/environment/adapters/__init__.py b/sphinx/environment/adapters/__init__.py
new file mode 100644
index 000000000..12a6fa490
--- /dev/null
+++ b/sphinx/environment/adapters/__init__.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.adapters
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Sphinx environment adapters
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
diff --git a/sphinx/environment/managers/indexentries.py b/sphinx/environment/adapters/indexentries.py
similarity index 79%
rename from sphinx/environment/managers/indexentries.py
rename to sphinx/environment/adapters/indexentries.py
index ef9c84d02..9eeb50833 100644
--- a/sphinx/environment/managers/indexentries.py
+++ b/sphinx/environment/adapters/indexentries.py
@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 """
-    sphinx.environment.managers.indexentries
+    sphinx.environment.adapters.indexentries
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    Index entries manager for sphinx.environment.
+    Index entries adapters for sphinx.environment.
 
     :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
@@ -15,59 +15,27 @@ import string
 from itertools import groupby
 
 from six import text_type
-from sphinx import addnodes
-from sphinx.util import iteritems, split_index_msg, split_into, logging
+
 from sphinx.locale import _
-from sphinx.environment.managers import EnvironmentManager
+from sphinx.util import iteritems, split_into, logging
 
 if False:
     # For type annotation
-    from typing import Pattern, Tuple  # NOQA
-    from docutils import nodes  # NOQA
+    from typing import Any, Pattern, Tuple  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA
 
 logger = logging.getLogger(__name__)
 
 
-class IndexEntries(EnvironmentManager):
-    name = 'indices'
-
+class IndexEntries(object):
     def __init__(self, env):
         # type: (BuildEnvironment) -> None
-        super(IndexEntries, self).__init__(env)
-        self.data = env.indexentries
-
-    def clear_doc(self, docname):
-        # type: (unicode) -> None
-        self.data.pop(docname, None)
-
-    def merge_other(self, docnames, other):
-        # type: (List[unicode], BuildEnvironment) -> None
-        for docname in docnames:
-            self.data[docname] = other.indexentries[docname]
-
-    def process_doc(self, docname, doctree):
-        # type: (unicode, nodes.Node) -> None
-        entries = self.data[docname] = []
-        for node in doctree.traverse(addnodes.index):
-            try:
-                for entry in node['entries']:
-                    split_index_msg(entry[0], entry[1])
-            except ValueError as exc:
-                logger.warning(str(exc), location=node)
-                node.parent.remove(node)
-            else:
-                for entry in node['entries']:
-                    if len(entry) == 5:
-                        # Since 1.4: new index structure including index_key (5th column)
-                        entries.append(entry)
-                    else:
-                        entries.append(entry + (None,))
+        self.env = env
 
     def create_index(self, builder, group_entries=True,
                      _fixre=re.compile(r'(.*) ([(][^()]*[)])')):
-        # type: (Builder, bool, Pattern) -> List[Tuple[unicode, List[Tuple[unicode, List[unicode]]]]]  # NOQA
+        # type: (Builder, bool, Pattern) -> List[Tuple[unicode, List[Tuple[unicode, Any]]]]  # NOQA
         """Create the real index from the collected index entries."""
         from sphinx.environment import NoUri
 
@@ -92,7 +60,7 @@ class IndexEntries(EnvironmentManager):
                     # maintain links in sorted/deterministic order
                     bisect.insort(entry[0], (main, uri))
 
-        for fn, entries in iteritems(self.data):
+        for fn, entries in iteritems(self.env.indexentries):
             # new entry types must be listed in directives/other.py!
             for type, value, tid, main, index_key in entries:
                 try:
diff --git a/sphinx/environment/managers/toctree.py b/sphinx/environment/adapters/toctree.py
similarity index 53%
rename from sphinx/environment/managers/toctree.py
rename to sphinx/environment/adapters/toctree.py
index 0cd011ae0..1ab3e229f 100644
--- a/sphinx/environment/managers/toctree.py
+++ b/sphinx/environment/adapters/toctree.py
@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 """
-    sphinx.environment.managers.toctree
+    sphinx.environment.adapters.toctree
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    Toctree manager for sphinx.environment.
+    Toctree adapter for sphinx.environment.
 
     :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
@@ -16,190 +16,39 @@ from docutils import nodes
 from sphinx import addnodes
 from sphinx.util import url_re, logging
 from sphinx.util.nodes import clean_astext, process_only_nodes
-from sphinx.transforms import SphinxContentsFilter
-from sphinx.environment.managers import EnvironmentManager
 
 if False:
     # For type annotation
-    from typing import Any, Tuple  # NOQA
+    from typing import Any  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.environment import BuildEnvironment  # NOQA
 
 logger = logging.getLogger(__name__)
 
 
-class Toctree(EnvironmentManager):
-    name = 'toctree'
-
+class TocTree(object):
     def __init__(self, env):
         # type: (BuildEnvironment) -> None
-        super(Toctree, self).__init__(env)
+        self.env = env
 
-        self.tocs = env.tocs
-        self.toc_num_entries = env.toc_num_entries
-        self.toc_secnumbers = env.toc_secnumbers
-        self.toc_fignumbers = env.toc_fignumbers
-        self.toctree_includes = env.toctree_includes
-        self.files_to_rebuild = env.files_to_rebuild
-        self.glob_toctrees = env.glob_toctrees
-        self.numbered_toctrees = env.numbered_toctrees
-
-    def clear_doc(self, docname):
-        # type: (unicode) -> None
-        self.tocs.pop(docname, None)
-        self.toc_secnumbers.pop(docname, None)
-        self.toc_fignumbers.pop(docname, None)
-        self.toc_num_entries.pop(docname, None)
-        self.toctree_includes.pop(docname, None)
-        self.glob_toctrees.discard(docname)
-        self.numbered_toctrees.discard(docname)
-
-        for subfn, fnset in list(self.files_to_rebuild.items()):
-            fnset.discard(docname)
-            if not fnset:
-                del self.files_to_rebuild[subfn]
-
-    def merge_other(self, docnames, other):
-        # type: (List[unicode], BuildEnvironment) -> None
-        for docname in docnames:
-            self.tocs[docname] = other.tocs[docname]
-            self.toc_num_entries[docname] = other.toc_num_entries[docname]
-            if docname in other.toctree_includes:
-                self.toctree_includes[docname] = other.toctree_includes[docname]
-            if docname in other.glob_toctrees:
-                self.glob_toctrees.add(docname)
-            if docname in other.numbered_toctrees:
-                self.numbered_toctrees.add(docname)
-
-        for subfn, fnset in other.files_to_rebuild.items():
-            self.files_to_rebuild.setdefault(subfn, set()).update(fnset & set(docnames))
-
-    def process_doc(self, docname, doctree):
-        # type: (unicode, nodes.Node) -> None
-        """Build a TOC from the doctree and store it in the inventory."""
-        numentries = [0]  # nonlocal again...
-
-        def traverse_in_section(node, cls):
-            """Like traverse(), but stay within the same section."""
-            result = []
-            if isinstance(node, cls):
-                result.append(node)
-            for child in node.children:
-                if isinstance(child, nodes.section):
-                    continue
-                result.extend(traverse_in_section(child, cls))
-            return result
-
-        def build_toc(node, depth=1):
-            entries = []
-            for sectionnode in node:
-                # find all toctree nodes in this section and add them
-                # to the toc (just copying the toctree node which is then
-                # resolved in self.get_and_resolve_doctree)
-                if isinstance(sectionnode, addnodes.only):
-                    onlynode = addnodes.only(expr=sectionnode['expr'])
-                    blist = build_toc(sectionnode, depth)
-                    if blist:
-                        onlynode += blist.children
-                        entries.append(onlynode)
-                    continue
-                if not isinstance(sectionnode, nodes.section):
-                    for toctreenode in traverse_in_section(sectionnode,
-                                                           addnodes.toctree):
-                        item = toctreenode.copy()
-                        entries.append(item)
-                        # important: do the inventory stuff
-                        self.note_toctree(docname, toctreenode)
-                    continue
-                title = sectionnode[0]
-                # copy the contents of the section title, but without references
-                # and unnecessary stuff
-                visitor = SphinxContentsFilter(doctree)
-                title.walkabout(visitor)
-                nodetext = visitor.get_entry_text()
-                if not numentries[0]:
-                    # for the very first toc entry, don't add an anchor
-                    # as it is the file's title anyway
-                    anchorname = ''
-                else:
-                    anchorname = '#' + sectionnode['ids'][0]
-                numentries[0] += 1
-                # make these nodes:
-                # list_item -> compact_paragraph -> reference
-                reference = nodes.reference(
-                    '', '', internal=True, refuri=docname,
-                    anchorname=anchorname, *nodetext)
-                para = addnodes.compact_paragraph('', '', reference)
-                item = nodes.list_item('', para)
-                sub_item = build_toc(sectionnode, depth + 1)
-                item += sub_item
-                entries.append(item)
-            if entries:
-                return nodes.bullet_list('', *entries)
-            return []
-        toc = build_toc(doctree)
-        if toc:
-            self.tocs[docname] = toc
-        else:
-            self.tocs[docname] = nodes.bullet_list('')
-        self.toc_num_entries[docname] = numentries[0]
-
-    def note_toctree(self, docname, toctreenode):
+    def note(self, docname, toctreenode):
         # type: (unicode, addnodes.toctree) -> None
         """Note a TOC tree directive in a document and gather information about
         file relations from it.
         """
         if toctreenode['glob']:
-            self.glob_toctrees.add(docname)
+            self.env.glob_toctrees.add(docname)
         if toctreenode.get('numbered'):
-            self.numbered_toctrees.add(docname)
+            self.env.numbered_toctrees.add(docname)
         includefiles = toctreenode['includefiles']
         for includefile in includefiles:
             # note that if the included file is rebuilt, this one must be
             # too (since the TOC of the included file could have changed)
-            self.files_to_rebuild.setdefault(includefile, set()).add(docname)
-        self.toctree_includes.setdefault(docname, []).extend(includefiles)
+            self.env.files_to_rebuild.setdefault(includefile, set()).add(docname)
+        self.env.toctree_includes.setdefault(docname, []).extend(includefiles)
 
-    def get_toc_for(self, docname, builder):
-        # type: (unicode, Builder) -> None
-        """Return a TOC nodetree -- for use on the same page only!"""
-        tocdepth = self.env.metadata[docname].get('tocdepth', 0)
-        try:
-            toc = self.tocs[docname].deepcopy()
-            self._toctree_prune(toc, 2, tocdepth)
-        except KeyError:
-            # the document does not exist anymore: return a dummy node that
-            # renders to nothing
-            return nodes.paragraph()
-        process_only_nodes(toc, builder.tags)
-        for node in toc.traverse(nodes.reference):
-            node['refuri'] = node['anchorname'] or '#'
-        return toc
-
-    def get_toctree_for(self, docname, builder, collapse, **kwds):
-        # type: (unicode, Builder, bool, Any) -> nodes.Node
-        """Return the global TOC nodetree."""
-        doctree = self.env.get_doctree(self.env.config.master_doc)
-        toctrees = []
-        if 'includehidden' not in kwds:
-            kwds['includehidden'] = True
-        if 'maxdepth' not in kwds:
-            kwds['maxdepth'] = 0
-        kwds['collapse'] = collapse
-        for toctreenode in doctree.traverse(addnodes.toctree):
-            toctree = self.env.resolve_toctree(docname, builder, toctreenode,
-                                               prune=True, **kwds)
-            if toctree:
-                toctrees.append(toctree)
-        if not toctrees:
-            return None
-        result = toctrees[0]
-        for toctree in toctrees[1:]:
-            result.extend(toctree.children)
-        return result
-
-    def resolve_toctree(self, docname, builder, toctree, prune=True, maxdepth=0,
-                        titles_only=False, collapse=False, includehidden=False):
+    def resolve(self, docname, builder, toctree, prune=True, maxdepth=0,
+                titles_only=False, collapse=False, includehidden=False):
         # type: (unicode, Builder, addnodes.toctree, bool, int, bool, bool, bool) -> nodes.Node
         """Resolve a *toctree* node into individual bullet lists with titles
         as items, returning None (if no containing titles are found) or
@@ -304,7 +153,7 @@ class Toctree(EnvironmentManager):
                                            location=ref)
                             continue
                         refdoc = ref
-                        toc = self.tocs[ref].deepcopy()
+                        toc = self.env.tocs[ref].deepcopy()
                         maxdepth = self.env.metadata[ref].get('tocdepth', 0)
                         if ref not in toctree_ancestors or (prune and maxdepth > 0):
                             self._toctree_prune(toc, 2, maxdepth, collapse)
@@ -405,7 +254,7 @@ class Toctree(EnvironmentManager):
     def get_toctree_ancestors(self, docname):
         # type: (unicode) -> List[unicode]
         parent = {}
-        for p, children in iteritems(self.toctree_includes):
+        for p, children in iteritems(self.env.toctree_includes):
             for child in children:
                 parent[child] = p
         ancestors = []  # type: List[unicode]
@@ -435,151 +284,41 @@ class Toctree(EnvironmentManager):
                         subnode.parent.remove(subnode)
                     else:
                         # recurse on visible children
-                        self._toctree_prune(subnode, depth + 1, maxdepth, collapse)
+                        self._toctree_prune(subnode, depth + 1, maxdepth,  collapse)
 
-    def assign_section_numbers(self):
-        # type: () -> List[unicode]
-        """Assign a section number to each heading under a numbered toctree."""
-        # a list of all docnames whose section numbers changed
-        rewrite_needed = []
+    def get_toc_for(self, docname, builder):
+        # type: (unicode, Builder) -> Dict[unicode, nodes.Node]
+        """Return a TOC nodetree -- for use on the same page only!"""
+        tocdepth = self.env.metadata[docname].get('tocdepth', 0)
+        try:
+            toc = self.env.tocs[docname].deepcopy()
+            self._toctree_prune(toc, 2, tocdepth)
+        except KeyError:
+            # the document does not exist anymore: return a dummy node that
+            # renders to nothing
+            return nodes.paragraph()
+        process_only_nodes(toc, builder.tags)
+        for node in toc.traverse(nodes.reference):
+            node['refuri'] = node['anchorname'] or '#'
+        return toc
 
-        assigned = set()  # type: Set[unicode]
-        old_secnumbers = self.toc_secnumbers
-        self.toc_secnumbers = self.env.toc_secnumbers = {}
-
-        def _walk_toc(node, secnums, depth, titlenode=None):
-            # titlenode is the title of the document, it will get assigned a
-            # secnumber too, so that it shows up in next/prev/parent rellinks
-            for subnode in node.children:
-                if isinstance(subnode, nodes.bullet_list):
-                    numstack.append(0)
-                    _walk_toc(subnode, secnums, depth - 1, titlenode)
-                    numstack.pop()
-                    titlenode = None
-                elif isinstance(subnode, nodes.list_item):
-                    _walk_toc(subnode, secnums, depth, titlenode)
-                    titlenode = None
-                elif isinstance(subnode, addnodes.only):
-                    # at this stage we don't know yet which sections are going
-                    # to be included; just include all of them, even if it leads
-                    # to gaps in the numbering
-                    _walk_toc(subnode, secnums, depth, titlenode)
-                    titlenode = None
-                elif isinstance(subnode, addnodes.compact_paragraph):
-                    numstack[-1] += 1
-                    if depth > 0:
-                        number = tuple(numstack)
-                    else:
-                        number = None
-                    secnums[subnode[0]['anchorname']] = \
-                        subnode[0]['secnumber'] = number
-                    if titlenode:
-                        titlenode['secnumber'] = number
-                        titlenode = None
-                elif isinstance(subnode, addnodes.toctree):
-                    _walk_toctree(subnode, depth)
-
-        def _walk_toctree(toctreenode, depth):
-            if depth == 0:
-                return
-            for (title, ref) in toctreenode['entries']:
-                if url_re.match(ref) or ref == 'self':
-                    # don't mess with those
-                    continue
-                elif ref in assigned:
-                    logger.warning('%s is already assigned section numbers '
-                                   '(nested numbered toctree?)', ref,
-                                   location=toctreenode, type='toc', subtype='secnum')
-                elif ref in self.tocs:
-                    secnums = self.toc_secnumbers[ref] = {}
-                    assigned.add(ref)
-                    _walk_toc(self.tocs[ref], secnums, depth,
-                              self.env.titles.get(ref))
-                    if secnums != old_secnumbers.get(ref):
-                        rewrite_needed.append(ref)
-
-        for docname in self.numbered_toctrees:
-            assigned.add(docname)
-            doctree = self.env.get_doctree(docname)
-            for toctreenode in doctree.traverse(addnodes.toctree):
-                depth = toctreenode.get('numbered', 0)
-                if depth:
-                    # every numbered toctree gets new numbering
-                    numstack = [0]
-                    _walk_toctree(toctreenode, depth)
-
-        return rewrite_needed
-
-    def assign_figure_numbers(self):
-        # type: () -> List[unicode]
-        """Assign a figure number to each figure under a numbered toctree."""
-
-        rewrite_needed = []
-
-        assigned = set()  # type: Set[unicode]
-        old_fignumbers = self.toc_fignumbers
-        self.toc_fignumbers = self.env.toc_fignumbers = {}
-        fignum_counter = {}  # type: Dict[unicode, Dict[Tuple[int], int]]
-
-        def get_section_number(docname, section):
-            anchorname = '#' + section['ids'][0]
-            secnumbers = self.toc_secnumbers.get(docname, {})
-            if anchorname in secnumbers:
-                secnum = secnumbers.get(anchorname)
-            else:
-                secnum = secnumbers.get('')
-
-            return secnum or tuple()
-
-        def get_next_fignumber(figtype, secnum):
-            counter = fignum_counter.setdefault(figtype, {})
-
-            secnum = secnum[:self.env.config.numfig_secnum_depth]
-            counter[secnum] = counter.get(secnum, 0) + 1
-            return secnum + (counter[secnum],)
-
-        def register_fignumber(docname, secnum, figtype, fignode):
-            self.toc_fignumbers.setdefault(docname, {})
-            fignumbers = self.toc_fignumbers[docname].setdefault(figtype, {})
-            figure_id = fignode['ids'][0]
-
-            fignumbers[figure_id] = get_next_fignumber(figtype, secnum)
-
-        def _walk_doctree(docname, doctree, secnum):
-            for subnode in doctree.children:
-                if isinstance(subnode, nodes.section):
-                    next_secnum = get_section_number(docname, subnode)
-                    if next_secnum:
-                        _walk_doctree(docname, subnode, next_secnum)
-                    else:
-                        _walk_doctree(docname, subnode, secnum)
-                    continue
-                elif isinstance(subnode, addnodes.toctree):
-                    for title, subdocname in subnode['entries']:
-                        if url_re.match(subdocname) or subdocname == 'self':
-                            # don't mess with those
-                            continue
-
-                        _walk_doc(subdocname, secnum)
-
-                    continue
-
-                figtype = self.env.get_domain('std').get_figtype(subnode)  # type: ignore
-                if figtype and subnode['ids']:
-                    register_fignumber(docname, secnum, figtype, subnode)
-
-                _walk_doctree(docname, subnode, secnum)
-
-        def _walk_doc(docname, secnum):
-            if docname not in assigned:
-                assigned.add(docname)
-                doctree = self.env.get_doctree(docname)
-                _walk_doctree(docname, doctree, secnum)
-
-        if self.env.config.numfig:
-            _walk_doc(self.env.config.master_doc, tuple())
-            for docname, fignums in iteritems(self.toc_fignumbers):
-                if fignums != old_fignumbers.get(docname):
-                    rewrite_needed.append(docname)
-
-        return rewrite_needed
+    def get_toctree_for(self, docname, builder, collapse, **kwds):
+        # type: (unicode, Builder, bool, Any) -> nodes.Node
+        """Return the global TOC nodetree."""
+        doctree = self.env.get_doctree(self.env.config.master_doc)
+        toctrees = []
+        if 'includehidden' not in kwds:
+            kwds['includehidden'] = True
+        if 'maxdepth' not in kwds:
+            kwds['maxdepth'] = 0
+        kwds['collapse'] = collapse
+        for toctreenode in doctree.traverse(addnodes.toctree):
+            toctree = self.resolve(docname, builder, toctreenode, prune=True, **kwds)
+            if toctree:
+                toctrees.append(toctree)
+        if not toctrees:
+            return None
+        result = toctrees[0]
+        for toctree in toctrees[1:]:
+            result.extend(toctree.children)
+        return result
diff --git a/sphinx/environment/collectors/__init__.py b/sphinx/environment/collectors/__init__.py
new file mode 100644
index 000000000..b8d73ad1f
--- /dev/null
+++ b/sphinx/environment/collectors/__init__.py
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.collectors
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    The data collector components for sphinx.environment.
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from six import itervalues
+
+if False:
+    # For type annotation
+    from docutils import nodes  # NOQA
+    from sphinx.sphinx import Sphinx  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+
+class EnvironmentCollector(object):
+    """An EnvironmentCollector is a specific data collector from each document.
+
+    It gathers data and stores :py:class:`BuildEnvironment
+    <sphinx.environment.BuildEnvironment>` as a database.  Examples of specific
+    data would be images, download files, section titles, metadatas, index
+    entries and toctrees, etc.
+    """
+
+    listener_ids = None  # type: Dict[unicode, int]
+
+    def enable(self, app):
+        # type: (Sphinx) -> None
+        assert self.listener_ids is None
+        self.listener_ids = {
+            'doctree-read':     app.connect('doctree-read', self.process_doc),
+            'env-merge-info':   app.connect('env-merge-info', self.merge_other),
+            'env-purge-doc':    app.connect('env-purge-doc', self.clear_doc),
+            'env-get-updated':  app.connect('env-get-updated', self.get_updated_docs),
+            'env-get-outdated': app.connect('env-get-outdated', self.get_outdated_docs),
+        }
+
+    def disable(self, app):
+        # type: (Sphinx) -> None
+        assert self.listener_ids is not None
+        for listener_id in itervalues(self.listener_ids):
+            app.disconnect(listener_id)
+        self.listener_ids = None
+
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        """Remove specified data of a document.
+
+        This method is called on the removal of the document."""
+        raise NotImplementedError
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        """Merge in specified data regarding docnames from a different `BuildEnvironment`
+        object which coming from a subprocess in parallel builds."""
+        raise NotImplementedError
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        """Process a document and gather specific data from it.
+
+        This method is called after the document is read."""
+        raise NotImplementedError
+
+    def get_updated_docs(self, app, env):
+        # type: (Sphinx, BuildEnvironment) -> List[unicode]
+        """Return a list of docnames to re-read.
+
+        This methods is called after reading the whole of documents (experimental).
+        """
+        return []
+
+    def get_outdated_docs(self, app, env, added, changed, removed):
+        # type: (Sphinx, BuildEnvironment, unicode, Set[unicode], Set[unicode], Set[unicode]) -> List[unicode]  # NOQA
+        """Return a list of docnames to re-read.
+
+        This methods is called before reading the documents.
+        """
+        return []
diff --git a/sphinx/environment/collectors/asset.py b/sphinx/environment/collectors/asset.py
new file mode 100644
index 000000000..54283790c
--- /dev/null
+++ b/sphinx/environment/collectors/asset.py
@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.collectors.asset
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    The image collector for sphinx.environment.
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+from os import path
+from glob import glob
+
+from six import iteritems, itervalues
+
+from docutils import nodes
+from docutils.utils import relative_path
+
+from sphinx import addnodes
+from sphinx.environment.collectors import EnvironmentCollector
+from sphinx.util import logging
+from sphinx.util.i18n import get_image_filename_for_language, search_image_for_language
+from sphinx.util.images import guess_mimetype
+
+if False:
+    # For type annotation
+    from typing import Tuple  # NOQA
+    from docutils import nodes  # NOQA
+    from sphinx.sphinx import Sphinx  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+logger = logging.getLogger(__name__)
+
+
+class ImageCollector(EnvironmentCollector):
+    """Image files collector for sphinx.environment."""
+
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        env.images.purge_doc(docname)
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        env.images.merge_other(docnames, other.images)
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        """Process and rewrite image URIs."""
+        docname = app.env.docname
+
+        for node in doctree.traverse(nodes.image):
+            # Map the mimetype to the corresponding image.  The writer may
+            # choose the best image from these candidates.  The special key * is
+            # set if there is only single candidate to be used by a writer.
+            # The special key ? is set for nonlocal URIs.
+            candidates = {}  # type: Dict[unicode, unicode]
+            node['candidates'] = candidates
+            imguri = node['uri']
+            if imguri.startswith('data:'):
+                logger.warning('image data URI found. some builders might not support',
+                               location=node, type='image', subtype='data_uri')
+                candidates['?'] = imguri
+                continue
+            elif imguri.find('://') != -1:
+                logger.warning('nonlocal image URI found: %s' % imguri,
+                               location=node,
+                               type='image', subtype='nonlocal_uri')
+                candidates['?'] = imguri
+                continue
+            rel_imgpath, full_imgpath = app.env.relfn2path(imguri, docname)
+            if app.config.language:
+                # substitute figures (ex. foo.png -> foo.en.png)
+                i18n_full_imgpath = search_image_for_language(full_imgpath, app.env)
+                if i18n_full_imgpath != full_imgpath:
+                    full_imgpath = i18n_full_imgpath
+                    rel_imgpath = relative_path(path.join(app.srcdir, 'dummy'),
+                                                i18n_full_imgpath)
+            # set imgpath as default URI
+            node['uri'] = rel_imgpath
+            if rel_imgpath.endswith(os.extsep + '*'):
+                if app.config.language:
+                    # Search language-specific figures at first
+                    i18n_imguri = get_image_filename_for_language(imguri, app.env)
+                    _, full_i18n_imgpath = app.env.relfn2path(i18n_imguri, docname)
+                    self.collect_candidates(app.env, full_i18n_imgpath, candidates, node)
+
+                self.collect_candidates(app.env, full_imgpath, candidates, node)
+            else:
+                candidates['*'] = rel_imgpath
+
+            # map image paths to unique image names (so that they can be put
+            # into a single directory)
+            for imgpath in itervalues(candidates):
+                app.env.dependencies[docname].add(imgpath)
+                if not os.access(path.join(app.srcdir, imgpath), os.R_OK):
+                    logger.warning('image file not readable: %s' % imgpath,
+                                   location=node)
+                    continue
+                app.env.images.add_file(docname, imgpath)
+
+    def collect_candidates(self, env, imgpath, candidates, node):
+        # type: (BuildEnvironment, unicode, Dict[unicode, unicode], nodes.Node) -> None
+        globbed = {}  # type: Dict[unicode, List[unicode]]
+        for filename in glob(imgpath):
+            new_imgpath = relative_path(path.join(env.srcdir, 'dummy'),
+                                        filename)
+            try:
+                mimetype = guess_mimetype(filename)
+                if mimetype not in candidates:
+                    globbed.setdefault(mimetype, []).append(new_imgpath)
+            except (OSError, IOError) as err:
+                logger.warning('image file %s not readable: %s' % (filename, err),
+                               location=node)
+        for key, files in iteritems(globbed):
+            candidates[key] = sorted(files, key=len)[0]  # select by similarity
+
+
+class DownloadFileCollector(EnvironmentCollector):
+    """Download files collector for sphinx.environment."""
+
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        env.dlfiles.purge_doc(docname)
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        env.dlfiles.merge_other(docnames, other.dlfiles)
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        """Process downloadable file paths. """
+        for node in doctree.traverse(addnodes.download_reference):
+            targetname = node['reftarget']
+            rel_filename, filename = app.env.relfn2path(targetname, app.env.docname)
+            app.env.dependencies[app.env.docname].add(rel_filename)
+            if not os.access(filename, os.R_OK):
+                logger.warning('download file not readable: %s' % filename,
+                               location=node)
+                continue
+            node['filename'] = app.env.dlfiles.add_file(app.env.docname, filename)
+
+
+def setup(app):
+    # type: (Sphinx) -> None
+    app.add_env_collector(ImageCollector)
+    app.add_env_collector(DownloadFileCollector)
diff --git a/sphinx/environment/collectors/dependencies.py b/sphinx/environment/collectors/dependencies.py
new file mode 100644
index 000000000..b4d35caf7
--- /dev/null
+++ b/sphinx/environment/collectors/dependencies.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.collectors.dependencies
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    The dependencies collector components for sphinx.environment.
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from os import path
+
+from docutils.utils import relative_path
+
+from sphinx.util.osutil import getcwd, fs_encoding
+from sphinx.environment.collectors import EnvironmentCollector
+
+if False:
+    # For type annotation
+    from docutils import nodes  # NOQA
+    from sphinx.sphinx import Sphinx  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+
+class DependenciesCollector(EnvironmentCollector):
+    """dependencies collector for sphinx.environment."""
+
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        env.dependencies.pop(docname, None)
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        for docname in docnames:
+            if docname in other.dependencies:
+                env.dependencies[docname] = other.dependencies[docname]
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        """Process docutils-generated dependency info."""
+        cwd = getcwd()
+        frompath = path.join(path.normpath(app.srcdir), 'dummy')
+        deps = doctree.settings.record_dependencies
+        if not deps:
+            return
+        for dep in deps.list:
+            # the dependency path is relative to the working dir, so get
+            # one relative to the srcdir
+            if isinstance(dep, bytes):
+                dep = dep.decode(fs_encoding)
+            relpath = relative_path(frompath,
+                                    path.normpath(path.join(cwd, dep)))
+            app.env.dependencies[app.env.docname].add(relpath)
+
+
+def setup(app):
+    # type: (Sphinx) -> None
+    app.add_env_collector(DependenciesCollector)
diff --git a/sphinx/environment/collectors/indexentries.py b/sphinx/environment/collectors/indexentries.py
new file mode 100644
index 000000000..c9aeda7e1
--- /dev/null
+++ b/sphinx/environment/collectors/indexentries.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.collectors.indexentries
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Index entries collector for sphinx.environment.
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from sphinx import addnodes
+from sphinx.util import split_index_msg, logging
+from sphinx.environment.collectors import EnvironmentCollector
+
+if False:
+    # For type annotation
+    from docutils import nodes  # NOQA
+    from sphinx.applicatin import Sphinx  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+logger = logging.getLogger(__name__)
+
+
+class IndexEntriesCollector(EnvironmentCollector):
+    name = 'indices'
+
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        env.indexentries.pop(docname, None)
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        for docname in docnames:
+            env.indexentries[docname] = other.indexentries[docname]
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        docname = app.env.docname
+        entries = app.env.indexentries[docname] = []
+        for node in doctree.traverse(addnodes.index):
+            try:
+                for entry in node['entries']:
+                    split_index_msg(entry[0], entry[1])
+            except ValueError as exc:
+                logger.warning(str(exc), location=node)
+                node.parent.remove(node)
+            else:
+                for entry in node['entries']:
+                    if len(entry) == 5:
+                        # Since 1.4: new index structure including index_key (5th column)
+                        entries.append(entry)
+                    else:
+                        entries.append(entry + (None,))
+
+
+def setup(app):
+    # type: (Sphinx) -> None
+    app.add_env_collector(IndexEntriesCollector)
diff --git a/sphinx/environment/collectors/metadata.py b/sphinx/environment/collectors/metadata.py
new file mode 100644
index 000000000..7a15cc614
--- /dev/null
+++ b/sphinx/environment/collectors/metadata.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.collectors.metadata
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    The metadata collector components for sphinx.environment.
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from docutils import nodes
+
+from sphinx.environment.collectors import EnvironmentCollector
+
+if False:
+    # For type annotation
+    from docutils import nodes  # NOQA
+    from sphinx.sphinx import Sphinx  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+
+class MetadataCollector(EnvironmentCollector):
+    """metadata collector for sphinx.environment."""
+
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        env.metadata.pop(docname, None)
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        for docname in docnames:
+            env.metadata[docname] = other.metadata[docname]
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        """Process the docinfo part of the doctree as metadata.
+
+        Keep processing minimal -- just return what docutils says.
+        """
+        md = app.env.metadata[app.env.docname]
+        try:
+            docinfo = doctree[0]
+        except IndexError:
+            # probably an empty document
+            return
+        if docinfo.__class__ is not nodes.docinfo:
+            # nothing to see here
+            return
+        for node in docinfo:
+            # nodes are multiply inherited...
+            if isinstance(node, nodes.authors):
+                md['authors'] = [author.astext() for author in node]
+            elif isinstance(node, nodes.TextElement):  # e.g. author
+                md[node.__class__.__name__] = node.astext()
+            else:
+                name, body = node
+                md[name.astext()] = body.astext()
+        for name, value in md.items():
+            if name in ('tocdepth',):
+                try:
+                    value = int(value)
+                except ValueError:
+                    value = 0
+                md[name] = value
+
+        del doctree[0]
+
+
+def setup(app):
+    # type: (Sphinx) -> None
+    app.add_env_collector(MetadataCollector)
diff --git a/sphinx/environment/collectors/title.py b/sphinx/environment/collectors/title.py
new file mode 100644
index 000000000..a5316fe94
--- /dev/null
+++ b/sphinx/environment/collectors/title.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.collectors.title
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    The title collector components for sphinx.environment.
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from docutils import nodes
+
+from sphinx.environment.collectors import EnvironmentCollector
+from sphinx.transforms import SphinxContentsFilter
+
+if False:
+    # For type annotation
+    from docutils import nodes  # NOQA
+    from sphinx.sphinx import Sphinx  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+
+class TitleCollector(EnvironmentCollector):
+    """title collector for sphinx.environment."""
+
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        env.titles.pop(docname, None)
+        env.longtitles.pop(docname, None)
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        for docname in docnames:
+            env.titles[docname] = other.titles[docname]
+            env.longtitles[docname] = other.longtitles[docname]
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        """Add a title node to the document (just copy the first section title),
+        and store that title in the environment.
+        """
+        titlenode = nodes.title()
+        longtitlenode = titlenode
+        # explicit title set with title directive; use this only for
+        # the <title> tag in HTML output
+        if 'title' in doctree:
+            longtitlenode = nodes.title()
+            longtitlenode += nodes.Text(doctree['title'])
+        # look for first section title and use that as the title
+        for node in doctree.traverse(nodes.section):
+            visitor = SphinxContentsFilter(doctree)
+            node[0].walkabout(visitor)
+            titlenode += visitor.get_entry_text()
+            break
+        else:
+            # document has no title
+            titlenode += nodes.Text('<no title>')
+        app.env.titles[app.env.docname] = titlenode
+        app.env.longtitles[app.env.docname] = longtitlenode
+
+
+def setup(app):
+    # type: (Sphinx) -> None
+    app.add_env_collector(TitleCollector)
diff --git a/sphinx/environment/collectors/toctree.py b/sphinx/environment/collectors/toctree.py
new file mode 100644
index 000000000..5c9ed5472
--- /dev/null
+++ b/sphinx/environment/collectors/toctree.py
@@ -0,0 +1,288 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.collectors.toctree
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Toctree collector for sphinx.environment.
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from six import iteritems
+
+from docutils import nodes
+
+from sphinx import addnodes
+from sphinx.util import url_re, logging
+from sphinx.transforms import SphinxContentsFilter
+from sphinx.environment.adapters.toctree import TocTree
+from sphinx.environment.collectors import EnvironmentCollector
+
+if False:
+    # For type annotation
+    from typing import Any, Tuple  # NOQA
+    from sphinx.application import Sphinx  # NOQA
+    from sphinx.builders import Builder  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+logger = logging.getLogger(__name__)
+
+
+class TocTreeCollector(EnvironmentCollector):
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        env.tocs.pop(docname, None)
+        env.toc_secnumbers.pop(docname, None)
+        env.toc_fignumbers.pop(docname, None)
+        env.toc_num_entries.pop(docname, None)
+        env.toctree_includes.pop(docname, None)
+        env.glob_toctrees.discard(docname)
+        env.numbered_toctrees.discard(docname)
+
+        for subfn, fnset in list(env.files_to_rebuild.items()):
+            fnset.discard(docname)
+            if not fnset:
+                del env.files_to_rebuild[subfn]
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        for docname in docnames:
+            env.tocs[docname] = other.tocs[docname]
+            env.toc_num_entries[docname] = other.toc_num_entries[docname]
+            if docname in other.toctree_includes:
+                env.toctree_includes[docname] = other.toctree_includes[docname]
+            if docname in other.glob_toctrees:
+                env.glob_toctrees.add(docname)
+            if docname in other.numbered_toctrees:
+                env.numbered_toctrees.add(docname)
+
+        for subfn, fnset in other.files_to_rebuild.items():
+            env.files_to_rebuild.setdefault(subfn, set()).update(fnset & set(docnames))
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        """Build a TOC from the doctree and store it in the inventory."""
+        docname = app.env.docname
+        numentries = [0]  # nonlocal again...
+
+        def traverse_in_section(node, cls):
+            """Like traverse(), but stay within the same section."""
+            result = []
+            if isinstance(node, cls):
+                result.append(node)
+            for child in node.children:
+                if isinstance(child, nodes.section):
+                    continue
+                result.extend(traverse_in_section(child, cls))
+            return result
+
+        def build_toc(node, depth=1):
+            entries = []
+            for sectionnode in node:
+                # find all toctree nodes in this section and add them
+                # to the toc (just copying the toctree node which is then
+                # resolved in self.get_and_resolve_doctree)
+                if isinstance(sectionnode, addnodes.only):
+                    onlynode = addnodes.only(expr=sectionnode['expr'])
+                    blist = build_toc(sectionnode, depth)
+                    if blist:
+                        onlynode += blist.children
+                        entries.append(onlynode)
+                    continue
+                if not isinstance(sectionnode, nodes.section):
+                    for toctreenode in traverse_in_section(sectionnode,
+                                                           addnodes.toctree):
+                        item = toctreenode.copy()
+                        entries.append(item)
+                        # important: do the inventory stuff
+                        TocTree(app.env).note(docname, toctreenode)
+                    continue
+                title = sectionnode[0]
+                # copy the contents of the section title, but without references
+                # and unnecessary stuff
+                visitor = SphinxContentsFilter(doctree)
+                title.walkabout(visitor)
+                nodetext = visitor.get_entry_text()
+                if not numentries[0]:
+                    # for the very first toc entry, don't add an anchor
+                    # as it is the file's title anyway
+                    anchorname = ''
+                else:
+                    anchorname = '#' + sectionnode['ids'][0]
+                numentries[0] += 1
+                # make these nodes:
+                # list_item -> compact_paragraph -> reference
+                reference = nodes.reference(
+                    '', '', internal=True, refuri=docname,
+                    anchorname=anchorname, *nodetext)
+                para = addnodes.compact_paragraph('', '', reference)
+                item = nodes.list_item('', para)
+                sub_item = build_toc(sectionnode, depth + 1)
+                item += sub_item
+                entries.append(item)
+            if entries:
+                return nodes.bullet_list('', *entries)
+            return []
+        toc = build_toc(doctree)
+        if toc:
+            app.env.tocs[docname] = toc
+        else:
+            app.env.tocs[docname] = nodes.bullet_list('')
+        app.env.toc_num_entries[docname] = numentries[0]
+
+    def get_updated_docs(self, app, env):
+        # type: (Sphinx, BuildEnvironment) -> List[unicode]
+        return self.assign_section_numbers(env) + self.assign_figure_numbers(env)
+
+    def assign_section_numbers(self, env):
+        # type: (BuildEnvironment) -> List[unicode]
+        """Assign a section number to each heading under a numbered toctree."""
+        # a list of all docnames whose section numbers changed
+        rewrite_needed = []
+
+        assigned = set()  # type: Set[unicode]
+        old_secnumbers = env.toc_secnumbers
+        env.toc_secnumbers = {}
+
+        def _walk_toc(node, secnums, depth, titlenode=None):
+            # titlenode is the title of the document, it will get assigned a
+            # secnumber too, so that it shows up in next/prev/parent rellinks
+            for subnode in node.children:
+                if isinstance(subnode, nodes.bullet_list):
+                    numstack.append(0)
+                    _walk_toc(subnode, secnums, depth - 1, titlenode)
+                    numstack.pop()
+                    titlenode = None
+                elif isinstance(subnode, nodes.list_item):
+                    _walk_toc(subnode, secnums, depth, titlenode)
+                    titlenode = None
+                elif isinstance(subnode, addnodes.only):
+                    # at this stage we don't know yet which sections are going
+                    # to be included; just include all of them, even if it leads
+                    # to gaps in the numbering
+                    _walk_toc(subnode, secnums, depth, titlenode)
+                    titlenode = None
+                elif isinstance(subnode, addnodes.compact_paragraph):
+                    numstack[-1] += 1
+                    if depth > 0:
+                        number = tuple(numstack)
+                    else:
+                        number = None
+                    secnums[subnode[0]['anchorname']] = \
+                        subnode[0]['secnumber'] = number
+                    if titlenode:
+                        titlenode['secnumber'] = number
+                        titlenode = None
+                elif isinstance(subnode, addnodes.toctree):
+                    _walk_toctree(subnode, depth)
+
+        def _walk_toctree(toctreenode, depth):
+            if depth == 0:
+                return
+            for (title, ref) in toctreenode['entries']:
+                if url_re.match(ref) or ref == 'self':
+                    # don't mess with those
+                    continue
+                elif ref in assigned:
+                    logger.warning('%s is already assigned section numbers '
+                                   '(nested numbered toctree?)', ref,
+                                   location=toctreenode, type='toc', subtype='secnum')
+                elif ref in env.tocs:
+                    secnums = env.toc_secnumbers[ref] = {}
+                    assigned.add(ref)
+                    _walk_toc(env.tocs[ref], secnums, depth,
+                              env.titles.get(ref))
+                    if secnums != old_secnumbers.get(ref):
+                        rewrite_needed.append(ref)
+
+        for docname in env.numbered_toctrees:
+            assigned.add(docname)
+            doctree = env.get_doctree(docname)
+            for toctreenode in doctree.traverse(addnodes.toctree):
+                depth = toctreenode.get('numbered', 0)
+                if depth:
+                    # every numbered toctree gets new numbering
+                    numstack = [0]
+                    _walk_toctree(toctreenode, depth)
+
+        return rewrite_needed
+
+    def assign_figure_numbers(self, env):
+        # type: (BuildEnvironment) -> List[unicode]
+        """Assign a figure number to each figure under a numbered toctree."""
+
+        rewrite_needed = []
+
+        assigned = set()  # type: Set[unicode]
+        old_fignumbers = env.toc_fignumbers
+        env.toc_fignumbers = {}
+        fignum_counter = {}  # type: Dict[unicode, Dict[Tuple[int], int]]
+
+        def get_section_number(docname, section):
+            anchorname = '#' + section['ids'][0]
+            secnumbers = env.toc_secnumbers.get(docname, {})
+            if anchorname in secnumbers:
+                secnum = secnumbers.get(anchorname)
+            else:
+                secnum = secnumbers.get('')
+
+            return secnum or tuple()
+
+        def get_next_fignumber(figtype, secnum):
+            counter = fignum_counter.setdefault(figtype, {})
+
+            secnum = secnum[:env.config.numfig_secnum_depth]
+            counter[secnum] = counter.get(secnum, 0) + 1
+            return secnum + (counter[secnum],)
+
+        def register_fignumber(docname, secnum, figtype, fignode):
+            env.toc_fignumbers.setdefault(docname, {})
+            fignumbers = env.toc_fignumbers[docname].setdefault(figtype, {})
+            figure_id = fignode['ids'][0]
+
+            fignumbers[figure_id] = get_next_fignumber(figtype, secnum)
+
+        def _walk_doctree(docname, doctree, secnum):
+            for subnode in doctree.children:
+                if isinstance(subnode, nodes.section):
+                    next_secnum = get_section_number(docname, subnode)
+                    if next_secnum:
+                        _walk_doctree(docname, subnode, next_secnum)
+                    else:
+                        _walk_doctree(docname, subnode, secnum)
+                    continue
+                elif isinstance(subnode, addnodes.toctree):
+                    for title, subdocname in subnode['entries']:
+                        if url_re.match(subdocname) or subdocname == 'self':
+                            # don't mess with those
+                            continue
+
+                        _walk_doc(subdocname, secnum)
+
+                    continue
+
+                figtype = env.get_domain('std').get_figtype(subnode)  # type: ignore
+                if figtype and subnode['ids']:
+                    register_fignumber(docname, secnum, figtype, subnode)
+
+                _walk_doctree(docname, subnode, secnum)
+
+        def _walk_doc(docname, secnum):
+            if docname not in assigned:
+                assigned.add(docname)
+                doctree = env.get_doctree(docname)
+                _walk_doctree(docname, doctree, secnum)
+
+        if env.config.numfig:
+            _walk_doc(env.config.master_doc, tuple())
+            for docname, fignums in iteritems(env.toc_fignumbers):
+                if fignums != old_fignumbers.get(docname):
+                    rewrite_needed.append(docname)
+
+        return rewrite_needed
+
+
+def setup(app):
+    # type: (Sphinx) -> None
+    app.add_env_collector(TocTreeCollector)
diff --git a/sphinx/environment/managers/__init__.py b/sphinx/environment/managers/__init__.py
deleted file mode 100644
index 0822f1091..000000000
--- a/sphinx/environment/managers/__init__.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    sphinx.environment.managers
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    Manager components for sphinx.environment.
-
-    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-if False:
-    # For type annotation
-    from typing import Any  # NOQA
-    from docutils import nodes  # NOQA
-    from sphinx.environment import BuildEnvironment  # NOQA
-
-
-class EnvironmentManager(object):
-    """Base class for sphinx.environment managers."""
-    name = None  # type: unicode
-    env = None  # type: BuildEnvironment
-
-    def __init__(self, env):
-        # type: (BuildEnvironment) -> None
-        self.env = env
-
-    def attach(self, env):
-        # type: (BuildEnvironment) -> None
-        self.env = env
-        if self.name:
-            setattr(env, self.name, self)
-
-    def detach(self, env):
-        # type: (BuildEnvironment) -> None
-        self.env = None
-        if self.name:
-            delattr(env, self.name)
-
-    def clear_doc(self, docname):
-        # type: (unicode) -> None
-        raise NotImplementedError
-
-    def merge_other(self, docnames, other):
-        # type: (List[unicode], Any) -> None
-        raise NotImplementedError
-
-    def process_doc(self, docname, doctree):
-        # type: (unicode, nodes.Node) -> None
-        raise NotImplementedError
diff --git a/sphinx/ext/autosummary/__init__.py b/sphinx/ext/autosummary/__init__.py
index 4995c0261..8d521a74d 100644
--- a/sphinx/ext/autosummary/__init__.py
+++ b/sphinx/ext/autosummary/__init__.py
@@ -69,6 +69,7 @@ from docutils import nodes
 
 import sphinx
 from sphinx import addnodes
+from sphinx.environment.adapters.toctree import TocTree
 from sphinx.util import import_object, rst, logging
 from sphinx.pycode import ModuleAnalyzer, PycodeError
 from sphinx.ext.autodoc import Options
@@ -104,7 +105,7 @@ def process_autosummary_toc(app, doctree):
             try:
                 if (isinstance(subnode, autosummary_toc) and
                         isinstance(subnode[0], addnodes.toctree)):
-                    env.note_toctree(env.docname, subnode[0])
+                    TocTree(env).note(env.docname, subnode[0])
                     continue
             except IndexError:
                 continue
diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py
index 8b03272cd..5fb42b9d5 100644
--- a/sphinx/util/__init__.py
+++ b/sphinx/util/__init__.py
@@ -147,7 +147,7 @@ class FilenameUniqDict(dict):
                 self._existing.discard(unique)
 
     def merge_other(self, docnames, other):
-        # type: (List[unicode], Dict[unicode, Tuple[Set[unicode], Any]]) -> None
+        # type: (Set[unicode], Dict[unicode, Tuple[Set[unicode], Any]]) -> None
         for filename, (docs, unique) in other.items():
             for doc in docs & set(docnames):
                 self.add_file(doc, filename)
diff --git a/tests/test_environment_indexentries.py b/tests/test_environment_indexentries.py
index 57a3cf52f..53e0ad65d 100644
--- a/tests/test_environment_indexentries.py
+++ b/tests/test_environment_indexentries.py
@@ -11,7 +11,7 @@
 
 from collections import namedtuple
 from sphinx import locale
-from sphinx.environment.managers.indexentries import IndexEntries
+from sphinx.environment.adapters.indexentries import IndexEntries
 
 import mock