diff --git a/sphinx/application.py b/sphinx/application.py
index 448b03672..6ada00b6d 100644
--- a/sphinx/application.py
+++ b/sphinx/application.py
@@ -105,6 +105,7 @@ builtin_extensions = (
     # collectors should be loaded by specific order
     'sphinx.environment.collectors.dependencies',
     'sphinx.environment.collectors.asset',
+    'sphinx.environment.collectors.metadata',
 )  # type: Tuple[unicode, ...]
 
 CONFIG_FILENAME = 'conf.py'
diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py
index 3b45c38b7..105c4e208 100644
--- a/sphinx/environment/__init__.py
+++ b/sphinx/environment/__init__.py
@@ -186,8 +186,8 @@ class BuildEnvironment(object):
                                     # next build
 
         # File metadata
-        self.metadata = {}          # type: Dict[unicode, Dict[unicode, Any]]
-                                    # docname -> dict of metadata items
+        self.metadata = defaultdict(dict)       # type: Dict[unicode, Dict[unicode, Any]]
+                                                # docname -> dict of metadata items
 
         # TOC inventory
         self.titles = {}            # type: Dict[unicode, nodes.Node]
@@ -312,7 +312,6 @@ class BuildEnvironment(object):
         if docname in self.all_docs:
             self.all_docs.pop(docname, None)
             self.reread_always.discard(docname)
-            self.metadata.pop(docname, None)
             self.titles.pop(docname, None)
             self.longtitles.pop(docname, None)
 
@@ -338,7 +337,6 @@ class BuildEnvironment(object):
             self.all_docs[docname] = other.all_docs[docname]
             if docname in other.reread_always:
                 self.reread_always.add(docname)
-            self.metadata[docname] = other.metadata[docname]
             self.titles[docname] = other.titles[docname]
             self.longtitles[docname] = other.longtitles[docname]
 
@@ -725,7 +723,6 @@ class BuildEnvironment(object):
             doctree = pub.document
 
         # post-processing
-        self.process_metadata(docname, doctree)
         self.create_title_from(docname, doctree)
         for manager in itervalues(self.managers):
             manager.process_doc(docname, doctree)
@@ -853,41 +850,6 @@ class BuildEnvironment(object):
 
     # post-processing of read doctrees
 
-    def process_metadata(self, docname, doctree):
-        # type: (unicode, nodes.Node) -> None
-        """Process the docinfo part of the doctree as metadata.
-
-        Keep processing minimal -- just return what docutils says.
-        """
-        self.metadata[docname] = {}
-        md = self.metadata[docname]
-        try:
-            docinfo = doctree[0]
-        except IndexError:
-            # probably an empty document
-            return
-        if docinfo.__class__ is not nodes.docinfo:
-            # nothing to see here
-            return
-        for node in docinfo:
-            # nodes are multiply inherited...
-            if isinstance(node, nodes.authors):
-                md['authors'] = [author.astext() for author in node]
-            elif isinstance(node, nodes.TextElement):  # e.g. author
-                md[node.__class__.__name__] = node.astext()
-            else:
-                name, body = node
-                md[name.astext()] = body.astext()
-        for name, value in md.items():
-            if name in ('tocdepth',):
-                try:
-                    value = int(value)
-                except ValueError:
-                    value = 0
-                md[name] = value
-
-        del doctree[0]
-
     def create_title_from(self, docname, document):
         # type: (unicode, nodes.Node) -> None
         """Add a title node to the document (just copy the first section title),
diff --git a/sphinx/environment/collectors/metadata.py b/sphinx/environment/collectors/metadata.py
new file mode 100644
index 000000000..7a15cc614
--- /dev/null
+++ b/sphinx/environment/collectors/metadata.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.environment.collectors.metadata
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    The metadata collector components for sphinx.environment.
+
+    :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from docutils import nodes
+
+from sphinx.environment.collectors import EnvironmentCollector
+
+if False:
+    # For type annotation
+    from docutils import nodes  # NOQA
+    from sphinx.sphinx import Sphinx  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+
+class MetadataCollector(EnvironmentCollector):
+    """metadata collector for sphinx.environment."""
+
+    def clear_doc(self, app, env, docname):
+        # type: (Sphinx, BuildEnvironment, unicode) -> None
+        env.metadata.pop(docname, None)
+
+    def merge_other(self, app, env, docnames, other):
+        # type: (Sphinx, BuildEnvironment, Set[unicode], BuildEnvironment) -> None
+        for docname in docnames:
+            env.metadata[docname] = other.metadata[docname]
+
+    def process_doc(self, app, doctree):
+        # type: (Sphinx, nodes.Node) -> None
+        """Process the docinfo part of the doctree as metadata.
+
+        Keep processing minimal -- just return what docutils says.
+        """
+        md = app.env.metadata[app.env.docname]
+        try:
+            docinfo = doctree[0]
+        except IndexError:
+            # probably an empty document
+            return
+        if docinfo.__class__ is not nodes.docinfo:
+            # nothing to see here
+            return
+        for node in docinfo:
+            # nodes are multiply inherited...
+            if isinstance(node, nodes.authors):
+                md['authors'] = [author.astext() for author in node]
+            elif isinstance(node, nodes.TextElement):  # e.g. author
+                md[node.__class__.__name__] = node.astext()
+            else:
+                name, body = node
+                md[name.astext()] = body.astext()
+        for name, value in md.items():
+            if name in ('tocdepth',):
+                try:
+                    value = int(value)
+                except ValueError:
+                    value = 0
+                md[name] = value
+
+        del doctree[0]
+
+
+def setup(app):
+    # type: (Sphinx) -> None
+    app.add_env_collector(MetadataCollector)