diff --git a/CHANGES b/CHANGES
index 3a5131e7b..46b12a15e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -7,6 +7,10 @@ Dependencies
 Incompatible changes
 --------------------
 
+* #4460: extensions which stores any data to environment should return the
+  version of its env data structure as metadata.  In detail, please see
+  :ref:`ext-metadata`.
+
 Deprecated
 ----------
 
diff --git a/doc/extdev/index.rst b/doc/extdev/index.rst
index b6c71cca6..3380c31ea 100644
--- a/doc/extdev/index.rst
+++ b/doc/extdev/index.rst
@@ -52,6 +52,8 @@ Note that it is still necessary to register the builder using
 
 .. _entry points: https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins
 
+.. _ext-metadata:
+
 Extension metadata
 ------------------
 
@@ -63,6 +65,11 @@ as metadata of the extension.  Metadata keys currently recognized are:
 * ``'version'``: a string that identifies the extension version.  It is used for
   extension version requirement checking (see :confval:`needs_extensions`) and
   informational purposes.  If not given, ``"unknown version"`` is substituted.
+* ``'env_version'``: an integer that identifies the version of env data
+  structure if the extension stores any data to environment.  It is used to
+  detect the data structure has been changed from last build.  The extensions
+  have to increment the version when data structure has changed.  If not given,
+  Sphinx considers the extension does not stores any data to environment.
 * ``'parallel_read_safe'``: a boolean that specifies if parallel reading of
   source files can be used when the extension is loaded.  It defaults to
   ``False``, i.e. you have to explicitly specify your extension to be
diff --git a/sphinx/application.py b/sphinx/application.py
index db4122b16..30b2505fd 100644
--- a/sphinx/application.py
+++ b/sphinx/application.py
@@ -305,6 +305,9 @@ class Sphinx(object):
                 logger.info(bold(__('loading pickled environment... ')), nonl=True)
                 filename = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
                 self.env = BuildEnvironment.frompickle(filename, self)
+                needed, reason = self.env.need_refresh(self)
+                if needed:
+                    raise IOError(reason)
                 self.env.domains = {}
                 for domain in self.registry.create_domains(self.env):
                     # this can raise if the data version doesn't fit
diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py
index 3030cff8a..1bfdf0b11 100644
--- a/sphinx/domains/c.py
+++ b/sphinx/domains/c.py
@@ -330,6 +330,7 @@ def setup(app):
 
     return {
         'version': 'builtin',
+        'env_version': 1,
         'parallel_read_safe': True,
         'parallel_write_safe': True,
     }
diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py
index d8aca3472..b8fa68056 100644
--- a/sphinx/domains/cpp.py
+++ b/sphinx/domains/cpp.py
@@ -6099,6 +6099,7 @@ def setup(app):
 
     return {
         'version': 'builtin',
+        'env_version': 1,
         'parallel_read_safe': True,
         'parallel_write_safe': True,
     }
diff --git a/sphinx/domains/javascript.py b/sphinx/domains/javascript.py
index 81f86f754..85954613a 100644
--- a/sphinx/domains/javascript.py
+++ b/sphinx/domains/javascript.py
@@ -415,6 +415,7 @@ def setup(app):
 
     return {
         'version': 'builtin',
+        'env_version': 1,
         'parallel_read_safe': True,
         'parallel_write_safe': True,
     }
diff --git a/sphinx/domains/python.py b/sphinx/domains/python.py
index fa96590b3..b0e783b33 100644
--- a/sphinx/domains/python.py
+++ b/sphinx/domains/python.py
@@ -912,6 +912,7 @@ def setup(app):
 
     return {
         'version': 'builtin',
+        'env_version': 1,
         'parallel_read_safe': True,
         'parallel_write_safe': True,
     }
diff --git a/sphinx/domains/rst.py b/sphinx/domains/rst.py
index 936dd1b9f..5d876568e 100644
--- a/sphinx/domains/rst.py
+++ b/sphinx/domains/rst.py
@@ -182,6 +182,7 @@ def setup(app):
 
     return {
         'version': 'builtin',
+        'env_version': 1,
         'parallel_read_safe': True,
         'parallel_write_safe': True,
     }
diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py
index a26109076..6ea01dd47 100644
--- a/sphinx/domains/std.py
+++ b/sphinx/domains/std.py
@@ -980,6 +980,7 @@ def setup(app):
 
     return {
         'version': 'builtin',
+        'env_version': 1,
         'parallel_read_safe': True,
         'parallel_write_safe': True,
     }
diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py
index 17f9667a1..330dda284 100644
--- a/sphinx/environment/__init__.py
+++ b/sphinx/environment/__init__.py
@@ -107,13 +107,9 @@ class BuildEnvironment(object):
             # This can happen for example when the pickle is from a
             # different version of Sphinx.
             raise IOError(exc)
-        if env.version != ENV_VERSION:
-            raise IOError('build environment version not current')
         if app:
             env.app = app
             env.config.values = app.config.values
-            if env.srcdir != app.srcdir:
-                raise IOError('source directory has changed')
         return env
 
     @classmethod
@@ -187,7 +183,7 @@ class BuildEnvironment(object):
         self._warnfunc = None  # type: Callable
 
         # this is to invalidate old pickles
-        self.version = ENV_VERSION
+        self.version = app.registry.get_envversion(app)
 
         # All "docnames" here are /-separated and relative and exclude
         # the source suffix.
@@ -304,6 +300,19 @@ class BuildEnvironment(object):
         """Like :meth:`warn`, but with source information taken from *node*."""
         self._warnfunc(msg, '%s:%s' % get_source_line(node), **kwargs)
 
+    def need_refresh(self, app):
+        # type: (Sphinx) -> Tuple[bool, unicode]
+        """Check refresh environment is needed.
+
+        If needed, this method returns the reason for refresh.
+        """
+        if self.version != app.registry.get_envversion(app):
+            return True, 'build environment version not current'
+        elif self.srcdir != app.srcdir:
+            return True, 'source directory has changed'
+        else:
+            return False, None
+
     def clear_doc(self, docname):
         # type: (unicode) -> None
         """Remove all traces of a source file in the inventory."""
diff --git a/sphinx/ext/intersphinx.py b/sphinx/ext/intersphinx.py
index 52683d309..8f56726b0 100644
--- a/sphinx/ext/intersphinx.py
+++ b/sphinx/ext/intersphinx.py
@@ -347,7 +347,11 @@ def setup(app):
     app.add_config_value('intersphinx_timeout', None, False)
     app.connect('missing-reference', missing_reference)
     app.connect('builder-inited', load_mappings)
-    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
+    return {
+        'version': sphinx.__display_version__,
+        'env_version': 1,
+        'parallel_read_safe': True
+    }
 
 
 def debug(argv):
diff --git a/sphinx/ext/todo.py b/sphinx/ext/todo.py
index e60620b5b..04076169f 100644
--- a/sphinx/ext/todo.py
+++ b/sphinx/ext/todo.py
@@ -258,4 +258,8 @@ def setup(app):
     app.connect('doctree-resolved', process_todo_nodes)
     app.connect('env-purge-doc', purge_todos)
     app.connect('env-merge-info', merge_info)
-    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
+    return {
+        'version': sphinx.__display_version__,
+        'env_version': 1,
+        'parallel_read_safe': True
+    }
diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py
index 757d7adc0..a2e8e64d3 100644
--- a/sphinx/ext/viewcode.py
+++ b/sphinx/ext/viewcode.py
@@ -241,4 +241,8 @@ def setup(app):
     app.connect('missing-reference', missing_reference)
     # app.add_config_value('viewcode_include_modules', [], 'env')
     # app.add_config_value('viewcode_exclude_modules', [], 'env')
-    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
+    return {
+        'version': sphinx.__display_version__,
+        'env_version': 1,
+        'parallel_read_safe': True
+    }
diff --git a/sphinx/registry.py b/sphinx/registry.py
index e48c12f96..eff5cfc6b 100644
--- a/sphinx/registry.py
+++ b/sphinx/registry.py
@@ -341,3 +341,11 @@ class SphinxComponentRegistry(object):
 
         app.extensions[extname] = Extension(extname, mod, **metadata)
         app._setting_up_extension.pop()
+
+    def get_envversion(self, app):
+        # type: (Sphinx) -> Dict[unicode, unicode]
+        from sphinx.environment import ENV_VERSION
+        envversion = {ext.name: ext.metadata['env_version'] for ext in app.extensions.values()
+                      if ext.metadata.get('env_version')}
+        envversion['sphinx'] = ENV_VERSION
+        return envversion