diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py
index 578068175..17f9667a1 100644
--- a/sphinx/environment/__init__.py
+++ b/sphinx/environment/__init__.py
@@ -20,11 +20,10 @@ from os import path
 from copy import copy
 from collections import defaultdict
 
-from six import BytesIO, itervalues, class_types, next, iteritems
+from six import BytesIO, itervalues, class_types, next
 from six.moves import cPickle as pickle
 
-from docutils.utils import Reporter, get_source_line, normalize_language_tag
-from docutils.utils.smartquotes import smartchars
+from docutils.utils import Reporter, get_source_line
 from docutils.frontend import OptionParser
 
 from sphinx import addnodes, versioning
@@ -643,29 +642,10 @@ class BuildEnvironment(object):
             self.config.trim_footnote_reference_space
         self.settings['gettext_compact'] = self.config.gettext_compact
 
-        language = self.config.language or 'en'
-        self.settings['language_code'] = language
-        if 'smart_quotes' not in self.settings:
-            self.settings['smart_quotes'] = self.config.smartquotes
+        self.settings['language_code'] = self.config.language or 'en'
 
-            # some conditions exclude smart quotes, overriding smart_quotes
-            for valname, vallist in iteritems(self.config.smartquotes_excludes):
-                if valname == 'builders':
-                    # this will work only for checking first build target
-                    if self.app.builder.name in vallist:
-                        self.settings['smart_quotes'] = False
-                        break
-                elif valname == 'languages':
-                    if self.config.language in vallist:
-                        self.settings['smart_quotes'] = False
-                        break
-
-        # confirm selected language supports smart_quotes or not
-        for tag in normalize_language_tag(language):
-            if tag in smartchars.quotes:
-                break
-        else:
-            self.settings['smart_quotes'] = False
+        # Allow to disable by 3rd party extension (workaround)
+        self.settings.setdefault('smart_quotes', True)
 
     def read_doc(self, docname, app=None):
         # type: (unicode, Sphinx) -> None
diff --git a/sphinx/io.py b/sphinx/io.py
index 471675e0e..61761f697 100644
--- a/sphinx/io.py
+++ b/sphinx/io.py
@@ -93,21 +93,14 @@ class SphinxStandaloneReader(SphinxBaseReader):
                   Locale, CitationReferences, DefaultSubstitutions, MoveModuleTargets,
                   HandleCodeBlocks, AutoNumbering, AutoIndexUpgrader, SortIds,
                   RemoveTranslatableInline, PreserveTranslatableMessages, FilterSystemMessages,
-                  RefOnlyBulletListTransform, UnreferencedFootnotesDetector
+                  RefOnlyBulletListTransform, UnreferencedFootnotesDetector, SphinxSmartQuotes,
                   ]  # type: List[Transform]
 
     def __init__(self, app, *args, **kwargs):
         # type: (Sphinx, Any, Any) -> None
         self.transforms = self.transforms + app.registry.get_transforms()
-        self.smart_quotes = app.env.settings['smart_quotes']
         SphinxBaseReader.__init__(self, app, *args, **kwargs)  # type: ignore
 
-    def get_transforms(self):
-        transforms = SphinxBaseReader.get_transforms(self)
-        if self.smart_quotes:
-            transforms.append(SphinxSmartQuotes)
-        return transforms
-
 
 class SphinxI18nReader(SphinxBaseReader):
     """
diff --git a/sphinx/transforms/__init__.py b/sphinx/transforms/__init__.py
index c2d8d4eb5..4e8c6f0bd 100644
--- a/sphinx/transforms/__init__.py
+++ b/sphinx/transforms/__init__.py
@@ -12,8 +12,9 @@
 from docutils import nodes
 from docutils.transforms import Transform, Transformer
 from docutils.transforms.parts import ContentsFilter
-from docutils.utils import new_document
 from docutils.transforms.universal import SmartQuotes
+from docutils.utils import new_document, normalize_language_tag
+from docutils.utils.smartquotes import smartchars
 
 from sphinx import addnodes
 from sphinx.locale import _
@@ -339,6 +340,39 @@ class SphinxSmartQuotes(SmartQuotes, SphinxTransform):
 
     refs: sphinx.parsers.RSTParser
     """
+    def apply(self):
+        # type: () -> None
+        if not self.is_available():
+            return
+
+        SmartQuotes.apply(self)
+
+    def is_available(self):
+        # type: () -> bool
+        builders = self.config.smartquotes_excludes.get('builders', [])
+        languages = self.config.smartquotes_excludes.get('languages', [])
+
+        if self.document.settings.smart_quotes is False:
+            # disabled by 3rd party extension (workaround)
+            return False
+        elif self.config.smartquotes is False:
+            # disabled by confval smartquotes
+            return False
+        elif self.app.builder.name in builders:
+            # disabled by confval smartquotes_excludes['builders']
+            return False
+        elif self.config.language in languages:
+            # disabled by confval smartquotes_excludes['languages']
+            return False
+
+        # confirm selected language supports smart_quotes or not
+        language = self.env.settings['language_code']
+        for tag in normalize_language_tag(language):
+            if tag in smartchars.quotes:
+                return True
+        else:
+            return False
+
     @property
     def smartquotes_action(self):
         # type: () -> unicode