Move SphinxSmartQuotes transform to SphinxStandaloneReader

closes #4142
closes #4357
closes #4359
refs: #3967

Adds ``smartquotes``, ``smartquotes_action``, ``smartquotes_excludes``
configuration variables.

- if ``smartquotes`` is set to False, then Smart Quotes transform is not
  applied even if a Docutils configuration file activates it,

- the current default of ``smartquotes_excludes`` deactivates Smart
  Quotes for Japanese language, and also for the ``man`` and ``text``
  builders.

  However, currently ``make text html`` deactivates Smart Quotes for
  ``html`` too, and ``make html text`` activates them for ``text`` too,
  because the picked environment is shared and already transformed.

- now Smart Quotes get applied also when source documents are in
  Markdown or other formats.

doc/config.rst

@@ -353,6 +353,63 @@ General configuration
 
    .. versionadded:: 1.3
 
+.. confval:: smartquotes
+
+   If true, the `Docutils Smart Quotes transform`__, originally based on
+   `SmartyPants`__ (limited to English) and currently applying to many
+   languages, will be used to convert quotes and dashes to typographically
+   correct entities.  Default: ``True``.
+
+   __ http://docutils.sourceforge.net/docs/user/smartquotes.html
+   __ https://daringfireball.net/projects/smartypants/
+
+   .. versionadded:: 1.6.6
+      It replaces deprecated :confval:`html_use_smartypants`.
+      It applies by default to all builders except ``man`` and ``text``
+      (see :confval:`smartquotes_excludes`.)
+
+   A `docutils.conf`__ file located in the configuration directory (or a
+   global :file:`~/.docutils` file) is obeyed unconditionally if it
+   *deactivates* smart quotes via the corresponding `Docutils option`__.  But
+   if it *activates* them, then :confval:`smartquotes` does prevail.
+
+   __ http://docutils.sourceforge.net/docs/user/config.html
+   __ http://docutils.sourceforge.net/docs/user/config.html#smart-quotes
+
+.. confval:: smartquotes_action
+
+   This string, for use with Docutils ``0.14`` or later, customizes the Smart
+   Quotes transform.  See the file :file:`smartquotes.py` at the `Docutils
+   repository`__ for details.  The default ``'qDe'`` educates normal **q**\
+   uote characters ``"``, ``'``, em- and en-**D**\ ashes ``---``, ``--``, and
+   **e**\ llipses ``...``.
+
+   .. versionadded:: 1.6.6
+
+   __ https://sourceforge.net/p/docutils/code/HEAD/tree/trunk/docutils/
+
+.. confval:: smartquotes_excludes
+
+   This is a ``dict`` whose default is::
+
+     {'languages': ['ja'], 'builders': ['man', 'text']}
+
+   Each entry gives a sufficient condition to ignore the
+   :confval:`smartquotes` setting and deactivate the Smart Quotes transform.
+   Accepted keys are as above ``'builders'`` or ``'languages'``.
+   The values are lists.
+
+   .. note:: Currently, in case of invocation of :program:`make` with multiple
+      targets, the first target name is the only one which is tested against
+      the ``'builders'`` entry and it decides for all.  Also, a ``make text``
+      following ``make html`` needs to be issued in the form ``make text
+      O="-E"`` to force re-parsing of source files, as the cached ones are
+      already transformed.  On the other hand the issue does not arise with
+      direct usage of :program:`sphinx-build` as it caches
+      (in its default usage) the parsed source files in per builder locations.
+
+   .. versionadded:: 1.6.6
+
 .. confval:: tls_verify
 
    If true, Sphinx verifies server certifications.  Default is ``True``.
@@ -784,15 +841,11 @@ that use Sphinx's HTMLWriter class.
 
 .. confval:: html_use_smartypants
 
-   If true, `SmartyPants <https://daringfireball.net/projects/smartypants/>`_
+   If true, quotes and dashes are converted to typographically correct
-   will be used to convert quotes and dashes to typographically correct
    entities.  Default: ``True``.
 
    .. deprecated:: 1.6
-      To disable or customize smart quotes, use the Docutils configuration file
+      To disable smart quotes, use rather :confval:`smartquotes`.
-      (``docutils.conf``) instead to set there its `smart_quotes option`_.
-
-      .. _`smart_quotes option`: http://docutils.sourceforge.net/docs/user/config.html#smart-quotes
 
 .. confval:: html_add_permalinks
 

sphinx/config.py

@@ -134,6 +134,11 @@ class Config(object):
 
         tls_verify = (True, 'env'),
         tls_cacerts = (None, 'env'),
+        smartquotes = (True, 'env'),
+        smartquotes_action = ('qDe', 'env'),
+        smartquotes_excludes = ({'languages': ['ja'],
+                                 'builders': ['man', 'text']},
+                                'env'),
     )  # type: Dict[unicode, Tuple]
 
     def __init__(self, dirname, filename, overrides, tags):

sphinx/environment/__init__.py

@@ -20,8 +20,9 @@ import warnings
 from os import path
 from copy import copy
 from collections import defaultdict
+from contextlib import contextmanager
 
-from six import BytesIO, itervalues, class_types, next
+from six import BytesIO, itervalues, class_types, next, iteritems
 from six.moves import cPickle as pickle
 
 from docutils.io import NullOutput
@@ -46,7 +47,7 @@ from sphinx.util.parallel import ParallelTasks, parallel_available, make_chunks
 from sphinx.util.websupport import is_commentable
 from sphinx.errors import SphinxError, ExtensionError
 from sphinx.locale import __
-from sphinx.transforms import SphinxTransformer
+from sphinx.transforms import SphinxTransformer, SphinxSmartQuotes
 from sphinx.versioning import add_uids, merge_doctrees
 from sphinx.deprecation import RemovedInSphinx17Warning, RemovedInSphinx20Warning
 from sphinx.environment.adapters.indexentries import IndexEntries
@@ -54,7 +55,7 @@ from sphinx.environment.adapters.toctree import TocTree
 
 if False:
     # For type annotation
-    from typing import Any, Callable, Dict, IO, Iterator, List, Pattern, Set, Tuple, Type, Union  # NOQA
+    from typing import Any, Callable, Dict, IO, Iterator, List, Pattern, Set, Tuple, Type, Union, Generator  # NOQA
     from docutils import nodes  # NOQA
     from sphinx.application import Sphinx  # NOQA
     from sphinx.builders import Builder  # NOQA
@@ -91,6 +92,22 @@ versioning_conditions = {
 }  # type: Dict[unicode, Union[bool, Callable]]
 
 
+@contextmanager
+def sphinx_smartquotes_action(env):
+    # type: (BuildEnvironment) -> Generator
+    if not hasattr(SphinxSmartQuotes, 'smartquotes_action'):
+        # less than docutils-0.14
+        yield
+    else:
+        # docutils-0.14 or above
+        try:
+            original = SphinxSmartQuotes.smartquotes_action
+            SphinxSmartQuotes.smartquotes_action = env.config.smartquotes_action
+            yield
+        finally:
+            SphinxSmartQuotes.smartquotes_action = original
+
+
 class NoUri(Exception):
     """Raised by get_relative_uri if there is no URI available."""
     pass
@@ -600,7 +617,8 @@ class BuildEnvironment(object):
             # remove all inventory entries for that file
             app.emit('env-purge-doc', self, docname)
             self.clear_doc(docname)
-            self.read_doc(docname, app)
+            with sphinx_smartquotes_action(self):
+                self.read_doc(docname, app)
 
     def _read_parallel(self, docnames, app, nproc):
         # type: (List[unicode], Sphinx, int) -> None
@@ -612,8 +630,9 @@ class BuildEnvironment(object):
         def read_process(docs):
             # type: (List[unicode]) -> unicode
             self.app = app
-            for docname in docs:
+            with sphinx_smartquotes_action(self):
-                self.read_doc(docname, app)
+                for docname in docs:
+                    self.read_doc(docname, app)
             # allow pickling self to send it back
             return BuildEnvironment.dumps(self)
 
@@ -677,15 +696,26 @@ class BuildEnvironment(object):
         language = self.config.language or 'en'
         self.settings['language_code'] = language
         if 'smart_quotes' not in self.settings:
-            self.settings['smart_quotes'] = True
+            self.settings['smart_quotes'] = self.config.smartquotes
             if self.config.html_use_smartypants is not None:
                 warnings.warn("html_use_smartypants option is deprecated. Smart "
                               "quotes are on by default; if you want to disable "
-                              "or customize them, use the smart_quotes option in "
+                              "them, use the smartquotes option.",
-                              "docutils.conf.",
                               RemovedInSphinx17Warning)
                 self.settings['smart_quotes'] = self.config.html_use_smartypants
 
+            # some conditions exclude smart quotes, overriding smart_quotes
+            for valname, vallist in iteritems(self.config.smartquotes_excludes):
+                if valname == 'builders':
+                    # this will work only for checking first build target
+                    if self.app.builder.name in vallist:
+                        self.settings['smart_quotes'] = False
+                        break
+                elif valname == 'languages':
+                    if self.config.language in vallist:
+                        self.settings['smart_quotes'] = False
+                        break
+
         # confirm selected language supports smart_quotes or not
         for tag in normalize_language_tag(language):
             if tag in smartchars.quotes:

sphinx/io.py

@@ -18,7 +18,7 @@ from sphinx.transforms import (
     ApplySourceWorkaround, ExtraTranslatableNodes, CitationReferences,
     DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, SortIds,
     AutoNumbering, AutoIndexUpgrader, FilterSystemMessages,
-    UnreferencedFootnotesDetector
+    UnreferencedFootnotesDetector, SphinxSmartQuotes
 )
 from sphinx.transforms.compact_bullet_list import RefOnlyBulletListTransform
 from sphinx.transforms.i18n import (
@@ -98,6 +98,16 @@ class SphinxStandaloneReader(SphinxBaseReader):
                   RemoveTranslatableInline, PreserveTranslatableMessages, FilterSystemMessages,
                   RefOnlyBulletListTransform, UnreferencedFootnotesDetector]
 
+    def __init__(self, app, parsers={}, *args, **kwargs):
+        SphinxBaseReader.__init__(self, app, parsers, *args, **kwargs)
+        self.smart_quotes = app.env.settings['smart_quotes']
+
+    def get_transforms(self):
+        transforms = SphinxBaseReader.get_transforms(self)
+        if self.smart_quotes:
+            transforms.append(SphinxSmartQuotes)
+        return transforms
+
 
 class SphinxI18nReader(SphinxBaseReader):
     """

sphinx/parsers.py

@@ -13,8 +13,6 @@ import docutils.parsers
 import docutils.parsers.rst
 from docutils.transforms.universal import SmartQuotes
 
-from sphinx.transforms import SphinxSmartQuotes
-
 if False:
     # For type annotation
     from typing import Any, Dict, List, Type  # NOQA
@@ -60,10 +58,11 @@ class RSTParser(docutils.parsers.rst.Parser):
 
     def get_transforms(self):
         # type: () -> List[Type[Transform]]
-        """Sphinx's reST parser replaces a transform class for smart-quotes by own's"""
+        """Sphinx's reST parser replaces a transform class for smart-quotes by own's
+
+        refs: sphinx.io.SphinxStandaloneReader"""
         transforms = docutils.parsers.rst.Parser.get_transforms(self)
         transforms.remove(SmartQuotes)
-        transforms.append(SphinxSmartQuotes)
         return transforms