Bundle sphinx/pycode/Grammar-*.pickle (ref: #2765)

So far, sphinx generates the grammar files on demand.
It causes uninstallation does not remove whole of files.
With this fix, Sphinx bundles the files in the package.

MANIFEST.in

@@ -20,8 +20,7 @@ recursive-include sphinx/search/non-minified-js *.js
 recursive-include sphinx/ext/autosummary/templates *
 recursive-include tests *
 recursive-include utils *
-include sphinx/pycode/Grammar-py2.txt
-include sphinx/pycode/Grammar-py3.txt
+include sphinx/pycode/Grammar-py*
 
 recursive-include doc *
 prune doc/_build

setup.py

@@ -6,6 +6,7 @@ import sys
 from distutils import log
 
 import sphinx
+from sphinx.pycode.pgen2.driver import compile_grammar
 
 long_desc = '''
 Sphinx is a tool that makes it easy to create intelligent and beautiful
@@ -72,6 +73,10 @@ extras_require = {
 if sys.platform == 'win32':
     requires.append('colorama>=0.3.5')
 
+# Compile grammars before packaging
+compile_grammar('sphinx/pycode/Grammar-py2.txt')
+compile_grammar('sphinx/pycode/Grammar-py3.txt')
+
 # Provide a "compile_catalog" command that also creates the translated
 # JavaScript files if Babel is available.
 

sphinx/pycode/pgen2/driver.py

@@ -109,27 +109,37 @@ def generate_lines(text):
         yield ""
 
 
-def load_grammar(gt="Grammar.txt", gp=None,
-                 save=True, force=False, logger=None):
+def get_compiled_path(filename):
+    head, tail = os.path.splitext(filename)
+    if tail == ".txt":
+        tail = ""
+    # embed Sphinx major version for the case we ever change the grammar...
+    return "%s%s-sphinx%s.pickle" % (head, tail, ".".join(map(str, sphinx.version_info[:2])))
+
+
+def compile_grammar(gt='Grammar.txt', logger=None):
+    """Compile the grammer."""
+    if logger is None:
+        logger = logging.getLogger()
+
+    logger.info("Generating grammar tables from %s", gt)
+    g = pgen.generate_grammar(gt)
+    gp = get_compiled_path(gt)
+    logger.info("Writing grammar tables to %s", gp)
+    try:
+        g.dump(gp)
+    except IOError as e:
+        logger.info("Writing failed:"+str(e))
+
+
+def load_grammar(gt="Grammar.txt", logger=None):
     """Load the grammar (maybe from a pickle)."""
     if logger is None:
         logger = logging.getLogger()
-    if gp is None:
-        head, tail = os.path.splitext(gt)
-        if tail == ".txt":
-            tail = ""
-        # embed Sphinx major version for the case we ever change the grammar...
-        gp = head + tail + "-sphinx" + \
-             ".".join(map(str, sphinx.version_info[:2])) + ".pickle"
-    if force or not _newer(gp, gt):
+    gp = get_compiled_path(gt)
+    if not os.path.exists(gp):
         logger.info("Generating grammar tables from %s", gt)
         g = pgen.generate_grammar(gt)
-        if save:
-            logger.info("Writing grammar tables to %s", gp)
-            try:
-                g.dump(gp)
-            except IOError as e:
-                logger.info("Writing failed:"+str(e))
     else:
         g = grammar.Grammar()
         g.load(gp)

tox.ini

@@ -3,7 +3,9 @@ envlist=flake8,py26,py27,py33,py34,py35,pypy,du12,du11,du10
 
 [testenv]
 deps=
+    six
     nose
+    docutils
     sqlalchemy
     whoosh
     html5lib