Implemented JSONHTMLBuilder and improved JSON handling (it now prefers json from the 2.6 stdlib or simplejson).

CHANGES

@@ -9,6 +9,9 @@ New features added
   `PickleHTMLBuilder` is a concrete subclass of it that uses pickle as
   serialization implementation.
 
+* `JSONHTMLBuilder` was added that similarily to `PickleHTMLBuilder`
+  dumps the generated HTML into JSON files for further processing.
+
 
 Release 0.4 (Jun 23, 2008)
 ==========================

doc/builders.rst

@@ -34,8 +34,7 @@ The builder's "name" must be given to the **-b** command-line option of
 
    This builder produces a directory with pickle files containing mostly HTML
    fragments and TOC information, for use of a web application (or custom
-   postprocessing tool) that doesn't use the standard HTML templates.  It also
-   is the format used by the Sphinx Web application.
+   postprocessing tool) that doesn't use the standard HTML templates.
 
    See :ref:`serialization-details` for details about the output format.
 
@@ -44,6 +43,21 @@ The builder's "name" must be given to the **-b** command-line option of
    The file suffix is ``.fpickle``.  The global context is called
    ``globalcontext.pickle``, the search index ``searchindex.pickle``.
 
+.. class:: JSONHTMLBuilder
+
+   This builder produces a directory with JSON files containing mostly HTML
+   fragments and TOC information, for use of a web application (or custom
+   postprocessing tool) that doesn't use the standard HTML templates.
+
+   See :ref:`serialization-details` for details about the output format.
+
+   Its name is ``json``.
+
+   The file suffix is ``.fjson``.  The global context is called
+   ``globalcontext.json``, the search index ``searchindex.json``.
+
+   .. versionadded:: 0.5
+
 .. class:: LaTeXBuilder
 
    This builder produces a bunch of LaTeX files in the output directory.  You
@@ -70,17 +84,19 @@ The builder's "name" must be given to the **-b** command-line option of
    (`pickle`, `simplejson`, `phpserialize`, and others) to dump the generated
    HTML documentation.  The pickle builder is a subclass of it.
 
-   A concreate subclass of this builder serializing to JSON could look like
-   this::
+   A concreate subclass of this builder serializing to the `PHP serialization`_
+   format could look like this::
 
-        import simplejson
+        import phpserialize
 
-        classs JSONBuilder(SerializingHTMLBuilder):
-            name = 'json'
-            implementation = simplejson
-            out_suffix = '.fjson'
-            globalcontext_filename = 'globalcontext.json'
-            searchindex_filename = 'searchindex.json'
+        classs PHPSerializedBuilder(SerializingHTMLBuilder):
+            name = 'phpserialized'
+            implementation = phpserialize
+            out_suffix = '.file.phpdump'
+            globalcontext_filename = 'globalcontext.phpdump'
+            searchindex_filename = 'searchindex.phpdump'
+
+   .. _PHP serialization: http://pypi.python.org/pypi/phpserialize
 
    .. attribute:: implementation
     

sphinx/builder.py

@@ -25,7 +25,7 @@ from docutils.frontend import OptionParser
 from docutils.readers.doctree import Reader as DoctreeReader
 
 from sphinx import addnodes
-from sphinx.util import ensuredir, relative_uri, SEP, os_path
+from sphinx.util import ensuredir, relative_uri, SEP, os_path, json
 from sphinx.htmlhelp import build_hhx
 from sphinx.htmlwriter import HTMLWriter, HTMLTranslator, SmartyPantsHTMLTranslator
 from sphinx.textwriter import TextWriter
@@ -294,7 +294,7 @@ class StandaloneHTMLBuilder(Builder):
     name = 'html'
     copysource = True
     out_suffix = '.html'
-    indexer_format = 'json'
+    indexer_format = json
     supported_image_types = ['image/svg+xml', 'image/png', 'image/gif',
                              'image/jpeg']
     searchindex_filename = 'searchindex.json'
@@ -379,8 +379,7 @@ class StandaloneHTMLBuilder(Builder):
             builder = self.name,
             parents = [],
             logo = logo,
-            favicon = favicon,
-            len = len, # the built-in
+            favicon = favicon
         )
 
     def get_doc_context(self, docname, body):
@@ -624,12 +623,14 @@ class StandaloneHTMLBuilder(Builder):
 
     def load_indexer(self, docnames):
         try:
-            f = open(path.join(self.outdir, self.searchindex_filename), 'r')
+            f = open(path.join(self.outdir, self.searchindex_filename), 'rb')
             try:
                 self.indexer.load(f, self.indexer_format)
             finally:
                 f.close()
-        except (IOError, OSError):
+        except (IOError, OSError, NotImplementedError):
+            # we catch NotImplementedError here because if no simplejson
+            # is installed the searchindex can't be loaded
             pass
         # delete all entries for files that will be rebuilt
         self.indexer.prune(set(self.env.all_docs) - set(docnames))
@@ -683,9 +684,9 @@ class StandaloneHTMLBuilder(Builder):
     def handle_finish(self):
         self.info(bold('dumping search index...'))
         self.indexer.prune(self.env.all_docs)
-        f = open(path.join(self.outdir, 'searchindex.json'), 'w')
+        f = open(path.join(self.outdir, self.searchindex_filename), 'wb')
         try:
-            self.indexer.dump(f, 'json')
+            self.indexer.dump(f, self.indexer_format)
         finally:
             f.close()
 
@@ -702,9 +703,6 @@ class SerializingHTMLBuilder(StandaloneHTMLBuilder):
     #: the filename for the global context file
     globalcontext_filename = None
 
-    #: If set to `None` the indexer uses the serialization implementation
-    indexer_format = None
-
     supported_image_types = ('image/svg+xml', 'image/png', 'image/gif',
                              'image/jpeg')
 
@@ -712,6 +710,9 @@ class SerializingHTMLBuilder(StandaloneHTMLBuilder):
         self.init_translator_class()
         self.templates = None   # no template bridge necessary
 
+    indexer_format = property(lambda x: x.implementation, doc='''
+        Alias the indexer format to the serilization implementation''')
+
     def get_target_uri(self, docname, typ=None):
         if docname == 'index':
             return ''
@@ -754,13 +755,8 @@ class SerializingHTMLBuilder(StandaloneHTMLBuilder):
         finally:
             f.close()
 
-        self.info(bold('dumping search index...'))
-        self.indexer.prune(self.env.all_docs)
-        f = open(path.join(self.outdir, self.searchindex_filename), 'wb')
-        try:
-            self.indexer.dump(f, self.indexer_format or self.implementation)
-        finally:
-            f.close()
+        # super here to dump the search index
+        StandaloneHTMLBuilder.handle_finish(self)
 
         # copy the environment file from the doctree dir to the output dir
         # as needed by the web app
@@ -773,6 +769,9 @@ class SerializingHTMLBuilder(StandaloneHTMLBuilder):
 
 
 class PickleHTMLBuilder(SerializingHTMLBuilder):
+    """
+    A Builder that dumps the generated HTML into pickle files.
+    """
     implementation = pickle
     name = 'pickle'
     out_suffix = '.fpickle'
@@ -780,6 +779,17 @@ class PickleHTMLBuilder(SerializingHTMLBuilder):
     searchindex_filename = 'searchindex.pickle'
 
 
+class JSONHTMLBuilder(SerializingHTMLBuilder):
+    """
+    A builder that dumps the generated HTML into JSON files.
+    """
+    implementation = json
+    name = 'json'
+    out_suffix = '.fjson'
+    globalcontext_filename = 'globalcontext.json'
+    searchindex_filename = 'searchindex.json'
+
+
 class HTMLHelpBuilder(StandaloneHTMLBuilder):
     """
     Builder that also outputs Windows HTML help project, contents and index files.
@@ -1128,6 +1138,7 @@ from sphinx.linkcheck import CheckExternalLinksBuilder
 builtin_builders = {
     'html': StandaloneHTMLBuilder,
     'pickle': PickleHTMLBuilder,
+    'json': JSONHTMLBuilder,
     'web': PickleHTMLBuilder,
     'htmlhelp': HTMLHelpBuilder,
     'latex': LaTeXBuilder,

sphinx/quickstart.py

@@ -238,7 +238,8 @@ ALLSPHINXOPTS   = -d %(rbuilddir)s/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) %
 help:
 \t@echo "Please use \\`make <target>' where <target> is one of"
 \t@echo "  html      to make standalone HTML files"
-\t@echo "  pickle    to make pickle files (usable by e.g. sphinx-web)"
+\t@echo "  pickle    to make pickle files"
+\t@echo "  json      to make JSON files"
 \t@echo "  htmlhelp  to make HTML files and a HTML help project"
 \t@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 \t@echo "  changes   to make an overview over all changed/added/deprecated items"
@@ -261,6 +262,12 @@ pickle:
 
 web: pickle
 
+json:
+\tmkdir -p %(rbuilddir)s/json %(rbuilddir)s/doctrees
+\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) %(rbuilddir)s/json
+\t@echo
+\t@echo "Build finished; now you can process the JSON files."
+
 htmlhelp:
 \tmkdir -p %(rbuilddir)s/htmlhelp %(rbuilddir)s/doctrees
 \t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) %(rbuilddir)s/htmlhelp

sphinx/search.py

@@ -9,7 +9,7 @@
     :license: BSD.
 """
 import re
-import pickle
+import cPickle as pickle
 
 from docutils.nodes import Text, NodeVisitor
 

sphinx/templates/genindex-single.html

@@ -24,7 +24,7 @@
     {%- endfor %}
   </dl></dd>
   {%- endif -%}
-{%- set numitems = numitems + 1 + len(subitems) -%}
+{%- set numitems = numitems + 1 + subitems|length -%}
 {%- if numcols < 2 and numitems > breakat -%}
 {%- set numcols = numcols+1 -%}
 </dl></td><td width="33%" valign="top"><dl>

sphinx/templates/genindex.html

@@ -32,7 +32,7 @@
     {%- endfor %}
   </dl></dd>
   {%- endif -%}
-{%- set numitems = numitems + 1 + len(subitems) -%}
+{%- set numitems = numitems + 1 + subitems|length -%}
 {%- if numcols < 2 and numitems > breakat -%}
 {%- set numcols = numcols+1 -%}
 </dl></td><td width="33%" valign="top"><dl>

sphinx/util/_json.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.util._json
+    ~~~~~~~~~~~~~~~~~
+
+    This module implements a simple JSON serializer if simplejson is
+    unavailable.
+
+    This is not fully JSON compliant but enough for the searchindex.
+    And the generated files are smaller than the simplejson ones.
+
+    Uses the basestring encode function from simplejson.
+
+    :copyright: Copyright 2008 by Armin Ronacher, Bob Ippolito.
+    :license: BSD.
+"""
+import re
+
+
+# escape \, ", control characters and everything outside ASCII
+ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
+ESCAPE_DICT = {
+    '\\': '\\\\',
+    '"': '\\"',
+    '\b': '\\b',
+    '\f': '\\f',
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+}
+
+
+def encode_basestring_ascii(s):
+    def replace(match):
+        s = match.group(0)
+        try:
+            return ESCAPE_DICT[s]
+        except KeyError:
+            n = ord(s)
+            if n < 0x10000:
+                return '\\u%04x' % (n,)
+            else:
+                # surrogate pair
+                n -= 0x10000
+                s1 = 0xd800 | ((n >> 10) & 0x3ff)
+                s2 = 0xdc00 | (n & 0x3ff)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
+
+
+def dumps(obj, key=False):
+    if key:
+        if not isinstance(obj, basestring):
+            obj = str(obj)
+        return encode_basestring_ascii(obj)
+    if obj is None:
+        return 'null'
+    elif obj is True or obj is False:
+        return obj and 'true' or 'false'
+    elif isinstance(obj, (int, long, float)):
+        return str(obj)
+    elif isinstance(obj, dict):
+        return '{%s}' % ','.join('%s:%s' % (
+            dumps(key, True),
+            dumps(value)
+        ) for key, value in obj.iteritems())
+    elif isinstance(obj, (tuple, list, set)):
+        return '[%s]' % ','.join(dumps(x) for x in obj)
+    elif isinstance(obj, basestring):
+        return encode_basestring_ascii(obj)
+    raise TypeError(type(obj))
+
+
+def dump(obj, f):
+    f.write(dumps(obj))

sphinx/util/json.py

@@ -3,87 +3,32 @@
     sphinx.util.json
     ~~~~~~~~~~~~~~~~
 
-    Minimal JSON module that generates small dumps.
+    This module imports JSON functions from various locations.
 
-    This is not fully JSON compliant but enough for the searchindex.
-    And the generated files are smaller than the simplejson ones.
-
-    Uses the basestring encode function from simplejson.
-
-    :copyright: 2007-2008 by Armin Ronacher, Bob Ippolito.
+    :copyright: Copyright 2008 by Armin Ronacher.
     :license: BSD.
 """
 
-import re
+# if no simplejson is available this module can not load json files.
+can_load = True
 
-# escape \, ", control characters and everything outside ASCII
-ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
-ESCAPE_DICT = {
-    '\\': '\\\\',
-    '"': '\\"',
-    '\b': '\\b',
-    '\f': '\\f',
-    '\n': '\\n',
-    '\r': '\\r',
-    '\t': '\\t',
-}
+# unset __name__ for a moment so that the import goes straight into
+# the stdlib for python 2.4.
+_old_name = __name__
+del __name__
 
+try:
+    from simplejson import dumps, dump, loads, load
+except ImportError:
+    try:
+        from json import dumps, dump, loads, load
+    except ImportError:
+        from sphinx.util._json import dumps, dump
+        def _dummy(x):
+            raise NotImplementedError('simplejson unavailable, can\'t load')
+        load = loads = _dummy
+        can_load = False
+        del _dummy
 
-def encode_basestring_ascii(s):
-    def replace(match):
-        s = match.group(0)
-        try:
-            return ESCAPE_DICT[s]
-        except KeyError:
-            n = ord(s)
-            if n < 0x10000:
-                return '\\u%04x' % (n,)
-            else:
-                # surrogate pair
-                n -= 0x10000
-                s1 = 0xd800 | ((n >> 10) & 0x3ff)
-                s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-def dump_json(obj, key=False):
-    if key:
-        if not isinstance(obj, basestring):
-            obj = str(obj)
-        return encode_basestring_ascii(obj)
-    if obj is None:
-        return 'null'
-    elif obj is True or obj is False:
-        return obj and 'true' or 'false'
-    elif isinstance(obj, (int, long, float)):
-        return str(obj)
-    elif isinstance(obj, dict):
-        return '{%s}' % ','.join('%s:%s' % (
-            dump_json(key, True),
-            dump_json(value)
-        ) for key, value in obj.iteritems())
-    elif isinstance(obj, (tuple, list, set)):
-        return '[%s]' % ','.join(dump_json(x) for x in obj)
-    elif isinstance(obj, basestring):
-        return encode_basestring_ascii(obj)
-    raise TypeError(type(obj))
-
-
-STRING = re.compile(r'("(\\\\|\\"|[^"])*")')
-
-def load_json(s):
-    d = {'null': None, 'true': True, 'false': False}
-    s = STRING.sub(r'u\1', s)
-    return eval(s, d)
-
-
-# serializer interface
-dumps = dump_json
-loads = load_json
-
-def dump(obj, f):
-    f.write(dumps(obj))
-
-def load(f):
-    return loads(f.read())
+__name__ = _old_name
+del _old_name