Improved search system. The search index is now a regular javascript file which should speed things up because browsers can cache it. Removed unused code from doctools.js

sphinx/builder.py

@@ -34,6 +34,7 @@ from sphinx.latexwriter import LaTeXWriter
 from sphinx.environment import BuildEnvironment, NoUri
 from sphinx.highlighting import PygmentsBridge
 from sphinx.util.console import bold, purple, darkgreen
+from sphinx.search import js_index
 
 # side effect: registers roles and directives
 from sphinx import roles
@@ -338,10 +339,10 @@ class StandaloneHTMLBuilder(Builder):
     name = 'html'
     copysource = True
     out_suffix = '.html'
-    indexer_format = json
+    indexer_format = js_index
     supported_image_types = ['image/svg+xml', 'image/png', 'image/gif',
                              'image/jpeg']
-    searchindex_filename = 'searchindex.json'
+    searchindex_filename = 'searchindex.js'
     add_header_links = True
     add_definition_links = True
 

sphinx/search.py

@@ -10,6 +10,7 @@
 """
 import re
 import cPickle as pickle
+from cStringIO import StringIO
 
 from docutils.nodes import Text, NodeVisitor
 
@@ -20,6 +21,37 @@ from sphinx.util import json
 word_re = re.compile(r'\w+(?u)')
 
 
+class _JavaScriptIndex(object):
+    """
+    The search index as javascript file that calls a function
+    on the documentation search object to register the index.
+    This serializing system does not support chaining because
+    simplejson (which it depends on) doesn't support it either.
+    """
+
+    PREFIX = 'Search.setIndex('
+    SUFFIX = ')'
+
+    def dumps(self, data):
+        return self.PREFIX + json.dumps(data) + self.SUFFIX
+
+    def loads(self, s):
+        data = s[len(self.PREFIX):-len(self.SUFFIX)]
+        if not data or not s.startswith(self.PREFIX) or not \
+           s.endswith(self.SUFFIX):
+            raise ValueError('invalid data')
+        return json.loads(data)
+
+    def dump(self, data, f):
+        f.write(self.dumps(data))
+
+    def load(self, f):
+        return self.loads(f.read())
+
+
+js_index = _JavaScriptIndex()
+
+
 class Stemmer(PorterStemmer):
     """
     All those porter stemmer implementations look hideous.

sphinx/static/doctools.js

@@ -94,11 +94,9 @@ jQuery.fn.highlightText = function(text, className) {
 var Documentation = {
 
   init : function() {
-    /* this.addContextElements(); -- now done statically */
     this.fixFirefoxAnchorBug();
     this.highlightSearchWords();
     this.initModIndex();
-    this.initComments();
   },
 
   /**
@@ -108,6 +106,8 @@ var Documentation = {
   PLURAL_EXPR : function(n) { return n == 1 ? 0 : 1; },
   LOCALE : 'unknown',
 
+  // gettext and ngettext don't access this so that the functions
+  // can savely bound to a different name (_ = Documentation.gettext)
   gettext : function(string) {
     var translated = Documentation.TRANSLATIONS[string];
     if (typeof translated == 'undefined')
@@ -133,14 +133,12 @@ var Documentation = {
    * add context elements like header anchor links
    */
   addContextElements : function() {
-    for (var i = 1; i <= 6; i++) {
-      $('h' + i + '[@id]').each(function() {
-        $('<a class="headerlink">\u00B6</a>').
-        attr('href', '#' + this.id).
-        attr('title', _('Permalink to this headline')).
-        appendTo(this);
-      });
-    }
+    $('div[@id] > :header:first').each(function() {
+      $('<a class="headerlink">\u00B6</a>').
+      attr('href', '#' + this.id).
+      attr('title', _('Permalink to this headline')).
+      appendTo(this);
+    });
     $('dt[@id]').each(function() {
       $('<a class="headerlink">\u00B6</a>').
       attr('href', '#' + this.id).
@@ -196,37 +194,6 @@ var Documentation = {
     }
   },
 
-  /**
-   * init the inline comments
-   */
-  initComments : function() {
-    $('.inlinecomments div.actions').each(function() {
-      this.innerHTML += ' | ';
-      $(this).append($('<a href="#">hide comments</a>').click(function() {
-        $(this).parent().parent().toggle();
-        return false;
-      }));
-    });
-    $('.inlinecomments .comments').hide();
-    $('.inlinecomments a.bubble').each(function() {
-      $(this).click($(this).is('.emptybubble') ? function() {
-          var params = $.getQueryParameters(this.href);
-          Documentation.newComment(params.target[0]);
-          return false;
-        } : function() {
-          $('.comments', $(this).parent().parent()[0]).toggle();
-          return false;
-      });
-    });
-    $('#comments div.actions a.newcomment').click(function() {
-      Documentation.newComment();
-      return false;
-    });
-    if (document.location.hash.match(/^#comment-/))
-      $('.inlinecomments .comments ' + document.location.hash)
-        .parent().toggle();
-  },
-
   /**
    * helper function to hide the search marks again
    */
@@ -235,22 +202,6 @@ var Documentation = {
     $('span.highlight').removeClass('highlight');
   },
 
-  /**
-   * show the comment window for a certain id or the whole page.
-   */
-  newComment : function(id) {
-    Documentation.CommentWindow.openFor(id || '');
-  },
-
-  /**
-   * write a new comment from within a comment view box
-   */
-  newCommentFromBox : function(link) {
-    var params = $.getQueryParameters(link.href);
-    $(link).parent().parent().fadeOut('slow');
-    this.newComment(params.target);
-  },
-
   /**
    * make the url absolute
    */
@@ -270,108 +221,7 @@ var Documentation = {
     });
     var url = parts.join('/');
     return path.substring(url.lastIndexOf('/') + 1, path.length - 1);
-  },
-
-  /**
-   * class that represents the comment window
-   */
-  CommentWindow : (function() {
-    var openWindows = {};
-
-    var Window = function(sectionID) {
-      this.url = Documentation.makeURL('@comments/' + Documentation.getCurrentURL()
-        + '/?target=' + $.urlencode(sectionID) + '&mode=ajax');
-      this.sectionID = sectionID;
-
-      this.root = $('<div class="commentwindow"></div>');
-      this.root.appendTo($('body'));
-      this.title = $('<h3>New Comment</h3>').appendTo(this.root);
-      this.body = $('<div class="form">please wait...</div>').appendTo(this.root);
-      this.resizeHandle = $('<div class="resizehandle"></div>').appendTo(this.root);
-
-      this.root.Draggable({
-        handle:       this.title[0]
-      });
-
-      this.root.css({
-        left:         window.innerWidth / 2 - $(this.root).width() / 2,
-        top:          window.scrollY + (window.innerHeight / 2 - 150)
-      });
-      this.root.fadeIn('slow');
-      this.updateView();
-    };
-
-    Window.prototype.updateView = function(data) {
-      var self = this;
-      function update(data) {
-        if (data.posted) {
-          document.location.hash = '#comment-' + data.commentID;
-          document.location.reload();
-        }
-        else {
-          self.body.html(data.body);
-          $('div.actions', self.body).append($('<input>')
-            .attr('type', 'button')
-            .attr('value', 'Close')
-            .click(function() { self.close(); })
-          );
-          $('div.actions input[@name="preview"]')
-            .attr('type', 'button')
-            .click(function() { self.submitForm($('form', self.body)[0], true); });
-          $('form', self.body).bind("submit", function() {
-            self.submitForm(this);
-            return false;
-          });
-
-          if (data.error) {
-            self.root.Highlight(1000, '#aadee1');
-            $('div.error', self.root).slideDown(500);
-          }
-        }
-      }
-
-      if (typeof data == 'undefined')
-        $.getJSON(this.url, function(json) { update(json); });
-      else
-        $.ajax({
-          url:      this.url,
-          type:     'POST',
-          dataType: 'json',
-          data:     data,
-          success:  function(json) { update(json); }
-        });
-    }
-
-    Window.prototype.getFormValue = function(name) {
-      return $('*[@name="' + name + '"]', this.body)[0].value;
-    }
-
-    Window.prototype.submitForm = function(form, previewMode) {
-      this.updateView({
-        author:         form.author.value,
-        author_mail:    form.author_mail.value,
-        title:          form.title.value,
-        comment_body:   form.comment_body.value,
-        preview:        previewMode ? 'yes' : ''
-      });
-    }
-
-    Window.prototype.close = function() {
-      var self = this;
-      delete openWindows[this.sectionID];
-      this.root.fadeOut('slow', function() {
-        self.root.remove();
-      });
-    }
-
-    Window.openFor = function(sectionID) {
-      if (sectionID in openWindows)
-        return openWindows[sectionID];
-      return new Window(sectionID);
-    }
-
-    return Window;
-  })()
+  }
 };
 
 // quick alias for translations

sphinx/static/searchtools.js

@@ -224,6 +224,10 @@ var PorterStemmer = function() {
  */
 var Search = {
 
+    _index : null,
+    _queued_query : null,
+    _pulse_status : -1,
+
     init : function() {
         var params = $.getQueryParameters();
         if (params.q) {
@@ -234,33 +238,68 @@ var Search = {
     },
 
     /**
-     * perform a search for something
+     * Sets the index
      */
-    performSearch : function(query) {
-        // create the required interface elements
-        var out = $('#search-results');
-        var title = $('<h2>' + _('Searching') + '</h2>').appendTo(out);
-        var dots = $('<span></span>').appendTo(title);
-        var status = $('<p style="display: none"></p>').appendTo(out);
-        var output = $('<ul class="search"/>').appendTo(out);
-        $('#search-progress').text(_('Getting search index...'));
+    setIndex : function(index) {
+        var q;
+        this._index = index;
+        if ((q = this._queued_query) !== null) {
+            this._queued_query = null;
+            Search.query(q);
+        }
+    },
 
-        // spawn a background runner for updating the dots
-        // until the search has finished
-        var pulseStatus = 0;
+    hasIndex : function() {
+        return self._index !== null;
+    },
+
+    deferQuery : function(query) {
+        this._queued_query = query;
+    },
+
+    stopPulse : function() {
+        this._pulse_status = 0;
+    },
+
+    startPulse : function() {
+        if (this._pulse_status >= 0)
+            return;
         function pulse() {
-            pulseStatus = (pulseStatus + 1) % 4;
+            Search._pulse_status = (Search._pulse_status + 1) % 4;
             var dotString = '';
-            for (var i = 0; i < pulseStatus; i++) {
+            for (var i = 0; i < Search._pulse_status; i++) {
                 dotString += '.';
             }
-            dots.text(dotString);
-            if (pulseStatus > -1) {
+            Search.dots.text(dotString);
+            if (Search._pulse_status > -1) {
                 window.setTimeout(pulse, 500);
             }
         };
         pulse();
+    },
 
+    /**
+     * perform a search for something
+     */
+    performSearch : function(query) {
+        // create the required interface elements
+        this.out = $('#search-results');
+        this.title = $('<h2>' + _('Searching') + '</h2>').appendTo(this.out);
+        this.dots = $('<span></span>').appendTo(this.title);
+        this.status = $('<p style="display: none"></p>').appendTo(this.out);
+        this.output = $('<ul class="search"/>').appendTo(this.out);
+
+        $('#search-progress').text(_('Preparing search...'));
+        this.startPulse();
+
+        // index already loaded, the browser was quick!
+        if (this.hasIndex())
+            this.query(query);
+        else
+            this.setQuery(query);
+    },
+
+    query : function(query) {
         // stem the searchwords and add them to the
         // correct list
         var stemmer = new PorterStemmer();
@@ -291,112 +330,102 @@ var Search = {
         console.info('required: ', searchwords);
         console.info('excluded: ', excluded);
 
-        // fetch searchindex and perform search
-        $.getJSON('searchindex.json', function(data) {
+        // prepare search
+        var filenames = this._index[0];
+        var titles = this._index[1];
+        var words = this._index[2];
+        var fileMap = {};
+        var files = null;
+        $('#search-progress').empty();
 
-                // prepare search
-                var filenames = data[0];
-                var titles = data[1]
-                var words = data[2];
-                var fileMap = {};
-                var files = null;
+        // perform the search on the required words
+        for (var i = 0; i < searchwords.length; i++) {
+            var word = searchwords[i];
+            // no match but word was a required one
+            if ((files = words[word]) == null)
+                break;
+            // create the mapping
+            for (var j = 0; j < files.length; j++) {
+                var file = files[j];
+                if (file in fileMap)
+                    fileMap[file].push(word);
+                else
+                    fileMap[file] = [word];
+            }
+        }
 
-                $('#search-progress').empty()
+        // now check if the files are in the correct
+        // areas and if the don't contain excluded words
+        var results = [];
+        for (var file in fileMap) {
+            var valid = true;
 
-                // perform the search on the required words
-                for (var i = 0; i < searchwords.length; i++) {
-                    var word = searchwords[i];
-                    // no match but word was a required one
-                    if ((files = words[word]) == null) {
-                        break;
-                    }
-                    // create the mapping
-                    for (var j = 0; j < files.length; j++) {
-                        var file = files[j];
-                        if (file in fileMap) {
-                            fileMap[file].push(word);
-                        }
-                        else {
-                            fileMap[file] = [word];
-                        }
-                    }
+            // check if all requirements are matched
+            if (fileMap[file].length != searchwords.length)
+                continue;
+
+            // ensure that none of the excluded words is in the
+            // search result.
+            for (var i = 0; i < excluded.length; i++) {
+                if ($.contains(words[excluded[i]] || [], file)) {
+                    valid = false;
+                    break;
                 }
+            }
 
-                // now check if the files are in the correct
-                // areas and if the don't contain excluded words
-                var results = [];
-                for (var file in fileMap) {
-                    var valid = true;
+            // if we have still a valid result we can add it
+            // to the result list
+            if (valid)
+                results.push([filenames[file], titles[file]]);
+        }
 
-                    // check if all requirements are matched
-                    if (fileMap[file].length != searchwords.length) {
-                        continue;
-                    }
-                    // ensure that none of the excluded words is in the
-                    // search result.
-                    for (var i = 0; i < excluded.length; i++) {
-                        if ($.contains(words[excluded[i]] || [], file)) {
-                            valid = false;
-                            break;
-                        }
-                    }
+        // delete unused variables in order to not waste
+        // memory until list is retrieved completely
+        delete filenames, titles, words;
 
-                    // if we have still a valid result we can add it
-                    // to the result list
-                    if (valid) {
-                        results.push([filenames[file], titles[file]]);
-                    }
-                }
+        // now sort the results by title
+        results.sort(function(a, b) {
+            var left = a[1].toLowerCase();
+            var right = b[1].toLowerCase();
+            return (left > right) ? -1 : ((left < right) ? 1 : 0);
+        });
 
-                // delete unused variables in order to not waste
-                // memory until list is retrieved completely
-                delete filenames, titles, words, data;
-
-                // now sort the results by title
-                results.sort(function(a, b) {
-                        var left = a[1].toLowerCase();
-                        var right = b[1].toLowerCase();
-                        return (left > right) ? -1 : ((left < right) ? 1 : 0);
+        // print the results
+        var resultCount = results.length;
+        function displayNextItem() {
+            // results left, load the summary and display it
+            if (results.length) {
+                var item = results.pop();
+                var listItem = $('<li style="display:none"></li>');
+                listItem.append($('<a/>').attr(
+                    'href',
+                    item[0] + DOCUMENTATION_OPTIONS.FILE_SUFFIX +
+                    highlightstring).html(item[1]));
+                $.get('_sources/' + item[0] + '.txt', function(data) {
+                    listItem.append($.makeSearchSummary(data, searchwords, hlwords));
+                    Search.output.append(listItem);
+                    listItem.slideDown(10, function() {
+                        displayNextItem();
                     });
-
-                // print the results
-                var resultCount = results.length;
-                function displayNextItem() {
-                    // results left, load the summary and display it
-                    if (results.length) {
-                        var item = results.pop();
-                        var listItem = $('<li style="display:none"></li>');
-                        listItem.append($('<a/>').attr(
-                            'href',
-                            item[0] + DOCUMENTATION_OPTIONS.FILE_SUFFIX +
-                            highlightstring).html(item[1]));
-                        $.get('_sources/' + item[0] + '.txt', function(data) {
-                                listItem.append($.makeSearchSummary(data, searchwords, hlwords));
-                                output.append(listItem);
-                                listItem.slideDown(10, function() {
-                                        displayNextItem();
-                                    });
-                            });
-                    }
-                    // search finished, update title and status message
-                    else {
-                        pulseStatus = -1;
-                        title.text(_('Search Results'));
-                        if (!resultCount) {
-                            status.text(_('Your search did not match any documents. Please make sure that all words are spelled correctly and that you\'ve selected enough categories.'));
-                        }
-                        else {
-                            status.text(_('Search finished, found %s page(s) matching the search query.').replace('%s', resultCount));
-                        }
-                        status.fadeIn(500);
-                    }
+                });
+            }
+            // search finished, update title and status message
+            else {
+                Search.stopPulse();
+                Search.title.text(_('Search Results'));
+                if (!resultCount) {
+                    Search.status.text(_('Your search did not match any documents. Please make sure that all words are spelled correctly and that you\'ve selected enough categories.'));
                 }
-                displayNextItem();
-            });
+                else {
+                    Search.status.text(_('Search finished, found %s page(s) matching the search query.').replace('%s', resultCount));
+                }
+                Search.status.fadeIn(500);
+            }
+        }
+        displayNextItem();
     }
-
 }
 
 $(document).ready(function() {
-        Search.init();
-    });
+    Search.init();
+});

sphinx/templates/search.html

@@ -1,6 +1,6 @@
 {% extends "layout.html" %}
 {% set title = _('Search') %}
-{% set script_files = script_files + ['_static/searchtools.js'] %}
+{% set script_files = script_files + ['_static/searchtools.js', 'searchindex.js'] %}
 {% block body %}
   <h1 id="search-documentation">{{ _('Search') }}</h1>
   <p>