[tests] JavaScript: refactor test fixtures (#12102)

This PR allows to serve JavaScript test fixtures using a fixture-based logic as for Python tests.

Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>

.gitignore

@@ -31,6 +31,7 @@ doc/_build/
 doc/locale/
 tests/.coverage
 tests/build/
+tests/js/roots/*/_build
 tests/test-server.lock
 utils/regression_test.js
 

.ruff.toml

@@ -4,6 +4,7 @@ output-format = "full"
 
 extend-exclude = [
     "tests/roots/*",
+    "tests/js/roots/*",
     "build/*",
     "doc/_build/*",
     "sphinx/search/*",

CHANGES.rst

@@ -49,6 +49,9 @@ Bugs fixed
 Testing
 -------
 
+* karma: refactor HTML search tests to use fixtures generated by Sphinx.
+  Patch by James Addison.
+
 Release 7.3.7 (released Apr 19, 2024)
 =====================================
 

doc/internals/contributing.rst

@@ -338,3 +338,9 @@ Debugging tips
   Minified files in ``sphinx/search/minified-js/*.js`` are generated from
   non-minified ones using ``uglifyjs`` (installed via npm), with ``-m``
   option to enable mangling.
+
+* The ``searchindex.js`` files found in the ``tests/js/fixtures/*`` directories
+  are generated by using the standard Sphinx HTML builder on the corresponding
+  input projects found in ``tests/js/roots/*``.  The fixtures provide test data
+  used by the Sphinx JavaScript unit tests, and can be regenerated by running
+  the ``utils/generate_js_fixtures.py`` script.

karma.conf.js

@@ -15,7 +15,9 @@ module.exports = function(config) {
 
     // list of files / patterns to load in the browser
     files: [
+      { pattern: 'tests/js/fixtures/**/*.js', included: false, served: true },
       'tests/js/documentation_options.js',
+      'tests/js/language_data.js',
       'sphinx/themes/basic/static/doctools.js',
       'sphinx/themes/basic/static/searchtools.js',
       'sphinx/themes/basic/static/sphinx_highlight.js',

tests/js/fixtures/cpp/searchindex.js

@@ -0,0 +1 @@
+Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx", false]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "becaus": 0, "c": 0, "can": 0, "cardin": 0, "challeng": 0, "charact": 0, "class": 0, "descript": 0, "drop": 0, "engin": 0, "fixtur": 0, "frequent": 0, "gener": 0, "i": 0, "index": 0, "inflat": 0, "mathemat": 0, "occur": 0, "often": 0, "project": 0, "punctuat": 0, "queri": 0, "relat": 0, "sampl": 0, "search": 0, "size": 0, "sphinx": 0, "term": 0, "thei": 0, "thi": 0, "token": 0, "us": 0, "web": 0, "would": 0}, "titles": ["<no title>"], "titleterms": {}})

tests/js/fixtures/multiterm/searchindex.js

@@ -0,0 +1 @@
+Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"At": 0, "adjac": 0, "all": 0, "an": 0, "appear": 0, "applic": 0, "ar": 0, "built": 0, "can": 0, "check": 0, "contain": 0, "do": 0, "document": 0, "doesn": 0, "each": 0, "fixtur": 0, "format": 0, "function": 0, "futur": 0, "html": 0, "i": 0, "includ": 0, "match": 0, "messag": 0, "multipl": 0, "multiterm": 0, "order": 0, "other": 0, "output": 0, "perform": 0, "perhap": 0, "phrase": 0, "project": 0, "queri": 0, "requir": 0, "same": 0, "search": 0, "successfulli": 0, "support": 0, "t": 0, "term": 0, "test": 0, "thi": 0, "time": 0, "us": 0, "when": 0, "write": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}})

tests/js/fixtures/partial/searchindex.js

@@ -0,0 +1 @@
+Search.setIndex({"alltitles": {"sphinx_utils module": [[0, "sphinx-utils-module"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"also": 0, "ar": 0, "built": 0, "confirm": 0, "document": 0, "function": 0, "html": 0, "i": 0, "includ": 0, "input": 0, "javascript": 0, "known": 0, "match": 0, "partial": 0, "possibl": 0, "prefix": 0, "project": 0, "provid": 0, "restructuredtext": 0, "sampl": 0, "search": 0, "should": 0, "thi": 0, "titl": 0, "us": 0, "when": 0}, "titles": ["sphinx_utils module"], "titleterms": {"modul": 0, "sphinx_util": 0}})

tests/js/language_data.js

@@ -0,0 +1,26 @@
+/*
+ * language_data.js
+ * ~~~~~~~~~~~~~~~~
+ *
+ * This script contains the language-specific data used by searchtools.js,
+ * namely the list of stopwords, stemmer, scorer and splitter.
+ *
+ * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ */
+
+var stopwords = [];
+
+
+/* Non-minified version is copied as a separate JS file, if available */
+
+/**
+ * Dummy stemmer for languages without stemming rules.
+ */
+var Stemmer = function() {
+  this.stemWord = function(w) {
+    return w;
+  }
+}
+

tests/js/roots/cpp/index.rst

@@ -0,0 +1,10 @@
+This is a sample C++ project used to generate a search engine index fixture.
+
+.. cpp:class:: public Sphinx
+
+   The description of Sphinx class.
+
+Indexing and querying the term C++ can be challenging, because search-related
+tokenization often drops punctuation and mathematical characters (they occur
+frequently on the web and would inflate the cardinality and size of web search
+indexes).

tests/js/roots/multiterm/index.rst

@@ -0,0 +1,13 @@
+Main Page
+=========
+
+This is the main page of the ``multiterm`` test project.
+
+This document is used as a test fixture to check that the search functionality
+included when projects are built into an HTML output format can successfully
+match this document when a search query containing multiple terms is performed.
+
+At the time-of-writing this message, the application doesn't support "phrase
+queries" -- queries that require all of the contained terms to appear adjacent
+to each other and in the same order in the document as in the query; perhaps it
+will do in future?

tests/js/roots/partial/index.rst

@@ -0,0 +1,9 @@
+sphinx_utils module
+===================
+
+Partial (also known as "prefix") matches on document titles should be possible
+using the JavaScript search functionality included when HTML documentation
+projects are built.
+
+This document provides a sample reStructuredText input to confirm that partial
+title matching is possible.

tests/js/searchtools.js

@@ -1,20 +1,20 @@
 describe('Basic html theme search', function() {
 
+  function loadFixture(name) {
+      req = new XMLHttpRequest();
+      req.open("GET", `base/tests/js/fixtures/${name}`, false);
+      req.send(null);
+      return req.responseText;
+  }
+
   describe('terms search', function() {
 
     it('should find "C++" when in index', function() {
-      index = {
-        docnames:["index"],
-        filenames:["index.rst"],
-        terms:{'c++':0},
-        titles:["<no title>"],
-        titleterms:{}
-      }
-      Search.setIndex(index);
-      searchterms = ['c++'];
-      excluded = [];
-      terms = index.terms;
-      titleterms = index.titleterms;
+      eval(loadFixture("cpp/searchindex.js"));
+
+      [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('C++');
+      terms = Search._index.terms;
+      titleterms = Search._index.titleterms;
 
       hits = [[
         "index",
@@ -28,22 +28,11 @@ describe('Basic html theme search', function() {
     });
 
     it('should be able to search for multiple terms', function() {
-      index = {
-        alltitles: {
-          'Main Page': [[0, 'main-page']],
-        },
-        docnames:["index"],
-        filenames:["index.rst"],
-        terms:{main:0, page:0},
-        titles:["Main Page"],
-        titleterms:{ main:0, page:0 }
-      }
-      Search.setIndex(index);
+      eval(loadFixture("multiterm/searchindex.js"));
 
-      searchterms = ['main', 'page'];
-      excluded = [];
-      terms = index.terms;
-      titleterms = index.titleterms;
+      [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('main page');
+      terms = Search._index.terms;
+      titleterms = Search._index.titleterms;
       hits = [[
         'index',
         'Main Page',
@@ -55,18 +44,11 @@ describe('Basic html theme search', function() {
     });
 
     it('should partially-match "sphinx" when in title index', function() {
-      index = {
-        docnames:["index"],
-        filenames:["index.rst"],
-        terms:{'useful': 0, 'utilities': 0},
-        titles:["sphinx_utils module"],
-        titleterms:{'sphinx_utils': 0}
-      }
-      Search.setIndex(index);
-      searchterms = ['sphinx'];
-      excluded = [];
-      terms = index.terms;
-      titleterms = index.titleterms;
+      eval(loadFixture("partial/searchindex.js"));
+
+      [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('sphinx');
+      terms = Search._index.terms;
+      titleterms = Search._index.titleterms;
 
       hits = [[
         "index",
@@ -81,6 +63,37 @@ describe('Basic html theme search', function() {
 
   });
 
+  describe('aggregation of search results', function() {
+
+    it('should combine document title and document term matches', function() {
+      eval(loadFixture("multiterm/searchindex.js"));
+
+      searchParameters = Search._parseQuery('main page');
+
+      // fixme: duplicate result due to https://github.com/sphinx-doc/sphinx/issues/11961
+      hits = [
+        [
+          'index',
+          'Main Page',
+          '',
+          null,
+          15,
+          'index.rst'
+        ],
+        [
+          'index',
+          'Main Page',
+          '#main-page',
+          null,
+          100,
+          'index.rst'
+        ]
+      ];
+      expect(Search._performSearch(...searchParameters)).toEqual(hits);
+    });
+
+  });
+
 });
 
 describe("htmlToText", function() {

tests/test_search.py

@@ -11,6 +11,10 @@ from docutils.parsers import rst
 
 from sphinx.search import IndexBuilder
 
+from tests.utils import TESTS_ROOT
+
+JAVASCRIPT_TEST_ROOTS = list((TESTS_ROOT / 'js' / 'roots').iterdir())
+
 
 class DummyEnvironment:
     def __init__(self, version, domains):
@@ -346,3 +350,15 @@ def assert_is_sorted(item, path: str):
             assert item == sorted(item), f'{err_path} is not sorted'
         for i, child in enumerate(item):
             assert_is_sorted(child, f'{path}[{i}]')
+
+
+@pytest.mark.parametrize('directory', JAVASCRIPT_TEST_ROOTS)
+def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory):
+    app = make_app('html', srcdir=directory, builddir=sphinx_test_tempdir / directory.name)
+    app.build()
+
+    fresh_searchindex = (app.outdir / 'searchindex.js')
+    existing_searchindex = (TESTS_ROOT / 'js' / 'fixtures' / directory.name / 'searchindex.js')
+
+    msg = f"Search index fixture {existing_searchindex} does not match regenerated copy."
+    assert fresh_searchindex.read_bytes() == existing_searchindex.read_bytes(), msg

utils/generate_js_fixtures.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import subprocess
+from pathlib import Path
+
+SPHINX_ROOT = Path(__file__).resolve().parent.parent
+TEST_JS_FIXTURES = SPHINX_ROOT / 'tests' / 'js' / 'fixtures'
+TEST_JS_ROOTS = SPHINX_ROOT / 'tests' / 'js' / 'roots'
+
+
+def build(srcdir: Path) -> None:
+    cmd = (
+        'sphinx-build',
+        '--fresh-env',
+        '--quiet',
+        *('--builder', 'html'),
+        f'{srcdir}',
+        f'{srcdir}/_build',
+    )
+    subprocess.run(cmd, check=True, capture_output=True)
+
+
+for directory in TEST_JS_ROOTS.iterdir():
+    searchindex = directory / '_build' / 'searchindex.js'
+    destination = TEST_JS_FIXTURES / directory.name / 'searchindex.js'
+
+    print(f'Building {directory} ... ', end='')
+    build(directory)
+    print('done')
+
+    print(f'Moving {searchindex} to {destination} ... ', end='')
+    destination.parent.mkdir(exist_ok=True)
+    searchindex.replace(destination)
+    print('done')