From b01a7be5c689e0b0de5fc465649c17d5b6b51c6f Mon Sep 17 00:00:00 2001 From: Timotheus Kampik Date: Sun, 1 May 2016 17:01:53 +0200 Subject: [PATCH 1/4] add keywords of meta tag to search index #2463 --- sphinx/search/__init__.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/sphinx/search/__init__.py b/sphinx/search/__init__.py index 13fb4d528e..83da9e6549 100644 --- a/sphinx/search/__init__.py +++ b/sphinx/search/__init__.py @@ -180,6 +180,17 @@ class WordCollector(NodeVisitor): self.found_title_words = [] self.lang = lang + def is_meta_keywords(self, node, nodetype): + is_meta = str(nodetype) == '' + if is_meta: + language_match = re.search(r'lang\=\"(.*?)\"', str(node)) + is_correct_language =\ + language_match == None \ + or self.lang.lang == language_match.group(1) + return is_meta and is_correct_language + else: + return False + def dispatch_visit(self, node): nodetype = type(node) if issubclass(nodetype, comment): @@ -197,6 +208,9 @@ class WordCollector(NodeVisitor): self.found_words.extend(self.lang.split(node.astext())) elif issubclass(nodetype, title): self.found_title_words.extend(self.lang.split(node.astext())) + elif self.is_meta_keywords(node, nodetype): + keywords = re.search(r'content\=\"(.*?)\"', str(node)).group(1) + self.found_words.extend(self.lang.split(keywords)) class IndexBuilder(object): @@ -353,7 +367,6 @@ class IndexBuilder(object): def feed(self, filename, title, doctree): """Feed a doctree to the index.""" self._titles[filename] = title - visitor = WordCollector(doctree, self.lang) doctree.walk(visitor) From a8dabf334d24f13afd37eb9c2f32b5a14cd125ef Mon Sep 17 00:00:00 2001 From: Timotheus Kampik Date: Mon, 2 May 2016 00:11:48 +0200 Subject: [PATCH 2/4] add tests for meta-tag search index generation #2463 --- tests/roots/test-search/conf.py | 11 +++++++++++ tests/roots/test-search/index.rst | 8 ++++++++ tests/test_search.py | 8 ++++++++ 3 files changed, 27 insertions(+) create mode 100644 tests/roots/test-search/conf.py create mode 100644 tests/roots/test-search/index.rst diff --git a/tests/roots/test-search/conf.py b/tests/roots/test-search/conf.py new file mode 100644 index 0000000000..50088e3dfc --- /dev/null +++ b/tests/roots/test-search/conf.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- + +import sys +import os + +sys.path.insert(0, os.path.abspath('.')) +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +master_doc = 'index' +exclude_patterns = ['_build'] + +html_search_language = 'en' diff --git a/tests/roots/test-search/index.rst b/tests/roots/test-search/index.rst new file mode 100644 index 0000000000..f08a3c1e80 --- /dev/null +++ b/tests/roots/test-search/index.rst @@ -0,0 +1,8 @@ +meta keywords +============= + +.. meta:: + :keywords lang=en: findthiskey + :keywords lang=de: findnotthiskey + :keywords: thisonetoo + diff --git a/tests/test_search.py b/tests/test_search.py index cd2ff76f29..143a8a2736 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -52,3 +52,11 @@ def test_objects_are_escaped(app, status, warning): index = jsdump.loads(searchindex[16:-2]) assert 'n::Array<T, d>' in index.get('objects').get('') # n::Array is escaped + +@with_app(testroot='search') +def test_meta_keys_are_handled_for_language(app, status, warning): + app.builder.build_all() + searchindex = (app.outdir / 'searchindex.js').text() + assert 'findnotthiskey' not in searchindex + assert 'findthiskei' in searchindex + assert 'thisonetoo' in searchindex \ No newline at end of file From 6f6b28d3e69425d61678c09cf82ee225a7b75d83 Mon Sep 17 00:00:00 2001 From: Timotheus Kampik Date: Mon, 2 May 2016 07:54:44 +0200 Subject: [PATCH 3/4] refactor, add more tests #2516 --- AUTHORS | 2 +- sphinx/search/__init__.py | 14 +++++++------- tests/roots/test-search-de/conf.py | 3 +++ tests/roots/test-search-de/index.rst | 8 ++++++++ tests/roots/test-search/conf.py | 8 -------- tests/roots/test-search/index.rst | 8 ++++---- tests/test_search.py | 18 +++++++++++++++--- 7 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 tests/roots/test-search-de/conf.py create mode 100644 tests/roots/test-search-de/index.rst diff --git a/AUTHORS b/AUTHORS index e1e4e8a597..dc8e824822 100644 --- a/AUTHORS +++ b/AUTHORS @@ -29,7 +29,7 @@ Other contributors, listed alphabetically, are: * Horst Gutmann -- internationalization support * Martin Hans -- autodoc improvements * Doug Hellmann -- graphviz improvements -* Timotheus Kampik - JS enhancements, stop words language fix +* Timotheus Kampik - JS theme & search enhancements * Takeshi Komiya -- numref feature * Dave Kuhlman -- original LaTeX writer * Blaise Laflamme -- pyramid theme diff --git a/sphinx/search/__init__.py b/sphinx/search/__init__.py index 83da9e6549..defcad8a7f 100644 --- a/sphinx/search/__init__.py +++ b/sphinx/search/__init__.py @@ -182,11 +182,10 @@ class WordCollector(NodeVisitor): def is_meta_keywords(self, node, nodetype): is_meta = str(nodetype) == '' - if is_meta: - language_match = re.search(r'lang\=\"(.*?)\"', str(node)) - is_correct_language =\ - language_match == None \ - or self.lang.lang == language_match.group(1) + if is_meta and node.get('name', None) == 'keywords': + node_lang = node.get('lang', None) + is_correct_language = node_lang == None \ + or node_lang == self.lang.lang return is_meta and is_correct_language else: return False @@ -209,8 +208,9 @@ class WordCollector(NodeVisitor): elif issubclass(nodetype, title): self.found_title_words.extend(self.lang.split(node.astext())) elif self.is_meta_keywords(node, nodetype): - keywords = re.search(r'content\=\"(.*?)\"', str(node)).group(1) - self.found_words.extend(self.lang.split(keywords)) + keywords = node['content'] + keywords = [keyword.strip() for keyword in keywords.split(',')] + self.found_words.extend(keywords) class IndexBuilder(object): diff --git a/tests/roots/test-search-de/conf.py b/tests/roots/test-search-de/conf.py new file mode 100644 index 0000000000..3c902c292a --- /dev/null +++ b/tests/roots/test-search-de/conf.py @@ -0,0 +1,3 @@ +master_doc = 'index' +exclude_patterns = ['_build'] +html_search_language = 'de' diff --git a/tests/roots/test-search-de/index.rst b/tests/roots/test-search-de/index.rst new file mode 100644 index 0000000000..57a9216bbe --- /dev/null +++ b/tests/roots/test-search-de/index.rst @@ -0,0 +1,8 @@ +meta keywords +============= + +.. meta:: + :keywords lang=de: findthiskey, thistoo + :keywords: thisonetoo + :keywords lang=en: findnotthiskey + :description: thisnoteither diff --git a/tests/roots/test-search/conf.py b/tests/roots/test-search/conf.py index 50088e3dfc..38b8b28c5a 100644 --- a/tests/roots/test-search/conf.py +++ b/tests/roots/test-search/conf.py @@ -1,11 +1,3 @@ -# -*- coding: utf-8 -*- - -import sys -import os - -sys.path.insert(0, os.path.abspath('.')) -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] master_doc = 'index' exclude_patterns = ['_build'] - html_search_language = 'en' diff --git a/tests/roots/test-search/index.rst b/tests/roots/test-search/index.rst index f08a3c1e80..930789c79f 100644 --- a/tests/roots/test-search/index.rst +++ b/tests/roots/test-search/index.rst @@ -2,7 +2,7 @@ meta keywords ============= .. meta:: - :keywords lang=en: findthiskey - :keywords lang=de: findnotthiskey - :keywords: thisonetoo - + :keywords lang=en: findthiskey, thistoo + :keywords: thisonetoo + :keywords lang=de: findnotthiskey + :description: thisnoteither diff --git a/tests/test_search.py b/tests/test_search.py index 143a8a2736..033e7ccb4f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -53,10 +53,22 @@ def test_objects_are_escaped(app, status, warning): index = jsdump.loads(searchindex[16:-2]) assert 'n::Array<T, d>' in index.get('objects').get('') # n::Array is escaped +def assert_lang_agnostic_key_words(searchindex): + assert 'findnotthiskey' not in searchindex + assert 'thisnoteith' not in searchindex + assert 'thistoo' in searchindex + assert 'thisonetoo' in searchindex + @with_app(testroot='search') -def test_meta_keys_are_handled_for_language(app, status, warning): +def test_meta_keys_are_handled_for_language_en(app, status, warning): app.builder.build_all() searchindex = (app.outdir / 'searchindex.js').text() - assert 'findnotthiskey' not in searchindex + assert_lang_agnostic_key_words(searchindex) assert 'findthiskei' in searchindex - assert 'thisonetoo' in searchindex \ No newline at end of file + +@with_app(testroot='search-de') +def test_meta_keys_are_handled_for_language_de(app, status, warning): + app.builder.build_all() + searchindex = (app.outdir / 'searchindex.js').text() + assert_lang_agnostic_key_words(searchindex) + assert 'findthiskey' in searchindex \ No newline at end of file From 01797faedec6346f3bfb574e963ef3c91e8fbbf5 Mon Sep 17 00:00:00 2001 From: Timotheus Kampik Date: Thu, 5 May 2016 18:09:22 +0200 Subject: [PATCH 4/4] refactor code/tests: keyword support for search index #2516 --- sphinx/search/__init__.py | 23 ++++++++++++----------- tests/roots/test-search-de/conf.py | 3 --- tests/roots/test-search-de/index.rst | 8 -------- tests/roots/test-search/index.rst | 6 +++--- tests/test_search.py | 12 ++++++++---- 5 files changed, 23 insertions(+), 29 deletions(-) delete mode 100644 tests/roots/test-search-de/conf.py delete mode 100644 tests/roots/test-search-de/index.rst diff --git a/sphinx/search/__init__.py b/sphinx/search/__init__.py index defcad8a7f..2ba0c38029 100644 --- a/sphinx/search/__init__.py +++ b/sphinx/search/__init__.py @@ -15,6 +15,7 @@ from six.moves import cPickle as pickle from docutils.nodes import raw, comment, title, Text, NodeVisitor, SkipNode from os import path +import sphinx from sphinx.util import jsdump, rpartition from sphinx.util.pycompat import htmlescape @@ -181,14 +182,14 @@ class WordCollector(NodeVisitor): self.lang = lang def is_meta_keywords(self, node, nodetype): - is_meta = str(nodetype) == '' - if is_meta and node.get('name', None) == 'keywords': - node_lang = node.get('lang', None) - is_correct_language = node_lang == None \ - or node_lang == self.lang.lang - return is_meta and is_correct_language - else: - return False + if isinstance(node, sphinx.addnodes.meta) and node.get('name') == 'keywords': + meta_lang = node.get('lang') + if meta_lang is None: # lang not specified + return True + elif meta_lang == self.lang.lang: # matched to html_search_language + return True + + return False def dispatch_visit(self, node): nodetype = type(node) @@ -208,9 +209,9 @@ class WordCollector(NodeVisitor): elif issubclass(nodetype, title): self.found_title_words.extend(self.lang.split(node.astext())) elif self.is_meta_keywords(node, nodetype): - keywords = node['content'] - keywords = [keyword.strip() for keyword in keywords.split(',')] - self.found_words.extend(keywords) + keywords = node['content'] + keywords = [keyword.strip() for keyword in keywords.split(',')] + self.found_words.extend(keywords) class IndexBuilder(object): diff --git a/tests/roots/test-search-de/conf.py b/tests/roots/test-search-de/conf.py deleted file mode 100644 index 3c902c292a..0000000000 --- a/tests/roots/test-search-de/conf.py +++ /dev/null @@ -1,3 +0,0 @@ -master_doc = 'index' -exclude_patterns = ['_build'] -html_search_language = 'de' diff --git a/tests/roots/test-search-de/index.rst b/tests/roots/test-search-de/index.rst deleted file mode 100644 index 57a9216bbe..0000000000 --- a/tests/roots/test-search-de/index.rst +++ /dev/null @@ -1,8 +0,0 @@ -meta keywords -============= - -.. meta:: - :keywords lang=de: findthiskey, thistoo - :keywords: thisonetoo - :keywords lang=en: findnotthiskey - :description: thisnoteither diff --git a/tests/roots/test-search/index.rst b/tests/roots/test-search/index.rst index 930789c79f..f2536ce9a8 100644 --- a/tests/roots/test-search/index.rst +++ b/tests/roots/test-search/index.rst @@ -2,7 +2,7 @@ meta keywords ============= .. meta:: - :keywords lang=en: findthiskey, thistoo + :keywords lang=en: findthiskey, thistoo, notgerman :keywords: thisonetoo - :keywords lang=de: findnotthiskey - :description: thisnoteither + :keywords lang=de: onlygerman, onlytoogerman + :description: thisnoteither \ No newline at end of file diff --git a/tests/test_search.py b/tests/test_search.py index 033e7ccb4f..160dd18278 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -8,6 +8,7 @@ :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ +import os from docutils import frontend, utils from docutils.parsers import rst @@ -54,21 +55,24 @@ def test_objects_are_escaped(app, status, warning): assert 'n::Array<T, d>' in index.get('objects').get('') # n::Array is escaped def assert_lang_agnostic_key_words(searchindex): - assert 'findnotthiskey' not in searchindex assert 'thisnoteith' not in searchindex - assert 'thistoo' in searchindex assert 'thisonetoo' in searchindex @with_app(testroot='search') def test_meta_keys_are_handled_for_language_en(app, status, warning): + os.remove(app.outdir / 'searchindex.js') app.builder.build_all() searchindex = (app.outdir / 'searchindex.js').text() assert_lang_agnostic_key_words(searchindex) assert 'findthiskei' in searchindex + assert 'onlygerman' not in searchindex + assert 'thistoo' in searchindex -@with_app(testroot='search-de') +@with_app(testroot='search', confoverrides={'html_search_language': 'de'}) def test_meta_keys_are_handled_for_language_de(app, status, warning): app.builder.build_all() searchindex = (app.outdir / 'searchindex.js').text() assert_lang_agnostic_key_words(searchindex) - assert 'findthiskey' in searchindex \ No newline at end of file + assert 'onlygerman' in searchindex + assert 'notgerman' not in searchindex + assert 'onlytoogerman' in searchindex \ No newline at end of file