From 7886ecc99483d7415a85cf65af41095678e0627c Mon Sep 17 00:00:00 2001 From: Matthias Geier Date: Fri, 22 Apr 2016 15:44:34 +0200 Subject: [PATCH 1/8] Add test cases for HTML source links --- tests/test_build_html.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_build_html.py b/tests/test_build_html.py index 4fac8e59f..177fa71b5 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -68,6 +68,7 @@ HTML_XPATH = { (".//img[@src='_images/img1.png']", ''), (".//img[@src='_images/simg.png']", ''), (".//img[@src='_images/svgimg.svg']", ''), + (".//a[@href='_sources/images.txt']", ''), ], 'subdir/images.html': [ (".//img[@src='../_images/img1.png']", ''), @@ -318,6 +319,7 @@ HTML_XPATH = { ], 'otherext.html': [ (".//h1", "Generated section"), + (".//a[@href='_sources/otherext.txt']", ''), ] } From 86615e1f091b2bf08ec5b5b1242cb26bb64569e0 Mon Sep 17 00:00:00 2001 From: Matthias Geier Date: Tue, 29 Mar 2016 12:52:21 +0200 Subject: [PATCH 2/8] Add option html_sourcelink_txt --- doc/config.rst | 7 +++++++ sphinx/builders/html.py | 14 ++++++++++---- sphinx/config.py | 1 + tests/test_build_html.py | 14 +++++++++++++- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/doc/config.rst b/doc/config.rst index 891f080b5..9654e7856 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -870,6 +870,13 @@ that use Sphinx's HTMLWriter class. .. versionadded:: 0.6 +.. confval:: html_sourcelink_txt + + If true, ``.txt`` is appended to source links (see + :confval:`html_show_sourcelink`). Default is ``True``. + + .. versionadded:: 1.5 + .. confval:: html_use_opensearch If nonempty, an `OpenSearch `_ description file will be diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 63ccdd66c..01a634bdd 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -402,15 +402,21 @@ class StandaloneHTMLBuilder(Builder): # title rendered as HTML title = self.env.longtitles.get(docname) title = title and self.render_partial(title)['title'] or '' + + # Suffix for the document + source_suffix = path.splitext(self.env.doc2path(docname))[1] + # the name for the copied source - sourcename = self.config.html_copy_source and docname + '.txt' or '' + if self.config.html_copy_source: + sourcename = docname + source_suffix + if self.config.html_sourcelink_txt and source_suffix != '.txt': + sourcename += '.txt' + else: + sourcename = '' # metadata for the document meta = self.env.metadata.get(docname) - # Suffix for the document - source_suffix = '.' + self.env.doc2path(docname).split('.')[-1] - # local TOC and global TOC tree self_toc = self.env.get_toc_for(docname, self) toc = self.render_partial(self_toc)['fragment'] diff --git a/sphinx/config.py b/sphinx/config.py index ef57334fe..cd05db45e 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -121,6 +121,7 @@ class Config(object): html_split_index = (False, 'html'), html_copy_source = (True, 'html'), html_show_sourcelink = (True, 'html'), + html_sourcelink_txt = (True, 'html'), html_use_opensearch = ('', 'html'), html_file_suffix = (None, 'html', string_classes), html_link_suffix = (None, 'html', string_classes), diff --git a/tests/test_build_html.py b/tests/test_build_html.py index 177fa71b5..df10c458c 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -319,7 +319,7 @@ HTML_XPATH = { ], 'otherext.html': [ (".//h1", "Generated section"), - (".//a[@href='_sources/otherext.txt']", ''), + (".//a[@href='_sources/otherext.foo.txt']", ''), ] } @@ -987,3 +987,15 @@ def test_html_extra_path(app, status, warning): assert (app.outdir / 'rimg.png').exists() assert not (app.outdir / '_build/index.html').exists() assert (app.outdir / 'background.png').exists() + + +@with_app(buildername='html', confoverrides={'html_sourcelink_txt': False}) +def test_html_sourcelink_txt(app, status, warning): + app.builder.build_all() + content_otherext = (app.outdir / 'otherext.html').text() + content_images = (app.outdir / 'images.html').text() + + assert ' Date: Mon, 16 May 2016 21:11:17 +0900 Subject: [PATCH 3/8] Pass original filenames to search indecies (ref: #2399) * Not tested carefully * Always adds .txt to source filename even if html_sourcelink_txt is False --- sphinx/builders/html.py | 3 ++- sphinx/search/__init__.py | 25 ++++++++++++--------- sphinx/themes/basic/static/searchtools.js_t | 8 ++++--- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 01a634bdd..6f1bcf2db 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -718,7 +718,8 @@ class StandaloneHTMLBuilder(Builder): def index_page(self, pagename, doctree, title): # only index pages with title if self.indexer is not None and title: - self.indexer.feed(pagename, title, doctree) + filename = self.env.doc2path(pagename, base=None) + self.indexer.feed(pagename, filename, title, doctree) def _get_local_toctree(self, docname, collapse=True, **kwds): if 'includehidden' not in kwds: diff --git a/sphinx/search/__init__.py b/sphinx/search/__init__.py index 8bbce360e..704532eb2 100644 --- a/sphinx/search/__init__.py +++ b/sphinx/search/__init__.py @@ -226,11 +226,13 @@ class IndexBuilder(object): def __init__(self, env, lang, options, scoring): self.env = env - # filename -> title + # docname -> title self._titles = {} - # stemmed word -> set(filenames) + # docname -> filename + self._filenames = {} + # stemmed word -> set(docname) self._mapping = {} - # stemmed words in titles -> set(filenames) + # stemmed words in titles -> set(docname) self._title_mapping = {} # word -> stemmed word self._stem_cache = {} @@ -338,15 +340,16 @@ class IndexBuilder(object): def freeze(self): """Create a usable data structure for serializing.""" - filenames, titles = zip(*sorted(self._titles.items())) - fn2index = dict((f, i) for (i, f) in enumerate(filenames)) + docnames, titles = zip(*sorted(self._titles.items())) + filenames = [self._filenames.get(docname) for docname in docnames] + fn2index = dict((f, i) for (i, f) in enumerate(docnames)) terms, title_terms = self.get_terms(fn2index) objects = self.get_objects(fn2index) # populates _objtypes objtypes = dict((v, k[0] + ':' + k[1]) for (k, v) in iteritems(self._objtypes)) objnames = self._objnames - return dict(filenames=filenames, titles=titles, terms=terms, + return dict(docnames=docnames, filenames=filenames, titles=titles, terms=terms, objects=objects, objtypes=objtypes, objnames=objnames, titleterms=title_terms, envversion=self.env.version) @@ -365,9 +368,11 @@ class IndexBuilder(object): for wordnames in itervalues(self._title_mapping): wordnames.intersection_update(filenames) - def feed(self, filename, title, doctree): + def feed(self, docname, filename, title, doctree): """Feed a doctree to the index.""" - self._titles[filename] = title + self._titles[docname] = title + self._filenames[docname] = filename + visitor = WordCollector(doctree, self.lang) doctree.walk(visitor) @@ -383,12 +388,12 @@ class IndexBuilder(object): for word in visitor.found_title_words: word = stem(word) if _filter(word): - self._title_mapping.setdefault(word, set()).add(filename) + self._title_mapping.setdefault(word, set()).add(docname) for word in visitor.found_words: word = stem(word) if word not in self._title_mapping and _filter(word): - self._mapping.setdefault(word, set()).add(filename) + self._mapping.setdefault(word, set()).add(docname) def context_for_searchtool(self): return dict( diff --git a/sphinx/themes/basic/static/searchtools.js_t b/sphinx/themes/basic/static/searchtools.js_t index 8a150b272..68a194820 100644 --- a/sphinx/themes/basic/static/searchtools.js_t +++ b/sphinx/themes/basic/static/searchtools.js_t @@ -256,7 +256,7 @@ var Search = { displayNextItem(); }); } else if (DOCUMENTATION_OPTIONS.HAS_SOURCE) { - $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[0] + '.txt', + $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + '.txt', dataType: "text", complete: function(jqxhr, textstatus) { var data = jqxhr.responseText; @@ -295,6 +295,7 @@ var Search = { */ performObjectSearch : function(object, otherterms) { var filenames = this._index.filenames; + var docnames = this._index.docnames; var objects = this._index.objects; var objnames = this._index.objnames; var titles = this._index.titles; @@ -348,7 +349,7 @@ var Search = { } else { score += Scorer.objPrioDefault; } - results.push([filenames[match[0]], fullname, '#'+anchor, descr, score]); + results.push([docnames[match[0]], fullname, '#'+anchor, descr, score, filenames[match[0]]]); } } } @@ -360,6 +361,7 @@ var Search = { * search for full-text terms in the index */ performTermsSearch : function(searchterms, excluded, terms, titleterms) { + var docnames = this._index.docnames; var filenames = this._index.filenames; var titles = this._index.titles; @@ -434,7 +436,7 @@ var Search = { // select one (max) score for the file. // for better ranking, we should calculate ranking by using words statistics like basic tf-idf... var score = $u.max($u.map(fileMap[file], function(w){return scoreMap[file][w]})); - results.push([filenames[file], titles[file], '', null, score]); + results.push([docnames[file], titles[file], '', null, score, filenames[file]]); } } return results; From 55591505787dbfac72743250dea0713ddbbe286c Mon Sep 17 00:00:00 2001 From: Matthias Geier Date: Tue, 17 May 2016 09:34:01 +0200 Subject: [PATCH 4/8] Change html_sourcelink_txt -> html_sourcelink_suffix --- doc/config.rst | 6 +++--- sphinx/builders/html.py | 4 ++-- sphinx/config.py | 2 +- tests/test_build_html.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/config.rst b/doc/config.rst index 9654e7856..9d38d7d18 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -870,10 +870,10 @@ that use Sphinx's HTMLWriter class. .. versionadded:: 0.6 -.. confval:: html_sourcelink_txt +.. confval:: html_sourcelink_suffix - If true, ``.txt`` is appended to source links (see - :confval:`html_show_sourcelink`). Default is ``True``. + Suffix to be appended to source links (see :confval:`html_show_sourcelink`), + unless they have this suffix already. Default is ``'.txt'``. .. versionadded:: 1.5 diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 6f1bcf2db..c48e6b3c2 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -409,8 +409,8 @@ class StandaloneHTMLBuilder(Builder): # the name for the copied source if self.config.html_copy_source: sourcename = docname + source_suffix - if self.config.html_sourcelink_txt and source_suffix != '.txt': - sourcename += '.txt' + if source_suffix != self.config.html_sourcelink_suffix: + sourcename += self.config.html_sourcelink_suffix else: sourcename = '' diff --git a/sphinx/config.py b/sphinx/config.py index cd05db45e..62c00e73c 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -121,7 +121,7 @@ class Config(object): html_split_index = (False, 'html'), html_copy_source = (True, 'html'), html_show_sourcelink = (True, 'html'), - html_sourcelink_txt = (True, 'html'), + html_sourcelink_suffix = ('.txt', 'html'), html_use_opensearch = ('', 'html'), html_file_suffix = (None, 'html', string_classes), html_link_suffix = (None, 'html', string_classes), diff --git a/tests/test_build_html.py b/tests/test_build_html.py index df10c458c..042f387fd 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -989,8 +989,8 @@ def test_html_extra_path(app, status, warning): assert (app.outdir / 'background.png').exists() -@with_app(buildername='html', confoverrides={'html_sourcelink_txt': False}) -def test_html_sourcelink_txt(app, status, warning): +@with_app(buildername='html', confoverrides={'html_sourcelink_suffix': ''}) +def test_html_sourcelink_suffix(app, status, warning): app.builder.build_all() content_otherext = (app.outdir / 'otherext.html').text() content_images = (app.outdir / 'images.html').text() From 3ecb08c5a9135d5ad4f11fba8294f9b2b48ae40a Mon Sep 17 00:00:00 2001 From: Matthias Geier Date: Tue, 7 Jun 2016 15:01:18 +0200 Subject: [PATCH 5/8] TESTS: Add missing docname in test_search.py --- tests/test_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_search.py b/tests/test_search.py index 212ce778c..d1a396577 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -51,7 +51,7 @@ def test_wordcollector(): parser.parse(FILE_CONTENTS, doc) ix = IndexBuilder(None, 'en', {}, None) - ix.feed('filename', 'title', doc) + ix.feed('docname', 'filename', 'title', doc) assert 'boson' not in ix._mapping assert 'fermion' in ix._mapping From d27386cc95772a7e7cc47941a0c08b27e0272f67 Mon Sep 17 00:00:00 2001 From: Matthias Geier Date: Wed, 8 Jun 2016 15:53:18 +0200 Subject: [PATCH 6/8] Updates related to new feed() signature --- sphinx/builders/html.py | 6 +++++- sphinx/websupport/search/__init__.py | 8 +++++--- sphinx/websupport/search/nullsearch.py | 2 +- sphinx/websupport/search/whooshsearch.py | 2 +- tests/test_searchadapters.py | 2 +- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index c48e6b3c2..880f021bc 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -719,7 +719,11 @@ class StandaloneHTMLBuilder(Builder): # only index pages with title if self.indexer is not None and title: filename = self.env.doc2path(pagename, base=None) - self.indexer.feed(pagename, filename, title, doctree) + try: + self.indexer.feed(pagename, filename, title, doctree) + except TypeError: + # fallback for old search-adapters + self.indexer.feed(pagename, title, doctree) def _get_local_toctree(self, docname, collapse=True, **kwds): if 'includehidden' not in kwds: diff --git a/sphinx/websupport/search/__init__.py b/sphinx/websupport/search/__init__.py index 844a3b468..80b5a3535 100644 --- a/sphinx/websupport/search/__init__.py +++ b/sphinx/websupport/search/__init__.py @@ -34,19 +34,20 @@ class BaseSearch(object): """ pass - def feed(self, pagename, title, doctree): + def feed(self, pagename, filename, title, doctree): """Called by the builder to add a doctree to the index. Converts the `doctree` to text and passes it to :meth:`add_document`. You probably won't want to override this unless you need access to the `doctree`. Override :meth:`add_document` instead. :param pagename: the name of the page to be indexed + :param filename: the name of the original source file :param title: the title of the page to be indexed :param doctree: is the docutils doctree representation of the page """ - self.add_document(pagename, title, doctree.astext()) + self.add_document(pagename, filename, title, doctree.astext()) - def add_document(self, pagename, title, text): + def add_document(self, pagename, filename, title, text): """Called by :meth:`feed` to add a document to the search index. This method should should do everything necessary to add a single document to the search index. @@ -59,6 +60,7 @@ class BaseSearch(object): query. :param pagename: the name of the page being indexed + :param filename: the name of the original source file :param title: the page's title :param text: the full text of the page """ diff --git a/sphinx/websupport/search/nullsearch.py b/sphinx/websupport/search/nullsearch.py index 9e990b1cf..4d0db9553 100644 --- a/sphinx/websupport/search/nullsearch.py +++ b/sphinx/websupport/search/nullsearch.py @@ -17,7 +17,7 @@ class NullSearch(BaseSearch): """A search adapter that does nothing. Used when no search adapter is specified. """ - def feed(self, pagename, title, doctree): + def feed(self, pagename, filename, title, doctree): pass def query(self, q): diff --git a/sphinx/websupport/search/whooshsearch.py b/sphinx/websupport/search/whooshsearch.py index 4b0769f50..f31dc4d52 100644 --- a/sphinx/websupport/search/whooshsearch.py +++ b/sphinx/websupport/search/whooshsearch.py @@ -44,7 +44,7 @@ class WhooshSearch(BaseSearch): def finish_indexing(self): self.index_writer.commit() - def add_document(self, pagename, title, text): + def add_document(self, pagename, filename, title, text): self.index_writer.add_document(path=text_type(pagename), title=title, text=text) diff --git a/tests/test_searchadapters.py b/tests/test_searchadapters.py index f6a389fea..e8c5c9364 100644 --- a/tests/test_searchadapters.py +++ b/tests/test_searchadapters.py @@ -41,7 +41,7 @@ def search_adapter_helper(adapter): # Make sure documents are properly updated by the search adapter. s.init_indexing(changed=['markup']) - s.add_document(u'markup', u'title', u'SomeLongRandomWord') + s.add_document(u'markup', u'filename', u'title', u'SomeLongRandomWord') s.finish_indexing() # Now a search for "Epigraph" should return zero results. results = s.query(u'Epigraph') From 71e1aaf3770831df1b5b7f36aaadb789176a2927 Mon Sep 17 00:00:00 2001 From: Matthias Geier Date: Sat, 25 Jun 2016 15:39:25 +0200 Subject: [PATCH 7/8] searchtools: Add .txt only if suffix isn't already .txt --- sphinx/themes/basic/static/searchtools.js_t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/themes/basic/static/searchtools.js_t b/sphinx/themes/basic/static/searchtools.js_t index 68a194820..000722f03 100644 --- a/sphinx/themes/basic/static/searchtools.js_t +++ b/sphinx/themes/basic/static/searchtools.js_t @@ -256,7 +256,7 @@ var Search = { displayNextItem(); }); } else if (DOCUMENTATION_OPTIONS.HAS_SOURCE) { - $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + '.txt', + $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + (item[5].endsWith('.txt') ? '' : '.txt'), dataType: "text", complete: function(jqxhr, textstatus) { var data = jqxhr.responseText; From 71dd8bfbf94417ad55b2444e1dbd219db266f335 Mon Sep 17 00:00:00 2001 From: Matthias Geier Date: Sat, 25 Jun 2016 16:46:25 +0200 Subject: [PATCH 8/8] HTML: Add sourcelink_suffix to globalcontext --- sphinx/builders/html.py | 1 + sphinx/themes/basic/layout.html | 3 ++- sphinx/themes/basic/static/searchtools.js_t | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 880f021bc..09fe96be0 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -339,6 +339,7 @@ class StandaloneHTMLBuilder(Builder): show_sphinx = self.config.html_show_sphinx, has_source = self.config.html_copy_source, show_source = self.config.html_show_sourcelink, + sourcelink_suffix = self.config.html_sourcelink_suffix, file_suffix = self.out_suffix, script_files = self.script_files, language = self.config.language, diff --git a/sphinx/themes/basic/layout.html b/sphinx/themes/basic/layout.html index 1afc4a0bf..f8ff477c7 100644 --- a/sphinx/themes/basic/layout.html +++ b/sphinx/themes/basic/layout.html @@ -91,7 +91,8 @@ VERSION: '{{ release|e }}', COLLAPSE_INDEX: false, FILE_SUFFIX: '{{ '' if no_search_suffix else file_suffix }}', - HAS_SOURCE: {{ has_source|lower }} + HAS_SOURCE: {{ has_source|lower }}, + SOURCELINK_SUFFIX: '{{ sourcelink_suffix }}' }; {%- for scriptfile in script_files %} diff --git a/sphinx/themes/basic/static/searchtools.js_t b/sphinx/themes/basic/static/searchtools.js_t index 000722f03..9d70f4303 100644 --- a/sphinx/themes/basic/static/searchtools.js_t +++ b/sphinx/themes/basic/static/searchtools.js_t @@ -256,7 +256,8 @@ var Search = { displayNextItem(); }); } else if (DOCUMENTATION_OPTIONS.HAS_SOURCE) { - $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + (item[5].endsWith('.txt') ? '' : '.txt'), + var suffix = DOCUMENTATION_OPTIONS.SOURCELINK_SUFFIX; + $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + (item[5].endsWith(suffix) ? '' : suffix), dataType: "text", complete: function(jqxhr, textstatus) { var data = jqxhr.responseText;