From 62fa80399d825fe9b1a96bb35fde7a0cb5f4d4c3 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 13 Mar 2018 17:23:40 +0900 Subject: [PATCH 01/24] Add Domain#get_enumerable_node_type() --- CHANGES | 2 ++ sphinx/domains/__init__.py | 8 ++++++++ sphinx/domains/std.py | 19 ++++++++++++++++--- sphinx/environment/collectors/toctree.py | 11 ++++++++++- sphinx/writers/html.py | 2 +- sphinx/writers/html5.py | 2 +- sphinx/writers/latex.py | 2 +- 7 files changed, 39 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index f79078244..8053140c5 100644 --- a/CHANGES +++ b/CHANGES @@ -47,6 +47,8 @@ Features added * ``sphinx-build`` command supports i18n console output * Add ``app.add_message_catalog()`` and ``sphinx.locale.get_translations()`` to support translation for 3rd party extensions +* Add ``Domain.enumerable_nodes`` to manage own enumerable nodes for domains + (experimental) Bugs fixed ---------- diff --git a/sphinx/domains/__init__.py b/sphinx/domains/__init__.py index bcb67d337..3aa0fb48f 100644 --- a/sphinx/domains/__init__.py +++ b/sphinx/domains/__init__.py @@ -150,6 +150,8 @@ class Domain(object): indices = [] # type: List[Type[Index]] #: role name -> a warning message if reference is missing dangling_warnings = {} # type: Dict[unicode, unicode] + #: node_class -> (enum_node_type, title_getter) + enumerable_nodes = {} # type: Dict[nodes.Node, Tuple[unicode, Callable]] #: data value for a fresh environment initial_data = {} # type: Dict @@ -333,6 +335,12 @@ class Domain(object): return type.lname return _('%s %s') % (self.label, type.lname) + def get_enumerable_node_type(self, node): + # type: (nodes.Node) -> unicode + """Get type of enumerable nodes (experimental).""" + enum_node_type, _ = self.enumerable_nodes.get(node.__class__, (None, None)) + return enum_node_type + def get_full_qualified_name(self, node): # type: (nodes.Node) -> unicode """Return full qualified name for given node.""" diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py index 4658c9995..2e34cfb32 100644 --- a/sphinx/domains/std.py +++ b/sphinx/domains/std.py @@ -11,6 +11,7 @@ import re import unicodedata +import warnings from copy import copy from typing import TYPE_CHECKING @@ -20,6 +21,7 @@ from docutils.statemachine import ViewList from six import iteritems from sphinx import addnodes +from sphinx.deprecation import RemovedInSphinx30Warning from sphinx.directives import ObjectDescription from sphinx.domains import Domain, ObjType from sphinx.locale import _, __ @@ -726,7 +728,7 @@ class StandardDomain(Domain): return None target_node = env.get_doctree(docname).ids.get(labelid) - figtype = self.get_figtype(target_node) + figtype = self.get_enumerable_node_type(target_node) if figtype is None: return None @@ -926,9 +928,9 @@ class StandardDomain(Domain): return None - def get_figtype(self, node): + def get_enumerable_node_type(self, node): # type: (nodes.Node) -> unicode - """Get figure type of nodes.""" + """Get type of enumerable nodes.""" def has_child(node, cls): # type: (nodes.Node, Type) -> bool return any(isinstance(child, cls) for child in node) @@ -944,6 +946,17 @@ class StandardDomain(Domain): figtype, _ = self.enumerable_nodes.get(node.__class__, (None, None)) return figtype + def get_figtype(self, node): + # type: (nodes.Node) -> unicode + """Get figure type of nodes. + + .. deprecated:: 1.8 + """ + warnings.warn('StandardDomain.get_figtype() is deprecated. ' + 'Please use get_enumerable_node_type() instead.', + RemovedInSphinx30Warning) + return self.get_enumerable_node_type(node) + def get_fignumber(self, env, builder, figtype, docname, target_node): # type: (BuildEnvironment, Builder, unicode, unicode, nodes.Node) -> Tuple[int, ...] if figtype == 'section': diff --git a/sphinx/environment/collectors/toctree.py b/sphinx/environment/collectors/toctree.py index 5ff03da39..aceb0d715 100644 --- a/sphinx/environment/collectors/toctree.py +++ b/sphinx/environment/collectors/toctree.py @@ -224,6 +224,15 @@ class TocTreeCollector(EnvironmentCollector): env.toc_fignumbers = {} fignum_counter = {} # type: Dict[unicode, Dict[Tuple[int, ...], int]] + def get_figtype(node): + # type: (nodes.Node) -> unicode + for domain in env.domains.values(): + figtype = domain.get_enumerable_node_type(node) + if figtype: + return figtype + + return None + def get_section_number(docname, section): # type: (unicode, nodes.Node) -> Tuple[int, ...] anchorname = '#' + section['ids'][0] @@ -271,7 +280,7 @@ class TocTreeCollector(EnvironmentCollector): continue - figtype = env.get_domain('std').get_figtype(subnode) + figtype = get_figtype(subnode) if figtype and subnode['ids']: register_fignumber(docname, secnum, figtype, subnode) diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py index 26abdcd12..be051ce07 100644 --- a/sphinx/writers/html.py +++ b/sphinx/writers/html.py @@ -342,7 +342,7 @@ class HTMLTranslator(BaseTranslator): self.body.append(prefix % '.'.join(map(str, numbers)) + ' ') self.body.append('') - figtype = self.builder.env.domains['std'].get_figtype(node) # type: ignore + figtype = self.builder.env.domains['std'].get_enumerable_node_type(node) if figtype: if len(node['ids']) == 0: msg = __('Any IDs not assigned for %s node') % node.tagname diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py index b2acf8570..697dbabd4 100644 --- a/sphinx/writers/html5.py +++ b/sphinx/writers/html5.py @@ -310,7 +310,7 @@ class HTML5Translator(BaseTranslator): self.body.append(prefix % '.'.join(map(str, numbers)) + ' ') self.body.append('') - figtype = self.builder.env.domains['std'].get_figtype(node) # type: ignore + figtype = self.builder.env.domains['std'].get_enumerable_node_type(node) if figtype: if len(node['ids']) == 0: msg = __('Any IDs not assigned for %s node') % node.tagname diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 3cd5022b9..9755f4248 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -1990,7 +1990,7 @@ class LaTeXTranslator(nodes.NodeVisitor): return else: domain = self.builder.env.get_domain('std') - figtype = domain.get_figtype(next) + figtype = domain.get_enumerable_node_type(next) if figtype and domain.get_numfig_title(next): ids = set() # labels for figures go in the figure body, not before From b69a61c3ceb834f816cffc7e258293389686a282 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 13 Mar 2018 21:20:28 +0900 Subject: [PATCH 02/24] refactor: move displaymath node to MathDomain --- sphinx/ext/mathbase.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sphinx/ext/mathbase.py b/sphinx/ext/mathbase.py index 54c300d67..362b0a2e7 100644 --- a/sphinx/ext/mathbase.py +++ b/sphinx/ext/mathbase.py @@ -63,6 +63,9 @@ class MathDomain(Domain): dangling_warnings = { 'eq': 'equation not found: %(target)s', } + enumerable_nodes = { # node_class -> (figtype, title_getter) + displaymath: ('displaymath', None), + } # type: Dict[nodes.Node, Tuple[unicode, Callable]] def clear_doc(self, docname): # type: (unicode) -> None @@ -378,12 +381,12 @@ def setup_math(app, htmlinlinevisitors, htmldisplayvisitors): man=(man_visit_math, None), texinfo=(texinfo_visit_math, None), html=htmlinlinevisitors) - app.add_enumerable_node(displaymath, 'displaymath', - latex=(latex_visit_displaymath, None), - text=(text_visit_displaymath, None), - man=(man_visit_displaymath, man_depart_displaymath), - texinfo=(texinfo_visit_displaymath, texinfo_depart_displaymath), - html=htmldisplayvisitors) + app.add_node(displaymath, + latex=(latex_visit_displaymath, None), + text=(text_visit_displaymath, None), + man=(man_visit_displaymath, man_depart_displaymath), + texinfo=(texinfo_visit_displaymath, texinfo_depart_displaymath), + html=htmldisplayvisitors) app.add_node(eqref, latex=(latex_visit_eqref, None)) app.add_role('math', math_role) app.add_role('eq', EqXRefRole(warn_dangling=True)) From 27abce6306645ce14a4f8affa8f43a665559c9b0 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 17 Mar 2018 13:38:34 +0900 Subject: [PATCH 03/24] Show warning on creating directory for remote image (refs: #4688) --- sphinx/transforms/post_transforms/images.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sphinx/transforms/post_transforms/images.py b/sphinx/transforms/post_transforms/images.py index b8f4b9a5d..8c75b6aa0 100644 --- a/sphinx/transforms/post_transforms/images.py +++ b/sphinx/transforms/post_transforms/images.py @@ -66,16 +66,17 @@ class ImageDownloader(BaseImageConverter): def handle(self, node): # type: (nodes.Node) -> None - basename = os.path.basename(node['uri']) - if '?' in basename: - basename = basename.split('?')[0] - if basename == '': - basename = sha1(node['uri'].encode("utf-8")).hexdigest() - dirname = node['uri'].replace('://', '/').translate({ord("?"): u"/", - ord("&"): u"/"}) - ensuredir(os.path.join(self.imagedir, dirname)) - path = os.path.join(self.imagedir, dirname, basename) try: + basename = os.path.basename(node['uri']) + if '?' in basename: + basename = basename.split('?')[0] + if basename == '': + basename = sha1(node['uri'].encode("utf-8")).hexdigest() + dirname = node['uri'].replace('://', '/').translate({ord("?"): u"/", + ord("&"): u"/"}) + ensuredir(os.path.join(self.imagedir, dirname)) + path = os.path.join(self.imagedir, dirname, basename) + headers = {} if os.path.exists(path): timestamp = ceil(os.stat(path).st_mtime) From 0eb5e0865ac7b5bba131311e4d108174c32a3398 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sat, 17 Mar 2018 13:43:42 +0900 Subject: [PATCH 04/24] Fix #4688: Error to download remote images having long URL --- CHANGES | 1 + sphinx/transforms/post_transforms/images.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index fb9c429a8..e9ee71765 100644 --- a/CHANGES +++ b/CHANGES @@ -27,6 +27,7 @@ Bugs fixed * #4725: Sphinx does not work with python 3.5.0 and 3.5.1 * #4716: Generation PDF file with TexLive on Windows, file not found error * #4574: vertical space before equation in latex +* #4688: Error to download remote images having long URL Testing -------- diff --git a/sphinx/transforms/post_transforms/images.py b/sphinx/transforms/post_transforms/images.py index 8c75b6aa0..6dd135e1e 100644 --- a/sphinx/transforms/post_transforms/images.py +++ b/sphinx/transforms/post_transforms/images.py @@ -30,6 +30,8 @@ if False: logger = logging.getLogger(__name__) +MAX_FILENAME_LEN = 32 + class BaseImageConverter(SphinxTransform): def apply(self): @@ -70,10 +72,14 @@ class ImageDownloader(BaseImageConverter): basename = os.path.basename(node['uri']) if '?' in basename: basename = basename.split('?')[0] - if basename == '': - basename = sha1(node['uri'].encode("utf-8")).hexdigest() + if basename == '' or len(basename) > MAX_FILENAME_LEN: + filename, ext = os.path.splitext(node['uri']) + basename = sha1(filename.encode("utf-8")).hexdigest() + ext + dirname = node['uri'].replace('://', '/').translate({ord("?"): u"/", ord("&"): u"/"}) + if len(dirname) > MAX_FILENAME_LEN: + dirname = sha1(dirname.encode('utf-8')).hexdigest() ensuredir(os.path.join(self.imagedir, dirname)) path = os.path.join(self.imagedir, dirname, basename) From f3b50ebef0357f49b779784f4e0ab8d8d163ec4a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 18 Mar 2018 12:32:22 +0900 Subject: [PATCH 05/24] Add testcase for qthelp --- tests/test_build_qthelp.py | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_build_qthelp.py b/tests/test_build_qthelp.py index de676e6e0..e2e2322d4 100644 --- a/tests/test_build_qthelp.py +++ b/tests/test_build_qthelp.py @@ -14,15 +14,61 @@ import pytest +@pytest.mark.sphinx('qthelp', testroot='basic') +def test_qthelp_basic(app, status, warning): + app.builder.build_all() + + qhcp = (app.outdir / 'Python.qhcp').text() + assert 'Python documentation' in qhcp + assert 'qthelp://org.sphinx.python/doc/index.html' in qhcp + assert 'qthelp://org.sphinx.python/doc/index.html' in qhcp + assert 'Python.qhp' in qhcp + assert 'Python.qch' in qhcp + assert 'Python.qch' in qhcp + + @pytest.mark.sphinx('qthelp', testroot='basic') def test_qthelp_namespace(app, status, warning): # default namespace app.builder.build_all() + qhp = (app.outdir / 'Python.qhp').text() assert 'org.sphinx.python' in qhp + qhcp = (app.outdir / 'Python.qhcp').text() + assert 'qthelp://org.sphinx.python/doc/index.html' in qhcp + assert 'qthelp://org.sphinx.python/doc/index.html' in qhcp + # give a namespace app.config.qthelp_namespace = 'org.sphinx-doc.sphinx' app.builder.build_all() + qhp = (app.outdir / 'Python.qhp').text() assert 'org.sphinxdoc.sphinx' in qhp + + qhcp = (app.outdir / 'Python.qhcp').text() + assert 'qthelp://org.sphinxdoc.sphinx/doc/index.html' in qhcp + assert 'qthelp://org.sphinxdoc.sphinx/doc/index.html' in qhcp + + +@pytest.mark.sphinx('qthelp', testroot='basic') +def test_qthelp_title(app, status, warning): + # default title + app.builder.build_all() + + qhp = (app.outdir / 'Python.qhp').text() + assert '
' in qhp + + qhcp = (app.outdir / 'Python.qhcp').text() + assert 'Python documentation' in qhcp + + # give a title + app.config.html_title = 'Sphinx "full" title' + app.config.html_short_title = 'Sphinx "short" title' + app.builder.build_all() + + qhp = (app.outdir / 'Python.qhp').text() + assert '
' in qhp + + qhcp = (app.outdir / 'Python.qhcp').text() + assert 'Sphinx <b>"short"</b> title' in qhcp From dfd550eca6c1fe66d9801f90be509fc869c4b8a5 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 18 Mar 2018 12:32:22 +0900 Subject: [PATCH 06/24] builder: Use template for generating .qhcp file --- sphinx/builders/qthelp.py | 45 ++++++++-------------------- sphinx/templates/qthelp/project.qhcp | 19 ++++++++++++ tests/test_build_qthelp.py | 2 +- 3 files changed, 32 insertions(+), 34 deletions(-) create mode 100644 sphinx/templates/qthelp/project.qhcp diff --git a/sphinx/builders/qthelp.py b/sphinx/builders/qthelp.py index 776a2d142..cf9776131 100644 --- a/sphinx/builders/qthelp.py +++ b/sphinx/builders/qthelp.py @@ -19,12 +19,14 @@ from docutils import nodes from six import text_type from sphinx import addnodes +from sphinx import package_dir from sphinx.builders.html import StandaloneHTMLBuilder from sphinx.config import string_classes from sphinx.environment.adapters.indexentries import IndexEntries from sphinx.util import force_decode, logging from sphinx.util.osutil import make_filename from sphinx.util.pycompat import htmlescape +from sphinx.util.template import SphinxRenderer if False: # For type annotation @@ -38,34 +40,6 @@ logger = logging.getLogger(__name__) _idpattern = re.compile( r'(?P.+) (\((class in )?(?P<id>[\w\.]+)( (?P<descr>\w+))?\))$') - -# Qt Help Collection Project (.qhcp). -# Is the input file for the help collection generator. -# It contains references to compressed help files which should be -# included in the collection. -# It may contain various other information for customizing Qt Assistant. -collection_template = u'''\ -<?xml version="1.0" encoding="utf-8" ?> -<QHelpCollectionProject version="1.0"> - <assistant> - <title>%(title)s - %(homepage)s - %(startpage)s - - - - - %(outname)s.qhp - %(outname)s.qch - - - - %(outname)s.qch - - - -''' - # Qt Help Project (.qhp) # This is the input file for the help generator. # It contains the table of contents, indices and references to the @@ -102,6 +76,12 @@ section_template = '
' file_template = ' ' * 12 + '%(filename)s' +def render_file(filename, **kwargs): + # type: (unicode, Any) -> unicode + pathname = os.path.join(package_dir, 'templates', 'qthelp', filename) + return SphinxRenderer.render_from_file(pathname, kwargs) + + class QtHelpBuilder(StandaloneHTMLBuilder): """ Builder that also outputs Qt help project, contents and index files. @@ -232,11 +212,10 @@ class QtHelpBuilder(StandaloneHTMLBuilder): logger.info('writing collection project file...') with codecs.open(path.join(outdir, outname + '.qhcp'), 'w', 'utf-8') as f: # type: ignore # NOQA - f.write(collection_template % { - 'outname': htmlescape(outname), - 'title': htmlescape(self.config.html_short_title), - 'homepage': htmlescape(homepage), - 'startpage': htmlescape(startpage)}) + content = render_file('project.qhcp', outname=outname, + title=self.config.html_short_title, + homepage=homepage, startpage=startpage) + f.write(content) def isdocnode(self, node): # type: (nodes.Node) -> bool diff --git a/sphinx/templates/qthelp/project.qhcp b/sphinx/templates/qthelp/project.qhcp new file mode 100644 index 000000000..fe12eaa14 --- /dev/null +++ b/sphinx/templates/qthelp/project.qhcp @@ -0,0 +1,19 @@ + + + + {{ title|e }} + {{ homepage|e }} + {{ startpage|e }} + + + + + {{ outname|e }}.qhp + {{ outname|e }}.qch + + + + {{ outname|e }}.qch + + + diff --git a/tests/test_build_qthelp.py b/tests/test_build_qthelp.py index e2e2322d4..541e03b75 100644 --- a/tests/test_build_qthelp.py +++ b/tests/test_build_qthelp.py @@ -71,4 +71,4 @@ def test_qthelp_title(app, status, warning): assert '
' in qhp qhcp = (app.outdir / 'Python.qhcp').text() - assert 'Sphinx <b>"short"</b> title' in qhcp + assert 'Sphinx <b>"short"</b> title' in qhcp From dc3faa57b47c0c0e1ded6740d9a8912d26ded76c Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 18 Mar 2018 12:32:23 +0900 Subject: [PATCH 07/24] Add testcase for qthelp (.qhp files) --- tests/test_build_qthelp.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_build_qthelp.py b/tests/test_build_qthelp.py index 541e03b75..3beddf554 100644 --- a/tests/test_build_qthelp.py +++ b/tests/test_build_qthelp.py @@ -13,11 +13,23 @@ import pytest +from sphinx.testing.util import etree_parse + @pytest.mark.sphinx('qthelp', testroot='basic') def test_qthelp_basic(app, status, warning): app.builder.build_all() + qhp = (app.outdir / 'Python.qhp').text() + assert '' in qhp + assert 'Python' in qhp + assert '' in qhp + assert '
' in qhp + assert 'genindex.html' in qhp + assert 'index.html' in qhp + assert '_static/basic.css' in qhp + assert '_static/down.png' in qhp + qhcp = (app.outdir / 'Python.qhcp').text() assert 'Python documentation' in qhcp assert 'qthelp://org.sphinx.python/doc/index.html' in qhcp @@ -27,6 +39,27 @@ def test_qthelp_basic(app, status, warning): assert 'Python.qch' in qhcp +@pytest.mark.sphinx('qthelp', testroot='toctree') +def test_qthelp_toctree(app, status, warning): + app.builder.build_all() + + et = etree_parse(app.outdir / 'Python.qhp') + toc = et.find('.//toc') + assert len(toc) == 1 + assert toc[0].attrib == {'title': 'Python documentation', + 'ref': 'index.html'} + assert len(toc[0]) == 4 + assert toc[0][0].attrib == {'title': 'foo', 'ref': 'foo.html'} + assert toc[0][1].attrib == {'title': 'bar', 'ref': 'bar.html'} + assert toc[0][0][0].attrib == {'title': 'quux', 'ref': 'quux.html'} + assert toc[0][0][1].attrib == {'title': 'foo.1', 'ref': 'foo.html#foo-1'} + assert toc[0][0][1][0].attrib == {'title': 'foo.1-1', 'ref': 'foo.html#foo-1-1'} + assert toc[0][0][2].attrib == {'title': 'foo.2', 'ref': 'foo.html#foo-2'} + assert toc[0][2].attrib == {'title': 'http://sphinx-doc.org/', + 'ref': 'http://sphinx-doc.org/'} + assert toc[0][3].attrib == {'title': 'baz', 'ref': 'baz.html'} + + @pytest.mark.sphinx('qthelp', testroot='basic') def test_qthelp_namespace(app, status, warning): # default namespace From c271cc45424bdc75b280fb6311cfcb439ab27e73 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 18 Mar 2018 12:32:23 +0900 Subject: [PATCH 08/24] Use template for generating .qhp file --- sphinx/builders/qthelp.py | 53 +++++-------------------- sphinx/templates/qthelp/project.qhp | 24 +++++++++++ tests/roots/test-need-escaped/bar.rst | 2 + tests/roots/test-need-escaped/baz.rst | 2 + tests/roots/test-need-escaped/conf.py | 5 +++ tests/roots/test-need-escaped/foo.rst | 15 +++++++ tests/roots/test-need-escaped/index.rst | 30 ++++++++++++++ tests/roots/test-need-escaped/quux.rst | 2 + tests/roots/test-need-escaped/qux.rst | 1 + tests/test_build_qthelp.py | 28 +++++++++---- 10 files changed, 110 insertions(+), 52 deletions(-) create mode 100644 sphinx/templates/qthelp/project.qhp create mode 100644 tests/roots/test-need-escaped/bar.rst create mode 100644 tests/roots/test-need-escaped/baz.rst create mode 100644 tests/roots/test-need-escaped/conf.py create mode 100644 tests/roots/test-need-escaped/foo.rst create mode 100644 tests/roots/test-need-escaped/index.rst create mode 100644 tests/roots/test-need-escaped/quux.rst create mode 100644 tests/roots/test-need-escaped/qux.rst diff --git a/sphinx/builders/qthelp.py b/sphinx/builders/qthelp.py index cf9776131..ef729e420 100644 --- a/sphinx/builders/qthelp.py +++ b/sphinx/builders/qthelp.py @@ -40,37 +40,6 @@ logger = logging.getLogger(__name__) _idpattern = re.compile( r'(?P.+) (\((class in )?(?P<id>[\w\.]+)( (?P<descr>\w+))?\))$') -# Qt Help Project (.qhp) -# This is the input file for the help generator. -# It contains the table of contents, indices and references to the -# actual documentation files (*.html). -# In addition it defines a unique namespace for the documentation. -project_template = u'''\ -<?xml version="1.0" encoding="utf-8" ?> -<QtHelpProject version="1.0"> - <namespace>%(namespace)s</namespace> - <virtualFolder>doc</virtualFolder> - <customFilter name="%(project)s %(version)s"> - <filterAttribute>%(outname)s</filterAttribute> - <filterAttribute>%(version)s</filterAttribute> - </customFilter> - <filterSection> - <filterAttribute>%(outname)s</filterAttribute> - <filterAttribute>%(version)s</filterAttribute> - <toc> - <section title="%(title)s" ref="%(masterdoc)s.html"> -%(sections)s - </section> - </toc> - <keywords> -%(keywords)s - </keywords> - <files> -%(files)s - </files> - </filterSection> -</QtHelpProject> -''' section_template = '<section title="%(title)s" ref="%(ref)s"/>' file_template = ' ' * 12 + '<file>%(filename)s</file>' @@ -195,16 +164,12 @@ class QtHelpBuilder(StandaloneHTMLBuilder): # write the project file with codecs.open(path.join(outdir, outname + '.qhp'), 'w', 'utf-8') as f: # type: ignore # NOQA - f.write(project_template % { - 'outname': htmlescape(outname), - 'title': htmlescape(self.config.html_title), - 'version': htmlescape(self.config.version), - 'project': htmlescape(self.config.project), - 'namespace': htmlescape(nspace), - 'masterdoc': htmlescape(self.config.master_doc), - 'sections': sections, - 'keywords': keywords, - 'files': projectfiles}) + content = render_file('project.qhp', outname=outname, + title=self.config.html_title, version=self.config.version, + project=self.config.project, namespace=nspace, + master_doc=self.config.master_doc, + sections=sections, keywords=keywords, files=projectfiles) + f.write(content) homepage = 'qthelp://' + posixpath.join( nspace, 'doc', self.get_target_uri(self.config.master_doc)) @@ -279,9 +244,9 @@ class QtHelpBuilder(StandaloneHTMLBuilder): if id: item = ' ' * 12 + '<keyword name="%s" id="%s" ref="%s"/>' % ( - name, id, ref[1]) + name, id, htmlescape(ref[1])) else: - item = ' ' * 12 + '<keyword name="%s" ref="%s"/>' % (name, ref[1]) + item = ' ' * 12 + '<keyword name="%s" ref="%s"/>' % (name, htmlescape(ref[1])) item.encode('ascii', 'xmlcharrefreplace') return item @@ -289,7 +254,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder): # type: (unicode, List[Any], Any) -> List[unicode] keywords = [] # type: List[unicode] - title = htmlescape(title) + title = htmlescape(title, quote=True) # if len(refs) == 0: # XXX # write_param('See Also', title) if len(refs) == 1: diff --git a/sphinx/templates/qthelp/project.qhp b/sphinx/templates/qthelp/project.qhp new file mode 100644 index 000000000..92a7af0e8 --- /dev/null +++ b/sphinx/templates/qthelp/project.qhp @@ -0,0 +1,24 @@ +<?xml version="1.0" encoding="utf-8" ?> +<QtHelpProject version="1.0"> + <namespace>{{ namespace|e }}</namespace> + <virtualFolder>doc</virtualFolder> + <customFilter name="{{ project|e }} {{ version|e }}"> + <filterAttribute>{{ outname|e }}</filterAttribute> + <filterAttribute>{{ version|e }}</filterAttribute> + </customFilter> + <filterSection> + <filterAttribute>{{ outname|e }}</filterAttribute> + <filterAttribute>{{ version|e }}</filterAttribute> + <toc> + <section title="{{ title|e }}" ref="{{ master_doc|e }}.html"> +{{ sections }} + </section> + </toc> + <keywords> +{{ keywords }} + </keywords> + <files> +{{ files }} + </files> + </filterSection> +</QtHelpProject> diff --git a/tests/roots/test-need-escaped/bar.rst b/tests/roots/test-need-escaped/bar.rst new file mode 100644 index 000000000..1cccd3cb7 --- /dev/null +++ b/tests/roots/test-need-escaped/bar.rst @@ -0,0 +1,2 @@ +bar +=== diff --git a/tests/roots/test-need-escaped/baz.rst b/tests/roots/test-need-escaped/baz.rst new file mode 100644 index 000000000..52e2e72ac --- /dev/null +++ b/tests/roots/test-need-escaped/baz.rst @@ -0,0 +1,2 @@ +baz +=== diff --git a/tests/roots/test-need-escaped/conf.py b/tests/roots/test-need-escaped/conf.py new file mode 100644 index 000000000..d65a22e07 --- /dev/null +++ b/tests/roots/test-need-escaped/conf.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +master_doc = 'index' +project = 'need <b>"escaped"</b> project' +smartquotes = False diff --git a/tests/roots/test-need-escaped/foo.rst b/tests/roots/test-need-escaped/foo.rst new file mode 100644 index 000000000..70859b3fc --- /dev/null +++ b/tests/roots/test-need-escaped/foo.rst @@ -0,0 +1,15 @@ +<foo> +===== + +.. toctree:: + + quux + +foo "1" +------- + +foo.1-1 +^^^^^^^ + +foo.2 +----- diff --git a/tests/roots/test-need-escaped/index.rst b/tests/roots/test-need-escaped/index.rst new file mode 100644 index 000000000..9ef74e00a --- /dev/null +++ b/tests/roots/test-need-escaped/index.rst @@ -0,0 +1,30 @@ +.. Sphinx Tests documentation master file, created by sphinx-quickstart on Wed Jun 4 23:49:58 2008. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Sphinx Tests's documentation! +======================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + :numbered: + :caption: Table of Contents + :name: mastertoc + + foo + bar + http://sphinx-doc.org/ + baz + qux + +.. index:: + pair: "subsection"; <subsection> + +---------- +subsection +---------- + +subsubsection +------------- diff --git a/tests/roots/test-need-escaped/quux.rst b/tests/roots/test-need-escaped/quux.rst new file mode 100644 index 000000000..07dd0a0a3 --- /dev/null +++ b/tests/roots/test-need-escaped/quux.rst @@ -0,0 +1,2 @@ +quux +==== diff --git a/tests/roots/test-need-escaped/qux.rst b/tests/roots/test-need-escaped/qux.rst new file mode 100644 index 000000000..26176b947 --- /dev/null +++ b/tests/roots/test-need-escaped/qux.rst @@ -0,0 +1 @@ +qux.rst has no section title diff --git a/tests/test_build_qthelp.py b/tests/test_build_qthelp.py index 3beddf554..a0d4fcf2a 100644 --- a/tests/test_build_qthelp.py +++ b/tests/test_build_qthelp.py @@ -39,26 +39,37 @@ def test_qthelp_basic(app, status, warning): assert '<file>Python.qch</file>' in qhcp -@pytest.mark.sphinx('qthelp', testroot='toctree') -def test_qthelp_toctree(app, status, warning): +@pytest.mark.sphinx('qthelp', testroot='need-escaped') +def test_qthelp_escaped(app, status, warning): app.builder.build_all() - et = etree_parse(app.outdir / 'Python.qhp') + et = etree_parse(app.outdir / 'needbescapedbproject.qhp') + customFilter = et.find('.//customFilter') + assert len(customFilter) == 2 + assert customFilter.attrib == {'name': 'need <b>"escaped"</b> project '} + assert customFilter[0].text == 'needbescapedbproject' + assert customFilter[1].text is None + toc = et.find('.//toc') assert len(toc) == 1 - assert toc[0].attrib == {'title': 'Python documentation', + assert toc[0].attrib == {'title': 'need <b>"escaped"</b> project documentation', 'ref': 'index.html'} assert len(toc[0]) == 4 - assert toc[0][0].attrib == {'title': 'foo', 'ref': 'foo.html'} - assert toc[0][1].attrib == {'title': 'bar', 'ref': 'bar.html'} + assert toc[0][0].attrib == {'title': '<foo>', 'ref': 'foo.html'} assert toc[0][0][0].attrib == {'title': 'quux', 'ref': 'quux.html'} - assert toc[0][0][1].attrib == {'title': 'foo.1', 'ref': 'foo.html#foo-1'} + assert toc[0][0][1].attrib == {'title': 'foo "1"', 'ref': 'foo.html#foo-1'} assert toc[0][0][1][0].attrib == {'title': 'foo.1-1', 'ref': 'foo.html#foo-1-1'} assert toc[0][0][2].attrib == {'title': 'foo.2', 'ref': 'foo.html#foo-2'} + assert toc[0][1].attrib == {'title': 'bar', 'ref': 'bar.html'} assert toc[0][2].attrib == {'title': 'http://sphinx-doc.org/', 'ref': 'http://sphinx-doc.org/'} assert toc[0][3].attrib == {'title': 'baz', 'ref': 'baz.html'} + keywords = et.find('.//keywords') + assert len(keywords) == 2 + assert keywords[0].attrib == {'name': '<subsection>', 'ref': 'index.html#index-0'} + assert keywords[1].attrib == {'name': '"subsection"', 'ref': 'index.html#index-0'} + @pytest.mark.sphinx('qthelp', testroot='basic') def test_qthelp_namespace(app, status, warning): @@ -101,7 +112,8 @@ def test_qthelp_title(app, status, warning): app.builder.build_all() qhp = (app.outdir / 'Python.qhp').text() - assert '<section title="Sphinx <b>"full"</b> title" ref="index.html">' in qhp + assert ('<section title="Sphinx <b>"full"</b> title" ref="index.html">' + in qhp) qhcp = (app.outdir / 'Python.qhcp').text() assert '<title>Sphinx <b>"short"</b> title' in qhcp From ba8569f131f0031b3785657751c684e5df59948e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 18 Mar 2018 12:32:23 +0900 Subject: [PATCH 09/24] Do not construct tag by python code --- sphinx/builders/qthelp.py | 39 ++++++++++++++--------------- sphinx/templates/qthelp/project.qhp | 4 ++- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/sphinx/builders/qthelp.py b/sphinx/builders/qthelp.py index ef729e420..4c1016708 100644 --- a/sphinx/builders/qthelp.py +++ b/sphinx/builders/qthelp.py @@ -42,7 +42,6 @@ _idpattern = re.compile( section_template = '
' -file_template = ' ' * 12 + '%(filename)s' def render_file(filename, **kwargs): @@ -132,24 +131,6 @@ class QtHelpBuilder(StandaloneHTMLBuilder): keywords.extend(self.build_keywords(title, refs, subitems)) keywords = u'\n'.join(keywords) # type: ignore - # files - if not outdir.endswith(os.sep): - outdir += os.sep - olen = len(outdir) - projectfiles = [] - staticdir = path.join(outdir, '_static') - imagesdir = path.join(outdir, self.imagedir) - for root, dirs, files in os.walk(outdir): - resourcedir = root.startswith(staticdir) or \ - root.startswith(imagesdir) - for fn in sorted(files): - if (resourcedir and not fn.endswith('.js')) or \ - fn.endswith('.html'): - filename = path.join(root, fn)[olen:] - projectfiles.append(file_template % - {'filename': htmlescape(filename)}) - projectfiles = '\n'.join(projectfiles) # type: ignore - # it seems that the "namespace" may not contain non-alphanumeric # characters, and more than one successive dot, or leading/trailing # dots, are also forbidden @@ -168,7 +149,8 @@ class QtHelpBuilder(StandaloneHTMLBuilder): title=self.config.html_title, version=self.config.version, project=self.config.project, namespace=nspace, master_doc=self.config.master_doc, - sections=sections, keywords=keywords, files=projectfiles) + sections=sections, keywords=keywords, + files=self.get_project_files(outdir)) f.write(content) homepage = 'qthelp://' + posixpath.join( @@ -274,6 +256,23 @@ class QtHelpBuilder(StandaloneHTMLBuilder): return keywords + def get_project_files(self, outdir): + # type: (unicode) -> List[unicode] + if not outdir.endswith(os.sep): + outdir += os.sep + olen = len(outdir) + project_files = [] + staticdir = path.join(outdir, '_static') + imagesdir = path.join(outdir, self.imagedir) + for root, dirs, files in os.walk(outdir): + resourcedir = root.startswith((staticdir, imagesdir)) + for fn in sorted(files): + if (resourcedir and not fn.endswith('.js')) or fn.endswith('.html'): + filename = path.join(root, fn)[olen:] + project_files.append(filename) + + return project_files + def setup(app): # type: (Sphinx) -> Dict[unicode, Any] diff --git a/sphinx/templates/qthelp/project.qhp b/sphinx/templates/qthelp/project.qhp index 92a7af0e8..53f999043 100644 --- a/sphinx/templates/qthelp/project.qhp +++ b/sphinx/templates/qthelp/project.qhp @@ -18,7 +18,9 @@ {{ keywords }} -{{ files }} + {%- for filename in files %} + {{ filename|e }} + {%- endfor %} From a709adf23338c58588440dde98b63970b616c864 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 18 Mar 2018 12:32:02 +0900 Subject: [PATCH 10/24] qthelp: escape keywords in .qhp file --- CHANGES | 2 ++ sphinx/builders/qthelp.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index a105fa084..f1e0d3c7b 100644 --- a/CHANGES +++ b/CHANGES @@ -28,6 +28,8 @@ Bugs fixed * #4716: Generation PDF file with TexLive on Windows, file not found error * #4574: vertical space before equation in latex * #4720: message when an image is mismatched for builder is not clear +* #1435: qthelp builder should htmlescape keywords + Testing -------- diff --git a/sphinx/builders/qthelp.py b/sphinx/builders/qthelp.py index 4c1016708..7a5c6c0d6 100644 --- a/sphinx/builders/qthelp.py +++ b/sphinx/builders/qthelp.py @@ -224,11 +224,12 @@ class QtHelpBuilder(StandaloneHTMLBuilder): else: id = None + nameattr = htmlescape(name, quote=True) + refattr = htmlescape(ref[1], quote=True) if id: - item = ' ' * 12 + '' % ( - name, id, htmlescape(ref[1])) + item = ' ' * 12 + '' % (nameattr, id, refattr) else: - item = ' ' * 12 + '' % (name, htmlescape(ref[1])) + item = ' ' * 12 + '' % (nameattr, refattr) item.encode('ascii', 'xmlcharrefreplace') return item @@ -236,7 +237,6 @@ class QtHelpBuilder(StandaloneHTMLBuilder): # type: (unicode, List[Any], Any) -> List[unicode] keywords = [] # type: List[unicode] - title = htmlescape(title, quote=True) # if len(refs) == 0: # XXX # write_param('See Also', title) if len(refs) == 1: From 00c2a90cf89cfa85cfe21537e646eeddbe8556ac Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 18 Mar 2018 12:58:31 +0900 Subject: [PATCH 11/24] Fix mypy violations --- sphinx/builders/qthelp.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sphinx/builders/qthelp.py b/sphinx/builders/qthelp.py index 7a5c6c0d6..13ea256da 100644 --- a/sphinx/builders/qthelp.py +++ b/sphinx/builders/qthelp.py @@ -145,13 +145,13 @@ class QtHelpBuilder(StandaloneHTMLBuilder): # write the project file with codecs.open(path.join(outdir, outname + '.qhp'), 'w', 'utf-8') as f: # type: ignore # NOQA - content = render_file('project.qhp', outname=outname, - title=self.config.html_title, version=self.config.version, - project=self.config.project, namespace=nspace, - master_doc=self.config.master_doc, - sections=sections, keywords=keywords, - files=self.get_project_files(outdir)) - f.write(content) + body = render_file('project.qhp', outname=outname, + title=self.config.html_title, version=self.config.version, + project=self.config.project, namespace=nspace, + master_doc=self.config.master_doc, + sections=sections, keywords=keywords, + files=self.get_project_files(outdir)) + f.write(body) homepage = 'qthelp://' + posixpath.join( nspace, 'doc', self.get_target_uri(self.config.master_doc)) @@ -159,10 +159,10 @@ class QtHelpBuilder(StandaloneHTMLBuilder): logger.info('writing collection project file...') with codecs.open(path.join(outdir, outname + '.qhcp'), 'w', 'utf-8') as f: # type: ignore # NOQA - content = render_file('project.qhcp', outname=outname, - title=self.config.html_short_title, - homepage=homepage, startpage=startpage) - f.write(content) + body = render_file('project.qhcp', outname=outname, + title=self.config.html_short_title, + homepage=homepage, startpage=startpage) + f.write(body) def isdocnode(self, node): # type: (nodes.Node) -> bool From 869ac1efe2ed37c79c58f40931a28f399358bf93 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 19 Mar 2018 23:23:20 +0900 Subject: [PATCH 12/24] Fix #4754: sphinx/pycode/__init__.py raises AttributeError --- CHANGES | 1 + sphinx/pycode/__init__.py | 20 -------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/CHANGES b/CHANGES index 029b9efcb..b293f12b6 100644 --- a/CHANGES +++ b/CHANGES @@ -29,6 +29,7 @@ Bugs fixed * #4574: vertical space before equation in latex * #4720: message when an image is mismatched for builder is not clear * #4655, #4684: Incomplete localization strings in Polish and Chinese +* #4754: sphinx/pycode/__init__.py raises AttributeError Testing -------- diff --git a/sphinx/pycode/__init__.py b/sphinx/pycode/__init__.py index de951a19f..fca28817d 100644 --- a/sphinx/pycode/__init__.py +++ b/sphinx/pycode/__init__.py @@ -117,23 +117,3 @@ class ModuleAnalyzer(object): self.parse() return self.tags - - -if __name__ == '__main__': - import time - import pprint - x0 = time.time() - # ma = ModuleAnalyzer.for_file(__file__.rstrip('c'), 'sphinx.builders.html') - ma = ModuleAnalyzer.for_file('sphinx/environment.py', - 'sphinx.environment') - ma.tokenize() - x1 = time.time() - ma.parse() - x2 = time.time() - # for (ns, name), doc in iteritems(ma.find_attr_docs()): - # print '>>', ns, name - # print '\n'.join(doc) - pprint.pprint(ma.find_tags()) - x3 = time.time() - # print nodes.nice_repr(ma.parsetree, number2name) - print("tokenizing %.4f, parsing %.4f, finding %.4f" % (x1 - x0, x2 - x1, x3 - x2)) From 049df5d5e179764d9d565a6bcbd8e92f5d63f85e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 20 Mar 2018 20:26:45 +0900 Subject: [PATCH 13/24] test: Suppress DeprecationWarning --- tests/test_autodoc.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_autodoc.py b/tests/test_autodoc.py index 5505891b2..881df1fda 100644 --- a/tests/test_autodoc.py +++ b/tests/test_autodoc.py @@ -11,13 +11,15 @@ """ import sys +from warnings import catch_warnings import pytest from docutils.statemachine import ViewList from six import PY3 -from sphinx.ext.autodoc import AutoDirective, add_documenter, \ - ModuleLevelDocumenter, FunctionDocumenter, cut_lines, between, ALL +from sphinx.ext.autodoc import ( + AutoDirective, ModuleLevelDocumenter, FunctionDocumenter, cut_lines, between, ALL +) from sphinx.testing.util import SphinxTestApp, Struct # NOQA from sphinx.util import logging @@ -550,7 +552,7 @@ def test_new_documenter(): def document_members(self, all_members=False): return - add_documenter(MyDocumenter) + app.add_autodocumenter(MyDocumenter) def assert_result_contains(item, objtype, name, **kw): app._warning.truncate(0) @@ -591,12 +593,13 @@ def test_attrgetter_using(): assert fullname not in documented_members, \ '%r was not hooked by special_attrgetter function' % fullname - options.members = ALL - options.inherited_members = False - assert_getter_works('class', 'target.Class', Class, ['meth']) + with catch_warnings(record=True): + options.members = ALL + options.inherited_members = False + assert_getter_works('class', 'target.Class', Class, ['meth']) - options.inherited_members = True - assert_getter_works('class', 'target.Class', Class, ['meth', 'inheritedmeth']) + options.inherited_members = True + assert_getter_works('class', 'target.Class', Class, ['meth', 'inheritedmeth']) @pytest.mark.usefixtures('setup_test') From 98919244851f2ea71622d22fa6b73b3379c7f5e2 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 20 Mar 2018 20:27:55 +0900 Subject: [PATCH 14/24] epub: Fix docTitle elements of toc.ncx is not escaped --- CHANGES | 1 + sphinx/builders/_epub_base.py | 2 +- tests/test_build_epub.py | 58 ++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 48ded796f..b226db3a3 100644 --- a/CHANGES +++ b/CHANGES @@ -33,6 +33,7 @@ Bugs fixed * #4688: Error to download remote images having long URL * #4754: sphinx/pycode/__init__.py raises AttributeError * #1435: qthelp builder should htmlescape keywords +* epub: Fix docTitle elements of toc.ncx is not escaped Testing -------- diff --git a/sphinx/builders/_epub_base.py b/sphinx/builders/_epub_base.py index d44e7f2be..2c0c0f25e 100644 --- a/sphinx/builders/_epub_base.py +++ b/sphinx/builders/_epub_base.py @@ -672,7 +672,7 @@ class EpubBuilder(StandaloneHTMLBuilder): """ metadata = {} # type: Dict[unicode, Any] metadata['uid'] = self.config.epub_uid - metadata['title'] = self.config.epub_title + metadata['title'] = self.esc(self.config.epub_title) metadata['level'] = level metadata['navpoints'] = navpoints return metadata diff --git a/tests/test_build_epub.py b/tests/test_build_epub.py index 3256fcb9f..52a6e3dfe 100644 --- a/tests/test_build_epub.py +++ b/tests/test_build_epub.py @@ -29,7 +29,7 @@ def runnable(command): class EPUBElementTree(object): - """Test helper for content.opf and tox.ncx""" + """Test helper for content.opf and toc.ncx""" namespaces = { 'idpf': 'http://www.idpf.org/2007/opf', 'dc': 'http://purl.org/dc/elements/1.1/', @@ -226,6 +226,62 @@ def test_nested_toc(app): assert navinfo(grandchild[0]) == ('foo.xhtml#foo-1-1', 'foo.1-1') +@pytest.mark.sphinx('epub', testroot='need-escaped') +def test_escaped_toc(app): + app.build() + + # toc.ncx + toc = EPUBElementTree.fromstring((app.outdir / 'toc.ncx').bytes()) + assert toc.find("./ncx:docTitle/ncx:text").text == ('need "escaped" ' + 'project documentation') + + # toc.ncx / navPoint + def navinfo(elem): + label = elem.find("./ncx:navLabel/ncx:text") + content = elem.find("./ncx:content") + return (elem.get('id'), elem.get('playOrder'), + content.get('src'), label.text) + + navpoints = toc.findall("./ncx:navMap/ncx:navPoint") + assert len(navpoints) == 4 + assert navinfo(navpoints[0]) == ('navPoint1', '1', 'index.xhtml', + u"Welcome to Sphinx Tests's documentation!") + assert navpoints[0].findall("./ncx:navPoint") == [] + + # toc.ncx / nested navPoints + assert navinfo(navpoints[1]) == ('navPoint2', '2', 'foo.xhtml', '') + navchildren = navpoints[1].findall("./ncx:navPoint") + assert len(navchildren) == 4 + assert navinfo(navchildren[0]) == ('navPoint3', '2', 'foo.xhtml', '') + assert navinfo(navchildren[1]) == ('navPoint4', '3', 'quux.xhtml', 'quux') + assert navinfo(navchildren[2]) == ('navPoint5', '4', 'foo.xhtml#foo-1', u'foo “1”') + assert navinfo(navchildren[3]) == ('navPoint8', '6', 'foo.xhtml#foo-2', 'foo.2') + + # nav.xhtml / nav + def navinfo(elem): + anchor = elem.find("./xhtml:a") + return (anchor.get('href'), anchor.text) + + nav = EPUBElementTree.fromstring((app.outdir / 'nav.xhtml').bytes()) + toc = nav.findall("./xhtml:body/xhtml:nav/xhtml:ol/xhtml:li") + assert len(toc) == 4 + assert navinfo(toc[0]) == ('index.xhtml', + "Welcome to Sphinx Tests's documentation!") + assert toc[0].findall("./xhtml:ol") == [] + + # nav.xhtml / nested toc + assert navinfo(toc[1]) == ('foo.xhtml', '') + tocchildren = toc[1].findall("./xhtml:ol/xhtml:li") + assert len(tocchildren) == 3 + assert navinfo(tocchildren[0]) == ('quux.xhtml', 'quux') + assert navinfo(tocchildren[1]) == ('foo.xhtml#foo-1', u'foo “1”') + assert navinfo(tocchildren[2]) == ('foo.xhtml#foo-2', 'foo.2') + + grandchild = tocchildren[1].findall("./xhtml:ol/xhtml:li") + assert len(grandchild) == 1 + assert navinfo(grandchild[0]) == ('foo.xhtml#foo-1-1', 'foo.1-1') + + @pytest.mark.sphinx('epub', testroot='basic') def test_epub_writing_mode(app): # horizontal (default) From 0a8e3f5087a332a17ccb34e2d138f3323eaffa33 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 20 Mar 2018 22:58:21 +0900 Subject: [PATCH 15/24] Fix #4724: doc: an example of autogen is incorrect --- doc/man/sphinx-autogen.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/man/sphinx-autogen.rst b/doc/man/sphinx-autogen.rst index 49a8220d0..ef84afcb9 100644 --- a/doc/man/sphinx-autogen.rst +++ b/doc/man/sphinx-autogen.rst @@ -73,7 +73,7 @@ If you run the following: .. code-block:: bash - $ sphinx-autodoc doc/index.rst + $ PYTHONPATH=. sphinx-autodoc doc/index.rst then the following stub files will be created in ``docs``:: From cd08872cc0d3453c01290d75e6b32ef54c872a59 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 20 Mar 2018 23:35:32 +0900 Subject: [PATCH 16/24] Fix existence check for JRE was broken --- tests/test_build_epub.py | 6 +++--- tox.ini | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_build_epub.py b/tests/test_build_epub.py index 52a6e3dfe..2f09f6d5a 100644 --- a/tests/test_build_epub.py +++ b/tests/test_build_epub.py @@ -19,13 +19,13 @@ import pytest # check given command is runnable def runnable(command): try: - p = Popen(command, stdout=PIPE) + p = Popen(command, stdout=PIPE, stderr=PIPE) except OSError: # command not found return False else: p.communicate() - return p.returncode + return p.returncode == 0 class EPUBElementTree(object): @@ -322,7 +322,7 @@ def test_run_epubcheck(app): app.build() epubcheck = os.environ.get('EPUBCHECK_PATH', '/usr/share/java/epubcheck.jar') - if runnable('java') and os.path.exists(epubcheck): + if runnable(['java', '-version']) and os.path.exists(epubcheck): p = Popen(['java', '-jar', epubcheck, app.outdir / 'SphinxTests.epub'], stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() diff --git a/tox.ini b/tox.ini index b8d6a0e32..16ea8f9ba 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = docs,flake8,mypy,coverage,py{27,34,35,36,py},du{11,12,13,14} [testenv] usedevelop = True passenv = - https_proxy http_proxy no_proxy PERL PERL5LIB PYTEST_ADDOPTS + https_proxy http_proxy no_proxy PERL PERL5LIB PYTEST_ADDOPTS EPUBCHECK_PATH description = py{27,34,35,36,py}: Run unit tests against {envname}. du{11,12,13,14}: Run unit tests with the given version of docutils. From d8c107a61b30bdf8d9f3e4e8b183c8e34ef7fb23 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 21 Mar 2018 00:01:44 +0900 Subject: [PATCH 17/24] Fix #4744: Add canonical URL to docs --- doc/_themes/sphinx13/layout.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/_themes/sphinx13/layout.html b/doc/_themes/sphinx13/layout.html index ce6f08daa..597b6261f 100644 --- a/doc/_themes/sphinx13/layout.html +++ b/doc/_themes/sphinx13/layout.html @@ -13,6 +13,11 @@ {% block sidebar1 %}{{ sidebar() }}{% endblock %} {% block sidebar2 %}{% endblock %} +{% block linktags %} +{{ super() }} + +{% endblock %} + {% block extrahead %} From cef28eedfee424b16a0f61c06d0f5657b1bb0869 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Mon, 19 Mar 2018 12:22:58 +0000 Subject: [PATCH 18/24] codecov: Disable status checks We don't currently care if a commit changes our coverage metrics, so we can disable the status checks that would enforce that. Signed-off-by: Stephen Finucane Fixes: #4738 --- .codecov.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index 2ce4fda70..f6272f5f1 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -2,9 +2,7 @@ coverage: status: project: default: - # allowed to drop X% and still result in a "success" commit status - threshold: 0.05 + enabled: no patch: default: - # allowed to drop X% and still result in a "success" commit status - threshold: 0.05 + enabled: no From a7aac6956d795e4a5aebf61a2a0d09920a58772f Mon Sep 17 00:00:00 2001 From: Christer Bystrom Date: Sat, 24 Feb 2018 11:44:29 +0100 Subject: [PATCH 19/24] Closes #4520 - acidic: Subpackage not in toc. The rule of skipping folders with only an empty __init__.py has been removed. The reason for this is that it was never working consistently in the first place and made the code unnecessary hard to reason about. Tests for the TOC generation have been added, as well as tests for the exclude mechanism since they are coupled. One test (test_ext_apidoc.py::test_exclude) has also been modified to reflect the new behaviour. --- CHANGES | 5 ++ sphinx/ext/apidoc.py | 19 +++--- .../parent/__init__.py | 0 .../parent/child/__init__.py | 0 .../parent/child/foo.py | 1 + tests/test_ext_apidoc.py | 67 ++++++++++++++++++- 6 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 tests/roots/test-apidoc-subpackage-in-toc/parent/__init__.py create mode 100644 tests/roots/test-apidoc-subpackage-in-toc/parent/child/__init__.py create mode 100644 tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py diff --git a/CHANGES b/CHANGES index b226db3a3..34d5a5bd8 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,7 @@ Dependencies Incompatible changes -------------------- +* apidoc: As a consequence of a bug fix (#4520#) and cleaning up the code, folders with an empty __init__.py are no longer excluded from TOC. Deprecated ---------- @@ -34,6 +35,7 @@ Bugs fixed * #4754: sphinx/pycode/__init__.py raises AttributeError * #1435: qthelp builder should htmlescape keywords * epub: Fix docTitle elements of toc.ncx is not escaped +* #4520#: apidoc: Subpackage not in toc (introduced in 1.6.6) now fixed. Testing -------- @@ -41,6 +43,9 @@ Testing Release 1.7.1 (released Feb 23, 2018) ===================================== +Dependencies +------------ + Deprecated ---------- diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py index 6704b150d..3ea563d50 100644 --- a/sphinx/ext/apidoc.py +++ b/sphinx/ext/apidoc.py @@ -194,16 +194,15 @@ def shall_skip(module, opts, excludes=[]): if not opts.implicit_namespaces and not path.exists(module): return True - # skip it if there is nothing (or just \n or \r\n) in the file - if path.exists(module) and path.getsize(module) <= 2: - if os.path.basename(module) == '__init__.py': - # We only want to skip packages if they do not contain any - # .py files other than __init__.py. - basemodule = path.dirname(module) - for module in glob.glob(path.join(basemodule, '*.py')): - if not is_excluded(path.join(basemodule, module), excludes): - return True - else: + # Are we a package (here defined as __init__.py, not the folder in itself) + if os.path.basename(module) == INITPY: + # Yes, check if we have any non-excluded modules at all here + basemodule = path.dirname(module) + for module in glob.glob(path.join(basemodule, '*.py')): + if not is_excluded(path.join(basemodule, module), excludes): + # There's a non-excluded module here, we won't skip + all_skipped = False + if all_skipped: return True # skip if it has a "private" name and this is selected diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/__init__.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/child/__init__.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py new file mode 100644 index 000000000..810c96eee --- /dev/null +++ b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py @@ -0,0 +1 @@ +"foo" diff --git a/tests/test_ext_apidoc.py b/tests/test_ext_apidoc.py index 836c20b04..d3d61d1e0 100644 --- a/tests/test_ext_apidoc.py +++ b/tests/test_ext_apidoc.py @@ -211,7 +211,7 @@ def test_trailing_underscore(make_app, apidoc): @pytest.mark.apidoc( coderoot='test-apidoc-pep420/a', - excludes=["b/c/d.py", "b/e/f.py"], + excludes=["b/c/d.py", "b/e/f.py", "b/e/__init__.py"], options=["--implicit-namespaces", "--separate"], ) def test_excludes(apidoc): @@ -223,6 +223,45 @@ def test_excludes(apidoc): assert (outdir / 'a.b.x.y.rst').isfile() +@pytest.mark.apidoc( + coderoot='test-apidoc-pep420/a', + excludes=["b/e"], + options=["--implicit-namespaces", "--separate"], +) +def test_excludes_subpackage_should_be_skipped(apidoc): + """Subpackage exclusion should work.""" + outdir = apidoc.outdir + assert (outdir / 'conf.py').isfile() + assert (outdir / 'a.b.c.rst').isfile() # generated because not empty + assert not (outdir / 'a.b.e.f.rst').isfile() # skipped because 'b/e' subpackage is skipped + + +@pytest.mark.apidoc( + coderoot='test-apidoc-pep420/a', + excludes=["b/e/f.py"], + options=["--implicit-namespaces", "--separate"], +) +def test_excludes_module_should_be_skipped(apidoc): + """Module exclusion should work.""" + outdir = apidoc.outdir + assert (outdir / 'conf.py').isfile() + assert (outdir / 'a.b.c.rst').isfile() # generated because not empty + assert not (outdir / 'a.b.e.f.rst').isfile() # skipped because of empty after excludes + + +@pytest.mark.apidoc( + coderoot='test-apidoc-pep420/a', + excludes=[], + options=["--implicit-namespaces", "--separate"], +) +def test_excludes_module_should_not_be_skipped(apidoc): + """Module should be included if no excludes are used.""" + outdir = apidoc.outdir + assert (outdir / 'conf.py').isfile() + assert (outdir / 'a.b.c.rst').isfile() # generated because not empty + assert (outdir / 'a.b.e.f.rst').isfile() # skipped because of empty after excludes + + @pytest.mark.apidoc( coderoot='test-root', options=[ @@ -339,3 +378,29 @@ def extract_toc(path): toctree = rst[start_idx + len(toctree_start):end_idx] return toctree + + +@pytest.mark.apidoc( + coderoot='test-apidoc-subpackage-in-toc', + options=['--separate'] +) + + +def test_subpackage_in_toc(make_app, apidoc): + """Make sure that empty subpackages with non-empty subpackages in them + are not skipped (issue #4520) + """ + outdir = apidoc.outdir + assert (outdir / 'conf.py').isfile() + + assert (outdir / 'parent.rst').isfile() + with open(outdir / 'parent.rst') as f: + parent = f.read() + assert 'parent.child' in parent + + assert (outdir / 'parent.child.rst').isfile() + with open(outdir / 'parent.child.rst') as f: + parent_child = f.read() + assert 'parent.child.foo' in parent_child + + assert (outdir / 'parent.child.foo.rst').isfile() From 75eccc86d7c9460f7aea478831342ea6aa1c91fe Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 21 Mar 2018 14:30:53 +0900 Subject: [PATCH 20/24] apidoc: Fix local variable is not initialized --- sphinx/ext/apidoc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py index 3ea563d50..921152ef1 100644 --- a/sphinx/ext/apidoc.py +++ b/sphinx/ext/apidoc.py @@ -197,6 +197,7 @@ def shall_skip(module, opts, excludes=[]): # Are we a package (here defined as __init__.py, not the folder in itself) if os.path.basename(module) == INITPY: # Yes, check if we have any non-excluded modules at all here + all_skipped = True basemodule = path.dirname(module) for module in glob.glob(path.join(basemodule, '*.py')): if not is_excluded(path.join(basemodule, module), excludes): From a35be4cc665d4bbf6bd593da908cdb0cfe40ce37 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 21 Mar 2018 14:37:13 +0900 Subject: [PATCH 21/24] Update CHANGES --- CHANGES | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 34d5a5bd8..b7f7b3506 100644 --- a/CHANGES +++ b/CHANGES @@ -6,7 +6,8 @@ Dependencies Incompatible changes -------------------- -* apidoc: As a consequence of a bug fix (#4520#) and cleaning up the code, folders with an empty __init__.py are no longer excluded from TOC. +* #4520: apidoc: folders with an empty __init__.py are no longer excluded from + TOC Deprecated ---------- @@ -35,7 +36,7 @@ Bugs fixed * #4754: sphinx/pycode/__init__.py raises AttributeError * #1435: qthelp builder should htmlescape keywords * epub: Fix docTitle elements of toc.ncx is not escaped -* #4520#: apidoc: Subpackage not in toc (introduced in 1.6.6) now fixed. +* #4520: apidoc: Subpackage not in toc (introduced in 1.6.6) now fixed Testing -------- @@ -43,9 +44,6 @@ Testing Release 1.7.1 (released Feb 23, 2018) ===================================== -Dependencies ------------- - Deprecated ---------- From d029b3d5c94aeeadbdbf197356d2a4c6a0335b8d Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 21 Mar 2018 21:03:54 +0900 Subject: [PATCH 22/24] Bump to 1.7.2 final --- CHANGES | 16 ++-------------- sphinx/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/CHANGES b/CHANGES index b7f7b3506..66271df6a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,20 +1,11 @@ -Release 1.7.2 (in development) -============================== - -Dependencies ------------- +Release 1.7.2 (released Mar 21, 2018) +===================================== Incompatible changes -------------------- * #4520: apidoc: folders with an empty __init__.py are no longer excluded from TOC -Deprecated ----------- - -Features added --------------- - Bugs fixed ---------- @@ -38,9 +29,6 @@ Bugs fixed * epub: Fix docTitle elements of toc.ncx is not escaped * #4520: apidoc: Subpackage not in toc (introduced in 1.6.6) now fixed -Testing --------- - Release 1.7.1 (released Feb 23, 2018) ===================================== diff --git a/sphinx/__init__.py b/sphinx/__init__.py index b819b1d5f..e93c6da09 100644 --- a/sphinx/__init__.py +++ b/sphinx/__init__.py @@ -31,13 +31,13 @@ if 'PYTHONWARNINGS' not in os.environ: warnings.filterwarnings('ignore', "'U' mode is deprecated", DeprecationWarning, module='docutils.io') -__version__ = '1.7.2+' +__version__ = '1.7.2' __released__ = '1.7.2' # used when Sphinx builds its own docs # version info for better programmatic use # possible values for 3rd element: 'alpha', 'beta', 'rc', 'final' # 'final' has 0 as the last element -version_info = (1, 7, 2, 'beta', 0) +version_info = (1, 7, 2, 'final', 0) package_dir = path.abspath(path.dirname(__file__)) From 991b471025ca6211f01a8cf10996d1e5e64b0016 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Wed, 21 Mar 2018 21:06:48 +0900 Subject: [PATCH 23/24] Bump version --- CHANGES | 21 +++++++++++++++++++++ sphinx/__init__.py | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 66271df6a..18149fed9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,24 @@ +Release 1.7.3 (in development) +============================== + +Dependencies +------------ + +Incompatible changes +-------------------- + +Deprecated +---------- + +Features added +-------------- + +Bugs fixed +---------- + +Testing +-------- + Release 1.7.2 (released Mar 21, 2018) ===================================== diff --git a/sphinx/__init__.py b/sphinx/__init__.py index e93c6da09..67278f4c0 100644 --- a/sphinx/__init__.py +++ b/sphinx/__init__.py @@ -31,13 +31,13 @@ if 'PYTHONWARNINGS' not in os.environ: warnings.filterwarnings('ignore', "'U' mode is deprecated", DeprecationWarning, module='docutils.io') -__version__ = '1.7.2' -__released__ = '1.7.2' # used when Sphinx builds its own docs +__version__ = '1.7.3+' +__released__ = '1.7.3' # used when Sphinx builds its own docs # version info for better programmatic use # possible values for 3rd element: 'alpha', 'beta', 'rc', 'final' # 'final' has 0 as the last element -version_info = (1, 7, 2, 'final', 0) +version_info = (1, 7, 3, 'beta', 0) package_dir = path.abspath(path.dirname(__file__)) From c0ec1ab23e1844cf197363d547a9b5965cc377f0 Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Thu, 22 Mar 2018 23:20:02 +0900 Subject: [PATCH 24/24] Minor typo --- doc/setuptools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/setuptools.rst b/doc/setuptools.rst index 8d759f985..10f732559 100644 --- a/doc/setuptools.rst +++ b/doc/setuptools.rst @@ -62,7 +62,7 @@ Options for setuptools integration A boolean that determines whether the saved environment should be discarded on build. Default is false. - This can also be set by passing the `-E` flag to ``setup.py``. + This can also be set by passing the `-E` flag to ``setup.py``: .. code-block:: bash