From 82e05e7a9c7865c5f609e31ba70bab02ab521616 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 2 Aug 2015 00:07:23 +0900 Subject: [PATCH 01/11] Fixes #1725: On py2 environment, doctest with using non-ASCII characters causes `'ascii' codec can't decode byte` exception. --- CHANGES | 2 + sphinx/ext/doctest.py | 60 +++++++++++++++++++++------- tests/roots/test-doctest/doctest.txt | 9 +++++ 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/CHANGES b/CHANGES index 9ede62739..145260e26 100644 --- a/CHANGES +++ b/CHANGES @@ -47,6 +47,8 @@ Bugs fixed * #1869: Fix problems when dealing with files containing non-ASCII characters. Thanks to Marvin Schmidt. * #1798: Fix building LaTeX with references in titles. +* #1725: On py2 environment, doctest with using non-ASCII characters causes + ``'ascii' codec can't decode byte`` exception. Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py index fda4ea83b..e22024d42 100644 --- a/sphinx/ext/doctest.py +++ b/sphinx/ext/doctest.py @@ -9,14 +9,16 @@ :copyright: Copyright 2007-2015 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ +from __future__ import absolute_import import re import sys import time import codecs from os import path +import doctest -from six import itervalues, StringIO, binary_type +from six import itervalues, StringIO, binary_type, text_type, PY2 from docutils import nodes from docutils.parsers.rst import directives @@ -26,13 +28,30 @@ from sphinx.util import force_decode from sphinx.util.nodes import set_source_info from sphinx.util.compat import Directive from sphinx.util.console import bold - -# circumvent relative import -doctest = __import__('doctest') +from sphinx.util.osutil import fs_encoding blankline_re = re.compile(r'^\s*', re.MULTILINE) doctestopt_re = re.compile(r'#\s*doctest:.+$', re.MULTILINE) +if PY2: + def doctest_encode(text, encoding): + if isinstance(text, text_type): + text = text.encode(encoding) + if text.startswith(codecs.BOM_UTF8): + text = text[len(codecs.BOM_UTF8):] + return text +else: + def doctest_encode(text, encoding): + return text + + +class _SpoofOutSphinx(doctest._SpoofOut): + # override: convert console encoding to unicode + if PY2: + def getvalue(self): + result = doctest._SpoofOut.getvalue(self) + return result.decode('string_escape') + # set up the necessary directives @@ -165,6 +184,11 @@ class TestCode(object): class SphinxDocTestRunner(doctest.DocTestRunner): + def __init__(self, *args, **kw): + doctest.DocTestRunner.__init__(self, *args, **kw) + # Override a fake output target for capturing doctest output. + self._fakeout = _SpoofOutSphinx() + def summarize(self, out, verbose=None): string_io = StringIO() old_stdout = sys.stdout @@ -358,19 +382,25 @@ Doctest summary return compile(code, name, self.type, flags, dont_inherit) def test_group(self, group, filename): + if PY2: + filename_str = filename.encode(fs_encoding) + else: + filename_str = filename + ns = {} def run_setup_cleanup(runner, testcodes, what): examples = [] for testcode in testcodes: - examples.append(doctest.Example(testcode.code, '', - lineno=testcode.lineno)) + examples.append(doctest.Example( + doctest_encode(testcode.code, self.env.config.source_encoding), '', + lineno=testcode.lineno)) if not examples: return True # simulate a doctest with the code sim_doctest = doctest.DocTest(examples, {}, '%s (%s code)' % (group.name, what), - filename, 0, None) + filename_str, 0, None) sim_doctest.globs = ns old_f = runner.failures self.type = 'exec' # the snippet may contain multiple statements @@ -389,8 +419,9 @@ Doctest summary if len(code) == 1: # ordinary doctests (code/output interleaved) try: - test = parser.get_doctest(code[0].code, {}, group.name, - filename, code[0].lineno) + test = parser.get_doctest( + doctest_encode(code[0].code, self.env.config.source_encoding), {}, + group.name, filename_str, code[0].lineno) except Exception: self.warn('ignoring invalid doctest code: %r' % code[0].code, @@ -416,12 +447,13 @@ Doctest summary exc_msg = m.group('msg') else: exc_msg = None - example = doctest.Example(code[0].code, output, - exc_msg=exc_msg, - lineno=code[0].lineno, - options=options) + example = doctest.Example( + doctest_encode(code[0].code, self.env.config.source_encoding), output, + exc_msg=exc_msg, + lineno=code[0].lineno, + options=options) test = doctest.DocTest([example], {}, group.name, - filename, code[0].lineno, None) + filename_str, code[0].lineno, None) self.type = 'exec' # multiple statements again # DocTest.__init__ copies the globs namespace, which we don't want test.globs = ns diff --git a/tests/roots/test-doctest/doctest.txt b/tests/roots/test-doctest/doctest.txt index ce4d88bd8..ac5da0d15 100644 --- a/tests/roots/test-doctest/doctest.txt +++ b/tests/roots/test-doctest/doctest.txt @@ -127,3 +127,12 @@ Special directives import test_ext_doctest test_ext_doctest.cleanup_call() + +non-ASCII result +---------------- + +>>> print('umlauts: äöü.') +umlauts: äöü. +>>> print('Japanese: 日本語') +Japanese: 日本語 + From 49b812b856f3c81c6edee5ba793405c7ed927cf3 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 9 Aug 2015 21:08:14 +0900 Subject: [PATCH 02/11] Fix #1540: Fix RuntimeError with circular referenced toctree --- CHANGES | 1 + sphinx/builders/html.py | 2 +- sphinx/builders/latex.py | 2 +- sphinx/builders/manpage.py | 2 +- sphinx/builders/texinfo.py | 2 +- sphinx/util/nodes.py | 38 ++++++++++++++++++++------------------ 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/CHANGES b/CHANGES index 145260e26..3311e571e 100644 --- a/CHANGES +++ b/CHANGES @@ -49,6 +49,7 @@ Bugs fixed * #1798: Fix building LaTeX with references in titles. * #1725: On py2 environment, doctest with using non-ASCII characters causes ``'ascii' codec can't decode byte`` exception. +* #1540: Fix RuntimeError with circular referenced toctree Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index b0ae48205..7006eff79 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -938,7 +938,7 @@ class SingleFileHTMLBuilder(StandaloneHTMLBuilder): def assemble_doctree(self): master = self.config.master_doc tree = self.env.get_doctree(master) - tree = inline_all_toctrees(self, set(), master, tree, darkgreen) + tree = inline_all_toctrees(self, set(), master, tree, darkgreen, [master]) tree['docname'] = master self.env.resolve_references(tree, master, self) self.fix_refuris(tree) diff --git a/sphinx/builders/latex.py b/sphinx/builders/latex.py index f0c1d3a7c..44cab2cfd 100644 --- a/sphinx/builders/latex.py +++ b/sphinx/builders/latex.py @@ -124,7 +124,7 @@ class LaTeXBuilder(Builder): new_sect += node tree = new_tree largetree = inline_all_toctrees(self, self.docnames, indexfile, tree, - darkgreen) + darkgreen, [indexfile]) largetree['docname'] = indexfile for docname in appendices: appendix = self.env.get_doctree(docname) diff --git a/sphinx/builders/manpage.py b/sphinx/builders/manpage.py index 0176621f5..2af853dd8 100644 --- a/sphinx/builders/manpage.py +++ b/sphinx/builders/manpage.py @@ -70,7 +70,7 @@ class ManualPageBuilder(Builder): tree = self.env.get_doctree(docname) docnames = set() largetree = inline_all_toctrees(self, docnames, docname, tree, - darkgreen) + darkgreen, [docname]) self.info('} ', nonl=True) self.env.resolve_references(largetree, docname, self) # remove pending_xref nodes diff --git a/sphinx/builders/texinfo.py b/sphinx/builders/texinfo.py index 3adeec404..7528c0777 100644 --- a/sphinx/builders/texinfo.py +++ b/sphinx/builders/texinfo.py @@ -180,7 +180,7 @@ class TexinfoBuilder(Builder): new_sect += node tree = new_tree largetree = inline_all_toctrees(self, self.docnames, indexfile, tree, - darkgreen) + darkgreen, [indexfile]) largetree['docname'] = indexfile for docname in appendices: appendix = self.env.get_doctree(docname) diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index ea26ecdbe..e3f1f5172 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -228,7 +228,7 @@ def process_index_entry(entry, targetid): return indexentries -def inline_all_toctrees(builder, docnameset, docname, tree, colorfunc): +def inline_all_toctrees(builder, docnameset, docname, tree, colorfunc, traversed): """Inline all toctrees in the *tree*. Record all docnames in *docnameset*, and output docnames with *colorfunc*. @@ -238,23 +238,25 @@ def inline_all_toctrees(builder, docnameset, docname, tree, colorfunc): newnodes = [] includefiles = map(text_type, toctreenode['includefiles']) for includefile in includefiles: - try: - builder.info(colorfunc(includefile) + " ", nonl=1) - subtree = inline_all_toctrees(builder, docnameset, includefile, - builder.env.get_doctree(includefile), - colorfunc) - docnameset.add(includefile) - except Exception: - builder.warn('toctree contains ref to nonexisting ' - 'file %r' % includefile, - builder.env.doc2path(docname)) - else: - sof = addnodes.start_of_file(docname=includefile) - sof.children = subtree.children - for sectionnode in sof.traverse(nodes.section): - if 'docname' not in sectionnode: - sectionnode['docname'] = includefile - newnodes.append(sof) + if includefile not in traversed: + try: + traversed.append(includefile) + builder.info(colorfunc(includefile) + " ", nonl=1) + subtree = inline_all_toctrees(builder, docnameset, includefile, + builder.env.get_doctree(includefile), + colorfunc, traversed) + docnameset.add(includefile) + except Exception: + builder.warn('toctree contains ref to nonexisting ' + 'file %r' % includefile, + builder.env.doc2path(docname)) + else: + sof = addnodes.start_of_file(docname=includefile) + sof.children = subtree.children + for sectionnode in sof.traverse(nodes.section): + if 'docname' not in sectionnode: + sectionnode['docname'] = includefile + newnodes.append(sof) toctreenode.parent.replace(toctreenode, newnodes) return tree From a3109f697647fc880a2b0886106000ed73783c07 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 9 Aug 2015 23:41:47 +0900 Subject: [PATCH 03/11] Fixes #1983: i18n translation feature breaks references which uses section name. --- CHANGES | 1 + sphinx/transforms.py | 6 ++- tests/roots/test-intl/contents.txt | 1 + tests/roots/test-intl/refs.po | 85 ++++++++++++++++++++++++++++++ tests/roots/test-intl/refs.txt | 47 +++++++++++++++++ tests/test_intl.py | 8 +++ 6 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 tests/roots/test-intl/refs.po create mode 100644 tests/roots/test-intl/refs.txt diff --git a/CHANGES b/CHANGES index 3311e571e..4ddb209f2 100644 --- a/CHANGES +++ b/CHANGES @@ -50,6 +50,7 @@ Bugs fixed * #1725: On py2 environment, doctest with using non-ASCII characters causes ``'ascii' codec can't decode byte`` exception. * #1540: Fix RuntimeError with circular referenced toctree +* #1983: i18n translation feature breaks references which uses section name. Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/transforms.py b/sphinx/transforms.py index 3d12eb230..afb91d935 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -277,8 +277,10 @@ class Locale(Transform): # document nameids mapping with new name. names = section_node.setdefault('names', []) names.append(new_name) - if old_name in names: - names.remove(old_name) + # Original section name (reference target name) should be kept to refer + # from other nodes which is still not translated or uses explicit target + # name like "`text to display `_".. + # So, `old_name` is still exist in `names`. _id = self.document.nameids.get(old_name, None) explicit = self.document.nametypes.get(old_name, None) diff --git a/tests/roots/test-intl/contents.txt b/tests/roots/test-intl/contents.txt index 418b3022c..8882137f3 100644 --- a/tests/roots/test-intl/contents.txt +++ b/tests/roots/test-intl/contents.txt @@ -22,3 +22,4 @@ CONTENTS versionchange docfields raw + refs diff --git a/tests/roots/test-intl/refs.po b/tests/roots/test-intl/refs.po new file mode 100644 index 000000000..510a5a7fc --- /dev/null +++ b/tests/roots/test-intl/refs.po @@ -0,0 +1,85 @@ +# +msgid "" +msgstr "" +"Project-Id-Version: 1191 1.3\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2015-08-08 15:31+0900\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "Translation Tips" +msgstr "X TIPS" + +msgid "A-1. Here's how you can `download Sphinx`_." +msgstr "A-1. HERE'S HOW YOU CAN `download Sphinx`_." + +msgid "A-2. Here's how you can `download Sphinx`_." +msgstr "A-2. HERE'S HOW YOU CAN `A1 DOWNLOAD SPHINX`_." + +msgid "A-3. Here's how you can `download Sphinx`_." +msgstr "" +"A-3. HERE'S HOW YOU CAN `A3 DOWNLOAD SPHINX `_ AND `A3 DOWNLOAD " +"SPHINX `_." + +msgid "B-1. `Docutils site`_ and `Sphinx site`_." +msgstr "B-1. `Docutils site`_ and `Sphinx site`_." + +msgid "B-2. `Docutils site`_ and `Sphinx site`_." +msgstr "B-2. `B1 DOCUTILS SITE`_ AND `B1 SPHINX SITE`_." + +msgid "B-3. `Docutils site`_ and `Sphinx site`_." +msgstr "B-3. `B2 SPHINX SITE`_ AND `B2 DOCUTILS SITE`_." + +msgid "B-4. `Docutils site`_ and `Sphinx site`_." +msgstr "" +"B-4. `B4 SPHINX SITE `_ AND `B4 DOCUTILS SITE `_." + +msgid "B-5. `Docutils site`_ and `Sphinx site`_." +msgstr "" +"B-5. `B5 SPHINX SITE `_ AND `B5 DOCUTILS SITE `_\" AND `B5 SPHINX SITE `_." + +msgid "C-1. Link to `Translation Tips`_ section." +msgstr "C-1. LINK TO `Translation Tips`_ SECTION." + +msgid "C-2. Link to `Translation Tips`_ section." +msgstr "C-2. LINK TO `X TIPS`_ SECTION." + +msgid "C-3. Link to `Translation Tips`_ section." +msgstr "C-3. LINK TO `X TIPS `_ SECTION." + +msgid "C-4. Link to `Translation Tips`_ section." +msgstr "" +"C-4. LINK TO `X TIPS `_ x `X TIPS `_ " +"SECTION." + +msgid "C-5. Link to `Translation Tips`_ section." +msgstr "" +"C-5. LINK TO `TRANS `_ x `LATION `_ " + +msgid "D-1. Link to `Translation Tips`_ and `Next Section`_ section." +msgstr "D-1. LINK TO `Translation Tips`_ and `Next Section`_ SECTION." + +msgid "D-2. Link to `Translation Tips`_ and `Next Section`_ section." +msgstr "D-2. LINK TO `X TIPS`_ AND `N SECTION`_ SECTION." + +msgid "D-3. Link to `Translation Tips`_ and `Next Section`_ section." +msgstr "D-3. LINK TO `N SECTION`_ AND `X TIPS`_ SECTION." + +msgid "D-4. Link to `Translation Tips`_ and `Next Section`_ section." +msgstr "" +"D-4. LINK TO `N SECTION `_ AND `X TIPS `_ " +"SECTION." + +msgid "D-5. Link to `Translation Tips`_ and `Next Section`_ section." +msgstr "" +"D-5. LINK TO `Next `_ AND `Tips `_ " + +msgid "Next Section" +msgstr "N SECTION" + diff --git a/tests/roots/test-intl/refs.txt b/tests/roots/test-intl/refs.txt new file mode 100644 index 000000000..b093dbe60 --- /dev/null +++ b/tests/roots/test-intl/refs.txt @@ -0,0 +1,47 @@ +References +=========== + +Translation Tips +----------------- + +.. _download Sphinx: https://pypi.python.org/pypi/sphinx +.. _Docutils site: http://docutils.sourceforge.net/ +.. _Sphinx site: http://sphinx-doc.org/ + + +A-1. Here's how you can `download Sphinx`_. + +A-2. Here's how you can `download Sphinx`_. + +A-3. Here's how you can `download Sphinx`_. + +B-1. `Docutils site`_ and `Sphinx site`_. + +B-2. `Docutils site`_ and `Sphinx site`_. + +B-3. `Docutils site`_ and `Sphinx site`_. + +B-4. `Docutils site`_ and `Sphinx site`_. + +C-1. Link to `Translation Tips`_ section. + +C-2. Link to `Translation Tips`_ section. + +C-3. Link to `Translation Tips`_ section. + +C-4. Link to `Translation Tips`_ section. + +C-5. Link to `Translation Tips`_ section. + +D-1. Link to `Translation Tips`_ and `Next Section`_ section. + +D-2. Link to `Translation Tips`_ and `Next Section`_ section. + +D-3. Link to `Translation Tips`_ and `Next Section`_ section. + +D-4. Link to `Translation Tips`_ and `Next Section`_ section. + +D-5. Link to `Translation Tips`_ and `Next Section`_ section. + +Next Section +------------- diff --git a/tests/test_intl.py b/tests/test_intl.py index 965fdf187..dabb2a3f5 100644 --- a/tests/test_intl.py +++ b/tests/test_intl.py @@ -751,3 +751,11 @@ def test_additional_targets_should_be_translated(app, status, warning): expected_expr = """IMG -> I18N""" yield assert_count(expected_expr, result, 1) + +@gen_with_intl_app('text', freshenv=True) +def test_references(app, status, warning): + app.builder.build_specific([app.srcdir / 'refs.txt']) + + warnings = warning.getvalue().replace(os.sep, '/') + warning_expr = u'refs.txt:\\d+: ERROR: Unknown target name:' + yield assert_count(warning_expr, warnings, 0) From bf7b410f6a302b3c6c1106994e8cfeca91162a27 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 9 Aug 2015 23:46:08 +0900 Subject: [PATCH 04/11] Fix: "building [mo]" status indicates absolute path instead of relative from source directory --- sphinx/builders/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 7f5da17d2..eebd6af64 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -160,10 +160,14 @@ class Builder(object): def compile_catalogs(self, catalogs, message): if not self.config.gettext_auto_build: return + + def cat2relpath(cat): + return path.relpath(cat.mo_path, self.env.srcdir).replace(path.sep, SEP) + self.info(bold('building [mo]: ') + message) for catalog in self.app.status_iterator( catalogs, 'writing output... ', darkgreen, len(catalogs), - lambda c: c.mo_path): + cat2relpath): catalog.write_mo(self.config.language) def compile_all_catalogs(self): From 20d992cdf52c45a447947373e150b7d9f183084f Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 21 Aug 2015 12:59:54 +0900 Subject: [PATCH 05/11] Close #1990: Use caption of toctree to title of \tableofcontents in LaTeX --- sphinx/builders/latex.py | 11 +++++++++ sphinx/writers/latex.py | 33 +++++++++++++++++-------- tests/roots/test-contentsname/bar.rst | 4 +++ tests/roots/test-contentsname/conf.py | 4 +++ tests/roots/test-contentsname/foo.rst | 4 +++ tests/roots/test-contentsname/index.rst | 8 ++++++ tests/test_build_latex.py | 22 +++++++++++++++++ 7 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 tests/roots/test-contentsname/bar.rst create mode 100644 tests/roots/test-contentsname/conf.py create mode 100644 tests/roots/test-contentsname/foo.rst create mode 100644 tests/roots/test-contentsname/index.rst diff --git a/sphinx/builders/latex.py b/sphinx/builders/latex.py index 44cab2cfd..1eef58d66 100644 --- a/sphinx/builders/latex.py +++ b/sphinx/builders/latex.py @@ -102,11 +102,22 @@ class LaTeXBuilder(Builder): doctree.settings = docsettings doctree.settings.author = author doctree.settings.title = title + doctree.settings.contentsname = self.get_contentsname(docname) doctree.settings.docname = docname doctree.settings.docclass = docclass docwriter.write(doctree, destination) self.info("done") + def get_contentsname(self, indexfile): + tree = self.env.get_doctree(indexfile) + contentsname = None + for toctree in tree.traverse(addnodes.toctree): + if toctree['caption']: + contentsname = toctree['caption'] + break + + return contentsname + def assemble_doctree(self, indexfile, toctree_only, appendices): self.docnames = set([indexfile] + appendices) self.info(darkgreen(indexfile) + " ", nonl=1) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 54dfeb189..93592300b 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -43,6 +43,7 @@ HEADER = r'''%% Generated by Sphinx. \usepackage{sphinx} \usepackage{multirow} %(usepackages)s +%(contentsname)s %(numfig_format)s %(preamble)s @@ -159,6 +160,7 @@ class LaTeXTranslator(nodes.NodeVisitor): 'longtable': '\\usepackage{longtable}', 'usepackages': '', 'numfig_format': '', + 'contentsname': '', 'preamble': '', 'title': '', 'date': '', @@ -248,6 +250,8 @@ class LaTeXTranslator(nodes.NodeVisitor): return '\\usepackage{%s}' % (packagename,) usepackages = (declare_package(*p) for p in builder.usepackages) self.elements['usepackages'] += "\n".join(usepackages) + if getattr(document.settings, 'contentsname', None): + self.elements['contentsname'] = self.babel_renewcommand(builder, '\\contentsname', document.settings.contentsname) self.elements['numfig_format'] = self.generate_numfig_format(builder) # allow the user to override them all self.elements.update(builder.config.latex_elements) @@ -329,8 +333,7 @@ class LaTeXTranslator(nodes.NodeVisitor): encode('ascii', 'backslashreplace').decode('ascii').\ replace('\\', '_') - def generate_numfig_format(self, builder): - ret = [] + def babel_renewcommand(self, builder, command, definition): if builder.config.language == 'ja': babel_prefix = '' babel_suffix = '' @@ -349,15 +352,27 @@ class LaTeXTranslator(nodes.NodeVisitor): babel_prefix = '' babel_suffix = '' + return ('%s\\renewcommand{%s}{%s}%s\n' % + (babel_prefix, command, definition, babel_suffix)) + + def generate_contentsname(self, builder, document): + print '---' + print '---' + print '---' + for toctree in document.traverse(addnodes.toctree): + if toctree['caption']: + print toctree['caption'] + return '' + + def generate_numfig_format(self, builder): + ret = [] figure = self.builder.config.numfig_format['figure'].split('%s', 1) if len(figure) == 1: ret.append('\\def\\fnum@figure{%s}\n' % text_type(figure[0]).translate(tex_escape_map)) else: - ret.append('%s\\renewcommand{\\figurename}{%s}%s\n' % - (babel_prefix, - text_type(figure[0]).translate(tex_escape_map), - babel_suffix)) + definition = text_type(figure[0]).translate(tex_escape_map) + ret.append(self.babel_renewcommand(builder, '\\figurename', definition)) if figure[1]: ret.append('\\makeatletter\n') ret.append('\\def\\fnum@figure{\\figurename\\thefigure%s}\n' % @@ -369,10 +384,8 @@ class LaTeXTranslator(nodes.NodeVisitor): ret.append('\\def\\fnum@table{%s}\n' % text_type(table[0]).translate(tex_escape_map)) else: - ret.append('%s\\renewcommand{\\tablename}{%s}%s\n' % - (babel_prefix, - text_type(table[0]).translate(tex_escape_map), - babel_suffix)) + definition = text_type(table[0]).translate(tex_escape_map) + ret.append(self.babel_renewcommand(builder, '\\tablename', definition)) if table[1]: ret.append('\\makeatletter\n') ret.append('\\def\\fnum@table{\\tablename\\thetable%s}\n' % diff --git a/tests/roots/test-contentsname/bar.rst b/tests/roots/test-contentsname/bar.rst new file mode 100644 index 000000000..c1ddf304d --- /dev/null +++ b/tests/roots/test-contentsname/bar.rst @@ -0,0 +1,4 @@ +=== +Bar +=== + diff --git a/tests/roots/test-contentsname/conf.py b/tests/roots/test-contentsname/conf.py new file mode 100644 index 000000000..cf05c9b5c --- /dev/null +++ b/tests/roots/test-contentsname/conf.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +master_doc = 'index' +html_theme = 'classic' diff --git a/tests/roots/test-contentsname/foo.rst b/tests/roots/test-contentsname/foo.rst new file mode 100644 index 000000000..cecc672ce --- /dev/null +++ b/tests/roots/test-contentsname/foo.rst @@ -0,0 +1,4 @@ +=== +Foo +=== + diff --git a/tests/roots/test-contentsname/index.rst b/tests/roots/test-contentsname/index.rst new file mode 100644 index 000000000..7c19f9efd --- /dev/null +++ b/tests/roots/test-contentsname/index.rst @@ -0,0 +1,8 @@ +test-tocdepth +============= + +.. toctree:: + :caption: Table of content + + foo + bar diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py index f7c6fcc01..639204ca1 100644 --- a/tests/test_build_latex.py +++ b/tests/test_build_latex.py @@ -275,3 +275,25 @@ def test_latex_add_latex_package(app, status, warning): result = (app.outdir / 'SphinxTests.tex').text(encoding='utf8') assert '\\usepackage{foo}' in result assert '\\usepackage[baz]{bar}' in result + + +@with_app(buildername='latex', testroot='contentsname') +def test_contentsname(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'Python.tex').text(encoding='utf8') + print(result) + print(status.getvalue()) + print(warning.getvalue()) + assert ('\\addto\\captionsenglish{\\renewcommand{\\contentsname}{Table of content}}' + in result) + + +@with_app(buildername='latex', testroot='contentsname', + confoverrides={'language': 'ja'}) +def test_contentsname_with_language_ja(app, status, warning): + app.builder.build_all() + result = (app.outdir / 'Python.tex').text(encoding='utf8') + print(result) + print(status.getvalue()) + print(warning.getvalue()) + assert '\\renewcommand{\\contentsname}{Table of content}' in result From e1305d549d27d622fe6c945f55d388403ee6eeb6 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 21 Aug 2015 14:29:01 +0900 Subject: [PATCH 06/11] Fix flake8 violation --- CHANGES | 1 + sphinx/writers/latex.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 4ddb209f2..522213fe2 100644 --- a/CHANGES +++ b/CHANGES @@ -51,6 +51,7 @@ Bugs fixed ``'ascii' codec can't decode byte`` exception. * #1540: Fix RuntimeError with circular referenced toctree * #1983: i18n translation feature breaks references which uses section name. +* #1990: Use caption of toctree to title of \tableofcontents in LaTeX Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 93592300b..142e368be 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -251,7 +251,9 @@ class LaTeXTranslator(nodes.NodeVisitor): usepackages = (declare_package(*p) for p in builder.usepackages) self.elements['usepackages'] += "\n".join(usepackages) if getattr(document.settings, 'contentsname', None): - self.elements['contentsname'] = self.babel_renewcommand(builder, '\\contentsname', document.settings.contentsname) + self.elements['contentsname'] = \ + self.babel_renewcommand(builder, '\\contentsname', + document.settings.contentsname) self.elements['numfig_format'] = self.generate_numfig_format(builder) # allow the user to override them all self.elements.update(builder.config.latex_elements) From 3803ce7db49fd9b9823881bcbba12175e60d8173 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 21 Aug 2015 14:33:36 +0900 Subject: [PATCH 07/11] Remove debug print --- sphinx/writers/latex.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 142e368be..01a438738 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -358,9 +358,6 @@ class LaTeXTranslator(nodes.NodeVisitor): (babel_prefix, command, definition, babel_suffix)) def generate_contentsname(self, builder, document): - print '---' - print '---' - print '---' for toctree in document.traverse(addnodes.toctree): if toctree['caption']: print toctree['caption'] From 5be5b057a9bfc1270420589a76e66f2ef44826f4 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Fri, 21 Aug 2015 14:37:51 +0900 Subject: [PATCH 08/11] Remove unused method: LaTeXTranslator.generate_contentsname() --- sphinx/writers/latex.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 01a438738..920df5940 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -357,12 +357,6 @@ class LaTeXTranslator(nodes.NodeVisitor): return ('%s\\renewcommand{%s}{%s}%s\n' % (babel_prefix, command, definition, babel_suffix)) - def generate_contentsname(self, builder, document): - for toctree in document.traverse(addnodes.toctree): - if toctree['caption']: - print toctree['caption'] - return '' - def generate_numfig_format(self, builder): ret = [] figure = self.builder.config.numfig_format['figure'].split('%s', 1) From 2c5c54aa67869ca5d3ae36f8392b6ab170c20c9a Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Tue, 1 Sep 2015 00:28:33 +0900 Subject: [PATCH 09/11] Closes #1987: Fix ampersand is ignored in ``:menuselection:`` and ``:guilabel:`` on LaTeX builder --- CHANGES | 1 + sphinx/writers/latex.py | 3 +++ tests/test_markup.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 522213fe2..9955865de 100644 --- a/CHANGES +++ b/CHANGES @@ -52,6 +52,7 @@ Bugs fixed * #1540: Fix RuntimeError with circular referenced toctree * #1983: i18n translation feature breaks references which uses section name. * #1990: Use caption of toctree to title of \tableofcontents in LaTeX +* #1987: Fix ampersand is ignored in ``:menuselection:`` and ``:guilabel:`` on LaTeX builder Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index 920df5940..d82e7dd76 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -1713,6 +1713,9 @@ class LaTeXTranslator(nodes.NodeVisitor): if classes in [['menuselection'], ['guilabel']]: self.body.append(r'\emph{') self.context.append('}') + elif classes in [['accelerator']]: + self.body.append(r'\underline{') + self.context.append('}') elif classes and not self.in_title: self.body.append(r'\DUspan{%s}{' % ','.join(classes)) self.context.append('}') diff --git a/tests/test_markup.py b/tests/test_markup.py index e741571fb..a5804366f 100644 --- a/tests/test_markup.py +++ b/tests/test_markup.py @@ -109,7 +109,7 @@ def test_inline(): yield (verify, ':guilabel:`&Foo -&&- &Bar`', u'

Foo ' '-&- Bar

', - r'\emph{\DUspan{accelerator}{F}oo -\&- \DUspan{accelerator}{B}ar}') + r'\emph{\underline{F}oo -\&- \underline{B}ar}') # non-interpolation of dashes in option role yield (verify_re, ':option:`--with-option`', From f886c54263150b03b6df1dc64eb38ede72c42a89 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 16 Aug 2015 19:20:45 +0900 Subject: [PATCH 10/11] Fix #1994: More supporting non-standard parser (like recommonmark parser) for Translation and WebSupport feature. Now node.rawsource is fall backed to node.astext() during docutils transforming. --- CHANGES | 3 +++ sphinx/directives/other.py | 9 +++++---- sphinx/environment.py | 10 ++++++---- sphinx/transforms.py | 13 +++++++++++++ sphinx/util/nodes.py | 10 ++++++++-- tests/test_util_nodes.py | 36 +++++++++++++++++++++++++++++++++++- 6 files changed, 70 insertions(+), 11 deletions(-) diff --git a/CHANGES b/CHANGES index 9955865de..05ac85393 100644 --- a/CHANGES +++ b/CHANGES @@ -53,6 +53,9 @@ Bugs fixed * #1983: i18n translation feature breaks references which uses section name. * #1990: Use caption of toctree to title of \tableofcontents in LaTeX * #1987: Fix ampersand is ignored in ``:menuselection:`` and ``:guilabel:`` on LaTeX builder +* #1994: More supporting non-standard parser (like recommonmark parser) for Translation and + WebSupport feature. Now node.rawsource is fall backed to node.astext() during docutils + transforming. Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/directives/other.py b/sphinx/directives/other.py index 2b941c31f..794dbda5e 100644 --- a/sphinx/directives/other.py +++ b/sphinx/directives/other.py @@ -142,7 +142,7 @@ class Author(Directive): env = self.state.document.settings.env if not env.config.show_authors: return [] - para = nodes.paragraph() + para = nodes.paragraph(translatable=False) emph = nodes.emphasis() para += emph if self.name == 'sectionauthor': @@ -205,7 +205,7 @@ class VersionChange(Directive): if len(self.arguments) == 2: inodes, messages = self.state.inline_text(self.arguments[1], self.lineno+1) - para = nodes.paragraph(self.arguments[1], '', *inodes) + para = nodes.paragraph(self.arguments[1], '', *inodes, translatable=False) set_source_info(self, para) node.append(para) else: @@ -218,13 +218,14 @@ class VersionChange(Directive): content.source = node[0].source content.line = node[0].line content += node[0].children - node[0].replace_self(nodes.paragraph('', '', content)) + node[0].replace_self(nodes.paragraph('', '', content, translatable=False)) node[0].insert(0, nodes.inline('', '%s: ' % text, classes=['versionmodified'])) else: para = nodes.paragraph('', '', nodes.inline('', '%s.' % text, - classes=['versionmodified'])) + classes=['versionmodified']), + translatable=False) node.append(para) env = self.state.document.settings.env # XXX should record node.source as well diff --git a/sphinx/environment.py b/sphinx/environment.py index 47ebf7bd8..da92f2e81 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -49,9 +49,11 @@ from sphinx.util.websupport import is_commentable from sphinx.errors import SphinxError, ExtensionError from sphinx.locale import _ from sphinx.versioning import add_uids, merge_doctrees -from sphinx.transforms import DefaultSubstitutions, MoveModuleTargets, \ - HandleCodeBlocks, AutoNumbering, SortIds, CitationReferences, Locale, \ - RemoveTranslatableInline, SphinxContentsFilter, ExtraTranslatableNodes +from sphinx.transforms import ( + DefaultSubstitutions, MoveModuleTargets, ApplySourceWorkaround, + HandleCodeBlocks, AutoNumbering, SortIds, CitationReferences, Locale, + RemoveTranslatableInline, SphinxContentsFilter, ExtraTranslatableNodes, +) orig_role_function = roles.role @@ -99,7 +101,7 @@ class SphinxStandaloneReader(standalone.Reader): """ Add our own transforms. """ - transforms = [ExtraTranslatableNodes, Locale, CitationReferences, + transforms = [ApplySourceWorkaround, ExtraTranslatableNodes, Locale, CitationReferences, DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, AutoNumbering, SortIds, RemoveTranslatableInline] diff --git a/sphinx/transforms.py b/sphinx/transforms.py index afb91d935..d4ad4316e 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -22,6 +22,7 @@ from sphinx.locale import _, init as init_locale from sphinx.util import split_index_msg from sphinx.util.nodes import ( traverse_translatable_index, extract_messages, LITERAL_TYPE_NODES, IMAGE_TYPE_NODES, + apply_source_workaround, ) from sphinx.util.osutil import ustrftime from sphinx.util.i18n import find_catalog @@ -169,6 +170,18 @@ TRANSLATABLE_NODES = { } +class ApplySourceWorkaround(Transform): + """ + update source and rawsource attributes + """ + default_priority = 10 + + def apply(self): + for n in self.document.traverse(): + if isinstance(n, nodes.TextElement): + apply_source_workaround(n) + + class ExtraTranslatableNodes(Transform): """ make nodes translatable diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index e3f1f5172..2b0fff81d 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -50,6 +50,10 @@ def apply_source_workaround(node): # overwrite: ``term : classifier1 : classifier2`` -> ``term text`` node.rawsource = node.astext() + # workaround: recommonmark-0.2.0 doesn't set rawsource attribute + if not node.rawsource: + node.rawsource = node.astext() + if node.source and node.rawsource: return @@ -74,18 +78,20 @@ IGNORED_NODES = ( nodes.Inline, nodes.literal_block, nodes.doctest_block, + addnodes.versionmodified, # XXX there are probably more ) def is_translatable(node): if isinstance(node, nodes.TextElement): - apply_source_workaround(node) - if not node.source: return False # built-in message if isinstance(node, IGNORED_NODES) and 'translatable' not in node: return False + if not node.get('translatable', True): + # not(node['translatable'] == True or node['translatable'] is None) + return False # orphan # XXX ignore all metadata (== docinfo) if isinstance(node, nodes.field_name) and node.children[0] == 'orphan': diff --git a/tests/test_util_nodes.py b/tests/test_util_nodes.py index 5cdd20a19..a14e850fe 100644 --- a/tests/test_util_nodes.py +++ b/tests/test_util_nodes.py @@ -16,13 +16,24 @@ from docutils.utils import new_document from docutils import frontend from sphinx.util.nodes import extract_messages +from sphinx.transforms import ApplySourceWorkaround -def _get_doctree(text): +def _transform(doctree): + ApplySourceWorkaround(doctree).apply() + + +def create_new_document(): settings = frontend.OptionParser( components=(rst.Parser,)).get_default_values() document = new_document('dummy.txt', settings) + return document + + +def _get_doctree(text): + document = create_new_document() rst.Parser().parse(text, document) + _transform(document) return document @@ -119,3 +130,26 @@ def test_extract_messages(): extract_messages(_get_doctree(text)), nodes.line, 2, ) + + +def test_extract_messages_without_rawsource(): + """ + Check node.rawsource is fall-backed by using node.astext() value. + + `extract_message` which is used from Sphinx i18n feature drop ``not node.rawsource`` nodes. + So, all nodes which want to translate must have ``rawsource`` value. + However, sometimes node.rawsource is not set. + + For example: recommonmark-0.2.0 doesn't set rawsource to `paragraph` node. + + refs #1994: Fall back to node's astext() during i18n message extraction. + """ + p = nodes.paragraph() + p.append(nodes.Text('test')) + p.append(nodes.Text('sentence')) + assert not p.rawsource # target node must not have rawsource value + document = create_new_document() + document.append(p) + _transform(document) + assert_node_count(extract_messages(document), nodes.TextElement, 1) + assert [m for n, m in extract_messages(document)][0], 'text sentence' From 6bce0a1c10f7ebeda7b54afed88c27dca185f1ee Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sun, 13 Sep 2015 09:37:41 +0900 Subject: [PATCH 11/11] Avoid "2.0" version of Babel because it doesn't work with Windows environment. Closes #1976. see also: * https://github.com/mitsuhiko/babel/issues/174 * https://github.com/mitsuhiko/babel/pull/188 Version spec syntax "babel>=1.3,!=2.0" is following PEP440: https://www.python.org/dev/peps/pep-0440/#version-exclusion and it works with pip 6.0 or later (I didn't check before pip 6.0). --- CHANGES | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 05ac85393..9758dfc43 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,7 @@ Features added Bugs fixed ---------- +* #1976: Avoid "2.0" version of Babel because it doesn't work with Windows environment. * Add a "default.css" stylesheet (which imports "classic.css") for compatibility. * #1788: graphviz extension raises exception when caption option is present. * #1789: ``:pyobject:`` option of ``literalinclude`` directive includes following diff --git a/setup.py b/setup.py index 297891545..e722c909b 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ requires = [ 'Pygments>=2.0', 'docutils>=0.11', 'snowballstemmer>=1.1', - 'babel>=1.3', + 'babel>=1.3,!=2.0', 'alabaster>=0.7,<0.8', 'sphinx_rtd_theme>=0.1,<0.2', ]