From c1c0a03895c1bb73f364eca8605e8503b829ec4d Mon Sep 17 00:00:00 2001 From: Takayuki Shimizukawa Date: Mon, 4 Feb 2013 22:23:03 +0900 Subject: [PATCH 1/9] fix: reporting correct line number when translations have wrong reST syntax or else. --- sphinx/environment.py | 16 ++++++++++++++++ tests/roots/test-intl/contents.txt | 1 + tests/roots/test-intl/warnings.po | 23 +++++++++++++++++++++++ tests/roots/test-intl/warnings.txt | 5 +++++ tests/test_intl.py | 17 +++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 tests/roots/test-intl/warnings.po create mode 100644 tests/roots/test-intl/warnings.txt diff --git a/sphinx/environment.py b/sphinx/environment.py index 2995cbbc0..2e70569bc 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -189,6 +189,20 @@ class CitationReferences(Transform): citnode.parent.replace(citnode, refnode) +class CustomLocaleReporter(object): + """ + Replacer for document.reporter.get_source_and_line method. + + reST text lines for translation not have original source line number. + This class provide correct line number at reporting. + """ + def __init__(self, source, line): + self.source, self.line = source, line + + def get_source_and_line(self, lineno=None): + return self.source, self.line + + class Locale(Transform): """ Replace translatable nodes with their translated doctree. @@ -229,6 +243,8 @@ class Locale(Transform): # dummy literal node will discard by 'patch = patch[0]' patch = new_document(source, settings) + patch.reporter.get_source_and_line = CustomLocaleReporter( + node.source, node.line).get_source_and_line parser.parse(msgstr, patch) patch = patch[0] # XXX doctest and other block markup diff --git a/tests/roots/test-intl/contents.txt b/tests/roots/test-intl/contents.txt index a08cb0d3c..932a58dd6 100644 --- a/tests/roots/test-intl/contents.txt +++ b/tests/roots/test-intl/contents.txt @@ -4,6 +4,7 @@ subdir/contents bom + warnings footnote external_links refs_inconsistency diff --git a/tests/roots/test-intl/warnings.po b/tests/roots/test-intl/warnings.po new file mode 100644 index 000000000..bf82510ee --- /dev/null +++ b/tests/roots/test-intl/warnings.po @@ -0,0 +1,23 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2010, Georg Brandl & Team +# This file is distributed under the same license as the Sphinx package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: Sphinx 0.6\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2013-02-04 13:06\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "i18n with reST warnings" +msgstr "I18N WITH REST WARNINGS" + +msgid "line of ``literal`` markup." +msgstr "LINE OF ``BROKEN LITERAL MARKUP." diff --git a/tests/roots/test-intl/warnings.txt b/tests/roots/test-intl/warnings.txt new file mode 100644 index 000000000..a80fe183d --- /dev/null +++ b/tests/roots/test-intl/warnings.txt @@ -0,0 +1,5 @@ +i18n with reST warnings +======================== + +line of ``literal`` markup. + diff --git a/tests/test_intl.py b/tests/test_intl.py index 06181c3bc..9336beade 100644 --- a/tests/test_intl.py +++ b/tests/test_intl.py @@ -89,6 +89,23 @@ def test_subdir(app): assert result.startswith(u"\nsubdir contents\n***************\n") +@with_intl_app(buildername='text', warning=warnfile) +def test_i18n_warnings_in_translation(app): + app.builddir.rmtree(True) + app.builder.build(['warnings']) + result = (app.outdir / 'warnings.txt').text(encoding='utf-8') + expect = (u"\nI18N WITH REST WARNINGS" + u"\n***********************\n" + u"\nLINE OF >>``< Date: Mon, 4 Feb 2013 23:06:44 +0900 Subject: [PATCH 2/9] remove unusable 'SEVERE: Duplicate ID' message for footnote that the translation node has been recognized as a duplicate. --- sphinx/environment.py | 2 ++ tests/test_intl.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index 2e70569bc..f0f51fdd8 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -262,6 +262,8 @@ class Locale(Transform): 'translated message', node) for old, new in zip(old_foot_refs, new_foot_refs): new['ids'] = old['ids'] + for id in new['ids']: + self.document.ids[id] = new self.document.autofootnote_refs.remove(old) self.document.note_autofootnote_ref(new) diff --git a/tests/test_intl.py b/tests/test_intl.py index 9336beade..0cb43202e 100644 --- a/tests/test_intl.py +++ b/tests/test_intl.py @@ -114,9 +114,10 @@ def test_i18n_footnote_break_refid(app): # expect no error by build -@with_intl_app(buildername='text', cleanenv=True) +@with_intl_app(buildername='text', warning=warnfile) def test_i18n_footnote_regression(app): """regression test for fix #955""" + app.builddir.rmtree(True) app.builder.build(['footnote']) result = (app.outdir / 'footnote.txt').text(encoding='utf-8') expect = (u"\nI18N WITH FOOTNOTE" @@ -127,6 +128,10 @@ def test_i18n_footnote_regression(app): u"\n[100] THIS IS A NUMBERED FOOTNOTE.\n") assert result == expect + warnings = warnfile.getvalue().replace(os.sep, '/') + warning_expr = u'.*/footnote.txt:\\d*: SEVERE: Duplicate ID: ".*".\n' + assert not re.search(warning_expr, warnings) + @with_intl_app(buildername='html', cleanenv=True) def test_i18n_footnote_backlink(app): From 51deccfa5ca628291dc559f873abd2f506030ac3 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 4 Feb 2013 15:52:26 +0100 Subject: [PATCH 3/9] Move custom transforms from environment to their own module, to make sphinx.environment at least a little lighter. --- sphinx/environment.py | 295 ++--------------------------------------- sphinx/transforms.py | 300 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 286 deletions(-) create mode 100644 sphinx/transforms.py diff --git a/sphinx/environment.py b/sphinx/environment.py index f0f51fdd8..32e8eb5bd 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -26,28 +26,26 @@ from itertools import izip, groupby from docutils import nodes from docutils.io import FileInput, NullOutput from docutils.core import Publisher -from docutils.utils import Reporter, relative_path, new_document, \ - get_source_line +from docutils.utils import Reporter, relative_path, get_source_line from docutils.readers import standalone -from docutils.parsers.rst import roles, directives, Parser as RSTParser +from docutils.parsers.rst import roles, directives from docutils.parsers.rst.languages import en as english from docutils.parsers.rst.directives.html import MetaBody from docutils.writers import UnfilteredWriter -from docutils.transforms import Transform -from docutils.transforms.parts import ContentsFilter from sphinx import addnodes from sphinx.util import url_re, get_matching_docs, docname_join, split_into, \ - split_index_msg, FilenameUniqDict -from sphinx.util.nodes import clean_astext, make_refnode, extract_messages, \ - traverse_translatable_index, WarningStream -from sphinx.util.osutil import SEP, ustrftime, find_catalog, fs_encoding + FilenameUniqDict +from sphinx.util.nodes import clean_astext, make_refnode, WarningStream +from sphinx.util.osutil import SEP, fs_encoding from sphinx.util.matching import compile_matchers -from sphinx.util.pycompat import all, class_types +from sphinx.util.pycompat import class_types from sphinx.util.websupport import is_commentable from sphinx.errors import SphinxError, ExtensionError -from sphinx.locale import _, init as init_locale +from sphinx.locale import _ from sphinx.versioning import add_uids, merge_doctrees +from sphinx.transforms import DefaultSubstitutions, MoveModuleTargets, \ + HandleCodeBlocks, SortIds, CitationReferences, Locale, SphinxContentsFilter orig_role_function = roles.role @@ -73,12 +71,6 @@ default_settings = { ENV_VERSION = 42 + (sys.version_info[0] - 2) -default_substitutions = set([ - 'version', - 'release', - 'today', -]) - dummy_reporter = Reporter('', 4, 4) versioning_conditions = { @@ -93,261 +85,6 @@ class NoUri(Exception): pass -class DefaultSubstitutions(Transform): - """ - Replace some substitutions if they aren't defined in the document. - """ - # run before the default Substitutions - default_priority = 210 - - def apply(self): - config = self.document.settings.env.config - # only handle those not otherwise defined in the document - to_handle = default_substitutions - set(self.document.substitution_defs) - for ref in self.document.traverse(nodes.substitution_reference): - refname = ref['refname'] - if refname in to_handle: - text = config[refname] - if refname == 'today' and not text: - # special handling: can also specify a strftime format - text = ustrftime(config.today_fmt or _('%B %d, %Y')) - ref.replace_self(nodes.Text(text, text)) - - -class MoveModuleTargets(Transform): - """ - Move module targets that are the first thing in a section to the section - title. - - XXX Python specific - """ - default_priority = 210 - - def apply(self): - for node in self.document.traverse(nodes.target): - if not node['ids']: - continue - if (node.has_key('ismod') and - node.parent.__class__ is nodes.section and - # index 0 is the section title node - node.parent.index(node) == 1): - node.parent['ids'][0:0] = node['ids'] - node.parent.remove(node) - - -class HandleCodeBlocks(Transform): - """ - Several code block related transformations. - """ - default_priority = 210 - - def apply(self): - # move doctest blocks out of blockquotes - for node in self.document.traverse(nodes.block_quote): - if all(isinstance(child, nodes.doctest_block) for child - in node.children): - node.replace_self(node.children) - # combine successive doctest blocks - #for node in self.document.traverse(nodes.doctest_block): - # if node not in node.parent.children: - # continue - # parindex = node.parent.index(node) - # while len(node.parent) > parindex+1 and \ - # isinstance(node.parent[parindex+1], nodes.doctest_block): - # node[0] = nodes.Text(node[0] + '\n\n' + - # node.parent[parindex+1][0]) - # del node.parent[parindex+1] - - -class SortIds(Transform): - """ - Sort secion IDs so that the "id[0-9]+" one comes last. - """ - default_priority = 261 - - def apply(self): - for node in self.document.traverse(nodes.section): - if len(node['ids']) > 1 and node['ids'][0].startswith('id'): - node['ids'] = node['ids'][1:] + [node['ids'][0]] - - -class CitationReferences(Transform): - """ - Replace citation references by pending_xref nodes before the default - docutils transform tries to resolve them. - """ - default_priority = 619 - - def apply(self): - for citnode in self.document.traverse(nodes.citation_reference): - cittext = citnode.astext() - refnode = addnodes.pending_xref(cittext, reftype='citation', - reftarget=cittext, refwarn=True, - ids=citnode["ids"]) - refnode.line = citnode.line or citnode.parent.line - refnode += nodes.Text('[' + cittext + ']') - citnode.parent.replace(citnode, refnode) - - -class CustomLocaleReporter(object): - """ - Replacer for document.reporter.get_source_and_line method. - - reST text lines for translation not have original source line number. - This class provide correct line number at reporting. - """ - def __init__(self, source, line): - self.source, self.line = source, line - - def get_source_and_line(self, lineno=None): - return self.source, self.line - - -class Locale(Transform): - """ - Replace translatable nodes with their translated doctree. - """ - default_priority = 0 - - def apply(self): - env = self.document.settings.env - settings, source = self.document.settings, self.document['source'] - # XXX check if this is reliable - assert source.startswith(env.srcdir) - docname = path.splitext(relative_path(env.srcdir, source))[0] - textdomain = find_catalog(docname, - self.document.settings.gettext_compact) - - # fetch translations - dirs = [path.join(env.srcdir, directory) - for directory in env.config.locale_dirs] - catalog, has_catalog = init_locale(dirs, env.config.language, - textdomain) - if not has_catalog: - return - - parser = RSTParser() - - for node, msg in extract_messages(self.document): - msgstr = catalog.gettext(msg) - # XXX add marker to untranslated parts - if not msgstr or msgstr == msg: # as-of-yet untranslated - continue - - # Avoid "Literal block expected; none found." warnings. - # If msgstr ends with '::' then it cause warning message at - # parser.parse() processing. - # literal-block-warning is only appear in avobe case. - if msgstr.strip().endswith('::'): - msgstr += '\n\n dummy literal' - # dummy literal node will discard by 'patch = patch[0]' - - patch = new_document(source, settings) - patch.reporter.get_source_and_line = CustomLocaleReporter( - node.source, node.line).get_source_and_line - parser.parse(msgstr, patch) - patch = patch[0] - # XXX doctest and other block markup - if not isinstance(patch, nodes.paragraph): - continue # skip for now - - # auto-numbered foot note reference should use original 'ids'. - def is_autonumber_footnote_ref(node): - return isinstance(node, nodes.footnote_reference) and \ - node.get('auto') == 1 - old_foot_refs = node.traverse(is_autonumber_footnote_ref) - new_foot_refs = patch.traverse(is_autonumber_footnote_ref) - if len(old_foot_refs) != len(new_foot_refs): - env.warn_node('inconsistent footnote references in ' - 'translated message', node) - for old, new in zip(old_foot_refs, new_foot_refs): - new['ids'] = old['ids'] - for id in new['ids']: - self.document.ids[id] = new - self.document.autofootnote_refs.remove(old) - self.document.note_autofootnote_ref(new) - - # reference should use original 'refname'. - # * reference target ".. _Python: ..." is not translatable. - # * section refname is not translatable. - # * inline reference "`Python <...>`_" has no 'refname'. - def is_refnamed_ref(node): - return isinstance(node, nodes.reference) and \ - 'refname' in node - old_refs = node.traverse(is_refnamed_ref) - new_refs = patch.traverse(is_refnamed_ref) - applied_refname_map = {} - if len(old_refs) != len(new_refs): - env.warn_node('inconsistent references in ' - 'translated message', node) - for new in new_refs: - if new['refname'] in applied_refname_map: - # 2nd appearance of the reference - new['refname'] = applied_refname_map[new['refname']] - elif old_refs: - # 1st appearance of the reference in old_refs - old = old_refs.pop(0) - refname = old['refname'] - new['refname'] = refname - applied_refname_map[new['refname']] = refname - else: - # the reference is not found in old_refs - applied_refname_map[new['refname']] = new['refname'] - - self.document.note_refname(new) - - # refnamed footnote and citation should use original 'ids'. - def is_refnamed_footnote_ref(node): - footnote_ref_classes = (nodes.footnote_reference, - nodes.citation_reference) - return isinstance(node, footnote_ref_classes) and \ - 'refname' in node - old_refs = node.traverse(is_refnamed_footnote_ref) - new_refs = patch.traverse(is_refnamed_footnote_ref) - refname_ids_map = {} - if len(old_refs) != len(new_refs): - env.warn_node('inconsistent references in ' - 'translated message', node) - for old in old_refs: - refname_ids_map[old["refname"]] = old["ids"] - for new in new_refs: - refname = new["refname"] - if refname in refname_ids_map: - new["ids"] = refname_ids_map[refname] - - # Original pending_xref['reftarget'] contain not-translated - # target name, new pending_xref must use original one. - old_refs = node.traverse(addnodes.pending_xref) - new_refs = patch.traverse(addnodes.pending_xref) - if len(old_refs) != len(new_refs): - env.warn_node('inconsistent term references in ' - 'translated message', node) - for old, new in zip(old_refs, new_refs): - new['reftarget'] = old['reftarget'] - - # update leaves - for child in patch.children: - child.parent = node - node.children = patch.children - - # Extract and translate messages for index entries. - for node, entries in traverse_translatable_index(self.document): - new_entries = [] - for type, msg, tid, main in entries: - msg_parts = split_index_msg(type, msg) - msgstr_parts = [] - for part in msg_parts: - msgstr = catalog.gettext(part) - if not msgstr: - msgstr = part - msgstr_parts.append(msgstr) - - new_entries.append((type, ';'.join(msgstr_parts), tid, main)) - - node['raw_entries'] = entries - node['entries'] = new_entries - - class SphinxStandaloneReader(standalone.Reader): """ Add our own transforms. @@ -366,20 +103,6 @@ class SphinxDummyWriter(UnfilteredWriter): pass -class SphinxContentsFilter(ContentsFilter): - """ - Used with BuildEnvironment.add_toc_from() to discard cross-file links - within table-of-contents link nodes. - """ - def visit_pending_xref(self, node): - text = node.astext() - self.parent.append(nodes.literal(text, text)) - raise nodes.SkipNode - - def visit_image(self, node): - raise nodes.SkipNode - - class BuildEnvironment: """ The environment in which the ReST files are translated. diff --git a/sphinx/transforms.py b/sphinx/transforms.py new file mode 100644 index 000000000..43ec97cbc --- /dev/null +++ b/sphinx/transforms.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- +""" + sphinx.transforms + ~~~~~~~~~~~~~~~~~ + + Docutils transforms used by Sphinx when reading documents. + + :copyright: Copyright 2007-2013 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from os import path + +from docutils import nodes +from docutils.utils import new_document, relative_path +from docutils.parsers.rst import Parser as RSTParser +from docutils.transforms import Transform +from docutils.transforms.parts import ContentsFilter + +from sphinx import addnodes +from sphinx.locale import _, init as init_locale +from sphinx.util import split_index_msg +from sphinx.util.nodes import traverse_translatable_index, extract_messages +from sphinx.util.osutil import ustrftime, find_catalog +from sphinx.util.pycompat import all + + +default_substitutions = set([ + 'version', + 'release', + 'today', +]) + +class DefaultSubstitutions(Transform): + """ + Replace some substitutions if they aren't defined in the document. + """ + # run before the default Substitutions + default_priority = 210 + + def apply(self): + config = self.document.settings.env.config + # only handle those not otherwise defined in the document + to_handle = default_substitutions - set(self.document.substitution_defs) + for ref in self.document.traverse(nodes.substitution_reference): + refname = ref['refname'] + if refname in to_handle: + text = config[refname] + if refname == 'today' and not text: + # special handling: can also specify a strftime format + text = ustrftime(config.today_fmt or _('%B %d, %Y')) + ref.replace_self(nodes.Text(text, text)) + + +class MoveModuleTargets(Transform): + """ + Move module targets that are the first thing in a section to the section + title. + + XXX Python specific + """ + default_priority = 210 + + def apply(self): + for node in self.document.traverse(nodes.target): + if not node['ids']: + continue + if (node.has_key('ismod') and + node.parent.__class__ is nodes.section and + # index 0 is the section title node + node.parent.index(node) == 1): + node.parent['ids'][0:0] = node['ids'] + node.parent.remove(node) + + +class HandleCodeBlocks(Transform): + """ + Several code block related transformations. + """ + default_priority = 210 + + def apply(self): + # move doctest blocks out of blockquotes + for node in self.document.traverse(nodes.block_quote): + if all(isinstance(child, nodes.doctest_block) for child + in node.children): + node.replace_self(node.children) + # combine successive doctest blocks + #for node in self.document.traverse(nodes.doctest_block): + # if node not in node.parent.children: + # continue + # parindex = node.parent.index(node) + # while len(node.parent) > parindex+1 and \ + # isinstance(node.parent[parindex+1], nodes.doctest_block): + # node[0] = nodes.Text(node[0] + '\n\n' + + # node.parent[parindex+1][0]) + # del node.parent[parindex+1] + + +class SortIds(Transform): + """ + Sort secion IDs so that the "id[0-9]+" one comes last. + """ + default_priority = 261 + + def apply(self): + for node in self.document.traverse(nodes.section): + if len(node['ids']) > 1 and node['ids'][0].startswith('id'): + node['ids'] = node['ids'][1:] + [node['ids'][0]] + + +class CitationReferences(Transform): + """ + Replace citation references by pending_xref nodes before the default + docutils transform tries to resolve them. + """ + default_priority = 619 + + def apply(self): + for citnode in self.document.traverse(nodes.citation_reference): + cittext = citnode.astext() + refnode = addnodes.pending_xref(cittext, reftype='citation', + reftarget=cittext, refwarn=True, + ids=citnode["ids"]) + refnode.line = citnode.line or citnode.parent.line + refnode += nodes.Text('[' + cittext + ']') + citnode.parent.replace(citnode, refnode) + + +class CustomLocaleReporter(object): + """ + Replacer for document.reporter.get_source_and_line method. + + reST text lines for translation not have original source line number. + This class provide correct line number at reporting. + """ + def __init__(self, source, line): + self.source, self.line = source, line + + def get_source_and_line(self, lineno=None): + return self.source, self.line + + +class Locale(Transform): + """ + Replace translatable nodes with their translated doctree. + """ + default_priority = 0 + + def apply(self): + env = self.document.settings.env + settings, source = self.document.settings, self.document['source'] + # XXX check if this is reliable + assert source.startswith(env.srcdir) + docname = path.splitext(relative_path(env.srcdir, source))[0] + textdomain = find_catalog(docname, + self.document.settings.gettext_compact) + + # fetch translations + dirs = [path.join(env.srcdir, directory) + for directory in env.config.locale_dirs] + catalog, has_catalog = init_locale(dirs, env.config.language, + textdomain) + if not has_catalog: + return + + parser = RSTParser() + + for node, msg in extract_messages(self.document): + msgstr = catalog.gettext(msg) + # XXX add marker to untranslated parts + if not msgstr or msgstr == msg: # as-of-yet untranslated + continue + + # Avoid "Literal block expected; none found." warnings. + # If msgstr ends with '::' then it cause warning message at + # parser.parse() processing. + # literal-block-warning is only appear in avobe case. + if msgstr.strip().endswith('::'): + msgstr += '\n\n dummy literal' + # dummy literal node will discard by 'patch = patch[0]' + + patch = new_document(source, settings) + patch.reporter.get_source_and_line = CustomLocaleReporter( + node.source, node.line).get_source_and_line + parser.parse(msgstr, patch) + patch = patch[0] + # XXX doctest and other block markup + if not isinstance(patch, nodes.paragraph): + continue # skip for now + + # auto-numbered foot note reference should use original 'ids'. + def is_autonumber_footnote_ref(node): + return isinstance(node, nodes.footnote_reference) and \ + node.get('auto') == 1 + old_foot_refs = node.traverse(is_autonumber_footnote_ref) + new_foot_refs = patch.traverse(is_autonumber_footnote_ref) + if len(old_foot_refs) != len(new_foot_refs): + env.warn_node('inconsistent footnote references in ' + 'translated message', node) + for old, new in zip(old_foot_refs, new_foot_refs): + new['ids'] = old['ids'] + for id in new['ids']: + self.document.ids[id] = new + self.document.autofootnote_refs.remove(old) + self.document.note_autofootnote_ref(new) + + # reference should use original 'refname'. + # * reference target ".. _Python: ..." is not translatable. + # * section refname is not translatable. + # * inline reference "`Python <...>`_" has no 'refname'. + def is_refnamed_ref(node): + return isinstance(node, nodes.reference) and \ + 'refname' in node + old_refs = node.traverse(is_refnamed_ref) + new_refs = patch.traverse(is_refnamed_ref) + applied_refname_map = {} + if len(old_refs) != len(new_refs): + env.warn_node('inconsistent references in ' + 'translated message', node) + for new in new_refs: + if new['refname'] in applied_refname_map: + # 2nd appearance of the reference + new['refname'] = applied_refname_map[new['refname']] + elif old_refs: + # 1st appearance of the reference in old_refs + old = old_refs.pop(0) + refname = old['refname'] + new['refname'] = refname + applied_refname_map[new['refname']] = refname + else: + # the reference is not found in old_refs + applied_refname_map[new['refname']] = new['refname'] + + self.document.note_refname(new) + + # refnamed footnote and citation should use original 'ids'. + def is_refnamed_footnote_ref(node): + footnote_ref_classes = (nodes.footnote_reference, + nodes.citation_reference) + return isinstance(node, footnote_ref_classes) and \ + 'refname' in node + old_refs = node.traverse(is_refnamed_footnote_ref) + new_refs = patch.traverse(is_refnamed_footnote_ref) + refname_ids_map = {} + if len(old_refs) != len(new_refs): + env.warn_node('inconsistent references in ' + 'translated message', node) + for old in old_refs: + refname_ids_map[old["refname"]] = old["ids"] + for new in new_refs: + refname = new["refname"] + if refname in refname_ids_map: + new["ids"] = refname_ids_map[refname] + + # Original pending_xref['reftarget'] contain not-translated + # target name, new pending_xref must use original one. + old_refs = node.traverse(addnodes.pending_xref) + new_refs = patch.traverse(addnodes.pending_xref) + if len(old_refs) != len(new_refs): + env.warn_node('inconsistent term references in ' + 'translated message', node) + for old, new in zip(old_refs, new_refs): + new['reftarget'] = old['reftarget'] + + # update leaves + for child in patch.children: + child.parent = node + node.children = patch.children + + # Extract and translate messages for index entries. + for node, entries in traverse_translatable_index(self.document): + new_entries = [] + for type, msg, tid, main in entries: + msg_parts = split_index_msg(type, msg) + msgstr_parts = [] + for part in msg_parts: + msgstr = catalog.gettext(part) + if not msgstr: + msgstr = part + msgstr_parts.append(msgstr) + + new_entries.append((type, ';'.join(msgstr_parts), tid, main)) + + node['raw_entries'] = entries + node['entries'] = new_entries + + +class SphinxContentsFilter(ContentsFilter): + """ + Used with BuildEnvironment.add_toc_from() to discard cross-file links + within table-of-contents link nodes. + """ + def visit_pending_xref(self, node): + text = node.astext() + self.parent.append(nodes.literal(text, text)) + raise nodes.SkipNode + + def visit_image(self, node): + raise nodes.SkipNode From 7ebd7ab2a9e806a0479b006cec13fdff37407814 Mon Sep 17 00:00:00 2001 From: Takayuki Shimizukawa Date: Tue, 5 Feb 2013 12:47:08 +0900 Subject: [PATCH 4/9] fix: roles' reftarget ware swapped if there are some roles in 1 line and translation exchange rthat roles position. refs #1090 --- sphinx/transforms.py | 11 +++++++++-- tests/roots/test-intl/contents.txt | 3 +++ tests/roots/test-intl/role_xref.po | 23 +++++++++++++++++++++++ tests/roots/test-intl/role_xref.txt | 9 +++++++++ tests/test_intl.py | 18 ++++++++++++++++++ 5 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 tests/roots/test-intl/role_xref.po create mode 100644 tests/roots/test-intl/role_xref.txt diff --git a/sphinx/transforms.py b/sphinx/transforms.py index 43ec97cbc..845b17acc 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -255,13 +255,20 @@ class Locale(Transform): # Original pending_xref['reftarget'] contain not-translated # target name, new pending_xref must use original one. + # This code restricts to change ref-targets in the translation. old_refs = node.traverse(addnodes.pending_xref) new_refs = patch.traverse(addnodes.pending_xref) + xref_reftarget_map = {} if len(old_refs) != len(new_refs): env.warn_node('inconsistent term references in ' 'translated message', node) - for old, new in zip(old_refs, new_refs): - new['reftarget'] = old['reftarget'] + for old in old_refs: + key = old["reftype"], old["refdomain"] + xref_reftarget_map[key] = old["reftarget"] + for new in new_refs: + key = new["reftype"], new["refdomain"] + if key in xref_reftarget_map: + new['reftarget'] = xref_reftarget_map[key] # update leaves for child in patch.children: diff --git a/tests/roots/test-intl/contents.txt b/tests/roots/test-intl/contents.txt index 932a58dd6..88f7be5bd 100644 --- a/tests/roots/test-intl/contents.txt +++ b/tests/roots/test-intl/contents.txt @@ -1,3 +1,6 @@ +CONTENTS +======== + .. toctree:: :maxdepth: 2 :numbered: diff --git a/tests/roots/test-intl/role_xref.po b/tests/roots/test-intl/role_xref.po new file mode 100644 index 000000000..e7a348b36 --- /dev/null +++ b/tests/roots/test-intl/role_xref.po @@ -0,0 +1,23 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2012, foof +# This file is distributed under the same license as the foo package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: sphinx 1.0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2013-02-04 14:00\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "i18n role xref" +msgstr "I18N ROCK'N ROLE XREF" + +msgid "link to :term:`Some term`, :ref:`i18n-role-xref`, :doc:`contents`." +msgstr "LINK TO :ref:`i18n-role-xref`, :doc:`contents`, :term:`SOME NEW TERM`." diff --git a/tests/roots/test-intl/role_xref.txt b/tests/roots/test-intl/role_xref.txt new file mode 100644 index 000000000..382b740c9 --- /dev/null +++ b/tests/roots/test-intl/role_xref.txt @@ -0,0 +1,9 @@ +:tocdepth: 2 + +.. _i18n-role-xref: + +i18n role xref +============== + +link to :term:`Some term`, :ref:`i18n-role-xref`, :doc:`contents`. + diff --git a/tests/test_intl.py b/tests/test_intl.py index 0cb43202e..4dec84646 100644 --- a/tests/test_intl.py +++ b/tests/test_intl.py @@ -293,6 +293,24 @@ def test_i18n_glossary_terms(app): assert 'term not in glossary' not in warnings +@with_intl_app(buildername='text', warning=warnfile) +def test_i18n_role_xref(app): + # regression test for #1090 + app.builddir.rmtree(True) #for warnings acceleration + app.builder.build(['role_xref']) + result = (app.outdir / 'role_xref.txt').text(encoding='utf-8') + expect = (u"\nI18N ROCK'N ROLE XREF" + u"\n*********************\n" + u"\nLINK TO *I18N ROCK'N ROLE XREF*, *CONTENTS*, *SOME NEW TERM*.\n") + + warnings = warnfile.getvalue().replace(os.sep, '/') + assert 'term not in glossary' not in warnings + assert 'undefined label' not in warnings + assert 'unknown document' not in warnings + + assert result == expect + + @with_intl_app(buildername='text', warning=warnfile) def test_i18n_glossary_terms_inconsistency(app): # regression test for #1090 From 6667a7cbdf56e8707cbd837483a3bbd062907926 Mon Sep 17 00:00:00 2001 From: Takayuki Shimizukawa Date: Tue, 5 Feb 2013 03:54:35 +0000 Subject: [PATCH 5/9] fix toctree entry missing for testing. --- tests/roots/test-intl/contents.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/roots/test-intl/contents.txt b/tests/roots/test-intl/contents.txt index 88f7be5bd..298beb6ce 100644 --- a/tests/roots/test-intl/contents.txt +++ b/tests/roots/test-intl/contents.txt @@ -16,5 +16,6 @@ CONTENTS definition_terms figure_caption index_entries + role_xref glossary_terms glossary_terms_inconsistency From 0c7cc10a69adf89992c3bbb71316106e899e5f89 Mon Sep 17 00:00:00 2001 From: Takayuki Shimizukawa Date: Tue, 5 Feb 2013 04:40:47 +0000 Subject: [PATCH 6/9] fix: reporting correct line number implementation (6fc25e0e3a5c) was not worked with docutils 0.7, 0.8. --- sphinx/transforms.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/sphinx/transforms.py b/sphinx/transforms.py index 845b17acc..5b96300de 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -137,6 +137,19 @@ class CustomLocaleReporter(object): def __init__(self, source, line): self.source, self.line = source, line + try: + from docutils import __version__ as du_version + v = tuple([int(x) for x in du_version.split('.')[:2]]) + except ImportError: + v = (99, 99) + self.du_version = v + + def set_reporter(self, document): + if self.du_version < (0, 9): + document.reporter.locator = self.get_source_and_line + else: + document.reporter.get_source_and_line = self.get_source_and_line + def get_source_and_line(self, lineno=None): return self.source, self.line @@ -181,8 +194,7 @@ class Locale(Transform): # dummy literal node will discard by 'patch = patch[0]' patch = new_document(source, settings) - patch.reporter.get_source_and_line = CustomLocaleReporter( - node.source, node.line).get_source_and_line + CustomLocaleReporter(node.source, node.line).set_reporter(patch) parser.parse(msgstr, patch) patch = patch[0] # XXX doctest and other block markup From 5aaf533a39f27884c24dea3924c9e11c728d8019 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 5 Feb 2013 09:19:52 +0100 Subject: [PATCH 7/9] Add setup.py "upload" alias for automatic package signing. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 525e24ea3..5dac60b4d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,6 +4,7 @@ tag_date = true [aliases] release = egg_info -RDb '' +upload = upload --sign --identity=36580288 [extract_messages] mapping_file = babel.cfg From 529e45a980a8e936cabd5ceab6e3539d72834b1d Mon Sep 17 00:00:00 2001 From: Takayuki Shimizukawa Date: Tue, 5 Feb 2013 23:57:26 +0900 Subject: [PATCH 8/9] Fix text builder did not respect wide/fullwidth charactors for title line. --- CHANGES | 2 ++ sphinx/writers/text.py | 4 +++- tests/test_build_text.py | 44 ++++++++++++++++++++++++++++++++++++++++ tests/util.py | 7 +++++++ 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tests/test_build_text.py diff --git a/CHANGES b/CHANGES index 38fc749f0..8538e0519 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ Release 1.2 (in development) ============================ +* Fix text builder did not respect wide/fullwidth charactors. + * #1062: sphinx.ext.autodoc use __init__ method signature for class signature. * PR#111: Respect add_autodoc_attrgetter() even when inherited-members is set. diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py index 81c785ee6..21a27a179 100644 --- a/sphinx/writers/text.py +++ b/sphinx/writers/text.py @@ -13,6 +13,7 @@ import re import textwrap from docutils import nodes, writers +from docutils.utils import column_width from sphinx import addnodes from sphinx.locale import admonitionlabels, versionlabels, _ @@ -165,7 +166,8 @@ class TextTranslator(nodes.NodeVisitor): char = '^' text = ''.join(x[1] for x in self.states.pop() if x[0] == -1) self.stateindent.pop() - self.states[-1].append((0, ['', text, '%s' % (char * len(text)), ''])) + self.states[-1].append( + (0, ['', text, '%s' % (char * column_width(text)), ''])) def visit_subtitle(self, node): pass diff --git a/tests/test_build_text.py b/tests/test_build_text.py new file mode 100644 index 000000000..d0472b8f1 --- /dev/null +++ b/tests/test_build_text.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +""" + test_build_text + ~~~~~~~~~~~~~~~ + + Test the build process with Text builder with the test root. + + :copyright: Copyright 2007-2013 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from textwrap import dedent + +from docutils.utils import column_width + +from util import * + + +def with_text_app(*args, **kw): + default_kw = { + 'buildername': 'text', + 'srcdir': '(empty)', + 'confoverrides': { + 'project': 'text', + 'master_doc': 'contents', + }, + } + default_kw.update(kw) + return with_app(*args, **default_kw) + + +@with_text_app() +def test_multibyte_title_line(app): + title = u'\u65e5\u672c\u8a9e' + underline = u'=' * column_width(title) + content = u'\n'.join((title, underline, u'')) + + (app.srcdir / 'contents.rst').write_text(content, encoding='utf-8') + app.builder.build_all() + result = (app.outdir / 'contents.txt').text(encoding='utf-8') + + expect_underline = underline.replace('=', '*') + result_underline = result.splitlines()[2].strip() + assert expect_underline == result_underline diff --git a/tests/util.py b/tests/util.py index 3e17d3626..4ba89030a 100644 --- a/tests/util.py +++ b/tests/util.py @@ -142,6 +142,13 @@ class TestApp(application.Sphinx): temproot = tempdir / 'root' test_root.copytree(temproot) srcdir = temproot + elif srcdir == '(empty)': + tempdir = path(tempfile.mkdtemp()) + self.cleanup_trees.append(tempdir) + temproot = tempdir / 'root' + temproot.makedirs() + (temproot / 'conf.py').write_text('') + srcdir = temproot else: srcdir = path(srcdir) self.builddir = srcdir.joinpath('_build') From b8296ad11e4547e821ec8dbcee82ec1456c19bce Mon Sep 17 00:00:00 2001 From: Takayuki Shimizukawa Date: Wed, 6 Feb 2013 00:29:59 +0900 Subject: [PATCH 9/9] Fix text builder did not respect wide/fullwidth charactors for table layout. --- sphinx/writers/text.py | 6 ++++-- tests/test_build_text.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py index 21a27a179..f42d637a3 100644 --- a/sphinx/writers/text.py +++ b/sphinx/writers/text.py @@ -393,7 +393,7 @@ class TextTranslator(nodes.NodeVisitor): for i, cell in enumerate(line): par = my_wrap(cell, width=colwidths[i]) if par: - maxwidth = max(map(len, par)) + maxwidth = max(map(column_width, par)) else: maxwidth = 0 realwidths[i] = max(realwidths[i], maxwidth) @@ -413,7 +413,9 @@ class TextTranslator(nodes.NodeVisitor): out = ['|'] for i, cell in enumerate(line): if cell: - out.append(' ' + cell.ljust(realwidths[i]+1)) + adjust_len = len(cell) - column_width(cell) + out.append(' ' + cell.ljust( + realwidths[i] + 1 + adjust_len)) else: out.append(' ' * (realwidths[i] + 2)) out.append('|') diff --git a/tests/test_build_text.py b/tests/test_build_text.py index d0472b8f1..63df8ee0a 100644 --- a/tests/test_build_text.py +++ b/tests/test_build_text.py @@ -42,3 +42,24 @@ def test_multibyte_title_line(app): expect_underline = underline.replace('=', '*') result_underline = result.splitlines()[2].strip() assert expect_underline == result_underline + + +@with_text_app() +def test_multibyte_table(app): + text = u'\u65e5\u672c\u8a9e' + contents = (u"\n.. list-table::" + "\n" + "\n - - spam" + "\n - egg" + "\n" + "\n - - %(text)s" + "\n - %(text)s" + "\n" % locals()) + + (app.srcdir / 'contents.rst').write_text(contents, encoding='utf-8') + app.builder.build_all() + result = (app.outdir / 'contents.txt').text(encoding='utf-8') + + lines = [line.strip() for line in result.splitlines() if line.strip()] + line_widths = [column_width(line) for line in lines] + assert len(set(line_widths)) == 1 # same widths