From 8600ffeda21c834cead0b321f3988493b8290510 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sat, 25 Jul 2015 15:47:56 +0200 Subject: [PATCH 1/9] Fixed #1796: On py3, automated .mo building cause UnicodeDecodeError. --- CHANGES | 1 + sphinx/application.py | 3 ++- sphinx/builders/__init__.py | 3 +++ sphinx/locale/__init__.py | 5 +++-- sphinx/transforms.py | 3 ++- sphinx/util/i18n.py | 11 ++++++----- tests/test_util_i18n.py | 8 ++++---- 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/CHANGES b/CHANGES index 269401f96..3457994bd 100644 --- a/CHANGES +++ b/CHANGES @@ -33,6 +33,7 @@ Bugs fixed * #1949: Use ``safe_getattr`` in the coverage builder to avoid aborting with descriptors that have custom behavior. * #1915: Do not generate smart quotes in doc field type annotations. +* #1796, On py3, automated .mo building cause UnicodeDecodeError Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/application.py b/sphinx/application.py index 93bdeee60..16b1c8176 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -200,7 +200,8 @@ class Sphinx(object): else: locale_dirs = [] self.translator, has_translation = locale.init(locale_dirs, - self.config.language) + self.config.language, + charset=self.config.source_encoding) if self.config.language is not None: if has_translation or self.config.language == 'en': # "en" never needs to be translated diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 38cde6123..7f5da17d2 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -170,6 +170,7 @@ class Builder(object): catalogs = i18n.find_catalog_source_files( [path.join(self.srcdir, x) for x in self.config.locale_dirs], self.config.language, + charset=self.config.source_encoding, gettext_compact=self.config.gettext_compact, force_all=True) message = 'all of %d po files' % len(catalogs) @@ -186,6 +187,7 @@ class Builder(object): [path.join(self.srcdir, x) for x in self.config.locale_dirs], self.config.language, domains=list(specified_domains), + charset=self.config.source_encoding, gettext_compact=self.config.gettext_compact) message = 'targets for %d po files that are specified' % len(catalogs) self.compile_catalogs(catalogs, message) @@ -194,6 +196,7 @@ class Builder(object): catalogs = i18n.find_catalog_source_files( [path.join(self.srcdir, x) for x in self.config.locale_dirs], self.config.language, + charset=self.config.source_encoding, gettext_compact=self.config.gettext_compact) message = 'targets for %d po files that are out of date' % len(catalogs) self.compile_catalogs(catalogs, message) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 4847c0845..444ad5d0c 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -195,7 +195,7 @@ else: return translators['sphinx'].ugettext(message) -def init(locale_dirs, language, catalog='sphinx'): +def init(locale_dirs, language, catalog='sphinx', charset='utf-8'): """Look for message catalogs in `locale_dirs` and *ensure* that there is at least a NullTranslations catalog set in `translators`. If called multiple times or if several ``.mo`` files are found, their contents are merged @@ -212,7 +212,8 @@ def init(locale_dirs, language, catalog='sphinx'): # compile mo files if po file is updated # TODO: remove circular importing from sphinx.util.i18n import find_catalog_source_files - for catinfo in find_catalog_source_files(locale_dirs, language, domains=[catalog]): + for catinfo in find_catalog_source_files(locale_dirs, language, domains=[catalog], + charset=charset): catinfo.write_mo(language) # loading diff --git a/sphinx/transforms.py b/sphinx/transforms.py index 681b50e05..3d12eb230 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -225,7 +225,8 @@ class Locale(Transform): dirs = [path.join(env.srcdir, directory) for directory in env.config.locale_dirs] catalog, has_catalog = init_locale(dirs, env.config.language, - textdomain) + textdomain, + charset=env.config.source_encoding) if not has_catalog: return diff --git a/sphinx/util/i18n.py b/sphinx/util/i18n.py index efdc31828..a72e138c5 100644 --- a/sphinx/util/i18n.py +++ b/sphinx/util/i18n.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for details. """ import gettext +import io from os import path from collections import namedtuple @@ -19,7 +20,7 @@ from sphinx.util.osutil import walk from sphinx.util import SEP -LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain') +LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain,charset') class CatalogInfo(LocaleFileInfoBase): @@ -46,8 +47,8 @@ class CatalogInfo(LocaleFileInfoBase): path.getmtime(self.mo_path) < path.getmtime(self.po_path)) def write_mo(self, locale): - with open(self.po_path, 'rt') as po: - with open(self.mo_path, 'wb') as mo: + with io.open(self.po_path, 'rt', encoding=self.charset) as po: + with io.open(self.mo_path, 'wb') as mo: write_mo(mo, read_po(po, locale)) @@ -72,7 +73,7 @@ def find_catalog_files(docname, srcdir, locale_dirs, lang, compaction): def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact=False, - force_all=False): + charset='utf-8', force_all=False): """ :param list locale_dirs: list of path as `['locale_dir1', 'locale_dir2', ...]` to find @@ -112,7 +113,7 @@ def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact domain = domain.replace(path.sep, SEP) if domains and domain not in domains: continue - cat = CatalogInfo(base_dir, domain) + cat = CatalogInfo(base_dir, domain, charset) if force_all or cat.is_outdated(): catalogs.add(cat) diff --git a/tests/test_util_i18n.py b/tests/test_util_i18n.py index c74962fc5..47ef8ecce 100644 --- a/tests/test_util_i18n.py +++ b/tests/test_util_i18n.py @@ -20,7 +20,7 @@ from util import with_tempdir def test_catalog_info_for_file_and_path(): - cat = i18n.CatalogInfo('path', 'domain') + cat = i18n.CatalogInfo('path', 'domain', 'utf-8') assert cat.po_file == 'domain.po' assert cat.mo_file == 'domain.mo' assert cat.po_path == path.join('path', 'domain.po') @@ -28,7 +28,7 @@ def test_catalog_info_for_file_and_path(): def test_catalog_info_for_sub_domain_file_and_path(): - cat = i18n.CatalogInfo('path', 'sub/domain') + cat = i18n.CatalogInfo('path', 'sub/domain', 'utf-8') assert cat.po_file == 'sub/domain.po' assert cat.mo_file == 'sub/domain.mo' assert cat.po_path == path.join('path', 'sub/domain.po') @@ -38,7 +38,7 @@ def test_catalog_info_for_sub_domain_file_and_path(): @with_tempdir def test_catalog_outdated(dir): (dir / 'test.po').write_text('#') - cat = i18n.CatalogInfo(dir, 'test') + cat = i18n.CatalogInfo(dir, 'test', 'utf-8') assert cat.is_outdated() # if mo is not exist mo_file = (dir / 'test.mo') @@ -52,7 +52,7 @@ def test_catalog_outdated(dir): @with_tempdir def test_catalog_write_mo(dir): (dir / 'test.po').write_text('#') - cat = i18n.CatalogInfo(dir, 'test') + cat = i18n.CatalogInfo(dir, 'test', 'utf-8') cat.write_mo('en') assert path.exists(cat.mo_path) assert read_mo(open(cat.mo_path, 'rb')) is not None From 468568bdcffae8e1f093c12d960479175cd83c4e Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sat, 25 Jul 2015 15:55:43 +0200 Subject: [PATCH 2/9] fix obsoleted documentation --- doc/ext/ifconfig.rst | 4 ++-- doc/extdev/tutorial.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ext/ifconfig.rst b/doc/ext/ifconfig.rst index a11e30726..f64ca6c58 100644 --- a/doc/ext/ifconfig.rst +++ b/doc/ext/ifconfig.rst @@ -26,8 +26,8 @@ This extension is quite simple, and features only one directive: :file:`conf.py`, e.g.:: def setup(app): - app.add_config_value('releaselevel', '', True) + app.add_config_value('releaselevel', '', 'env') - The second argument is the default value, the third should always be ``True`` + The second argument is the default value, the third should always be ``'env'`` for such values (it selects if Sphinx re-reads the documents if the value changes). diff --git a/doc/extdev/tutorial.rst b/doc/extdev/tutorial.rst index 8241e109e..8edff7140 100644 --- a/doc/extdev/tutorial.rst +++ b/doc/extdev/tutorial.rst @@ -149,7 +149,7 @@ The new elements are added in the extension's setup function. Let us create a new Python module called :file:`todo.py` and add the setup function:: def setup(app): - app.add_config_value('todo_include_todos', False, False) + app.add_config_value('todo_include_todos', False, 'html') app.add_node(todolist) app.add_node(todo, @@ -171,7 +171,7 @@ the individual calls do is the following: new *config value* ``todo_include_todos``, whose default value should be ``False`` (this also tells Sphinx that it is a boolean value). - If the third argument was ``True``, all documents would be re-read if the + If the third argument was ``'html'``, HTML documents would be full rebuild if the config value changed its value. This is needed for config values that influence reading (build phase 1). From e4b42fe15fbba4882f2dd3b5b38a43db37d5947c Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 25 Jul 2015 16:51:45 +0200 Subject: [PATCH 3/9] Fixes #1923: Use babel features only if the babel latex element is nonempty. --- CHANGES | 3 ++- sphinx/writers/latex.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 3457994bd..d88d41aed 100644 --- a/CHANGES +++ b/CHANGES @@ -33,7 +33,8 @@ Bugs fixed * #1949: Use ``safe_getattr`` in the coverage builder to avoid aborting with descriptors that have custom behavior. * #1915: Do not generate smart quotes in doc field type annotations. -* #1796, On py3, automated .mo building cause UnicodeDecodeError +* #1796: On py3, automated .mo building caused UnicodeDecodeError. +* #1923: Use babel features only if the babel latex element is nonempty. Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py index c1ba04cab..5d175eae6 100644 --- a/sphinx/writers/latex.py +++ b/sphinx/writers/latex.py @@ -342,8 +342,12 @@ class LaTeXTranslator(nodes.NodeVisitor): else: language = 'english' - babel_prefix = '\\addto\\captions%s{' % language - babel_suffix = '}' + if self.elements['babel']: + babel_prefix = '\\addto\\captions%s{' % language + babel_suffix = '}' + else: + babel_prefix = '' + babel_suffix = '' figure = self.builder.config.numfig_format['figure'].split('%s', 1) if len(figure) == 1: From 3e742bb43a041c21805cbcc0580e206ccad14cf4 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 25 Jul 2015 16:54:51 +0200 Subject: [PATCH 4/9] Closes #1942: Fix a KeyError in websupport. --- CHANGES | 1 + sphinx/builders/websupport.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index d88d41aed..a4e854d76 100644 --- a/CHANGES +++ b/CHANGES @@ -35,6 +35,7 @@ Bugs fixed * #1915: Do not generate smart quotes in doc field type annotations. * #1796: On py3, automated .mo building caused UnicodeDecodeError. * #1923: Use babel features only if the babel latex element is nonempty. +* #1942: Fix a KeyError in websupport. Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/builders/websupport.py b/sphinx/builders/websupport.py index 4a33667c6..66af0ab80 100644 --- a/sphinx/builders/websupport.py +++ b/sphinx/builders/websupport.py @@ -114,6 +114,8 @@ class WebSupportBuilder(PickleHTMLBuilder): doc_ctx = { 'body': ctx.get('body', ''), 'title': ctx.get('title', ''), + 'css': ctx.get('css', ''), + 'script': ctx.get('script', ''), } # partially render the html template to get at interesting macros template = self.templates.environment.get_template(templatename) From bf7d0ccd75fed8cf8ba529744e08a71779c39401 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 25 Jul 2015 17:04:06 +0200 Subject: [PATCH 5/9] Fixes #1887: note that graph name needs quotes if non alphanumeric. --- doc/ext/graphviz.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/ext/graphviz.rst b/doc/ext/graphviz.rst index 9b34b48fe..d5a5969f4 100644 --- a/doc/ext/graphviz.rst +++ b/doc/ext/graphviz.rst @@ -53,6 +53,10 @@ It adds these directives: "bar" -- "baz"; + .. note:: The graph name is passed unchanged to Graphviz. If it contains + non-alphanumeric characters (e.g. a dash), you will have to double-quote + it. + .. rst:directive:: digraph From 886d1745347735a707c58ce1fdad8f8891a89ed5 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 25 Jul 2015 17:12:30 +0200 Subject: [PATCH 6/9] Closes #1903: Fix strange id generation for glossary terms. --- CHANGES | 1 + sphinx/domains/std.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index a4e854d76..cdc839925 100644 --- a/CHANGES +++ b/CHANGES @@ -36,6 +36,7 @@ Bugs fixed * #1796: On py3, automated .mo building caused UnicodeDecodeError. * #1923: Use babel features only if the babel latex element is nonempty. * #1942: Fix a KeyError in websupport. +* #1903: Fix strange id generation for glossary terms. Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py index c437005a8..9ef2fda80 100644 --- a/sphinx/domains/std.py +++ b/sphinx/domains/std.py @@ -224,7 +224,7 @@ def make_termnodes_from_paragraph_node(env, node, new_id=None): termtext = node.astext() if new_id is None: - new_id = 'term-' + nodes.make_id(termtext) + new_id = nodes.make_id('term-' + termtext) if new_id in gloss_entries: new_id = 'term-' + str(len(gloss_entries)) gloss_entries.add(new_id) From 18bb74274597a39f693ba4fdc6fcf53c047da479 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sat, 25 Jul 2015 17:18:34 +0200 Subject: [PATCH 7/9] remove old patch for docutils. it's fixed at docutils-0.10 http://repo.or.cz/w/docutils.git?a=commit;h=e3cc3cef7c1120d46ed9ef317583d08a51a452cc --- sphinx/util/nodes.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index 5112bec29..83269399b 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -39,17 +39,6 @@ def apply_source_workaround(node): if node.source and node.rawsource: return - # workaround: nodes.term doesn't have source, line and rawsource - # (fixed in Docutils r7495) - if isinstance(node, nodes.term): - definition_list_item = node.parent - if definition_list_item.line is not None: - node.source = definition_list_item.source - node.line = definition_list_item.line - 1 - node.rawsource = definition_list_item. \ - rawsource.split("\n", 2)[0] - return - # workaround: docutils-0.10.0 or older's nodes.caption for nodes.figure # and nodes.title for nodes.admonition doesn't have source, line. # this issue was filed to Docutils tracker: From 3cd7e14e75824009763d6596ef1fdeee23b150a2 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sat, 25 Jul 2015 17:30:13 +0200 Subject: [PATCH 8/9] Fix: ``make text`` will crush if a definition list item has more than 1 classifiers as: ``term : classifier1 : classifier2``. --- CHANGES | 3 +++ sphinx/writers/text.py | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index cdc839925..f266cce1f 100644 --- a/CHANGES +++ b/CHANGES @@ -37,6 +37,9 @@ Bugs fixed * #1923: Use babel features only if the babel latex element is nonempty. * #1942: Fix a KeyError in websupport. * #1903: Fix strange id generation for glossary terms. +* #1796, On py3, automated .mo building cause UnicodeDecodeError +* Fix: ``make text`` will crush if a definition list item has more than 1 classifiers as: + ``term : classifier1 : classifier2``. Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py index 5bab4b881..a4785a980 100644 --- a/sphinx/writers/text.py +++ b/sphinx/writers/text.py @@ -628,8 +628,7 @@ class TextTranslator(nodes.NodeVisitor): self.end_state(first='%s. ' % self.list_counter[-1]) def visit_definition_list_item(self, node): - self._li_has_classifier = len(node) >= 2 and \ - isinstance(node[1], nodes.classifier) + self._classifier_count_in_li = len(node.traverse(nodes.classifier)) def depart_definition_list_item(self, node): pass @@ -638,7 +637,7 @@ class TextTranslator(nodes.NodeVisitor): self.new_state(0) def depart_term(self, node): - if not self._li_has_classifier: + if not self._classifier_count_in_li: self.end_state(end=None) def visit_termsep(self, node): @@ -649,7 +648,9 @@ class TextTranslator(nodes.NodeVisitor): self.add_text(' : ') def depart_classifier(self, node): - self.end_state(end=None) + self._classifier_count_in_li -= 1 + if not self._classifier_count_in_li: + self.end_state(end=None) def visit_definition(self, node): self.new_state() From d3375761d27a11a23636673c2390c321f1754114 Mon Sep 17 00:00:00 2001 From: shimizukawa Date: Sat, 25 Jul 2015 17:37:21 +0200 Subject: [PATCH 9/9] Fixed #1855: make gettext generates broken po file for definition lists with classifier. --- CHANGES | 4 ++++ sphinx/util/nodes.py | 14 ++++++++++++++ tests/roots/test-intl/definition_terms.po | 10 ++++++++++ tests/roots/test-intl/definition_terms.txt | 3 +++ tests/test_intl.py | 5 ++++- 5 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index f266cce1f..9817fcb1e 100644 --- a/CHANGES +++ b/CHANGES @@ -39,7 +39,11 @@ Bugs fixed * #1903: Fix strange id generation for glossary terms. * #1796, On py3, automated .mo building cause UnicodeDecodeError * Fix: ``make text`` will crush if a definition list item has more than 1 classifiers as: +* #1796: On py3, automated .mo building cause UnicodeDecodeError +* ``make text`` will crush if a definition list item has more than 1 classifiers as: +* Fixed #1855: make gettext generates broken po file for definition lists with classifier. ``term : classifier1 : classifier2``. +* #1855: make gettext generates broken po file for definition lists with classifier. Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index 83269399b..ea26ecdbe 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -36,6 +36,20 @@ caption_ref_re = explicit_title_re # b/w compat alias def apply_source_workaround(node): + # workaround: nodes.term have wrong rawsource if classifier is specified. + # The behavior of docutils-0.11, 0.12 is: + # * when ``term text : classifier1 : classifier2`` is specified, + # * rawsource of term node will have: ``term text : classifier1 : classifier2`` + # * rawsource of classifier node will be None + if isinstance(node, nodes.classifier) and not node.rawsource: + definition_list_item = node.parent + node.source = definition_list_item.source + node.line = definition_list_item.line - 1 + node.rawsource = node.astext() # set 'classifier1' (or 'classifier2') + if isinstance(node, nodes.term): + # overwrite: ``term : classifier1 : classifier2`` -> ``term text`` + node.rawsource = node.astext() + if node.source and node.rawsource: return diff --git a/tests/roots/test-intl/definition_terms.po b/tests/roots/test-intl/definition_terms.po index a147fe5e3..6de1d1c42 100644 --- a/tests/roots/test-intl/definition_terms.po +++ b/tests/roots/test-intl/definition_terms.po @@ -30,3 +30,13 @@ msgstr "SOME OTHER TERM" msgid "The corresponding definition #2" msgstr "THE CORRESPONDING DEFINITION #2" + +msgid "Some term with" +msgstr "SOME TERM WITH" + +msgid "classifier1" +msgstr "CLASSIFIER1" + +msgid "classifier2" +msgstr "CLASSIFIER2" + diff --git a/tests/roots/test-intl/definition_terms.txt b/tests/roots/test-intl/definition_terms.txt index 9891401d1..66230f98f 100644 --- a/tests/roots/test-intl/definition_terms.txt +++ b/tests/roots/test-intl/definition_terms.txt @@ -9,3 +9,6 @@ Some term Some other term The corresponding definition #2 +Some term with : classifier1 : classifier2 + The corresponding definition + diff --git a/tests/test_intl.py b/tests/test_intl.py index ea506020f..965fdf187 100644 --- a/tests/test_intl.py +++ b/tests/test_intl.py @@ -183,7 +183,10 @@ def test_text_builder(app, status, warning): u"\nSOME TERM" u"\n THE CORRESPONDING DEFINITION\n" u"\nSOME OTHER TERM" - u"\n THE CORRESPONDING DEFINITION #2\n") + u"\n THE CORRESPONDING DEFINITION #2\n" + u"\nSOME TERM WITH : CLASSIFIER1 : CLASSIFIER2" + u"\n THE CORRESPONDING DEFINITION\n" + ) yield assert_equal, result, expect # --- glossary terms: regression test for #1090