diff --git a/CHANGES b/CHANGES index 269401f96..3457994bd 100644 --- a/CHANGES +++ b/CHANGES @@ -33,6 +33,7 @@ Bugs fixed * #1949: Use ``safe_getattr`` in the coverage builder to avoid aborting with descriptors that have custom behavior. * #1915: Do not generate smart quotes in doc field type annotations. +* #1796, On py3, automated .mo building cause UnicodeDecodeError Release 1.3.1 (released Mar 17, 2015) diff --git a/sphinx/application.py b/sphinx/application.py index 93bdeee60..16b1c8176 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -200,7 +200,8 @@ class Sphinx(object): else: locale_dirs = [] self.translator, has_translation = locale.init(locale_dirs, - self.config.language) + self.config.language, + charset=self.config.source_encoding) if self.config.language is not None: if has_translation or self.config.language == 'en': # "en" never needs to be translated diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 38cde6123..7f5da17d2 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -170,6 +170,7 @@ class Builder(object): catalogs = i18n.find_catalog_source_files( [path.join(self.srcdir, x) for x in self.config.locale_dirs], self.config.language, + charset=self.config.source_encoding, gettext_compact=self.config.gettext_compact, force_all=True) message = 'all of %d po files' % len(catalogs) @@ -186,6 +187,7 @@ class Builder(object): [path.join(self.srcdir, x) for x in self.config.locale_dirs], self.config.language, domains=list(specified_domains), + charset=self.config.source_encoding, gettext_compact=self.config.gettext_compact) message = 'targets for %d po files that are specified' % len(catalogs) self.compile_catalogs(catalogs, message) @@ -194,6 +196,7 @@ class Builder(object): catalogs = i18n.find_catalog_source_files( [path.join(self.srcdir, x) for x in self.config.locale_dirs], self.config.language, + charset=self.config.source_encoding, gettext_compact=self.config.gettext_compact) message = 'targets for %d po files that are out of date' % len(catalogs) self.compile_catalogs(catalogs, message) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 4847c0845..444ad5d0c 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -195,7 +195,7 @@ else: return translators['sphinx'].ugettext(message) -def init(locale_dirs, language, catalog='sphinx'): +def init(locale_dirs, language, catalog='sphinx', charset='utf-8'): """Look for message catalogs in `locale_dirs` and *ensure* that there is at least a NullTranslations catalog set in `translators`. If called multiple times or if several ``.mo`` files are found, their contents are merged @@ -212,7 +212,8 @@ def init(locale_dirs, language, catalog='sphinx'): # compile mo files if po file is updated # TODO: remove circular importing from sphinx.util.i18n import find_catalog_source_files - for catinfo in find_catalog_source_files(locale_dirs, language, domains=[catalog]): + for catinfo in find_catalog_source_files(locale_dirs, language, domains=[catalog], + charset=charset): catinfo.write_mo(language) # loading diff --git a/sphinx/transforms.py b/sphinx/transforms.py index 681b50e05..3d12eb230 100644 --- a/sphinx/transforms.py +++ b/sphinx/transforms.py @@ -225,7 +225,8 @@ class Locale(Transform): dirs = [path.join(env.srcdir, directory) for directory in env.config.locale_dirs] catalog, has_catalog = init_locale(dirs, env.config.language, - textdomain) + textdomain, + charset=env.config.source_encoding) if not has_catalog: return diff --git a/sphinx/util/i18n.py b/sphinx/util/i18n.py index efdc31828..a72e138c5 100644 --- a/sphinx/util/i18n.py +++ b/sphinx/util/i18n.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for details. """ import gettext +import io from os import path from collections import namedtuple @@ -19,7 +20,7 @@ from sphinx.util.osutil import walk from sphinx.util import SEP -LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain') +LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain,charset') class CatalogInfo(LocaleFileInfoBase): @@ -46,8 +47,8 @@ class CatalogInfo(LocaleFileInfoBase): path.getmtime(self.mo_path) < path.getmtime(self.po_path)) def write_mo(self, locale): - with open(self.po_path, 'rt') as po: - with open(self.mo_path, 'wb') as mo: + with io.open(self.po_path, 'rt', encoding=self.charset) as po: + with io.open(self.mo_path, 'wb') as mo: write_mo(mo, read_po(po, locale)) @@ -72,7 +73,7 @@ def find_catalog_files(docname, srcdir, locale_dirs, lang, compaction): def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact=False, - force_all=False): + charset='utf-8', force_all=False): """ :param list locale_dirs: list of path as `['locale_dir1', 'locale_dir2', ...]` to find @@ -112,7 +113,7 @@ def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact domain = domain.replace(path.sep, SEP) if domains and domain not in domains: continue - cat = CatalogInfo(base_dir, domain) + cat = CatalogInfo(base_dir, domain, charset) if force_all or cat.is_outdated(): catalogs.add(cat) diff --git a/tests/test_util_i18n.py b/tests/test_util_i18n.py index c74962fc5..47ef8ecce 100644 --- a/tests/test_util_i18n.py +++ b/tests/test_util_i18n.py @@ -20,7 +20,7 @@ from util import with_tempdir def test_catalog_info_for_file_and_path(): - cat = i18n.CatalogInfo('path', 'domain') + cat = i18n.CatalogInfo('path', 'domain', 'utf-8') assert cat.po_file == 'domain.po' assert cat.mo_file == 'domain.mo' assert cat.po_path == path.join('path', 'domain.po') @@ -28,7 +28,7 @@ def test_catalog_info_for_file_and_path(): def test_catalog_info_for_sub_domain_file_and_path(): - cat = i18n.CatalogInfo('path', 'sub/domain') + cat = i18n.CatalogInfo('path', 'sub/domain', 'utf-8') assert cat.po_file == 'sub/domain.po' assert cat.mo_file == 'sub/domain.mo' assert cat.po_path == path.join('path', 'sub/domain.po') @@ -38,7 +38,7 @@ def test_catalog_info_for_sub_domain_file_and_path(): @with_tempdir def test_catalog_outdated(dir): (dir / 'test.po').write_text('#') - cat = i18n.CatalogInfo(dir, 'test') + cat = i18n.CatalogInfo(dir, 'test', 'utf-8') assert cat.is_outdated() # if mo is not exist mo_file = (dir / 'test.mo') @@ -52,7 +52,7 @@ def test_catalog_outdated(dir): @with_tempdir def test_catalog_write_mo(dir): (dir / 'test.po').write_text('#') - cat = i18n.CatalogInfo(dir, 'test') + cat = i18n.CatalogInfo(dir, 'test', 'utf-8') cat.write_mo('en') assert path.exists(cat.mo_path) assert read_mo(open(cat.mo_path, 'rb')) is not None