IntenseWebs/sphinx
3
0
Fork 0
mirror of https://github.com/sphinx-doc/sphinx.git synced 2025-02-25 18:55:22 -06:00
Code Issues Packages Projects Releases Wiki Activity

Fixed #1796: On py3, automated .mo building cause UnicodeDecodeError.

Browse Source
This commit is contained in:
shimizukawa 2015-07-25 15:47:56 +02:00
parent 657f6b16f0
commit 8600ffeda2
7 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES
View File

@ -33,6 +33,7 @@ Bugs fixed
* #1949: Use ``safe_getattr`` in the coverage builder to avoid aborting with
descriptors that have custom behavior.
* #1915: Do not generate smart quotes in doc field type annotations.
* #1796, On py3, automated .mo building cause UnicodeDecodeError
Release 1.3.1 (released Mar 17, 2015)

3
sphinx/application.py
View File

@ -200,7 +200,8 @@ class Sphinx(object):
else:
locale_dirs = []
self.translator, has_translation = locale.init(locale_dirs,
self.config.language)
self.config.language,
charset=self.config.source_encoding)
if self.config.language is not None:
if has_translation or self.config.language == 'en':
# "en" never needs to be translated

3
sphinx/builders/__init__.py
View File

@ -170,6 +170,7 @@ class Builder(object):
catalogs = i18n.find_catalog_source_files(
[path.join(self.srcdir, x) for x in self.config.locale_dirs],
self.config.language,
charset=self.config.source_encoding,
gettext_compact=self.config.gettext_compact,
force_all=True)
message = 'all of %d po files' % len(catalogs)
@ -186,6 +187,7 @@ class Builder(object):
[path.join(self.srcdir, x) for x in self.config.locale_dirs],
self.config.language,
domains=list(specified_domains),
charset=self.config.source_encoding,
gettext_compact=self.config.gettext_compact)
message = 'targets for %d po files that are specified' % len(catalogs)
self.compile_catalogs(catalogs, message)
@ -194,6 +196,7 @@ class Builder(object):
catalogs = i18n.find_catalog_source_files(
[path.join(self.srcdir, x) for x in self.config.locale_dirs],
self.config.language,
charset=self.config.source_encoding,
gettext_compact=self.config.gettext_compact)
message = 'targets for %d po files that are out of date' % len(catalogs)
self.compile_catalogs(catalogs, message)

5
sphinx/locale/__init__.py
View File

@ -195,7 +195,7 @@ else:
return translators['sphinx'].ugettext(message)
def init(locale_dirs, language, catalog='sphinx'):
def init(locale_dirs, language, catalog='sphinx', charset='utf-8'):
"""Look for message catalogs in `locale_dirs` and *ensure* that there is at
least a NullTranslations catalog set in `translators`. If called multiple
times or if several ``.mo`` files are found, their contents are merged
@ -212,7 +212,8 @@ def init(locale_dirs, language, catalog='sphinx'):
# compile mo files if po file is updated
# TODO: remove circular importing
from sphinx.util.i18n import find_catalog_source_files
for catinfo in find_catalog_source_files(locale_dirs, language, domains=[catalog]):
for catinfo in find_catalog_source_files(locale_dirs, language, domains=[catalog],
charset=charset):
catinfo.write_mo(language)
# loading

3
sphinx/transforms.py
View File

@ -225,7 +225,8 @@ class Locale(Transform):
dirs = [path.join(env.srcdir, directory)
for directory in env.config.locale_dirs]
catalog, has_catalog = init_locale(dirs, env.config.language,
textdomain)
textdomain,
charset=env.config.source_encoding)
if not has_catalog:
return

11
sphinx/util/i18n.py
View File

@ -9,6 +9,7 @@
:license: BSD, see LICENSE for details.
"""
import gettext
import io
from os import path
from collections import namedtuple
@ -19,7 +20,7 @@ from sphinx.util.osutil import walk
from sphinx.util import SEP
LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain')
LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain,charset')
class CatalogInfo(LocaleFileInfoBase):
@ -46,8 +47,8 @@ class CatalogInfo(LocaleFileInfoBase):
path.getmtime(self.mo_path) < path.getmtime(self.po_path))
def write_mo(self, locale):
with open(self.po_path, 'rt') as po:
with open(self.mo_path, 'wb') as mo:
with io.open(self.po_path, 'rt', encoding=self.charset) as po:
with io.open(self.mo_path, 'wb') as mo:
write_mo(mo, read_po(po, locale))
@ -72,7 +73,7 @@ def find_catalog_files(docname, srcdir, locale_dirs, lang, compaction):
def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact=False,
force_all=False):
charset='utf-8', force_all=False):
"""
:param list locale_dirs:
list of path as `['locale_dir1', 'locale_dir2', ...]` to find
@ -112,7 +113,7 @@ def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact
domain = domain.replace(path.sep, SEP)
if domains and domain not in domains:
continue
cat = CatalogInfo(base_dir, domain)
cat = CatalogInfo(base_dir, domain, charset)
if force_all or cat.is_outdated():
catalogs.add(cat)

8
tests/test_util_i18n.py
View File

@ -20,7 +20,7 @@ from util import with_tempdir
def test_catalog_info_for_file_and_path():
cat = i18n.CatalogInfo('path', 'domain')
cat = i18n.CatalogInfo('path', 'domain', 'utf-8')
assert cat.po_file == 'domain.po'
assert cat.mo_file == 'domain.mo'
assert cat.po_path == path.join('path', 'domain.po')
@ -28,7 +28,7 @@ def test_catalog_info_for_file_and_path():
def test_catalog_info_for_sub_domain_file_and_path():
cat = i18n.CatalogInfo('path', 'sub/domain')
cat = i18n.CatalogInfo('path', 'sub/domain', 'utf-8')
assert cat.po_file == 'sub/domain.po'
assert cat.mo_file == 'sub/domain.mo'
assert cat.po_path == path.join('path', 'sub/domain.po')
@ -38,7 +38,7 @@ def test_catalog_info_for_sub_domain_file_and_path():
@with_tempdir
def test_catalog_outdated(dir):
(dir / 'test.po').write_text('#')
cat = i18n.CatalogInfo(dir, 'test')
cat = i18n.CatalogInfo(dir, 'test', 'utf-8')
assert cat.is_outdated() # if mo is not exist
mo_file = (dir / 'test.mo')
@ -52,7 +52,7 @@ def test_catalog_outdated(dir):
@with_tempdir
def test_catalog_write_mo(dir):
(dir / 'test.po').write_text('#')
cat = i18n.CatalogInfo(dir, 'test')
cat = i18n.CatalogInfo(dir, 'test', 'utf-8')
cat.write_mo('en')
assert path.exists(cat.mo_path)
assert read_mo(open(cat.mo_path, 'rb')) is not None