Fixed #1796: On py3, automated .mo building cause UnicodeDecodeError.

CHANGES

@@ -33,6 +33,7 @@ Bugs fixed
 * #1949: Use ``safe_getattr`` in the coverage builder to avoid aborting with
   descriptors that have custom behavior.
 * #1915: Do not generate smart quotes in doc field type annotations.
+* #1796, On py3, automated .mo building cause UnicodeDecodeError
 
 
 Release 1.3.1 (released Mar 17, 2015)

sphinx/application.py

@@ -200,7 +200,8 @@ class Sphinx(object):
         else:
             locale_dirs = []
         self.translator, has_translation = locale.init(locale_dirs,
-                                                       self.config.language)
+                                                       self.config.language,
+                                                       charset=self.config.source_encoding)
         if self.config.language is not None:
             if has_translation or self.config.language == 'en':
                 # "en" never needs to be translated

sphinx/builders/__init__.py

@@ -170,6 +170,7 @@ class Builder(object):
         catalogs = i18n.find_catalog_source_files(
             [path.join(self.srcdir, x) for x in self.config.locale_dirs],
             self.config.language,
+            charset=self.config.source_encoding,
             gettext_compact=self.config.gettext_compact,
             force_all=True)
         message = 'all of %d po files' % len(catalogs)
@@ -186,6 +187,7 @@ class Builder(object):
             [path.join(self.srcdir, x) for x in self.config.locale_dirs],
             self.config.language,
             domains=list(specified_domains),
+            charset=self.config.source_encoding,
             gettext_compact=self.config.gettext_compact)
         message = 'targets for %d po files that are specified' % len(catalogs)
         self.compile_catalogs(catalogs, message)
@@ -194,6 +196,7 @@ class Builder(object):
         catalogs = i18n.find_catalog_source_files(
             [path.join(self.srcdir, x) for x in self.config.locale_dirs],
             self.config.language,
+            charset=self.config.source_encoding,
             gettext_compact=self.config.gettext_compact)
         message = 'targets for %d po files that are out of date' % len(catalogs)
         self.compile_catalogs(catalogs, message)

sphinx/locale/__init__.py

@@ -195,7 +195,7 @@ else:
         return translators['sphinx'].ugettext(message)
 
 
-def init(locale_dirs, language, catalog='sphinx'):
+def init(locale_dirs, language, catalog='sphinx', charset='utf-8'):
     """Look for message catalogs in `locale_dirs` and *ensure* that there is at
     least a NullTranslations catalog set in `translators`.  If called multiple
     times or if several ``.mo`` files are found, their contents are merged
@@ -212,7 +212,8 @@ def init(locale_dirs, language, catalog='sphinx'):
     # compile mo files if po file is updated
     # TODO: remove circular importing
     from sphinx.util.i18n import find_catalog_source_files
-    for catinfo in find_catalog_source_files(locale_dirs, language, domains=[catalog]):
+    for catinfo in find_catalog_source_files(locale_dirs, language, domains=[catalog],
+                                             charset=charset):
         catinfo.write_mo(language)
 
     # loading

sphinx/transforms.py

@@ -225,7 +225,8 @@ class Locale(Transform):
         dirs = [path.join(env.srcdir, directory)
                 for directory in env.config.locale_dirs]
         catalog, has_catalog = init_locale(dirs, env.config.language,
-                                           textdomain)
+                                           textdomain,
+                                           charset=env.config.source_encoding)
         if not has_catalog:
             return
 

sphinx/util/i18n.py

@@ -9,6 +9,7 @@
     :license: BSD, see LICENSE for details.
 """
 import gettext
+import io
 from os import path
 from collections import namedtuple
 
@@ -19,7 +20,7 @@ from sphinx.util.osutil import walk
 from sphinx.util import SEP
 
 
-LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain')
+LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain,charset')
 
 
 class CatalogInfo(LocaleFileInfoBase):
@@ -46,8 +47,8 @@ class CatalogInfo(LocaleFileInfoBase):
             path.getmtime(self.mo_path) < path.getmtime(self.po_path))
 
     def write_mo(self, locale):
-        with open(self.po_path, 'rt') as po:
+        with io.open(self.po_path, 'rt', encoding=self.charset) as po:
-            with open(self.mo_path, 'wb') as mo:
+            with io.open(self.mo_path, 'wb') as mo:
                 write_mo(mo, read_po(po, locale))
 
 
@@ -72,7 +73,7 @@ def find_catalog_files(docname, srcdir, locale_dirs, lang, compaction):
 
 
 def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact=False,
-                              force_all=False):
+                              charset='utf-8', force_all=False):
     """
     :param list locale_dirs:
        list of path as `['locale_dir1', 'locale_dir2', ...]` to find
@@ -112,7 +113,7 @@ def find_catalog_source_files(locale_dirs, locale, domains=None, gettext_compact
                 domain = domain.replace(path.sep, SEP)
                 if domains and domain not in domains:
                     continue
-                cat = CatalogInfo(base_dir, domain)
+                cat = CatalogInfo(base_dir, domain, charset)
                 if force_all or cat.is_outdated():
                     catalogs.add(cat)
 

tests/test_util_i18n.py

@@ -20,7 +20,7 @@ from util import with_tempdir
 
 
 def test_catalog_info_for_file_and_path():
-    cat = i18n.CatalogInfo('path', 'domain')
+    cat = i18n.CatalogInfo('path', 'domain', 'utf-8')
     assert cat.po_file == 'domain.po'
     assert cat.mo_file == 'domain.mo'
     assert cat.po_path == path.join('path', 'domain.po')
@@ -28,7 +28,7 @@ def test_catalog_info_for_file_and_path():
 
 
 def test_catalog_info_for_sub_domain_file_and_path():
-    cat = i18n.CatalogInfo('path', 'sub/domain')
+    cat = i18n.CatalogInfo('path', 'sub/domain', 'utf-8')
     assert cat.po_file == 'sub/domain.po'
     assert cat.mo_file == 'sub/domain.mo'
     assert cat.po_path == path.join('path', 'sub/domain.po')
@@ -38,7 +38,7 @@ def test_catalog_info_for_sub_domain_file_and_path():
 @with_tempdir
 def test_catalog_outdated(dir):
     (dir / 'test.po').write_text('#')
-    cat = i18n.CatalogInfo(dir, 'test')
+    cat = i18n.CatalogInfo(dir, 'test', 'utf-8')
     assert cat.is_outdated()  # if mo is not exist
 
     mo_file = (dir / 'test.mo')
@@ -52,7 +52,7 @@ def test_catalog_outdated(dir):
 @with_tempdir
 def test_catalog_write_mo(dir):
     (dir / 'test.po').write_text('#')
-    cat = i18n.CatalogInfo(dir, 'test')
+    cat = i18n.CatalogInfo(dir, 'test', 'utf-8')
     cat.write_mo('en')
     assert path.exists(cat.mo_path)
     assert read_mo(open(cat.mo_path, 'rb')) is not None