Automatically compile `*.mo files from *.po` files.

CHANGES

@@ -40,6 +40,9 @@ Features added
 * #1434: Provide non-minified JS files for jquery.js and underscore.js to
   clarify the source of the minified files.
 * PR#252, #1291: Windows color console support. Thanks to meu31.
+* Automatically compile ``*.mo`` files from ``*.po`` files when
+  :confval:`gettext_auto_build` is True (default) and ``*.po`` is newer than
+  ``*.mo`` file.
 
 Bugs fixed
 ----------

doc/config.rst

@@ -417,6 +417,14 @@ documentation on :ref:`intl` for details.
 
    .. versionadded:: 1.3
 
+.. confval:: gettext_auto_build
+
+   If true, Sphinx builds mo file for each translation catalog files.
+
+   The default is ``True``.
+
+   .. versionadded:: 1.3
+
 
 .. _html-options:
 

setup.py

@@ -46,7 +46,12 @@ if sys.version_info < (2, 6) or (3, 0) <= sys.version_info < (3, 3):
     sys.exit(1)
 
 requires = [
-    'six>=1.4', 'Jinja2>=2.3', 'Pygments>=1.2', 'docutils>=0.10', 'snowballstemmer>=1.1'
+    'six>=1.4',
+    'Jinja2>=2.3',
+    'Pygments>=1.2',
+    'docutils>=0.10',
+    'snowballstemmer>=1.1',
+    'babel',
 ]
 
 if sys.platform == 'win32':

sphinx/application.py

@@ -216,10 +216,13 @@ class Sphinx(object):
     def build(self, force_all=False, filenames=None):
         try:
             if force_all:
+                self.builder.compile_all_catalogs()
                 self.builder.build_all()
             elif filenames:
+                self.builder.compile_specific_catalogs(filenames)
                 self.builder.build_specific(filenames)
             else:
+                self.builder.compile_update_catalogs()
                 self.builder.build_update()
         except Exception as err:
             # delete the saved env to force a fresh build next time

sphinx/builders/__init__.py

@@ -20,7 +20,8 @@ except ImportError:
 
 from docutils import nodes
 
-from sphinx.util.osutil import SEP, relative_uri
+from sphinx.util import i18n, path_stabilize
+from sphinx.util.osutil import SEP, relative_uri, find_catalog
 from sphinx.util.console import bold, purple, darkgreen, term_width_line
 
 # side effect: registers roles and directives
@@ -170,6 +171,46 @@ class Builder(object):
                 continue
             self.images[candidate] = self.env.images[candidate][1]
 
+    # compile po methods
+
+    def compile_catalogs(self, catalogs, message):
+        if not self.config.gettext_auto_build:
+            return
+        self.info(bold('building [mo]: '), nonl=1)
+        self.info(message)
+        for catalog in self.status_iterator(
+                catalogs, 'writing output... ', darkgreen, len(catalogs),
+                lambda c: c.mo_path):
+            catalog.write_mo(self.config.language)
+
+    def compile_all_catalogs(self):
+        catalogs = i18n.get_catalogs(
+            [path.join(self.srcdir, x) for x in self.config.locale_dirs],
+            self.config.language, True)
+        message = 'all of %d po files' % len(catalogs)
+        self.compile_catalogs(catalogs, message)
+
+    def compile_specific_catalogs(self, specified_files):
+        def to_domain(fpath):
+            docname, _ = path.splitext(path_stabilize(fpath))
+            dom = find_catalog(docname, self.config.gettext_compact)
+            return dom
+
+        specified_domains = set(map(to_domain, specified_files))
+        catalogs = i18n.get_catalogs(
+            [path.join(self.srcdir, x) for x in self.config.locale_dirs],
+            self.config.language, True)
+        catalogs = [f for f in catalogs if f.domain in specified_domains]
+        message = 'targets for %d po files that are specified' % len(catalogs)
+        self.compile_catalogs(catalogs, message)
+
+    def compile_update_catalogs(self):
+        catalogs = i18n.get_catalogs(
+            [path.join(self.srcdir, x) for x in self.config.locale_dirs],
+            self.config.language)
+        message = 'targets for %d po files that are out of date' % len(catalogs)
+        self.compile_catalogs(catalogs, message)
+
     # build methods
 
     def build_all(self):

sphinx/builders/gettext.py

@@ -99,6 +99,9 @@ class I18nBuilder(Builder):
     def prepare_writing(self, docnames):
         return
 
+    def compile_catalogs(self, catalogs, message):
+        return
+
     def write_doc(self, docname, doctree):
         catalog = self.catalogs[find_catalog(docname,
                                              self.config.gettext_compact)]

sphinx/config.py

@@ -204,6 +204,7 @@ class Config(object):
         gettext_compact = (True, 'gettext'),
         gettext_location = (True, 'gettext'),
         gettext_uuid = (True, 'gettext'),
+        gettext_auto_build = (True, 'env'),
 
         # XML options
         xml_pretty = (True, 'env'),

sphinx/util/i18n.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.util.i18n
+    ~~~~~~~~~~~~~~~~
+
+    Builder superclass for all builders.
+
+    :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from os import path
+from collections import namedtuple
+
+from babel.messages.pofile import read_po
+from babel.messages.mofile import write_mo
+
+from sphinx.util.osutil import walk
+
+
+LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain')
+
+
+class CatalogInfo(LocaleFileInfoBase):
+
+    @property
+    def po_file(self):
+        return self.domain + '.po'
+
+    @property
+    def mo_file(self):
+        return self.domain + '.mo'
+
+    @property
+    def po_path(self):
+        return path.join(self.base_dir, self.po_file)
+
+    @property
+    def mo_path(self):
+        return path.join(self.base_dir, self.mo_file)
+
+    def is_outdated(self):
+        return (
+            not path.exists(self.mo_path) or
+            path.getmtime(self.mo_path) < path.getmtime(self.po_path))
+
+    def write_mo(self, locale):
+        with open(self.po_path, 'rt') as p, open(self.mo_path, 'wb') as m:
+            write_mo(m, read_po(p, locale))
+
+
+def get_catalogs(locale_dirs, locale, gettext_compact=False, force_all=False):
+    """
+    :param list locale_dirs:
+       list of path as `['locale_dir1', 'locale_dir2', ...]` to find
+       translation catalogs. Each path contains a structure such as
+       `<locale>/LC_MESSAGES/domain.po`.
+    :param str locale: a language as `'en'`
+    :param boolean gettext_compact:
+       * False: keep domains directory structure (default).
+       * True: domains in the sub directory will be merged into 1 file.
+    :param boolean force_all:
+       Set True if you want to get all catalogs rather than updated catalogs.
+       default is False.
+    :return: [CatalogInfo(), ...]
+    """
+    if not locale:
+        return []  # locale is not specified
+
+    catalogs = set()
+    for locale_dir in locale_dirs:
+        base_dir = path.join(locale_dir, locale, 'LC_MESSAGES')
+
+        if not path.exists(base_dir):
+            continue  # locale path is not found
+
+        for dirpath, dirnames, filenames in walk(base_dir, followlinks=True):
+            filenames = [f for f in filenames if f.endswith('.po')]
+            for filename in filenames:
+                base = path.splitext(filename)[0]
+                domain = path.relpath(path.join(dirpath, base), base_dir)
+                if gettext_compact and path.sep in domain:
+                    domain = path.split(domain)[0]
+                cat = CatalogInfo(base_dir, domain)
+                if force_all or cat.is_outdated():
+                    catalogs.add(cat)
+
+    return catalogs

tests/test_build_base.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+    test_build_base
+    ~~~~~~~~~~~~~~~
+
+    Test the base build process.
+
+    :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import shutil
+
+from nose.tools import with_setup
+
+from util import test_roots, with_app, find_files
+
+root = test_roots / 'test-intl'
+build_dir = root / '_build'
+locale_dir = build_dir / 'locale'
+
+
+def setup_test():
+    # Delete remnants left over after failed build
+    locale_dir.rmtree(True)
+    # copy all catalogs into locale layout directory
+    for po in find_files(root, '.po'):
+        copy_po = (locale_dir / 'en' / 'LC_MESSAGES' / po)
+        if not copy_po.parent.exists():
+            copy_po.parent.makedirs()
+        shutil.copy(root / po, copy_po)
+
+
+def teardown_test():
+    build_dir.rmtree(True),
+
+
+@with_setup(setup_test, teardown_test)
+@with_app(buildername='html', srcdir=root,
+          confoverrides={'language': 'en', 'locale_dirs': [locale_dir]})
+def test_compile_all_catalogs(app):
+    app.builder.compile_all_catalogs()
+
+    catalog_dir = locale_dir / app.config.language / 'LC_MESSAGES'
+    expect = set([
+        x.replace('.po', '.mo')
+        for x in find_files(catalog_dir, '.po')
+    ])
+    actual = set(find_files(catalog_dir, '.mo'))
+    assert actual  # not empty
+    assert actual == expect
+
+
+@with_setup(setup_test, teardown_test)
+@with_app(buildername='html', srcdir=root,
+          confoverrides={'language': 'en', 'locale_dirs': [locale_dir]})
+def test_compile_specific_catalogs(app):
+    app.builder.compile_specific_catalogs(['admonitions'])
+
+    catalog_dir = locale_dir / app.config.language / 'LC_MESSAGES'
+    actual = set(find_files(catalog_dir, '.mo'))
+    assert actual == set(['admonitions.mo'])
+
+
+@with_setup(setup_test, teardown_test)
+@with_app(buildername='html', srcdir=root,
+          confoverrides={'language': 'en', 'locale_dirs': [locale_dir]})
+def test_compile_update_catalogs(app):
+    app.builder.compile_update_catalogs()
+
+    catalog_dir = locale_dir / app.config.language / 'LC_MESSAGES'
+    expect = set([
+        x.replace('.po', '.mo')
+        for x in find_files(catalog_dir, '.po')
+    ])
+    actual = set(find_files(catalog_dir, '.mo'))
+    assert actual  # not empty
+    assert actual == expect

tests/test_util_i18n.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+"""
+    test_util_i18n
+    ~~~~~~~~~~~~~~
+
+    Test i18n util.
+
+    :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+from __future__ import print_function
+
+import os
+from os import path
+
+from babel.messages.mofile import read_mo
+from sphinx.util import i18n
+
+from util import with_tempdir
+
+
+def test_catalog_info_for_file_and_path():
+    cat = i18n.CatalogInfo('path', 'domain')
+    assert cat.po_file == 'domain.po'
+    assert cat.mo_file == 'domain.mo'
+    assert cat.po_path == path.join('path', 'domain.po')
+    assert cat.mo_path == path.join('path', 'domain.mo')
+
+
+def test_catalog_info_for_sub_domain_file_and_path():
+    cat = i18n.CatalogInfo('path', 'sub/domain')
+    assert cat.po_file == 'sub/domain.po'
+    assert cat.mo_file == 'sub/domain.mo'
+    assert cat.po_path == path.join('path', 'sub/domain.po')
+    assert cat.mo_path == path.join('path', 'sub/domain.mo')
+
+
+@with_tempdir
+def test_catalog_outdated(dir):
+    (dir / 'test.po').write_text('#')
+    cat = i18n.CatalogInfo(dir, 'test')
+    assert cat.is_outdated()  # if mo is not exist
+
+    mo_file = (dir / 'test.mo')
+    mo_file.write_text('#')
+    assert not cat.is_outdated()  # if mo is exist and newer than po
+
+    os.utime(mo_file, (os.stat(mo_file).st_mtime - 10,) * 2) # to be outdate
+    assert cat.is_outdated()  # if mo is exist and older than po
+
+
+@with_tempdir
+def test_catalog_write_mo(dir):
+    (dir / 'test.po').write_text('#')
+    cat = i18n.CatalogInfo(dir, 'test')
+    cat.write_mo('en')
+    assert path.exists(cat.mo_path)
+    assert read_mo(open(cat.mo_path, 'rb')) is not None
+
+
+@with_tempdir
+def test_get_catalogs_for_xx(dir):
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test2.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test3.pot').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test4.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test5.po').write_text('#')
+    (dir / 'loc1' / 'en' / 'LC_MESSAGES').makedirs()
+    (dir / 'loc1' / 'en' / 'LC_MESSAGES' / 'test6.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_ALL').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_ALL' / 'test7.po').write_text('#')
+
+    catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx', force_all=False)
+    domains = set(c.domain for c in catalogs)
+    assert domains == set([
+        'test1',
+        'test2',
+        path.normpath('sub/test4'),
+        path.normpath('sub/test5'),
+    ])
+
+
+@with_tempdir
+def test_get_catalogs_for_en(dir):
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'xx_dom.po').write_text('#')
+    (dir / 'loc1' / 'en' / 'LC_MESSAGES').makedirs()
+    (dir / 'loc1' / 'en' / 'LC_MESSAGES' / 'en_dom.po').write_text('#')
+
+    catalogs = i18n.get_catalogs([dir / 'loc1'], 'en', force_all=False)
+    domains = set(c.domain for c in catalogs)
+    assert domains == set(['en_dom'])
+
+
+@with_tempdir
+def test_get_catalogs_with_non_existent_locale(dir):
+    catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx')
+    assert not catalogs
+
+    catalogs = i18n.get_catalogs([dir / 'loc1'], None)
+    assert not catalogs
+
+
+def test_get_catalogs_with_non_existent_locale_dirs():
+    catalogs = i18n.get_catalogs(['dummy'], 'xx')
+    assert not catalogs
+
+
+@with_tempdir
+def test_get_catalogs_for_xx_without_outdated(dir):
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.mo').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test2.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test2.mo').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test3.pot').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test3.mo').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test4.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test4.mo').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test5.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test5.mo').write_text('#')
+
+    catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx', force_all=False)
+    assert not catalogs
+
+    catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx', force_all=True)
+    domains = set(c.domain for c in catalogs)
+    assert domains == set([
+        'test1',
+        'test2',
+        path.normpath('sub/test4'),
+        path.normpath('sub/test5'),
+    ])
+
+
+@with_tempdir
+def test_get_catalogs_from_multiple_locale_dirs(dir):
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#')
+    (dir / 'loc2' / 'xx' / 'LC_MESSAGES').makedirs()
+    (dir / 'loc2' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#')
+    (dir / 'loc2' / 'xx' / 'LC_MESSAGES' / 'test2.po').write_text('#')
+
+    catalogs = i18n.get_catalogs([dir / 'loc1', dir / 'loc2'], 'xx')
+    domains = sorted(c.domain for c in catalogs)
+    assert domains == ['test1', 'test1', 'test2']
+
+
+@with_tempdir
+def test_get_catalogs_with_compact(dir):
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test2.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub').makedirs()
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test3.po').write_text('#')
+    (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test4.po').write_text('#')
+
+    catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx', gettext_compact=True)
+    domains = set(c.domain for c in catalogs)
+    assert domains == set(['test1', 'test2', 'sub'])

tests/util.py

@@ -7,6 +7,7 @@
     :license: BSD, see LICENSE for details.
 """
 
+import os
 import sys
 import tempfile
 import shutil
@@ -233,3 +234,11 @@ def sprint(*args):
 _unicode_literals_re = re.compile(r'u(".*?")|u(\'.*?\')')
 def remove_unicode_literals(s):
     return _unicode_literals_re.sub(lambda x: x.group(1) or x.group(2), s)
+
+
+def find_files(root, suffix=None):
+    for dirpath, dirs, files in os.walk(root, followlinks=True):
+        dirpath = path(dirpath)
+        for f in [f for f in files if not suffix or f.endswith(suffix)]:
+            fpath = dirpath / f
+            yield os.path.relpath(fpath, root)