diff --git a/CHANGES b/CHANGES index 30ca8a5cc..3c45ed4d2 100644 --- a/CHANGES +++ b/CHANGES @@ -40,6 +40,9 @@ Features added * #1434: Provide non-minified JS files for jquery.js and underscore.js to clarify the source of the minified files. * PR#252, #1291: Windows color console support. Thanks to meu31. +* Automatically compile ``*.mo`` files from ``*.po`` files when + :confval:`gettext_auto_build` is True (default) and ``*.po`` is newer than + ``*.mo`` file. Bugs fixed ---------- diff --git a/doc/config.rst b/doc/config.rst index 25135ef9a..d731e5deb 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -417,6 +417,14 @@ documentation on :ref:`intl` for details. .. versionadded:: 1.3 +.. confval:: gettext_auto_build + + If true, Sphinx builds mo file for each translation catalog files. + + The default is ``True``. + + .. versionadded:: 1.3 + .. _html-options: diff --git a/setup.py b/setup.py index b41ea075b..0696d6abf 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,12 @@ if sys.version_info < (2, 6) or (3, 0) <= sys.version_info < (3, 3): sys.exit(1) requires = [ - 'six>=1.4', 'Jinja2>=2.3', 'Pygments>=1.2', 'docutils>=0.10', 'snowballstemmer>=1.1' + 'six>=1.4', + 'Jinja2>=2.3', + 'Pygments>=1.2', + 'docutils>=0.10', + 'snowballstemmer>=1.1', + 'babel', ] if sys.platform == 'win32': diff --git a/sphinx/application.py b/sphinx/application.py index dc4563e0c..6683ba218 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -216,10 +216,13 @@ class Sphinx(object): def build(self, force_all=False, filenames=None): try: if force_all: + self.builder.compile_all_catalogs() self.builder.build_all() elif filenames: + self.builder.compile_specific_catalogs(filenames) self.builder.build_specific(filenames) else: + self.builder.compile_update_catalogs() self.builder.build_update() except Exception as err: # delete the saved env to force a fresh build next time diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 44c76fafb..760d88d99 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -20,7 +20,8 @@ except ImportError: from docutils import nodes -from sphinx.util.osutil import SEP, relative_uri +from sphinx.util import i18n, path_stabilize +from sphinx.util.osutil import SEP, relative_uri, find_catalog from sphinx.util.console import bold, purple, darkgreen, term_width_line # side effect: registers roles and directives @@ -170,6 +171,46 @@ class Builder(object): continue self.images[candidate] = self.env.images[candidate][1] + # compile po methods + + def compile_catalogs(self, catalogs, message): + if not self.config.gettext_auto_build: + return + self.info(bold('building [mo]: '), nonl=1) + self.info(message) + for catalog in self.status_iterator( + catalogs, 'writing output... ', darkgreen, len(catalogs), + lambda c: c.mo_path): + catalog.write_mo(self.config.language) + + def compile_all_catalogs(self): + catalogs = i18n.get_catalogs( + [path.join(self.srcdir, x) for x in self.config.locale_dirs], + self.config.language, True) + message = 'all of %d po files' % len(catalogs) + self.compile_catalogs(catalogs, message) + + def compile_specific_catalogs(self, specified_files): + def to_domain(fpath): + docname, _ = path.splitext(path_stabilize(fpath)) + dom = find_catalog(docname, self.config.gettext_compact) + return dom + + specified_domains = set(map(to_domain, specified_files)) + catalogs = i18n.get_catalogs( + [path.join(self.srcdir, x) for x in self.config.locale_dirs], + self.config.language, True) + catalogs = [f for f in catalogs if f.domain in specified_domains] + message = 'targets for %d po files that are specified' % len(catalogs) + self.compile_catalogs(catalogs, message) + + def compile_update_catalogs(self): + catalogs = i18n.get_catalogs( + [path.join(self.srcdir, x) for x in self.config.locale_dirs], + self.config.language) + message = 'targets for %d po files that are out of date' % len(catalogs) + self.compile_catalogs(catalogs, message) + # build methods def build_all(self): diff --git a/sphinx/builders/gettext.py b/sphinx/builders/gettext.py index f36d0202c..657ce9241 100644 --- a/sphinx/builders/gettext.py +++ b/sphinx/builders/gettext.py @@ -99,6 +99,9 @@ class I18nBuilder(Builder): def prepare_writing(self, docnames): return + def compile_catalogs(self, catalogs, message): + return + def write_doc(self, docname, doctree): catalog = self.catalogs[find_catalog(docname, self.config.gettext_compact)] diff --git a/sphinx/config.py b/sphinx/config.py index 1b67f89a2..7ab34eb3b 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -204,6 +204,7 @@ class Config(object): gettext_compact = (True, 'gettext'), gettext_location = (True, 'gettext'), gettext_uuid = (True, 'gettext'), + gettext_auto_build = (True, 'env'), # XML options xml_pretty = (True, 'env'), diff --git a/sphinx/util/i18n.py b/sphinx/util/i18n.py new file mode 100644 index 000000000..fa477a695 --- /dev/null +++ b/sphinx/util/i18n.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +""" + sphinx.util.i18n + ~~~~~~~~~~~~~~~~ + + Builder superclass for all builders. + + :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from os import path +from collections import namedtuple + +from babel.messages.pofile import read_po +from babel.messages.mofile import write_mo + +from sphinx.util.osutil import walk + + +LocaleFileInfoBase = namedtuple('CatalogInfo', 'base_dir,domain') + + +class CatalogInfo(LocaleFileInfoBase): + + @property + def po_file(self): + return self.domain + '.po' + + @property + def mo_file(self): + return self.domain + '.mo' + + @property + def po_path(self): + return path.join(self.base_dir, self.po_file) + + @property + def mo_path(self): + return path.join(self.base_dir, self.mo_file) + + def is_outdated(self): + return ( + not path.exists(self.mo_path) or + path.getmtime(self.mo_path) < path.getmtime(self.po_path)) + + def write_mo(self, locale): + with open(self.po_path, 'rt') as p, open(self.mo_path, 'wb') as m: + write_mo(m, read_po(p, locale)) + + +def get_catalogs(locale_dirs, locale, gettext_compact=False, force_all=False): + """ + :param list locale_dirs: + list of path as `['locale_dir1', 'locale_dir2', ...]` to find + translation catalogs. Each path contains a structure such as + `/LC_MESSAGES/domain.po`. + :param str locale: a language as `'en'` + :param boolean gettext_compact: + * False: keep domains directory structure (default). + * True: domains in the sub directory will be merged into 1 file. + :param boolean force_all: + Set True if you want to get all catalogs rather than updated catalogs. + default is False. + :return: [CatalogInfo(), ...] + """ + if not locale: + return [] # locale is not specified + + catalogs = set() + for locale_dir in locale_dirs: + base_dir = path.join(locale_dir, locale, 'LC_MESSAGES') + + if not path.exists(base_dir): + continue # locale path is not found + + for dirpath, dirnames, filenames in walk(base_dir, followlinks=True): + filenames = [f for f in filenames if f.endswith('.po')] + for filename in filenames: + base = path.splitext(filename)[0] + domain = path.relpath(path.join(dirpath, base), base_dir) + if gettext_compact and path.sep in domain: + domain = path.split(domain)[0] + cat = CatalogInfo(base_dir, domain) + if force_all or cat.is_outdated(): + catalogs.add(cat) + + return catalogs diff --git a/tests/test_build_base.py b/tests/test_build_base.py new file mode 100644 index 000000000..ee2706261 --- /dev/null +++ b/tests/test_build_base.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +""" + test_build_base + ~~~~~~~~~~~~~~~ + + Test the base build process. + + :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import shutil + +from nose.tools import with_setup + +from util import test_roots, with_app, find_files + +root = test_roots / 'test-intl' +build_dir = root / '_build' +locale_dir = build_dir / 'locale' + + +def setup_test(): + # Delete remnants left over after failed build + locale_dir.rmtree(True) + # copy all catalogs into locale layout directory + for po in find_files(root, '.po'): + copy_po = (locale_dir / 'en' / 'LC_MESSAGES' / po) + if not copy_po.parent.exists(): + copy_po.parent.makedirs() + shutil.copy(root / po, copy_po) + + +def teardown_test(): + build_dir.rmtree(True), + + +@with_setup(setup_test, teardown_test) +@with_app(buildername='html', srcdir=root, + confoverrides={'language': 'en', 'locale_dirs': [locale_dir]}) +def test_compile_all_catalogs(app): + app.builder.compile_all_catalogs() + + catalog_dir = locale_dir / app.config.language / 'LC_MESSAGES' + expect = set([ + x.replace('.po', '.mo') + for x in find_files(catalog_dir, '.po') + ]) + actual = set(find_files(catalog_dir, '.mo')) + assert actual # not empty + assert actual == expect + + +@with_setup(setup_test, teardown_test) +@with_app(buildername='html', srcdir=root, + confoverrides={'language': 'en', 'locale_dirs': [locale_dir]}) +def test_compile_specific_catalogs(app): + app.builder.compile_specific_catalogs(['admonitions']) + + catalog_dir = locale_dir / app.config.language / 'LC_MESSAGES' + actual = set(find_files(catalog_dir, '.mo')) + assert actual == set(['admonitions.mo']) + + +@with_setup(setup_test, teardown_test) +@with_app(buildername='html', srcdir=root, + confoverrides={'language': 'en', 'locale_dirs': [locale_dir]}) +def test_compile_update_catalogs(app): + app.builder.compile_update_catalogs() + + catalog_dir = locale_dir / app.config.language / 'LC_MESSAGES' + expect = set([ + x.replace('.po', '.mo') + for x in find_files(catalog_dir, '.po') + ]) + actual = set(find_files(catalog_dir, '.mo')) + assert actual # not empty + assert actual == expect diff --git a/tests/test_util_i18n.py b/tests/test_util_i18n.py new file mode 100644 index 000000000..afc9fb36d --- /dev/null +++ b/tests/test_util_i18n.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +""" + test_util_i18n + ~~~~~~~~~~~~~~ + + Test i18n util. + + :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +from __future__ import print_function + +import os +from os import path + +from babel.messages.mofile import read_mo +from sphinx.util import i18n + +from util import with_tempdir + + +def test_catalog_info_for_file_and_path(): + cat = i18n.CatalogInfo('path', 'domain') + assert cat.po_file == 'domain.po' + assert cat.mo_file == 'domain.mo' + assert cat.po_path == path.join('path', 'domain.po') + assert cat.mo_path == path.join('path', 'domain.mo') + + +def test_catalog_info_for_sub_domain_file_and_path(): + cat = i18n.CatalogInfo('path', 'sub/domain') + assert cat.po_file == 'sub/domain.po' + assert cat.mo_file == 'sub/domain.mo' + assert cat.po_path == path.join('path', 'sub/domain.po') + assert cat.mo_path == path.join('path', 'sub/domain.mo') + + +@with_tempdir +def test_catalog_outdated(dir): + (dir / 'test.po').write_text('#') + cat = i18n.CatalogInfo(dir, 'test') + assert cat.is_outdated() # if mo is not exist + + mo_file = (dir / 'test.mo') + mo_file.write_text('#') + assert not cat.is_outdated() # if mo is exist and newer than po + + os.utime(mo_file, (os.stat(mo_file).st_mtime - 10,) * 2) # to be outdate + assert cat.is_outdated() # if mo is exist and older than po + + +@with_tempdir +def test_catalog_write_mo(dir): + (dir / 'test.po').write_text('#') + cat = i18n.CatalogInfo(dir, 'test') + cat.write_mo('en') + assert path.exists(cat.mo_path) + assert read_mo(open(cat.mo_path, 'rb')) is not None + + +@with_tempdir +def test_get_catalogs_for_xx(dir): + (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs() + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test2.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test3.pot').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub').makedirs() + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test4.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test5.po').write_text('#') + (dir / 'loc1' / 'en' / 'LC_MESSAGES').makedirs() + (dir / 'loc1' / 'en' / 'LC_MESSAGES' / 'test6.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_ALL').makedirs() + (dir / 'loc1' / 'xx' / 'LC_ALL' / 'test7.po').write_text('#') + + catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx', force_all=False) + domains = set(c.domain for c in catalogs) + assert domains == set([ + 'test1', + 'test2', + path.normpath('sub/test4'), + path.normpath('sub/test5'), + ]) + + +@with_tempdir +def test_get_catalogs_for_en(dir): + (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs() + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'xx_dom.po').write_text('#') + (dir / 'loc1' / 'en' / 'LC_MESSAGES').makedirs() + (dir / 'loc1' / 'en' / 'LC_MESSAGES' / 'en_dom.po').write_text('#') + + catalogs = i18n.get_catalogs([dir / 'loc1'], 'en', force_all=False) + domains = set(c.domain for c in catalogs) + assert domains == set(['en_dom']) + + +@with_tempdir +def test_get_catalogs_with_non_existent_locale(dir): + catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx') + assert not catalogs + + catalogs = i18n.get_catalogs([dir / 'loc1'], None) + assert not catalogs + + +def test_get_catalogs_with_non_existent_locale_dirs(): + catalogs = i18n.get_catalogs(['dummy'], 'xx') + assert not catalogs + + +@with_tempdir +def test_get_catalogs_for_xx_without_outdated(dir): + (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs() + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.mo').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test2.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test2.mo').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test3.pot').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test3.mo').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub').makedirs() + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test4.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test4.mo').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test5.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test5.mo').write_text('#') + + catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx', force_all=False) + assert not catalogs + + catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx', force_all=True) + domains = set(c.domain for c in catalogs) + assert domains == set([ + 'test1', + 'test2', + path.normpath('sub/test4'), + path.normpath('sub/test5'), + ]) + + +@with_tempdir +def test_get_catalogs_from_multiple_locale_dirs(dir): + (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs() + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#') + (dir / 'loc2' / 'xx' / 'LC_MESSAGES').makedirs() + (dir / 'loc2' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#') + (dir / 'loc2' / 'xx' / 'LC_MESSAGES' / 'test2.po').write_text('#') + + catalogs = i18n.get_catalogs([dir / 'loc1', dir / 'loc2'], 'xx') + domains = sorted(c.domain for c in catalogs) + assert domains == ['test1', 'test1', 'test2'] + + +@with_tempdir +def test_get_catalogs_with_compact(dir): + (dir / 'loc1' / 'xx' / 'LC_MESSAGES').makedirs() + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test1.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'test2.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub').makedirs() + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test3.po').write_text('#') + (dir / 'loc1' / 'xx' / 'LC_MESSAGES' / 'sub' / 'test4.po').write_text('#') + + catalogs = i18n.get_catalogs([dir / 'loc1'], 'xx', gettext_compact=True) + domains = set(c.domain for c in catalogs) + assert domains == set(['test1', 'test2', 'sub']) diff --git a/tests/util.py b/tests/util.py index 24c548b2b..2e2f96655 100644 --- a/tests/util.py +++ b/tests/util.py @@ -7,6 +7,7 @@ :license: BSD, see LICENSE for details. """ +import os import sys import tempfile import shutil @@ -233,3 +234,11 @@ def sprint(*args): _unicode_literals_re = re.compile(r'u(".*?")|u(\'.*?\')') def remove_unicode_literals(s): return _unicode_literals_re.sub(lambda x: x.group(1) or x.group(2), s) + + +def find_files(root, suffix=None): + for dirpath, dirs, files in os.walk(root, followlinks=True): + dirpath = path(dirpath) + for f in [f for f in files if not suffix or f.endswith(suffix)]: + fpath = dirpath / f + yield os.path.relpath(fpath, root)