From 9222c314578b9307f2ffa376b70ff347c5f9d69b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 11:56:42 +0200 Subject: [PATCH 01/11] Fix copyright info --- sphinx/builders/intl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index de147c821..abb119a46 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -5,7 +5,7 @@ The MessageCatalogBuilder class. - :copyright: Copyright 2010 by the Sphinx team, see AUTHORS. + :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ From 57e83c6a94f21b76cf2700a62a34f3a029324fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 11:59:01 +0200 Subject: [PATCH 02/11] Fix line length --- sphinx/builders/intl.py | 3 ++- tests/test_build_gettext.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index abb119a46..7b01602e4 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -88,7 +88,8 @@ class MessageCatalogBuilder(Builder): pofile.write(POHEADER % data) for message in messages: # message contains *one* line of text ready for translation - message = message.replace(u'\\', ur'\\').replace(u'"', ur'\"') + message = message.replace(u'\\', ur'\\'). \ + replace(u'"', ur'\"') pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message pofile.write(pomsg.encode('utf-8')) finally: diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 9e98c36d6..3312f4f22 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -46,7 +46,8 @@ def test_gettext(app): if p.returncode != 0: print stdout print stderr - assert False, 'msginit exited with return code %s' % p.returncode + assert False, 'msginit exited with return code %s' % \ + p.returncode assert (app.outdir / 'en_US.po').isfile(), 'msginit failed' try: p = Popen(['msgfmt', 'en_US.po', '-o', @@ -59,8 +60,10 @@ def test_gettext(app): if p.returncode != 0: print stdout print stderr - assert False, 'msgfmt exited with return code %s' % p.returncode - assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), 'msgfmt failed' + assert False, 'msgfmt exited with return code %s' % \ + p.returncode + assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), \ + 'msgfmt failed' finally: os.chdir(cwd) @@ -95,7 +98,8 @@ def setup_patch(): print stdout print stderr assert False, 'msgfmt exited with return code %s' % p.returncode - assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), 'msgfmt failed' + assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), \ + 'msgfmt failed' def teardown_patch(): (test_root / 'xx').rmtree() test_patch.setup = setup_patch From 94d9644722601728d6ed12710ec5e7ff8a00d8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:04:27 +0200 Subject: [PATCH 03/11] Added a newline for readability --- tests/test_build_gettext.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 3312f4f22..772bba878 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -100,6 +100,7 @@ def setup_patch(): assert False, 'msgfmt exited with return code %s' % p.returncode assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), \ 'msgfmt failed' + def teardown_patch(): (test_root / 'xx').rmtree() test_patch.setup = setup_patch From 45285b678562b51115e9ba57c0760ffbcbc1209a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:13:04 +0200 Subject: [PATCH 04/11] Move i18n part of the MessageCatalogBuilder in a seperate one --- sphinx/builders/intl.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 7b01602e4..fa1dc82a3 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -9,7 +9,7 @@ :license: BSD, see LICENSE for details. """ -import collections +from collections import defaultdict from datetime import datetime from os import path @@ -41,14 +41,12 @@ msgstr "" """[1:] -class MessageCatalogBuilder(Builder): - """ - Builds gettext-style message catalogs (.pot files). - """ - name = 'gettext' +class I18NBuilder(Builder): + name = 'i18n' def init(self): - self.catalogs = collections.defaultdict(list) + Builder.init(self) + self.catalogs = defaultdict(list) def get_target_uri(self, docname, typ=None): return '' @@ -60,17 +58,17 @@ class MessageCatalogBuilder(Builder): return def write_doc(self, docname, doctree): - """ - Store a document's translatable strings in the message catalog of its - section. For this purpose a document's *top-level directory* -- or - otherwise its *name* -- is considered its section. - """ catalog = self.catalogs[docname.split(SEP, 1)[0]] for _, msg in extract_messages(doctree): - # XXX msgctxt for duplicate messages if msg not in catalog: catalog.append(msg) +class MessageCatalogBuilder(I18NBuilder): + """ + Builds gettext-style message catalogs (.pot files). + """ + name = 'gettext' + def finish(self): data = dict( version = self.config.version, From 817a7bb2c8c467c864b25e3e305f40ea0dce3a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:16:48 +0200 Subject: [PATCH 05/11] Fix test which was broken to change in the path object api --- tests/test_build_gettext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 772bba878..6a770869a 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -78,7 +78,7 @@ def test_all(app): confoverrides={'language': 'xx', 'locale_dirs': ['.']}) def test_patch(app): app.builder.build(['bom']) - result = (app.outdir / 'bom.txt').text('utf-8') + result = (app.outdir / 'bom.txt').text(encoding='utf-8') expect = (u"\nDatei mit UTF-8" u"\n***************\n" # underline matches new translation u"\nThis file has umlauts: äöü.\n") From 532a0de6010bf4b3573ed5119ef02629c59af96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:22:43 +0200 Subject: [PATCH 06/11] Monkey patch .gettext with .ugettext if possible (we use python 2.x) --- sphinx/environment.py | 2 +- sphinx/locale/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index 6339675b5..4809158d3 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -209,7 +209,7 @@ class Locale(Transform): for node, msg in extract_messages(self.document): ctx = node.parent patch = new_document(source, settings) - msgstr = catalog.ugettext(msg) + msgstr = catalog.gettext(msg) #XXX add marker to untranslated parts if not msgstr or msgstr == msg: # as-of-yet untranslated continue diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 481169918..2d3ab0269 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -217,4 +217,6 @@ def init(locale_dirs, language, catalog='sphinx'): translator = gettext.NullTranslations() has_translation = False translators[catalog] = translator + if hasattr(translator, 'ugettext'): + translator.gettext = translator.ugettext return translator, has_translation From f6680c85ca40d7f44f616d1f7d4919768077bb51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:26:37 +0200 Subject: [PATCH 07/11] Use codecs.open with python 2.x in the MessageCatalogBuilder --- sphinx/builders/intl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index fa1dc82a3..d4f5d837e 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -12,6 +12,7 @@ from collections import defaultdict from datetime import datetime from os import path +from codecs import open from docutils import nodes @@ -81,7 +82,8 @@ class MessageCatalogBuilder(I18NBuilder): self.catalogs.iteritems(), "writing message catalogs... ", lambda (section, _):darkgreen(section), len(self.catalogs)): - pofile = open(path.join(self.outdir, '%s.pot' % section), 'w') + pofp = path.join(self.outdir, section + '.pot') + pofile = open(pofp, 'w', encoding='utf-8') try: pofile.write(POHEADER % data) for message in messages: @@ -89,6 +91,6 @@ class MessageCatalogBuilder(I18NBuilder): message = message.replace(u'\\', ur'\\'). \ replace(u'"', ur'\"') pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message - pofile.write(pomsg.encode('utf-8')) + pofile.write(pomsg) finally: pofile.close() From 42279f17bd3ef43833b6d99bff3bd6e19c72ff6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:30:13 +0200 Subject: [PATCH 08/11] Fix test_gettext test for python 3.x --- tests/test_build_gettext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 6a770869a..581c1cb85 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -67,7 +67,7 @@ def test_gettext(app): finally: os.chdir(cwd) - _ = gettext.translation('test_root', app.outdir, languages=['en']).ugettext + _ = gettext.translation('test_root', app.outdir, languages=['en']).gettext assert _("Testing various markup") == u"Testing various markup" @with_app(buildername='gettext') From 0dcf347d0074d64cbd2fd9df8b32b516b386d562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:53:00 +0200 Subject: [PATCH 09/11] Added versioning support to i18n builder --- sphinx/builders/intl.py | 68 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index d4f5d837e..88c128f56 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -13,13 +13,18 @@ from collections import defaultdict from datetime import datetime from os import path from codecs import open +import os +import pickle from docutils import nodes +from docutils.utils import Reporter from sphinx.builders import Builder from sphinx.util.nodes import extract_messages -from sphinx.util.osutil import SEP +from sphinx.util.osutil import SEP, copyfile from sphinx.util.console import darkgreen +from sphinx.environment import WarningStream +from sphinx.versioning import add_uids, merge_doctrees POHEADER = ur""" # SOME DESCRIPTIVE TITLE. @@ -47,7 +52,42 @@ class I18NBuilder(Builder): def init(self): Builder.init(self) - self.catalogs = defaultdict(list) + self.catalogs = defaultdict(dict) + for root, dirs, files in os.walk(self.doctreedir): + for fn in files: + fp = path.join(root, fn) + if fp.endswith('.doctree'): + copyfile(fp, fp + '.old') + + def get_old_doctree(self, docname): + fp = self.env.doc2path(docname, self.doctreedir, '.doctree.old') + try: + f = open(fp, 'rb') + try: + doctree = pickle.load(f) + finally: + f.close() + except IOError: + return None + doctree.settings.env = self.env + doctree.reporter = Reporter(self.env.doc2path(docname), 2, 5, + stream=WarningStream(self.env._warnfunc)) + + def resave_doctree(self, docname, doctree): + reporter = doctree.reporter + doctree.reporter = None + doctree.settings.warning_stream = None + doctree.settings.env = None + doctree.settings.record_dependencies = None + + fp = self.env.doc2path(docname, self.doctreedir, '.doctree') + f = open(fp, 'wb') + try: + pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL) + finally: + f.close() + + doctree.reporter = reporter def get_target_uri(self, docname, typ=None): return '' @@ -60,9 +100,24 @@ class I18NBuilder(Builder): def write_doc(self, docname, doctree): catalog = self.catalogs[docname.split(SEP, 1)[0]] - for _, msg in extract_messages(doctree): - if msg not in catalog: - catalog.append(msg) + old_doctree = self.get_old_doctree(docname) + + if old_doctree: + list(merge_doctrees(old_doctree, doctree, nodes.TextElement)) + else: + list(add_uids(doctree, nodes.TextElement)) + self.resave_doctree(docname, doctree) + + for node, msg in extract_messages(doctree): + catalog.setdefault(node.uid, msg) + + def finish(self): + Builder.finish(self) + for root, dirs, files in os.walk(self.doctreedir): + for fn in files: + fp = path.join(root, fn) + if fp.endswith('.doctree.old'): + os.remove(fp) class MessageCatalogBuilder(I18NBuilder): """ @@ -71,6 +126,7 @@ class MessageCatalogBuilder(I18NBuilder): name = 'gettext' def finish(self): + I18NBuilder.finish(self) data = dict( version = self.config.version, copyright = self.config.copyright, @@ -86,7 +142,7 @@ class MessageCatalogBuilder(I18NBuilder): pofile = open(pofp, 'w', encoding='utf-8') try: pofile.write(POHEADER % data) - for message in messages: + for message in messages.itervalues(): # message contains *one* line of text ready for translation message = message.replace(u'\\', ur'\\'). \ replace(u'"', ur'\"') From 5e749197edc7c253e8ddfa6061bd61a80cf71d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 13:18:47 +0200 Subject: [PATCH 10/11] Before each id, str pair a comment with the uid can be found in the pot files --- sphinx/builders/intl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 88c128f56..19190926a 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -142,11 +142,11 @@ class MessageCatalogBuilder(I18NBuilder): pofile = open(pofp, 'w', encoding='utf-8') try: pofile.write(POHEADER % data) - for message in messages.itervalues(): + for uid, message in messages.iteritems(): # message contains *one* line of text ready for translation message = message.replace(u'\\', ur'\\'). \ replace(u'"', ur'\"') - pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message + pomsg = u'#%s\nmsgid "%s"\nmsgstr ""\n\n' % (uid, message) pofile.write(pomsg) finally: pofile.close() From d31e8c2f59487b6e1edac1c2f67ca45cdc9beb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 13:48:38 +0200 Subject: [PATCH 11/11] Switch VersioningBuilderMixin --- sphinx/builders/intl.py | 57 ++++------------------------------------- 1 file changed, 5 insertions(+), 52 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 19190926a..0af5b19a2 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -13,18 +13,14 @@ from collections import defaultdict from datetime import datetime from os import path from codecs import open -import os -import pickle from docutils import nodes -from docutils.utils import Reporter from sphinx.builders import Builder +from sphinx.builders.versioning import VersioningBuilderMixin from sphinx.util.nodes import extract_messages from sphinx.util.osutil import SEP, copyfile from sphinx.util.console import darkgreen -from sphinx.environment import WarningStream -from sphinx.versioning import add_uids, merge_doctrees POHEADER = ur""" # SOME DESCRIPTIVE TITLE. @@ -47,47 +43,13 @@ msgstr "" """[1:] -class I18NBuilder(Builder): +class I18NBuilder(Builder, VersioningBuilderMixin): name = 'i18n' def init(self): Builder.init(self) + VersioningBuilderMixin.init(self) self.catalogs = defaultdict(dict) - for root, dirs, files in os.walk(self.doctreedir): - for fn in files: - fp = path.join(root, fn) - if fp.endswith('.doctree'): - copyfile(fp, fp + '.old') - - def get_old_doctree(self, docname): - fp = self.env.doc2path(docname, self.doctreedir, '.doctree.old') - try: - f = open(fp, 'rb') - try: - doctree = pickle.load(f) - finally: - f.close() - except IOError: - return None - doctree.settings.env = self.env - doctree.reporter = Reporter(self.env.doc2path(docname), 2, 5, - stream=WarningStream(self.env._warnfunc)) - - def resave_doctree(self, docname, doctree): - reporter = doctree.reporter - doctree.reporter = None - doctree.settings.warning_stream = None - doctree.settings.env = None - doctree.settings.record_dependencies = None - - fp = self.env.doc2path(docname, self.doctreedir, '.doctree') - f = open(fp, 'wb') - try: - pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL) - finally: - f.close() - - doctree.reporter = reporter def get_target_uri(self, docname, typ=None): return '' @@ -100,24 +62,15 @@ class I18NBuilder(Builder): def write_doc(self, docname, doctree): catalog = self.catalogs[docname.split(SEP, 1)[0]] - old_doctree = self.get_old_doctree(docname) - if old_doctree: - list(merge_doctrees(old_doctree, doctree, nodes.TextElement)) - else: - list(add_uids(doctree, nodes.TextElement)) - self.resave_doctree(docname, doctree) + self.handle_versioning(docname, doctree, nodes.TextElement) for node, msg in extract_messages(doctree): catalog.setdefault(node.uid, msg) def finish(self): Builder.finish(self) - for root, dirs, files in os.walk(self.doctreedir): - for fn in files: - fp = path.join(root, fn) - if fp.endswith('.doctree.old'): - os.remove(fp) + VersioningBuilderMixin.finish(self) class MessageCatalogBuilder(I18NBuilder): """