From d405575620c4a69cbde7fb1db771e90d45708773 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sat, 29 May 2010 18:14:42 +0200 Subject: [PATCH 01/62] Skeleton for PO builder. --- sphinx/builders/intl.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 sphinx/builders/intl.py diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py new file mode 100644 index 000000000..5bc186971 --- /dev/null +++ b/sphinx/builders/intl.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" + sphinx.builders.intl + ~~~~~~~~~~~~~~~~~~~~ + + The MessageCatalogBuilder class. + + :copyright: Copyright 2010 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from sphinx.builders import Builder + +class MessageCatalogBuilder(Builder): + pass + + def get_target_uri(self, docname, typ=None): + return '' + + def get_outdated_docs(self): + return self.env.found_docs + + def prepare_writing(self, docnames): + return + + def write_doc(self, docname, doctree): + return + + def finish(self): + return From f0cbf2c4e19dba35fd0beb6595dd3e49b49de0fa Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sat, 29 May 2010 20:26:38 +0200 Subject: [PATCH 02/62] Add PO builder to sphinx-quickstart. --- sphinx/quickstart.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sphinx/quickstart.py b/sphinx/quickstart.py index 8479575a8..0aab261e7 100644 --- a/sphinx/quickstart.py +++ b/sphinx/quickstart.py @@ -321,7 +321,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) \ $(SPHINXOPTS) %(rsrcdir)s .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp \ -epub latex latexpdf text man changes linkcheck doctest +epub latex latexpdf text man changes linkcheck doctest gettext help: \t@echo "Please use \\`make ' where is one of" @@ -338,6 +338,7 @@ help: \t@echo " latexpdf to make LaTeX files and run them through pdflatex" \t@echo " text to make text files" \t@echo " man to make manual pages" +\t@echo " gettext to make PO message catalogs" \t@echo " changes to make an overview of all changed/added/deprecated items" \t@echo " linkcheck to check all external links for integrity" \t@echo " doctest to run all doctests embedded in the documentation \ @@ -424,6 +425,11 @@ man: \t@echo \t@echo "Build finished. The manual pages are in $(BUILDDIR)/man." +gettext: +\t$(SPHINXBUILD) -b gettext $(ALLSPHINXOPTS) $(BUILDDIR)/locale +\t@echo +\t@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + changes: \t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes \t@echo @@ -472,6 +478,7 @@ if "%%1" == "help" ( \techo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter \techo. text to make text files \techo. man to make manual pages +\techo. gettext to make PO message catalogs \techo. changes to make an overview over all changed/added/deprecated items \techo. linkcheck to check all external links for integrity \techo. doctest to run all doctests embedded in the documentation if enabled @@ -573,6 +580,13 @@ if "%%1" == "man" ( \tgoto end ) +if "%%1" == "gettext" ( +\t%%SPHINXBUILD%% -b gettext %%ALLSPHINXOPTS%% %%BUILDDIR%%/locale +\techo. +\techo.Build finished. The message catalogs are in %%BUILDDIR%%/locale. +\tgoto end +) + if "%%1" == "changes" ( \t%%SPHINXBUILD%% -b changes %%ALLSPHINXOPTS%% %%BUILDDIR%%/changes \techo. From ab16ae303a15e838fcf97dc477681da91eff0586 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sun, 30 May 2010 09:36:23 +0200 Subject: [PATCH 03/62] Add gettext build using intl.MessageCatalogBuilder. --- sphinx/builders/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index e345d570f..cf7f39557 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -329,4 +329,5 @@ BUILTIN_BUILDERS = { 'man': ('manpage', 'ManualPageBuilder'), 'changes': ('changes', 'ChangesBuilder'), 'linkcheck': ('linkcheck', 'CheckExternalLinksBuilder'), + 'gettext': ('intl', 'MessageCatalogBuilder'), } From a1932f62f0fe7121385b81842660688fdfa7bc4c Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sun, 30 May 2010 17:02:54 +0200 Subject: [PATCH 04/62] Add gettext builder to Sphinx docs. --- doc/Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/Makefile b/doc/Makefile index b873c7e5a..ff3cb22b3 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -29,6 +29,7 @@ help: @echo " epub to make an epub file" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run pdflatex" + @echo " gettext to make PO message catalogs" @echo " changes to make an overview over all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @@ -112,6 +113,11 @@ latexpdf: make -C _build/latex all-pdf @echo "pdflatex finished; the PDF files are in _build/latex." +gettext: + $(SPHINXBUILD) -b gettext $(ALLSPHINXOPTS) _build/locale + @echo + @echo "Build finished. The message catalogs are in _build/locale." + changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes @echo From 2cbe83cf5e07657bef7cc7b5fc0ec86799d35474 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 2 Jun 2010 09:17:42 +0200 Subject: [PATCH 05/62] Collect raw messages for translation. --- sphinx/builders/intl.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 5bc186971..0aecdcf3d 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -9,10 +9,14 @@ :license: BSD, see LICENSE for details. """ +import collections +from docutils import nodes + from sphinx.builders import Builder class MessageCatalogBuilder(Builder): - pass + def init(self): + self.catalogs = collections.defaultdict(list) def get_target_uri(self, docname, typ=None): return '' @@ -24,7 +28,9 @@ class MessageCatalogBuilder(Builder): return def write_doc(self, docname, doctree): - return + catalog = self.catalogs[docname.split('/')[0]] + for node in doctree.traverse(nodes.TextElement): + catalog.append(node.astext()) def finish(self): return From 8a87a46318f0deda6a29c389a394dc95de56e575 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 2 Jun 2010 09:18:30 +0200 Subject: [PATCH 06/62] Normalize messages for later rewrapping. --- sphinx/builders/intl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 0aecdcf3d..2cbb17cf3 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -30,7 +30,8 @@ class MessageCatalogBuilder(Builder): def write_doc(self, docname, doctree): catalog = self.catalogs[docname.split('/')[0]] for node in doctree.traverse(nodes.TextElement): - catalog.append(node.astext()) + msg = node.astext().replace('\n', ' ') + catalog.append(msg) def finish(self): return From 8413f2ce503f6f28e9454ae3256317de424280d7 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 2 Jun 2010 09:19:08 +0200 Subject: [PATCH 07/62] Ignore invisible and inline nodes during translation. --- sphinx/builders/intl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 2cbb17cf3..9373baae8 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -30,6 +30,8 @@ class MessageCatalogBuilder(Builder): def write_doc(self, docname, doctree): catalog = self.catalogs[docname.split('/')[0]] for node in doctree.traverse(nodes.TextElement): + if isinstance(node, (nodes.Invisible, nodes.Inline)): + continue msg = node.astext().replace('\n', ' ') catalog.append(msg) From 995c18609c4e68dd6b8450f8b7206952a78f189d Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 2 Jun 2010 09:35:33 +0200 Subject: [PATCH 08/62] Write message catalogs to POT files. --- sphinx/builders/intl.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 9373baae8..c40fff266 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -10,6 +10,8 @@ """ import collections +from os import path + from docutils import nodes from sphinx.builders import Builder @@ -36,4 +38,8 @@ class MessageCatalogBuilder(Builder): catalog.append(msg) def finish(self): - return + for section, messages in self.catalogs.iteritems(): + pofile = open(path.join(self.outdir, '%s.pot' % section), 'w') + for message in messages: + pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message + pofile.write(pomsg.encode('utf-8')) From 08381bb2ad2d17e3fc4d9faeba241640c80a412f Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 2 Jun 2010 09:45:13 +0200 Subject: [PATCH 09/62] Add meta information to PO headers. --- sphinx/builders/intl.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index c40fff266..3224d92f3 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -16,6 +16,27 @@ from docutils import nodes from sphinx.builders import Builder +POHEADER = r""" +# SOME DESCRIPTIVE TITLE. +# Copyright (C) %(copyright)s +# This file is distributed under the same license as the %(project)s package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: 1.0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2010-05-08 18:29+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +"""[1:] + class MessageCatalogBuilder(Builder): def init(self): self.catalogs = collections.defaultdict(list) @@ -40,6 +61,7 @@ class MessageCatalogBuilder(Builder): def finish(self): for section, messages in self.catalogs.iteritems(): pofile = open(path.join(self.outdir, '%s.pot' % section), 'w') + pofile.write(POHEADER % self.config) for message in messages: pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message pofile.write(pomsg.encode('utf-8')) From d60c41afa9585cc751b1240ad0d66acef8afaf9d Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Fri, 4 Jun 2010 16:54:37 +0200 Subject: [PATCH 10/62] Add public name to MessageCatalogBuilder. --- sphinx/builders/intl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 3224d92f3..8966cf8b3 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -38,6 +38,8 @@ msgstr "" """[1:] class MessageCatalogBuilder(Builder): + name = 'gettext' + def init(self): self.catalogs = collections.defaultdict(list) From 01e6e183035b8d7520a4741df093ba8b83b24b2b Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Fri, 4 Jun 2010 16:55:35 +0200 Subject: [PATCH 11/62] Escaped quotation marks in msgids. --- sphinx/builders/intl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 8966cf8b3..6291c4050 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -65,5 +65,6 @@ class MessageCatalogBuilder(Builder): pofile = open(path.join(self.outdir, '%s.pot' % section), 'w') pofile.write(POHEADER % self.config) for message in messages: + message = message.replace(u'"', ur'\"') pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message pofile.write(pomsg.encode('utf-8')) From 2801f52f15fa3051e903e3aaa4b7940378766141 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Fri, 4 Jun 2010 17:14:04 +0200 Subject: [PATCH 12/62] Properly close open .pot files. --- sphinx/builders/intl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 6291c4050..f388537d4 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -68,3 +68,4 @@ class MessageCatalogBuilder(Builder): message = message.replace(u'"', ur'\"') pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message pofile.write(pomsg.encode('utf-8')) + pofile.close() From 3f6fa966d000cc04d083926175e51087bb0826bc Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Fri, 4 Jun 2010 18:21:19 +0200 Subject: [PATCH 13/62] Use progress indicator for gettext builds. --- sphinx/builders/intl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index f388537d4..67937582a 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -15,6 +15,7 @@ from os import path from docutils import nodes from sphinx.builders import Builder +from sphinx.util.console import darkgreen POHEADER = r""" # SOME DESCRIPTIVE TITLE. @@ -61,7 +62,9 @@ class MessageCatalogBuilder(Builder): catalog.append(msg) def finish(self): - for section, messages in self.catalogs.iteritems(): + for section, messages in self.status_iterator( + self.catalogs.iteritems(), "writing message catalogs... ", + lambda (section, _):darkgreen(section), len(self.catalogs)): pofile = open(path.join(self.outdir, '%s.pot' % section), 'w') pofile.write(POHEADER % self.config) for message in messages: From 4a197b29ce36b5aa4274007967a2180326f673b6 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Fri, 4 Jun 2010 18:23:06 +0200 Subject: [PATCH 14/62] Safeguard file.close() against failure. --- sphinx/builders/intl.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 67937582a..dc6cbf25e 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -66,9 +66,11 @@ class MessageCatalogBuilder(Builder): self.catalogs.iteritems(), "writing message catalogs... ", lambda (section, _):darkgreen(section), len(self.catalogs)): pofile = open(path.join(self.outdir, '%s.pot' % section), 'w') - pofile.write(POHEADER % self.config) - for message in messages: - message = message.replace(u'"', ur'\"') - pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message - pofile.write(pomsg.encode('utf-8')) - pofile.close() + try: + pofile.write(POHEADER % self.config) + for message in messages: + message = message.replace(u'"', ur'\"') + pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message + pofile.write(pomsg.encode('utf-8')) + finally: + pofile.close() From 7c80750ee33eef4464e0c2f27f60349937381041 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Fri, 4 Jun 2010 18:26:47 +0200 Subject: [PATCH 15/62] Prepare msgid for escaped sequences. --- sphinx/builders/intl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index dc6cbf25e..506f90010 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -69,7 +69,8 @@ class MessageCatalogBuilder(Builder): try: pofile.write(POHEADER % self.config) for message in messages: - message = message.replace(u'"', ur'\"') + # message contains *one* line of text ready for translation + message = message.replace(u'\\', ur'\\').replace(u'"', ur'\"') pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message pofile.write(pomsg.encode('utf-8')) finally: From 0c21f913eb4fdb8e3ad5dcaa880fac6f560fddb6 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Fri, 4 Jun 2010 18:38:16 +0200 Subject: [PATCH 16/62] Document basic workflow in gettext builder. --- sphinx/builders/intl.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 506f90010..660155f38 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -39,6 +39,9 @@ msgstr "" """[1:] class MessageCatalogBuilder(Builder): + """ + Builds gettext-style message catalogs (.pot files). + """ name = 'gettext' def init(self): @@ -54,6 +57,11 @@ class MessageCatalogBuilder(Builder): return def write_doc(self, docname, doctree): + """ + Store a document's translatable strings in the message catalog of its + section. For this purpose a document's *top-level directory* -- or + otherwise its *name* -- is considered its section. + """ catalog = self.catalogs[docname.split('/')[0]] for node in doctree.traverse(nodes.TextElement): if isinstance(node, (nodes.Invisible, nodes.Inline)): From d3f161859842c59a3c16159ffc75975cb60076d3 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sun, 6 Jun 2010 20:54:01 +0200 Subject: [PATCH 17/62] Supply version information in PO header. --- sphinx/builders/intl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 660155f38..199a5a3e1 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -17,7 +17,7 @@ from docutils import nodes from sphinx.builders import Builder from sphinx.util.console import darkgreen -POHEADER = r""" +POHEADER = ur""" # SOME DESCRIPTIVE TITLE. # Copyright (C) %(copyright)s # This file is distributed under the same license as the %(project)s package. @@ -26,7 +26,7 @@ POHEADER = r""" #, fuzzy msgid "" msgstr "" -"Project-Id-Version: 1.0\n" +"Project-Id-Version: %(version)s\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2010-05-08 18:29+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" From e588fa8176538b2720be667a9e49fe79dfe60e90 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sun, 6 Jun 2010 20:55:57 +0200 Subject: [PATCH 18/62] Supply initial creation date in PO header. --- sphinx/builders/intl.py | 2 +- sphinx/config.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 199a5a3e1..2fc5011cb 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -28,7 +28,7 @@ msgid "" msgstr "" "Project-Id-Version: %(version)s\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2010-05-08 18:29+0200\n" +"POT-Creation-Date: %(gettext_ctime)s\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/sphinx/config.py b/sphinx/config.py index e25427828..ddaa3dfd3 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -11,6 +11,7 @@ import os import re +from datetime import datetime from os import path from sphinx.errors import ConfigError @@ -150,6 +151,11 @@ class Config(object): # manpage options man_pages = ([], None), + + # gettext options + gettext_ctime = (lambda self:datetime.now() # should supply tz + .strftime('%Y-%m-%d %H:%M%z'), + 'gettext'), ) def __init__(self, dirname, filename, overrides, tags): From ead9fa89b0a6dd5d45efbc2191a61d0c78e6f099 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Mon, 7 Jun 2010 13:58:04 +0200 Subject: [PATCH 19/62] Add generic test for gettext builder. --- tests/test_build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_build.py b/tests/test_build.py index f18ff1754..6d98e3995 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -57,3 +57,7 @@ else: @with_app(buildername='singlehtml', cleanenv=True) def test_singlehtml(app): app.builder.build_all() + +@with_app(buildername='gettext') +def test_gettext(app): + app.builder.build_all() From 0299008e55fc2f0e2c585d5e97139f90a1ac62b3 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Mon, 7 Jun 2010 14:04:16 +0200 Subject: [PATCH 20/62] Initial tests for gettext build. --- tests/test_build_gettext.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/test_build_gettext.py diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py new file mode 100644 index 000000000..caa775875 --- /dev/null +++ b/tests/test_build_gettext.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +""" + test_build_gettext + ~~~~~~~~~~~~~~~~ + + Test the build process with gettext builder with the test root. + + :copyright: Copyright 2010 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from util import * + + +def teardown_module(): + (test_root / '_build').rmtree(True) + + +@with_app(buildername='gettext', cleanenv=True) +def test_gettext(app): + app.builder.build_all() + assert (app.outdir / 'contents.pot').isfile() + # group into sections + assert (app.outdir / 'subdir.pot').isfile() From 5ac5f23f2b031eba4e9dba8f482b1bbe42cf291b Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 9 Jun 2010 06:46:30 +0200 Subject: [PATCH 21/62] Verify PO file format with msginit. --- tests/test_build_gettext.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index caa775875..3a992e12f 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -9,6 +9,9 @@ :license: BSD, see LICENSE for details. """ +import os +from subprocess import Popen, PIPE + from util import * @@ -22,3 +25,22 @@ def test_gettext(app): assert (app.outdir / 'contents.pot').isfile() # group into sections assert (app.outdir / 'subdir.pot').isfile() + + cwd = os.getcwd() + os.chdir(app.outdir) + try: + try: + p = Popen(['msginit', '--no-translator', '-i', 'contents.pot'], + stdout=PIPE, stderr=PIPE) + except OSError: + return # most likely msginit was not found + else: + stdout, stderr = p.communicate() + if p.returncode != 0: + print stdout + print stderr + del app.cleanup_trees[:] + assert False, 'msginit exited with return code %s' % p.returncode + assert (app.outdir / 'en_US.po').isfile(), 'msginit failed' + finally: + os.chdir(cwd) From 438c24e2226811299246b62fd2497a5cca899c11 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 9 Jun 2010 06:54:59 +0200 Subject: [PATCH 22/62] Prepare test root catalogs for gettext with msgfmt. --- tests/test_build_gettext.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 3a992e12f..72c00a0dd 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -26,6 +26,7 @@ def test_gettext(app): # group into sections assert (app.outdir / 'subdir.pot').isfile() + (app.outdir / 'en' / 'LC_MESSAGES').makedirs() cwd = os.getcwd() os.chdir(app.outdir) try: @@ -42,5 +43,19 @@ def test_gettext(app): del app.cleanup_trees[:] assert False, 'msginit exited with return code %s' % p.returncode assert (app.outdir / 'en_US.po').isfile(), 'msginit failed' + try: + p = Popen(['msgfmt', 'en_US.po', '-o', + os.path.join('en', 'LC_MESSAGES', 'test_root.mo')], + stdout=PIPE, stderr=PIPE) + except OSError: + return # most likely msgfmt was not found + else: + stdout, stderr = p.communicate() + if p.returncode != 0: + print stdout + print stderr + del app.cleanup_trees[:] + assert False, 'msgfmt exited with return code %s' % p.returncode + assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), 'msgfmt failed' finally: os.chdir(cwd) From 3e449e300531ce52bdce48a7ef268120e6330432 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 9 Jun 2010 07:46:40 +0200 Subject: [PATCH 23/62] Delete generated files on test failure. --- tests/test_build_gettext.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 72c00a0dd..5a7945150 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -40,7 +40,6 @@ def test_gettext(app): if p.returncode != 0: print stdout print stderr - del app.cleanup_trees[:] assert False, 'msginit exited with return code %s' % p.returncode assert (app.outdir / 'en_US.po').isfile(), 'msginit failed' try: @@ -54,7 +53,6 @@ def test_gettext(app): if p.returncode != 0: print stdout print stderr - del app.cleanup_trees[:] assert False, 'msgfmt exited with return code %s' % p.returncode assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), 'msgfmt failed' finally: From c9dd331c12c8200f6074d76b815eaa68ee2e81f4 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 9 Jun 2010 07:47:14 +0200 Subject: [PATCH 24/62] Fix empty and duplicate nodes. --- sphinx/builders/intl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 2fc5011cb..4943fb29c 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -66,7 +66,11 @@ class MessageCatalogBuilder(Builder): for node in doctree.traverse(nodes.TextElement): if isinstance(node, (nodes.Invisible, nodes.Inline)): continue - msg = node.astext().replace('\n', ' ') + msg = node.astext().replace('\n', ' ').strip() + # XXX nodes rendering empty are likely a bug in sphinx.addnodes + # XXX msgctxt for duplicate messages? + if not msg or msg in catalog: + continue catalog.append(msg) def finish(self): From 0109af80b56088e12325b0a499c67d3867fc33ef Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 9 Jun 2010 15:19:58 +0200 Subject: [PATCH 25/62] Refactor message extractor into utilities. --- sphinx/builders/intl.py | 14 +++++--------- sphinx/util/nodes.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 4943fb29c..72fb78964 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -15,6 +15,7 @@ from os import path from docutils import nodes from sphinx.builders import Builder +from sphinx.util.nodes import extract_messages from sphinx.util.console import darkgreen POHEADER = ur""" @@ -63,15 +64,10 @@ class MessageCatalogBuilder(Builder): otherwise its *name* -- is considered its section. """ catalog = self.catalogs[docname.split('/')[0]] - for node in doctree.traverse(nodes.TextElement): - if isinstance(node, (nodes.Invisible, nodes.Inline)): - continue - msg = node.astext().replace('\n', ' ').strip() - # XXX nodes rendering empty are likely a bug in sphinx.addnodes - # XXX msgctxt for duplicate messages? - if not msg or msg in catalog: - continue - catalog.append(msg) + for msg in extract_messages(doctree): + # XXX msgctxt for duplicate messages + if msg not in catalog: + catalog.append(msg) def finish(self): for section, messages in self.status_iterator( diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index 82427f134..84182b153 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -22,6 +22,17 @@ explicit_title_re = re.compile(r'^(.+?)\s*(?$', re.DOTALL) caption_ref_re = explicit_title_re # b/w compat alias +def extract_messages(doctree): + """Extract translatable messages from a document tree.""" + for node in doctree.traverse(nodes.TextElement): + if isinstance(node, (nodes.Invisible, nodes.Inline)): + continue + msg = node.astext().replace('\n', ' ').strip() + # XXX nodes rendering empty are likely a bug in sphinx.addnodes + if msg: + yield msg + + def nested_parse_with_titles(state, content, node): # hack around title style bookkeeping surrounding_title_styles = state.memo.title_styles From cd07472676920e16560d1ae85f9ef8ea845834f2 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 16 Jun 2010 20:27:27 +0200 Subject: [PATCH 26/62] Extract translatable strings alongside their doctree nodes. --- sphinx/builders/intl.py | 2 +- sphinx/util/nodes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 72fb78964..b760854b5 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -64,7 +64,7 @@ class MessageCatalogBuilder(Builder): otherwise its *name* -- is considered its section. """ catalog = self.catalogs[docname.split('/')[0]] - for msg in extract_messages(doctree): + for _, msg in extract_messages(doctree): # XXX msgctxt for duplicate messages if msg not in catalog: catalog.append(msg) diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index 84182b153..0b23d17fa 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -30,7 +30,7 @@ def extract_messages(doctree): msg = node.astext().replace('\n', ' ').strip() # XXX nodes rendering empty are likely a bug in sphinx.addnodes if msg: - yield msg + yield node, msg def nested_parse_with_titles(state, content, node): From e63bb0f4a98459138e1b853f960f9b716c7faa99 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Thu, 17 Jun 2010 11:46:49 +0200 Subject: [PATCH 27/62] Split up tests into logical units. --- tests/test_build_gettext.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 5a7945150..ee18d96e1 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -19,12 +19,17 @@ def teardown_module(): (test_root / '_build').rmtree(True) +@with_app(buildername='gettext', cleanenv=True) +def test_build(app): + app.builder.build_all() + # documents end up in a message catalog + assert (app.outdir / 'contents.pot').isfile() + # ..and are grouped into sections + assert (app.outdir / 'subdir.pot').isfile() + @with_app(buildername='gettext', cleanenv=True) def test_gettext(app): app.builder.build_all() - assert (app.outdir / 'contents.pot').isfile() - # group into sections - assert (app.outdir / 'subdir.pot').isfile() (app.outdir / 'en' / 'LC_MESSAGES').makedirs() cwd = os.getcwd() From b1c480f5c68b63f5c80c2577e615b0d668e4721f Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Thu, 17 Jun 2010 11:49:07 +0200 Subject: [PATCH 28/62] Remove cleanenv setting from tests. --- tests/test_build_gettext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index ee18d96e1..236558359 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -19,7 +19,7 @@ def teardown_module(): (test_root / '_build').rmtree(True) -@with_app(buildername='gettext', cleanenv=True) +@with_app(buildername='gettext') def test_build(app): app.builder.build_all() # documents end up in a message catalog @@ -27,7 +27,7 @@ def test_build(app): # ..and are grouped into sections assert (app.outdir / 'subdir.pot').isfile() -@with_app(buildername='gettext', cleanenv=True) +@with_app(buildername='gettext') def test_gettext(app): app.builder.build_all() From b5f29b2c7c7e6a054ac2f5deee9feafad2289801 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Thu, 17 Jun 2010 11:57:59 +0200 Subject: [PATCH 29/62] Strip down tests to build only critical parts. --- tests/test_build_gettext.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 236558359..c0dff938a 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -21,22 +21,22 @@ def teardown_module(): @with_app(buildername='gettext') def test_build(app): - app.builder.build_all() + app.builder.build(['extapi', 'subdir/includes']) # documents end up in a message catalog - assert (app.outdir / 'contents.pot').isfile() + assert (app.outdir / 'extapi.pot').isfile() # ..and are grouped into sections assert (app.outdir / 'subdir.pot').isfile() @with_app(buildername='gettext') def test_gettext(app): - app.builder.build_all() + app.builder.build(['markup']) (app.outdir / 'en' / 'LC_MESSAGES').makedirs() cwd = os.getcwd() os.chdir(app.outdir) try: try: - p = Popen(['msginit', '--no-translator', '-i', 'contents.pot'], + p = Popen(['msginit', '--no-translator', '-i', 'markup.pot'], stdout=PIPE, stderr=PIPE) except OSError: return # most likely msginit was not found From 804a92dad23363f32a9b6baef0cd6b37c86065ab Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Fri, 18 Jun 2010 10:15:08 +0200 Subject: [PATCH 30/62] Patch translatable messages with custom doctree. --- sphinx/environment.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index c8b3f018c..acf76f6eb 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -24,7 +24,7 @@ from itertools import izip, groupby from docutils import nodes from docutils.io import FileInput, NullOutput from docutils.core import Publisher -from docutils.utils import Reporter, relative_path +from docutils.utils import Reporter, relative_path, new_document from docutils.readers import standalone from docutils.parsers.rst import roles, directives from docutils.parsers.rst.languages import en as english @@ -36,7 +36,7 @@ from docutils.transforms.parts import ContentsFilter from sphinx import addnodes from sphinx.util import url_re, get_matching_docs, docname_join, \ FilenameUniqDict -from sphinx.util.nodes import clean_astext, make_refnode +from sphinx.util.nodes import clean_astext, make_refnode, extract_messages from sphinx.util.osutil import movefile, SEP, ustrftime from sphinx.util.matching import compile_matchers from sphinx.errors import SphinxError, ExtensionError @@ -168,13 +168,26 @@ class CitationReferences(Transform): refnode += nodes.Text('[' + cittext + ']') citnode.parent.replace(citnode, refnode) +class Locale(Transform): + """ + Replace translatable nodes with their translated doctree. + """ + default_priority = 0 + + def apply(self): + settings = self.document.settings + for node, msg in extract_messages(self.document): + ctx = node.parent + patch = new_document(msg, settings) + ctx.replace(node, patch.children) + class SphinxStandaloneReader(standalone.Reader): """ Add our own transforms. """ - transforms = [CitationReferences, DefaultSubstitutions, MoveModuleTargets, - HandleCodeBlocks, SortIds] + transforms = [Locale, CitationReferences, DefaultSubstitutions, + MoveModuleTargets, HandleCodeBlocks, SortIds] def get_transforms(self): return standalone.Reader.get_transforms(self) + self.transforms From 3a7ce4039ac8458f64f5a77cfc820fccc28633a2 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 23 Jun 2010 06:18:47 +0200 Subject: [PATCH 31/62] Check compiled message catalogs are processable with gettext. --- tests/test_build_gettext.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index c0dff938a..7041dcac8 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for details. """ +import gettext import os from subprocess import Popen, PIPE @@ -62,3 +63,6 @@ def test_gettext(app): assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), 'msgfmt failed' finally: os.chdir(cwd) + + _ = gettext.translation('test_root', app.outdir, languages=['en']).ugettext + assert _("Testing various markup") == u"Testing various markup" From 3c4cad50855d43ea28cf9242cdc962b5f74e3e2a Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 23 Jun 2010 07:28:58 +0200 Subject: [PATCH 32/62] Add parsing step to translation integration. --- sphinx/environment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index acf76f6eb..cab03cd23 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -26,7 +26,7 @@ from docutils.io import FileInput, NullOutput from docutils.core import Publisher from docutils.utils import Reporter, relative_path, new_document from docutils.readers import standalone -from docutils.parsers.rst import roles, directives +from docutils.parsers.rst import roles, directives, Parser as RSTParser from docutils.parsers.rst.languages import en as english from docutils.parsers.rst.directives.html import MetaBody from docutils.writers import UnfilteredWriter @@ -176,9 +176,12 @@ class Locale(Transform): def apply(self): settings = self.document.settings + parser = RSTParser() for node, msg in extract_messages(self.document): ctx = node.parent patch = new_document(msg, settings) + msgstr = "Insert translation **here**." + parser.parse(msgstr, patch) ctx.replace(node, patch.children) From 35ee258b2e391e62dd22900e5406194b64ac567c Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 23 Jun 2010 07:30:24 +0200 Subject: [PATCH 33/62] Fix source file reference in patched documents. --- sphinx/environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index cab03cd23..a2c30ddd2 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -175,11 +175,11 @@ class Locale(Transform): default_priority = 0 def apply(self): - settings = self.document.settings + settings, source = self.document.settings, self.document['source'] parser = RSTParser() for node, msg in extract_messages(self.document): ctx = node.parent - patch = new_document(msg, settings) + patch = new_document(source, settings) msgstr = "Insert translation **here**." parser.parse(msgstr, patch) ctx.replace(node, patch.children) From 8e12af649f9dd5355ec5f2b681b61aab414e0880 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 23 Jun 2010 07:36:33 +0200 Subject: [PATCH 34/62] Ignore orphan metadata field in translatable messages. --- sphinx/util/nodes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index 0b23d17fa..87fd362c7 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -27,6 +27,9 @@ def extract_messages(doctree): for node in doctree.traverse(nodes.TextElement): if isinstance(node, (nodes.Invisible, nodes.Inline)): continue + # orphan + if isinstance(node, nodes.field_name) and node.children[0] == 'orphan': + continue msg = node.astext().replace('\n', ' ').strip() # XXX nodes rendering empty are likely a bug in sphinx.addnodes if msg: From 10fc7cd73bb602e992443481750dcdf603d540cd Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sat, 26 Jun 2010 12:25:53 +0200 Subject: [PATCH 35/62] Move POT-Creation-Date configuration value back into code. --- sphinx/builders/intl.py | 13 +++++++++++-- sphinx/config.py | 6 ------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index b760854b5..e2aa0c320 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -10,6 +10,7 @@ """ import collections +from datetime import datetime from os import path from docutils import nodes @@ -29,7 +30,7 @@ msgid "" msgstr "" "Project-Id-Version: %(version)s\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: %(gettext_ctime)s\n" +"POT-Creation-Date: %(ctime)s\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -70,12 +71,20 @@ class MessageCatalogBuilder(Builder): catalog.append(msg) def finish(self): + data = dict( + version = self.config.version, + copyright = self.config.copyright, + project = self.config.project, + # XXX should supply tz + ctime = datetime.now().strftime('%Y-%m-%d %H:%M%z'), + ) for section, messages in self.status_iterator( self.catalogs.iteritems(), "writing message catalogs... ", lambda (section, _):darkgreen(section), len(self.catalogs)): + pofile = open(path.join(self.outdir, '%s.pot' % section), 'w') try: - pofile.write(POHEADER % self.config) + pofile.write(POHEADER % data) for message in messages: # message contains *one* line of text ready for translation message = message.replace(u'\\', ur'\\').replace(u'"', ur'\"') diff --git a/sphinx/config.py b/sphinx/config.py index 3b7fcd103..e1075ff6a 100644 --- a/sphinx/config.py +++ b/sphinx/config.py @@ -11,7 +11,6 @@ import os import re -from datetime import datetime from os import path from sphinx.errors import ConfigError @@ -151,11 +150,6 @@ class Config(object): # manpage options man_pages = ([], None), - - # gettext options - gettext_ctime = (lambda self:datetime.now() # should supply tz - .strftime('%Y-%m-%d %H:%M%z'), - 'gettext'), ) def __init__(self, dirname, filename, overrides, tags): From b14658199f2a8132e30a772512f7ff85ce0844cd Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sat, 26 Jun 2010 12:38:06 +0200 Subject: [PATCH 36/62] Document gettext builder and create i18n docs. --- doc/builders.rst | 12 ++++++++++++ doc/contents.rst | 1 + doc/intl.rst | 11 +++++++++++ 3 files changed, 24 insertions(+) create mode 100644 doc/intl.rst diff --git a/doc/builders.rst b/doc/builders.rst index 6e90ccc62..5aefac442 100644 --- a/doc/builders.rst +++ b/doc/builders.rst @@ -220,6 +220,18 @@ Note that a direct PDF builder using ReportLab is available in `rst2pdf .. versionadded:: 0.5 +.. module:: sphinx.builders.intl +.. class:: MessageCatalogBuilder + + This builder produces a message catalog file for each reST file or + subdirectory. + + See the documentation on :ref:`internationalization ` for further reference. + + Its name is ``gettext``. + + .. versionadded:: 1.XXX + .. module:: sphinx.builders.changes .. class:: ChangesBuilder diff --git a/doc/contents.rst b/doc/contents.rst index 079f93f26..c8c0988bc 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -14,6 +14,7 @@ Sphinx documentation contents domains builders config + intl theming templating extensions diff --git a/doc/intl.rst b/doc/intl.rst new file mode 100644 index 000000000..8722177de --- /dev/null +++ b/doc/intl.rst @@ -0,0 +1,11 @@ +.. _intl: + +Internationalization +==================== + +.. versionadded:: 1.XXX + +Complementary to translations provided for internal messages such as navigation +bars Sphinx provides mechanisms facilitating *document* translations in itself. +It relies on the existing configuration values :confval:`language` and +:confval:`locale_dirs`. From 7b3830a78a6c6c97e56c8a5467d233ba553b54ab Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sat, 26 Jun 2010 14:13:43 +0200 Subject: [PATCH 37/62] Pull gettext tests into one single location. --- tests/test_build.py | 4 ---- tests/test_build_gettext.py | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_build.py b/tests/test_build.py index 6d98e3995..f18ff1754 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -57,7 +57,3 @@ else: @with_app(buildername='singlehtml', cleanenv=True) def test_singlehtml(app): app.builder.build_all() - -@with_app(buildername='gettext') -def test_gettext(app): - app.builder.build_all() diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 7041dcac8..09dc0adb6 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -66,3 +66,7 @@ def test_gettext(app): _ = gettext.translation('test_root', app.outdir, languages=['en']).ugettext assert _("Testing various markup") == u"Testing various markup" + +@with_app(buildername='gettext') +def test_all(app): + app.builder.build_all() From ba5e496a5d86000dc6dc0bc33b241973170441e2 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sat, 26 Jun 2010 15:29:25 +0200 Subject: [PATCH 38/62] Hide canonical path separator. --- sphinx/builders/intl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index e2aa0c320..de147c821 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -17,6 +17,7 @@ from docutils import nodes from sphinx.builders import Builder from sphinx.util.nodes import extract_messages +from sphinx.util.osutil import SEP from sphinx.util.console import darkgreen POHEADER = ur""" @@ -64,7 +65,7 @@ class MessageCatalogBuilder(Builder): section. For this purpose a document's *top-level directory* -- or otherwise its *name* -- is considered its section. """ - catalog = self.catalogs[docname.split('/')[0]] + catalog = self.catalogs[docname.split(SEP, 1)[0]] for _, msg in extract_messages(doctree): # XXX msgctxt for duplicate messages if msg not in catalog: From 804b8df9bb4633b2de1df6f6c4a2ffdb4a3db74d Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Sat, 26 Jun 2010 15:30:35 +0200 Subject: [PATCH 39/62] Prepare locale initialization for multiple catalogs. --- sphinx/locale/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index b0b89720c..aadd99e61 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -176,18 +176,19 @@ pairindextypes = { 'builtin': l_('built-in function'), } -translator = None +translators = {} def _(message): - return translator.ugettext(message) + return translators['sphinx'].ugettext(message) -def init(locale_dirs, language): - global translator +def init(locale_dirs, language, catalog='sphinx'): + global translators + translator = translators.get(catalog) # the None entry is the system's default locale path has_translation = True for dir_ in locale_dirs: try: - trans = gettext.translation('sphinx', localedir=dir_, + trans = gettext.translation(catalog, localedir=dir_, languages=[language]) if translator is None: translator = trans @@ -199,4 +200,5 @@ def init(locale_dirs, language): if translator is None: translator = gettext.NullTranslations() has_translation = False + translators[catalog] = translator return translator, has_translation From bbc03c26e3862e04d69f2aec1275e105d52eef15 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Tue, 29 Jun 2010 08:05:28 +0200 Subject: [PATCH 40/62] Make locale.init reentrable. --- sphinx/locale/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index aadd99e61..02583ff84 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -182,8 +182,17 @@ def _(message): return translators['sphinx'].ugettext(message) def init(locale_dirs, language, catalog='sphinx'): + """ + Look for message catalogs in `locale_dirs` and *ensure* that there is at + least a NullTranslations catalog set in `translators`. If called multiple + times or several ``.mo`` files are found their contents are merged + together (thus making `init` reentrable). + """ global translators translator = translators.get(catalog) + # ignore previously failed attempts to find message catalogs + if isinstance(translator, gettext.NullTranslations): + translator = None # the None entry is the system's default locale path has_translation = True for dir_ in locale_dirs: @@ -197,6 +206,7 @@ def init(locale_dirs, language, catalog='sphinx'): except Exception: # Language couldn't be found in the specified path pass + # guarantee translations[catalog] exists if translator is None: translator = gettext.NullTranslations() has_translation = False From 6a715f5d1ab41b100c6f73c7440b41262c29aaad Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Tue, 29 Jun 2010 08:06:48 +0200 Subject: [PATCH 41/62] Move translation patching into read_doc for access to runtime settings. --- sphinx/environment.py | 45 ++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index 17d464257..50129e0cb 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -41,7 +41,7 @@ from sphinx.util.osutil import movefile, SEP, ustrftime from sphinx.util.matching import compile_matchers from sphinx.util.pycompat import all from sphinx.errors import SphinxError, ExtensionError -from sphinx.locale import _ +from sphinx.locale import _, init as init_locale orig_role_function = roles.role @@ -181,28 +181,12 @@ class CitationReferences(Transform): refnode += nodes.Text('[' + cittext + ']') citnode.parent.replace(citnode, refnode) -class Locale(Transform): - """ - Replace translatable nodes with their translated doctree. - """ - default_priority = 0 - - def apply(self): - settings, source = self.document.settings, self.document['source'] - parser = RSTParser() - for node, msg in extract_messages(self.document): - ctx = node.parent - patch = new_document(source, settings) - msgstr = "Insert translation **here**." - parser.parse(msgstr, patch) - ctx.replace(node, patch.children) - class SphinxStandaloneReader(standalone.Reader): """ Add our own transforms. """ - transforms = [Locale, CitationReferences, DefaultSubstitutions, + transforms = [CitationReferences, DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, SortIds] def get_transforms(self): @@ -611,6 +595,7 @@ class BuildEnvironment: Parse a file and add/update inventory entries for the doctree. If srcpath is given, read from a different source file. """ + section = docname.split(SEP, 1)[0] # remove all inventory entries for that file if app: app.emit('env-purge-doc', self, docname) @@ -675,6 +660,7 @@ class BuildEnvironment: # post-processing self.filter_messages(doctree) + self.process_translations(doctree, self.get_translation(section)) self.process_dependencies(docname, doctree) self.process_images(docname, doctree) self.process_downloads(docname, doctree) @@ -749,6 +735,13 @@ class BuildEnvironment: def note_dependency(self, filename): self.dependencies.setdefault(self.docname, set()).add(filename) + def get_translation(self, section): + translation, has_trans = init_locale(self.config.locale_dirs, + self.config.language, section) + if not has_trans: + return None + return translation + # post-processing of read doctrees def filter_messages(self, doctree): @@ -760,6 +753,22 @@ class BuildEnvironment: if node['level'] < filterlevel: node.parent.remove(node) + def process_translations(self, doctree, translation): + """ + Replace translatable nodes with their translated doctree. + """ + if not translation: + return + settings, source = doctree.settings, doctree['source'] + parser = RSTParser() + for node, msg in extract_messages(doctree): + ctx = node.parent + patch = new_document(source, settings) + msgstr = translation.gettext(msg) + parser.parse(msgstr, patch) + ctx.replace(node, patch.children) + + def process_dependencies(self, docname, doctree): """ Process docutils-generated dependency info. From 684ecc26788f309593f8f089eeeaeacf45cd0423 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Tue, 29 Jun 2010 08:08:31 +0200 Subject: [PATCH 42/62] Preliminary fix for broken translations in autodoc/viewcode. --- sphinx/ext/viewcode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py index 81881beb6..80b887c05 100644 --- a/sphinx/ext/viewcode.py +++ b/sphinx/ext/viewcode.py @@ -47,7 +47,7 @@ def doctree_read(app, doctree): for signode in objnode: if not isinstance(signode, addnodes.desc_signature): continue - modname = signode['module'] + modname = signode.get('module') if not modname: continue fullname = signode['fullname'] From 4b8249e6b11fb38ad2d49e06b787b8eaa9f6b470 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Tue, 29 Jun 2010 22:54:59 +0200 Subject: [PATCH 43/62] Fix translation lookup and propagation. --- sphinx/environment.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index 50129e0cb..205c62602 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -736,7 +736,8 @@ class BuildEnvironment: self.dependencies.setdefault(self.docname, set()).add(filename) def get_translation(self, section): - translation, has_trans = init_locale(self.config.locale_dirs, + dirs = [path.join(self.srcdir, x) for x in self.config.locale_dirs] + translation, has_trans = init_locale(dirs, self.config.language, section) if not has_trans: return None @@ -764,7 +765,7 @@ class BuildEnvironment: for node, msg in extract_messages(doctree): ctx = node.parent patch = new_document(source, settings) - msgstr = translation.gettext(msg) + msgstr = translation.ugettext(msg) parser.parse(msgstr, patch) ctx.replace(node, patch.children) From 1433b5cb11a00a72e36c47ba3fafb5e46c6b0df8 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Tue, 29 Jun 2010 23:30:22 +0200 Subject: [PATCH 44/62] Test translation patching in vitro. --- tests/root/bom.po | 12 ++++++++++++ tests/test_build_gettext.py | 27 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/root/bom.po diff --git a/tests/root/bom.po b/tests/root/bom.po new file mode 100644 index 000000000..61455f84f --- /dev/null +++ b/tests/root/bom.po @@ -0,0 +1,12 @@ +#, fuzzy +msgid "" +msgstr "" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "File with UTF-8 BOM" +msgstr "Datei mit UTF-8 BOM" + +msgid "This file has a UTF-8 \"BOM\"." +msgstr "This file has umlauts: äöü." diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 09dc0adb6..ede092894 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -70,3 +70,30 @@ def test_gettext(app): @with_app(buildername='gettext') def test_all(app): app.builder.build_all() + +@with_app(buildername='text', + confoverrides={'language': 'xx', 'locale_dirs': ['.']}) +def test_patch(app): + app.builder.build(['bom']) + res = (app.outdir / 'bom.txt').text('utf-8') + assert res == u"Datei mit UTF-8 BOM\n\nThis file has umlauts: äöü.\n" + +def setup_patch(): + (test_root / 'xx' / 'LC_MESSAGES').makedirs() + try: + p = Popen(['msgfmt', test_root / 'bom.po', '-o', + test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo'], + stdout=PIPE, stderr=PIPE) + except OSError: + return # most likely msgfmt was not found + else: + stdout, stderr = p.communicate() + if p.returncode != 0: + print stdout + print stderr + assert False, 'msgfmt exited with return code %s' % p.returncode + assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), 'msgfmt failed' +def teardown_patch(): + (test_root / 'xx').rmtree() +test_patch.setup = setup_patch +test_patch.teardown = teardown_patch From 8a5af56e62d6096c40c6431c27434159f94f7883 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 30 Jun 2010 12:50:34 +0200 Subject: [PATCH 45/62] Disallow structural changes to TextElements; replace their children instead. --- sphinx/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index 205c62602..1f00fa04a 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -767,7 +767,8 @@ class BuildEnvironment: patch = new_document(source, settings) msgstr = translation.ugettext(msg) parser.parse(msgstr, patch) - ctx.replace(node, patch.children) + assert isinstance(patch[0], nodes.paragraph) + node.children = patch[0].children def process_dependencies(self, docname, doctree): From 6e9b3c724960d30b0e9f7d5fe1d79617fef5720a Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 30 Jun 2010 12:57:10 +0200 Subject: [PATCH 46/62] Test if doctrees retain semantic structure. --- tests/root/bom.po | 2 +- tests/test_build_gettext.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/root/bom.po b/tests/root/bom.po index 61455f84f..c6025eb1e 100644 --- a/tests/root/bom.po +++ b/tests/root/bom.po @@ -6,7 +6,7 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" msgid "File with UTF-8 BOM" -msgstr "Datei mit UTF-8 BOM" +msgstr "Datei mit UTF-8" msgid "This file has a UTF-8 \"BOM\"." msgstr "This file has umlauts: äöü." diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index ede092894..9e98c36d6 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -75,8 +75,11 @@ def test_all(app): confoverrides={'language': 'xx', 'locale_dirs': ['.']}) def test_patch(app): app.builder.build(['bom']) - res = (app.outdir / 'bom.txt').text('utf-8') - assert res == u"Datei mit UTF-8 BOM\n\nThis file has umlauts: äöü.\n" + result = (app.outdir / 'bom.txt').text('utf-8') + expect = (u"\nDatei mit UTF-8" + u"\n***************\n" # underline matches new translation + u"\nThis file has umlauts: äöü.\n") + assert result == expect def setup_patch(): (test_root / 'xx' / 'LC_MESSAGES').makedirs() From 93f2924008f9b024017c5332652d27d08a9825f2 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 30 Jun 2010 12:57:26 +0200 Subject: [PATCH 47/62] Switch from normalized representation to rawsource. --- sphinx/util/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index 9dada0498..3728eac91 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -30,7 +30,7 @@ def extract_messages(doctree): # orphan if isinstance(node, nodes.field_name) and node.children[0] == 'orphan': continue - msg = node.astext().replace('\n', ' ').strip() + msg = node.rawsource.replace('\n', ' ').strip() # XXX nodes rendering empty are likely a bug in sphinx.addnodes if msg: yield node, msg From 4fa9fe329079660ae837ed5e483653935e636dc9 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 30 Jun 2010 12:59:12 +0200 Subject: [PATCH 48/62] Skip untranslated msgids. --- sphinx/environment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sphinx/environment.py b/sphinx/environment.py index 1f00fa04a..79b0f5d75 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -766,6 +766,9 @@ class BuildEnvironment: ctx = node.parent patch = new_document(source, settings) msgstr = translation.ugettext(msg) + #XXX add marker to untranslated parts + if not msgstr: # as-of-yet untranslated + continue parser.parse(msgstr, patch) assert isinstance(patch[0], nodes.paragraph) node.children = patch[0].children From 583f4a100e5ac431e2e993d92b7a708037e713e6 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 30 Jun 2010 13:11:22 +0200 Subject: [PATCH 49/62] Ignore translations which fall back to NullTranslations. --- sphinx/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index 79b0f5d75..ccdf7d15f 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -767,7 +767,7 @@ class BuildEnvironment: patch = new_document(source, settings) msgstr = translation.ugettext(msg) #XXX add marker to untranslated parts - if not msgstr: # as-of-yet untranslated + if not msgstr or msgstr == msg: # as-of-yet untranslated continue parser.parse(msgstr, patch) assert isinstance(patch[0], nodes.paragraph) From 38e1e7770f98f3e041d14f7b949f5f69909f60a4 Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 14 Jul 2010 22:01:08 +0200 Subject: [PATCH 50/62] Move translation patching back into transform for chronological order. --- sphinx/environment.py | 68 ++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index ccdf7d15f..d85ca27f9 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -16,6 +16,7 @@ import types import codecs import imghdr import string +import posixpath import cPickle as pickle from os import path from glob import glob @@ -181,12 +182,48 @@ class CitationReferences(Transform): refnode += nodes.Text('[' + cittext + ']') citnode.parent.replace(citnode, refnode) +class Locale(Transform): + """ + Replace translatable nodes with their translated doctree. + """ + default_priority = 0 + def apply(self): + env = self.document.settings.env + settings, source = self.document.settings, self.document['source'] + # XXX check if this is reliable + docname = posixpath.splitext(posixpath.basename(source))[0] + section = docname.split(SEP, 1)[0] + + # fetch translations + dirs = [path.join(env.srcdir, x) + for x in env.config.locale_dirs] + catalog, empty = init_locale(dirs, env.config.language, section) + if not empty: + return + + parser = RSTParser() + + for node, msg in extract_messages(self.document): + ctx = node.parent + patch = new_document(source, settings) + msgstr = catalog.ugettext(msg) + #XXX add marker to untranslated parts + if not msgstr or msgstr == msg: # as-of-yet untranslated + continue + parser.parse(msgstr, patch) + patch = patch[0] + assert isinstance(patch, nodes.paragraph) + for child in patch.children: # update leaves + child.parent = node + node.children = patch.children + + class SphinxStandaloneReader(standalone.Reader): """ Add our own transforms. """ - transforms = [CitationReferences, DefaultSubstitutions, + transforms = [Locale, CitationReferences, DefaultSubstitutions, MoveModuleTargets, HandleCodeBlocks, SortIds] def get_transforms(self): @@ -595,7 +632,6 @@ class BuildEnvironment: Parse a file and add/update inventory entries for the doctree. If srcpath is given, read from a different source file. """ - section = docname.split(SEP, 1)[0] # remove all inventory entries for that file if app: app.emit('env-purge-doc', self, docname) @@ -660,7 +696,6 @@ class BuildEnvironment: # post-processing self.filter_messages(doctree) - self.process_translations(doctree, self.get_translation(section)) self.process_dependencies(docname, doctree) self.process_images(docname, doctree) self.process_downloads(docname, doctree) @@ -735,14 +770,6 @@ class BuildEnvironment: def note_dependency(self, filename): self.dependencies.setdefault(self.docname, set()).add(filename) - def get_translation(self, section): - dirs = [path.join(self.srcdir, x) for x in self.config.locale_dirs] - translation, has_trans = init_locale(dirs, - self.config.language, section) - if not has_trans: - return None - return translation - # post-processing of read doctrees def filter_messages(self, doctree): @@ -754,25 +781,6 @@ class BuildEnvironment: if node['level'] < filterlevel: node.parent.remove(node) - def process_translations(self, doctree, translation): - """ - Replace translatable nodes with their translated doctree. - """ - if not translation: - return - settings, source = doctree.settings, doctree['source'] - parser = RSTParser() - for node, msg in extract_messages(doctree): - ctx = node.parent - patch = new_document(source, settings) - msgstr = translation.ugettext(msg) - #XXX add marker to untranslated parts - if not msgstr or msgstr == msg: # as-of-yet untranslated - continue - parser.parse(msgstr, patch) - assert isinstance(patch[0], nodes.paragraph) - node.children = patch[0].children - def process_dependencies(self, docname, doctree): """ From fda1d0985a6482a51f78ae78e60baa0609b1100d Mon Sep 17 00:00:00 2001 From: Robert Lehmann Date: Wed, 14 Jul 2010 23:41:38 +0200 Subject: [PATCH 51/62] Fixed docname resolution. --- sphinx/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index d85ca27f9..b03e46258 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -191,7 +191,8 @@ class Locale(Transform): env = self.document.settings.env settings, source = self.document.settings, self.document['source'] # XXX check if this is reliable - docname = posixpath.splitext(posixpath.basename(source))[0] + assert source.startswith(env.srcdir) + docname = posixpath.splitext(source[len(env.srcdir):].lstrip('/'))[0] section = docname.split(SEP, 1)[0] # fetch translations From 9222c314578b9307f2ffa376b70ff347c5f9d69b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 11:56:42 +0200 Subject: [PATCH 52/62] Fix copyright info --- sphinx/builders/intl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index de147c821..abb119a46 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -5,7 +5,7 @@ The MessageCatalogBuilder class. - :copyright: Copyright 2010 by the Sphinx team, see AUTHORS. + :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ From 57e83c6a94f21b76cf2700a62a34f3a029324fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 11:59:01 +0200 Subject: [PATCH 53/62] Fix line length --- sphinx/builders/intl.py | 3 ++- tests/test_build_gettext.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index abb119a46..7b01602e4 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -88,7 +88,8 @@ class MessageCatalogBuilder(Builder): pofile.write(POHEADER % data) for message in messages: # message contains *one* line of text ready for translation - message = message.replace(u'\\', ur'\\').replace(u'"', ur'\"') + message = message.replace(u'\\', ur'\\'). \ + replace(u'"', ur'\"') pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message pofile.write(pomsg.encode('utf-8')) finally: diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 9e98c36d6..3312f4f22 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -46,7 +46,8 @@ def test_gettext(app): if p.returncode != 0: print stdout print stderr - assert False, 'msginit exited with return code %s' % p.returncode + assert False, 'msginit exited with return code %s' % \ + p.returncode assert (app.outdir / 'en_US.po').isfile(), 'msginit failed' try: p = Popen(['msgfmt', 'en_US.po', '-o', @@ -59,8 +60,10 @@ def test_gettext(app): if p.returncode != 0: print stdout print stderr - assert False, 'msgfmt exited with return code %s' % p.returncode - assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), 'msgfmt failed' + assert False, 'msgfmt exited with return code %s' % \ + p.returncode + assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), \ + 'msgfmt failed' finally: os.chdir(cwd) @@ -95,7 +98,8 @@ def setup_patch(): print stdout print stderr assert False, 'msgfmt exited with return code %s' % p.returncode - assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), 'msgfmt failed' + assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), \ + 'msgfmt failed' def teardown_patch(): (test_root / 'xx').rmtree() test_patch.setup = setup_patch From 94d9644722601728d6ed12710ec5e7ff8a00d8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:04:27 +0200 Subject: [PATCH 54/62] Added a newline for readability --- tests/test_build_gettext.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 3312f4f22..772bba878 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -100,6 +100,7 @@ def setup_patch(): assert False, 'msgfmt exited with return code %s' % p.returncode assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), \ 'msgfmt failed' + def teardown_patch(): (test_root / 'xx').rmtree() test_patch.setup = setup_patch From 45285b678562b51115e9ba57c0760ffbcbc1209a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:13:04 +0200 Subject: [PATCH 55/62] Move i18n part of the MessageCatalogBuilder in a seperate one --- sphinx/builders/intl.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 7b01602e4..fa1dc82a3 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -9,7 +9,7 @@ :license: BSD, see LICENSE for details. """ -import collections +from collections import defaultdict from datetime import datetime from os import path @@ -41,14 +41,12 @@ msgstr "" """[1:] -class MessageCatalogBuilder(Builder): - """ - Builds gettext-style message catalogs (.pot files). - """ - name = 'gettext' +class I18NBuilder(Builder): + name = 'i18n' def init(self): - self.catalogs = collections.defaultdict(list) + Builder.init(self) + self.catalogs = defaultdict(list) def get_target_uri(self, docname, typ=None): return '' @@ -60,17 +58,17 @@ class MessageCatalogBuilder(Builder): return def write_doc(self, docname, doctree): - """ - Store a document's translatable strings in the message catalog of its - section. For this purpose a document's *top-level directory* -- or - otherwise its *name* -- is considered its section. - """ catalog = self.catalogs[docname.split(SEP, 1)[0]] for _, msg in extract_messages(doctree): - # XXX msgctxt for duplicate messages if msg not in catalog: catalog.append(msg) +class MessageCatalogBuilder(I18NBuilder): + """ + Builds gettext-style message catalogs (.pot files). + """ + name = 'gettext' + def finish(self): data = dict( version = self.config.version, From 817a7bb2c8c467c864b25e3e305f40ea0dce3a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:16:48 +0200 Subject: [PATCH 56/62] Fix test which was broken to change in the path object api --- tests/test_build_gettext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 772bba878..6a770869a 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -78,7 +78,7 @@ def test_all(app): confoverrides={'language': 'xx', 'locale_dirs': ['.']}) def test_patch(app): app.builder.build(['bom']) - result = (app.outdir / 'bom.txt').text('utf-8') + result = (app.outdir / 'bom.txt').text(encoding='utf-8') expect = (u"\nDatei mit UTF-8" u"\n***************\n" # underline matches new translation u"\nThis file has umlauts: äöü.\n") From 532a0de6010bf4b3573ed5119ef02629c59af96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:22:43 +0200 Subject: [PATCH 57/62] Monkey patch .gettext with .ugettext if possible (we use python 2.x) --- sphinx/environment.py | 2 +- sphinx/locale/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sphinx/environment.py b/sphinx/environment.py index 6339675b5..4809158d3 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -209,7 +209,7 @@ class Locale(Transform): for node, msg in extract_messages(self.document): ctx = node.parent patch = new_document(source, settings) - msgstr = catalog.ugettext(msg) + msgstr = catalog.gettext(msg) #XXX add marker to untranslated parts if not msgstr or msgstr == msg: # as-of-yet untranslated continue diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 481169918..2d3ab0269 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -217,4 +217,6 @@ def init(locale_dirs, language, catalog='sphinx'): translator = gettext.NullTranslations() has_translation = False translators[catalog] = translator + if hasattr(translator, 'ugettext'): + translator.gettext = translator.ugettext return translator, has_translation From f6680c85ca40d7f44f616d1f7d4919768077bb51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:26:37 +0200 Subject: [PATCH 58/62] Use codecs.open with python 2.x in the MessageCatalogBuilder --- sphinx/builders/intl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index fa1dc82a3..d4f5d837e 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -12,6 +12,7 @@ from collections import defaultdict from datetime import datetime from os import path +from codecs import open from docutils import nodes @@ -81,7 +82,8 @@ class MessageCatalogBuilder(I18NBuilder): self.catalogs.iteritems(), "writing message catalogs... ", lambda (section, _):darkgreen(section), len(self.catalogs)): - pofile = open(path.join(self.outdir, '%s.pot' % section), 'w') + pofp = path.join(self.outdir, section + '.pot') + pofile = open(pofp, 'w', encoding='utf-8') try: pofile.write(POHEADER % data) for message in messages: @@ -89,6 +91,6 @@ class MessageCatalogBuilder(I18NBuilder): message = message.replace(u'\\', ur'\\'). \ replace(u'"', ur'\"') pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message - pofile.write(pomsg.encode('utf-8')) + pofile.write(pomsg) finally: pofile.close() From 42279f17bd3ef43833b6d99bff3bd6e19c72ff6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:30:13 +0200 Subject: [PATCH 59/62] Fix test_gettext test for python 3.x --- tests/test_build_gettext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py index 6a770869a..581c1cb85 100644 --- a/tests/test_build_gettext.py +++ b/tests/test_build_gettext.py @@ -67,7 +67,7 @@ def test_gettext(app): finally: os.chdir(cwd) - _ = gettext.translation('test_root', app.outdir, languages=['en']).ugettext + _ = gettext.translation('test_root', app.outdir, languages=['en']).gettext assert _("Testing various markup") == u"Testing various markup" @with_app(buildername='gettext') From 0dcf347d0074d64cbd2fd9df8b32b516b386d562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 12:53:00 +0200 Subject: [PATCH 60/62] Added versioning support to i18n builder --- sphinx/builders/intl.py | 68 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index d4f5d837e..88c128f56 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -13,13 +13,18 @@ from collections import defaultdict from datetime import datetime from os import path from codecs import open +import os +import pickle from docutils import nodes +from docutils.utils import Reporter from sphinx.builders import Builder from sphinx.util.nodes import extract_messages -from sphinx.util.osutil import SEP +from sphinx.util.osutil import SEP, copyfile from sphinx.util.console import darkgreen +from sphinx.environment import WarningStream +from sphinx.versioning import add_uids, merge_doctrees POHEADER = ur""" # SOME DESCRIPTIVE TITLE. @@ -47,7 +52,42 @@ class I18NBuilder(Builder): def init(self): Builder.init(self) - self.catalogs = defaultdict(list) + self.catalogs = defaultdict(dict) + for root, dirs, files in os.walk(self.doctreedir): + for fn in files: + fp = path.join(root, fn) + if fp.endswith('.doctree'): + copyfile(fp, fp + '.old') + + def get_old_doctree(self, docname): + fp = self.env.doc2path(docname, self.doctreedir, '.doctree.old') + try: + f = open(fp, 'rb') + try: + doctree = pickle.load(f) + finally: + f.close() + except IOError: + return None + doctree.settings.env = self.env + doctree.reporter = Reporter(self.env.doc2path(docname), 2, 5, + stream=WarningStream(self.env._warnfunc)) + + def resave_doctree(self, docname, doctree): + reporter = doctree.reporter + doctree.reporter = None + doctree.settings.warning_stream = None + doctree.settings.env = None + doctree.settings.record_dependencies = None + + fp = self.env.doc2path(docname, self.doctreedir, '.doctree') + f = open(fp, 'wb') + try: + pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL) + finally: + f.close() + + doctree.reporter = reporter def get_target_uri(self, docname, typ=None): return '' @@ -60,9 +100,24 @@ class I18NBuilder(Builder): def write_doc(self, docname, doctree): catalog = self.catalogs[docname.split(SEP, 1)[0]] - for _, msg in extract_messages(doctree): - if msg not in catalog: - catalog.append(msg) + old_doctree = self.get_old_doctree(docname) + + if old_doctree: + list(merge_doctrees(old_doctree, doctree, nodes.TextElement)) + else: + list(add_uids(doctree, nodes.TextElement)) + self.resave_doctree(docname, doctree) + + for node, msg in extract_messages(doctree): + catalog.setdefault(node.uid, msg) + + def finish(self): + Builder.finish(self) + for root, dirs, files in os.walk(self.doctreedir): + for fn in files: + fp = path.join(root, fn) + if fp.endswith('.doctree.old'): + os.remove(fp) class MessageCatalogBuilder(I18NBuilder): """ @@ -71,6 +126,7 @@ class MessageCatalogBuilder(I18NBuilder): name = 'gettext' def finish(self): + I18NBuilder.finish(self) data = dict( version = self.config.version, copyright = self.config.copyright, @@ -86,7 +142,7 @@ class MessageCatalogBuilder(I18NBuilder): pofile = open(pofp, 'w', encoding='utf-8') try: pofile.write(POHEADER % data) - for message in messages: + for message in messages.itervalues(): # message contains *one* line of text ready for translation message = message.replace(u'\\', ur'\\'). \ replace(u'"', ur'\"') From 5e749197edc7c253e8ddfa6061bd61a80cf71d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 13:18:47 +0200 Subject: [PATCH 61/62] Before each id, str pair a comment with the uid can be found in the pot files --- sphinx/builders/intl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 88c128f56..19190926a 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -142,11 +142,11 @@ class MessageCatalogBuilder(I18NBuilder): pofile = open(pofp, 'w', encoding='utf-8') try: pofile.write(POHEADER % data) - for message in messages.itervalues(): + for uid, message in messages.iteritems(): # message contains *one* line of text ready for translation message = message.replace(u'\\', ur'\\'). \ replace(u'"', ur'\"') - pomsg = u'msgid "%s"\nmsgstr ""\n\n' % message + pomsg = u'#%s\nmsgid "%s"\nmsgstr ""\n\n' % (uid, message) pofile.write(pomsg) finally: pofile.close() From d31e8c2f59487b6e1edac1c2f67ca45cdc9beb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Neuh=C3=A4user?= Date: Sun, 15 Aug 2010 13:48:38 +0200 Subject: [PATCH 62/62] Switch VersioningBuilderMixin --- sphinx/builders/intl.py | 57 ++++------------------------------------- 1 file changed, 5 insertions(+), 52 deletions(-) diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py index 19190926a..0af5b19a2 100644 --- a/sphinx/builders/intl.py +++ b/sphinx/builders/intl.py @@ -13,18 +13,14 @@ from collections import defaultdict from datetime import datetime from os import path from codecs import open -import os -import pickle from docutils import nodes -from docutils.utils import Reporter from sphinx.builders import Builder +from sphinx.builders.versioning import VersioningBuilderMixin from sphinx.util.nodes import extract_messages from sphinx.util.osutil import SEP, copyfile from sphinx.util.console import darkgreen -from sphinx.environment import WarningStream -from sphinx.versioning import add_uids, merge_doctrees POHEADER = ur""" # SOME DESCRIPTIVE TITLE. @@ -47,47 +43,13 @@ msgstr "" """[1:] -class I18NBuilder(Builder): +class I18NBuilder(Builder, VersioningBuilderMixin): name = 'i18n' def init(self): Builder.init(self) + VersioningBuilderMixin.init(self) self.catalogs = defaultdict(dict) - for root, dirs, files in os.walk(self.doctreedir): - for fn in files: - fp = path.join(root, fn) - if fp.endswith('.doctree'): - copyfile(fp, fp + '.old') - - def get_old_doctree(self, docname): - fp = self.env.doc2path(docname, self.doctreedir, '.doctree.old') - try: - f = open(fp, 'rb') - try: - doctree = pickle.load(f) - finally: - f.close() - except IOError: - return None - doctree.settings.env = self.env - doctree.reporter = Reporter(self.env.doc2path(docname), 2, 5, - stream=WarningStream(self.env._warnfunc)) - - def resave_doctree(self, docname, doctree): - reporter = doctree.reporter - doctree.reporter = None - doctree.settings.warning_stream = None - doctree.settings.env = None - doctree.settings.record_dependencies = None - - fp = self.env.doc2path(docname, self.doctreedir, '.doctree') - f = open(fp, 'wb') - try: - pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL) - finally: - f.close() - - doctree.reporter = reporter def get_target_uri(self, docname, typ=None): return '' @@ -100,24 +62,15 @@ class I18NBuilder(Builder): def write_doc(self, docname, doctree): catalog = self.catalogs[docname.split(SEP, 1)[0]] - old_doctree = self.get_old_doctree(docname) - if old_doctree: - list(merge_doctrees(old_doctree, doctree, nodes.TextElement)) - else: - list(add_uids(doctree, nodes.TextElement)) - self.resave_doctree(docname, doctree) + self.handle_versioning(docname, doctree, nodes.TextElement) for node, msg in extract_messages(doctree): catalog.setdefault(node.uid, msg) def finish(self): Builder.finish(self) - for root, dirs, files in os.walk(self.doctreedir): - for fn in files: - fp = path.join(root, fn) - if fp.endswith('.doctree.old'): - os.remove(fp) + VersioningBuilderMixin.finish(self) class MessageCatalogBuilder(I18NBuilder): """