Automated merge with ssh://bitbucket.org/DasIch/sphinx-version-tracking

doc/Makefile

@@ -19,18 +19,19 @@ help:
 	@echo "  html       to make standalone HTML files"
 	@echo "  dirhtml    to make HTML files called index.html in directories"
 	@echo "  singlehtml to make one big HTML file"
-	@echo "  text       to make text files"
+	@echo "  text      to make text files"
-	@echo "  man        to make manual pages"
+	@echo "  man       to make manual pages"
-	@echo "  pickle     to make pickle files"
+	@echo "  pickle    to make pickle files"
-	@echo "  json       to make json files"
+	@echo "  json      to make json files"
-	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  htmlhelp  to make HTML files and a HTML help project"
-	@echo "  qthelp     to make Qt help files and project"
+	@echo "  qthelp    to make Qt help files and project"
-	@echo "  devhelp    to make Devhelp files and project"
+	@echo "  devhelp   to make Devhelp files and project"
-	@echo "  epub       to make an epub file"
+	@echo "  epub      to make an epub file"
-	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  latexpdf   to make LaTeX files and run pdflatex"
+	@echo "  latexpdf  to make LaTeX files and run pdflatex"
-	@echo "  changes    to make an overview over all changed/added/deprecated items"
+	@echo "  gettext   to make PO message catalogs"
-	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  changes   to make an overview over all changed/added/deprecated items"
+	@echo "  linkcheck to check all external links for integrity"
 
 clean:
 	-rm -rf _build/*
@@ -112,6 +113,11 @@ latexpdf:
 	make -C _build/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in _build/latex."
 
+gettext:
+	$(SPHINXBUILD) -b gettext $(ALLSPHINXOPTS) _build/locale
+	@echo
+	@echo "Build finished. The message catalogs are in _build/locale."
+
 changes:
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes
 	@echo

doc/builders.rst

@@ -220,6 +220,18 @@ Note that a direct PDF builder using ReportLab is available in `rst2pdf
 
    .. versionadded:: 0.5
 
+.. module:: sphinx.builders.intl
+.. class:: MessageCatalogBuilder
+
+   This builder produces a message catalog file for each reST file or
+   subdirectory.
+
+   See the documentation on :ref:`internationalization <intl>` for further reference.
+
+   Its name is ``gettext``.
+
+   .. versionadded:: 1.XXX
+
 .. module:: sphinx.builders.changes
 .. class:: ChangesBuilder
 

doc/contents.rst

@@ -14,6 +14,7 @@ Sphinx documentation contents
    domains
    builders
    config
+   intl
    theming
    templating
    extensions

doc/intl.rst

@@ -0,0 +1,11 @@
+.. _intl:
+
+Internationalization
+====================
+
+.. versionadded:: 1.XXX
+
+Complementary to translations provided for internal messages such as navigation
+bars Sphinx provides mechanisms facilitating *document* translations in itself.
+It relies on the existing configuration values :confval:`language` and
+:confval:`locale_dirs`.

sphinx/builders/__init__.py

@@ -329,4 +329,5 @@ BUILTIN_BUILDERS = {
     'man':        ('manpage', 'ManualPageBuilder'),
     'changes':    ('changes', 'ChangesBuilder'),
     'linkcheck':  ('linkcheck', 'CheckExternalLinksBuilder'),
+    'gettext':    ('intl', 'MessageCatalogBuilder'),
 }

sphinx/builders/intl.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.builders.intl
+    ~~~~~~~~~~~~~~~~~~~~
+
+    The MessageCatalogBuilder class.
+
+    :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from collections import defaultdict
+from datetime import datetime
+from os import path
+from codecs import open
+
+from docutils import nodes
+
+from sphinx.builders import Builder
+from sphinx.builders.versioning import VersioningBuilderMixin
+from sphinx.util.nodes import extract_messages
+from sphinx.util.osutil import SEP, copyfile
+from sphinx.util.console import darkgreen
+
+POHEADER = ur"""
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) %(copyright)s
+# This file is distributed under the same license as the %(project)s package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: %(version)s\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: %(ctime)s\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+"""[1:]
+
+class I18NBuilder(Builder, VersioningBuilderMixin):
+    name = 'i18n'
+
+    def init(self):
+        Builder.init(self)
+        VersioningBuilderMixin.init(self)
+        self.catalogs = defaultdict(dict)
+
+    def get_target_uri(self, docname, typ=None):
+        return ''
+
+    def get_outdated_docs(self):
+        return self.env.found_docs
+
+    def prepare_writing(self, docnames):
+        return
+
+    def write_doc(self, docname, doctree):
+        catalog = self.catalogs[docname.split(SEP, 1)[0]]
+
+        self.handle_versioning(docname, doctree, nodes.TextElement)
+
+        for node, msg in extract_messages(doctree):
+            catalog.setdefault(node.uid, msg)
+
+    def finish(self):
+        Builder.finish(self)
+        VersioningBuilderMixin.finish(self)
+
+class MessageCatalogBuilder(I18NBuilder):
+    """
+    Builds gettext-style message catalogs (.pot files).
+    """
+    name = 'gettext'
+
+    def finish(self):
+        I18NBuilder.finish(self)
+        data = dict(
+            version = self.config.version,
+            copyright = self.config.copyright,
+            project = self.config.project,
+            # XXX should supply tz
+            ctime = datetime.now().strftime('%Y-%m-%d %H:%M%z'),
+        )
+        for section, messages in self.status_iterator(
+                self.catalogs.iteritems(), "writing message catalogs... ",
+                lambda (section, _):darkgreen(section), len(self.catalogs)):
+
+            pofp = path.join(self.outdir, section + '.pot')
+            pofile = open(pofp, 'w', encoding='utf-8')
+            try:
+                pofile.write(POHEADER % data)
+                for uid, message in messages.iteritems():
+                    # message contains *one* line of text ready for translation
+                    message = message.replace(u'\\', ur'\\'). \
+                                      replace(u'"', ur'\"')
+                    pomsg = u'#%s\nmsgid "%s"\nmsgstr ""\n\n' % (uid, message)
+                    pofile.write(pomsg)
+            finally:
+                pofile.close()

sphinx/environment.py

@@ -17,6 +17,7 @@ import types
 import codecs
 import imghdr
 import string
+import posixpath
 import cPickle as pickle
 from os import path
 from glob import glob
@@ -25,9 +26,9 @@ from itertools import izip, groupby
 from docutils import nodes
 from docutils.io import FileInput, NullOutput
 from docutils.core import Publisher
-from docutils.utils import Reporter, relative_path
+from docutils.utils import Reporter, relative_path, new_document
 from docutils.readers import standalone
-from docutils.parsers.rst import roles, directives
+from docutils.parsers.rst import roles, directives, Parser as RSTParser
 from docutils.parsers.rst.languages import en as english
 from docutils.parsers.rst.directives.html import MetaBody
 from docutils.writers import UnfilteredWriter
@@ -37,12 +38,12 @@ from docutils.transforms.parts import ContentsFilter
 from sphinx import addnodes
 from sphinx.util import url_re, get_matching_docs, docname_join, \
      FilenameUniqDict
-from sphinx.util.nodes import clean_astext, make_refnode
+from sphinx.util.nodes import clean_astext, make_refnode, extract_messages
 from sphinx.util.osutil import movefile, SEP, ustrftime
 from sphinx.util.matching import compile_matchers
 from sphinx.util.pycompat import all, class_types
 from sphinx.errors import SphinxError, ExtensionError
-from sphinx.locale import _
+from sphinx.locale import _, init as init_locale
 
 fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
 
@@ -183,13 +184,50 @@ class CitationReferences(Transform):
             refnode += nodes.Text('[' + cittext + ']')
             citnode.parent.replace(citnode, refnode)
 
+class Locale(Transform):
+    """
+    Replace translatable nodes with their translated doctree.
+    """
+    default_priority = 0
+    def apply(self):
+        env = self.document.settings.env
+        settings, source = self.document.settings, self.document['source']
+        # XXX check if this is reliable
+        assert source.startswith(env.srcdir)
+        docname = posixpath.splitext(source[len(env.srcdir):].lstrip('/'))[0]
+        section = docname.split(SEP, 1)[0]
+
+        # fetch translations
+        dirs = [path.join(env.srcdir, x)
+                for x in env.config.locale_dirs]
+        catalog, empty = init_locale(dirs, env.config.language, section)
+        if not empty:
+            return
+
+        parser = RSTParser()
+
+        for node, msg in extract_messages(self.document):
+            ctx = node.parent
+            patch = new_document(source, settings)
+            msgstr = catalog.gettext(msg)
+            #XXX add marker to untranslated parts
+            if not msgstr or msgstr == msg: # as-of-yet untranslated
+                continue
+            parser.parse(msgstr, patch)
+            patch = patch[0]
+            assert isinstance(patch, nodes.paragraph)
+            for child in patch.children: # update leaves
+                child.parent = node
+            node.children = patch.children
+
+
 
 class SphinxStandaloneReader(standalone.Reader):
     """
     Add our own transforms.
     """
-    transforms = [CitationReferences, DefaultSubstitutions, MoveModuleTargets,
+    transforms = [Locale, CitationReferences, DefaultSubstitutions,
-                  HandleCodeBlocks, SortIds]
+                  MoveModuleTargets, HandleCodeBlocks, SortIds]
 
     def get_transforms(self):
         return standalone.Reader.get_transforms(self) + self.transforms
@@ -754,6 +792,7 @@ class BuildEnvironment:
             if node['level'] < filterlevel:
                 node.parent.remove(node)
 
+
     def process_dependencies(self, docname, doctree):
         """
         Process docutils-generated dependency info.

sphinx/ext/viewcode.py

@@ -51,7 +51,7 @@ def doctree_read(app, doctree):
         for signode in objnode:
             if not isinstance(signode, addnodes.desc_signature):
                 continue
-            modname = signode['module']
+            modname = signode.get('module')
             if not modname:
                 continue
             fullname = signode['fullname']

sphinx/locale/__init__.py

@@ -178,22 +178,32 @@ pairindextypes = {
     'builtin':   l_('built-in function'),
 }
 
-translator = None
+translators = {}
 
 if sys.version_info >= (3, 0):
     def _(message):
-        return translator.gettext(message)
+        return translators['sphinx'].gettext(message)
 else:
     def _(message):
-        return translator.ugettext(message)
+        return translators['sphinx'].ugettext(message)
 
-def init(locale_dirs, language):
+def init(locale_dirs, language, catalog='sphinx'):
-    global translator
+    """
+    Look for message catalogs in `locale_dirs` and *ensure* that there is at
+    least a NullTranslations catalog set in `translators`. If called multiple
+    times or several ``.mo`` files are found their contents are merged
+    together (thus making `init` reentrable).
+    """
+    global translators
+    translator = translators.get(catalog)
+    # ignore previously failed attempts to find message catalogs
+    if isinstance(translator, gettext.NullTranslations):
+        translator = None
     # the None entry is the system's default locale path
     has_translation = True
     for dir_ in locale_dirs:
         try:
-            trans = gettext.translation('sphinx', localedir=dir_,
+            trans = gettext.translation(catalog, localedir=dir_,
                     languages=[language])
             if translator is None:
                 translator = trans
@@ -202,7 +212,11 @@ def init(locale_dirs, language):
         except Exception:
             # Language couldn't be found in the specified path
             pass
+    # guarantee translations[catalog] exists
     if translator is None:
         translator = gettext.NullTranslations()
         has_translation = False
+    translators[catalog] = translator
+    if hasattr(translator, 'ugettext'):
+        translator.gettext = translator.ugettext
     return translator, has_translation

sphinx/quickstart.py

@@ -344,7 +344,7 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) \
 $(SPHINXOPTS) %(rsrcdir)s
 
 .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp \
-epub latex latexpdf text man changes linkcheck doctest
+epub latex latexpdf text man changes linkcheck doctest gettext
 
 help:
 \t@echo "Please use \\`make <target>' where <target> is one of"
@@ -361,6 +361,7 @@ help:
 \t@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
 \t@echo "  text       to make text files"
 \t@echo "  man        to make manual pages"
+\t@echo "  gettext    to make PO message catalogs"
 \t@echo "  changes    to make an overview of all changed/added/deprecated items"
 \t@echo "  linkcheck  to check all external links for integrity"
 \t@echo "  doctest    to run all doctests embedded in the documentation \
@@ -447,6 +448,11 @@ man:
 \t@echo
 \t@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
 
+gettext:
+\t$(SPHINXBUILD) -b gettext $(ALLSPHINXOPTS) $(BUILDDIR)/locale
+\t@echo
+\t@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
 changes:
 \t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 \t@echo
@@ -495,6 +501,7 @@ if "%%1" == "help" (
 \techo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
 \techo.  text       to make text files
 \techo.  man        to make manual pages
+\techo.  gettext    to make PO message catalogs
 \techo.  changes    to make an overview over all changed/added/deprecated items
 \techo.  linkcheck  to check all external links for integrity
 \techo.  doctest    to run all doctests embedded in the documentation if enabled
@@ -596,6 +603,13 @@ if "%%1" == "man" (
 \tgoto end
 )
 
+if "%%1" == "gettext" (
+\t%%SPHINXBUILD%% -b gettext %%ALLSPHINXOPTS%% %%BUILDDIR%%/locale
+\techo.
+\techo.Build finished. The message catalogs are in %%BUILDDIR%%/locale.
+\tgoto end
+)
+
 if "%%1" == "changes" (
 \t%%SPHINXBUILD%% -b changes %%ALLSPHINXOPTS%% %%BUILDDIR%%/changes
 \techo.

sphinx/util/nodes.py

@@ -22,6 +22,20 @@ explicit_title_re = re.compile(r'^(.+?)\s*(?<!\x00)<(.*?)>$', re.DOTALL)
 caption_ref_re = explicit_title_re  # b/w compat alias
 
 
+def extract_messages(doctree):
+    """Extract translatable messages from a document tree."""
+    for node in doctree.traverse(nodes.TextElement):
+        if isinstance(node, (nodes.Invisible, nodes.Inline)):
+            continue
+        # <field_name>orphan</field_name>
+        if isinstance(node, nodes.field_name) and node.children[0] == 'orphan':
+            continue
+        msg = node.rawsource.replace('\n', ' ').strip()
+        # XXX nodes rendering empty are likely a bug in sphinx.addnodes
+        if msg:
+            yield node, msg
+
+
 def nested_parse_with_titles(state, content, node):
     # hack around title style bookkeeping
     surrounding_title_styles = state.memo.title_styles

tests/root/bom.po

@@ -0,0 +1,12 @@
+#, fuzzy
+msgid ""
+msgstr ""
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "File with UTF-8 BOM"
+msgstr "Datei mit UTF-8"
+
+msgid "This file has a UTF-8 \"BOM\"."
+msgstr "This file has umlauts: äöü."

tests/test_build_gettext.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+"""
+    test_build_gettext
+    ~~~~~~~~~~~~~~~~
+
+    Test the build process with gettext builder with the test root.
+
+    :copyright: Copyright 2010 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import gettext
+import os
+from subprocess import Popen, PIPE
+
+from util import *
+
+
+def teardown_module():
+    (test_root / '_build').rmtree(True)
+
+
+@with_app(buildername='gettext')
+def test_build(app):
+    app.builder.build(['extapi', 'subdir/includes'])
+    # documents end up in a message catalog
+    assert (app.outdir / 'extapi.pot').isfile()
+    # ..and are grouped into sections
+    assert (app.outdir / 'subdir.pot').isfile()
+
+@with_app(buildername='gettext')
+def test_gettext(app):
+    app.builder.build(['markup'])
+
+    (app.outdir / 'en' / 'LC_MESSAGES').makedirs()
+    cwd = os.getcwd()
+    os.chdir(app.outdir)
+    try:
+        try:
+            p = Popen(['msginit', '--no-translator', '-i', 'markup.pot'],
+                        stdout=PIPE, stderr=PIPE)
+        except OSError:
+            return  # most likely msginit was not found
+        else:
+            stdout, stderr = p.communicate()
+            if p.returncode != 0:
+                print stdout
+                print stderr
+                assert False, 'msginit exited with return code %s' % \
+                        p.returncode
+        assert (app.outdir / 'en_US.po').isfile(), 'msginit failed'
+        try:
+            p = Popen(['msgfmt', 'en_US.po', '-o',
+                os.path.join('en', 'LC_MESSAGES', 'test_root.mo')],
+                stdout=PIPE, stderr=PIPE)
+        except OSError:
+            return  # most likely msgfmt was not found
+        else:
+            stdout, stderr = p.communicate()
+            if p.returncode != 0:
+                print stdout
+                print stderr
+                assert False, 'msgfmt exited with return code %s' % \
+                        p.returncode
+        assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), \
+                'msgfmt failed'
+    finally:
+        os.chdir(cwd)
+
+    _ = gettext.translation('test_root', app.outdir, languages=['en']).gettext
+    assert _("Testing various markup") == u"Testing various markup"
+
+@with_app(buildername='gettext')
+def test_all(app):
+    app.builder.build_all()
+
+@with_app(buildername='text',
+        confoverrides={'language': 'xx', 'locale_dirs': ['.']})
+def test_patch(app):
+    app.builder.build(['bom'])
+    result = (app.outdir / 'bom.txt').text(encoding='utf-8')
+    expect = (u"\nDatei mit UTF-8"
+              u"\n***************\n" # underline matches new translation
+              u"\nThis file has umlauts: äöü.\n")
+    assert result == expect
+
+def setup_patch():
+    (test_root / 'xx' / 'LC_MESSAGES').makedirs()
+    try:
+        p = Popen(['msgfmt', test_root / 'bom.po', '-o',
+            test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo'],
+            stdout=PIPE, stderr=PIPE)
+    except OSError:
+        return  # most likely msgfmt was not found
+    else:
+        stdout, stderr = p.communicate()
+        if p.returncode != 0:
+            print stdout
+            print stderr
+            assert False, 'msgfmt exited with return code %s' % p.returncode
+    assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), \
+            'msgfmt failed'
+
+def teardown_patch():
+    (test_root / 'xx').rmtree()
+test_patch.setup = setup_patch
+test_patch.teardown = teardown_patch