diff --git a/doc/Makefile b/doc/Makefile index 90fb5af25..a6662cda8 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -19,18 +19,19 @@ help: @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files called index.html in directories" @echo " singlehtml to make one big HTML file" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " pickle to make pickle files" - @echo " json to make json files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make Qt help files and project" - @echo " devhelp to make Devhelp files and project" - @echo " epub to make an epub file" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run pdflatex" - @echo " changes to make an overview over all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " pickle to make pickle files" + @echo " json to make json files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make Qt help files and project" + @echo " devhelp to make Devhelp files and project" + @echo " epub to make an epub file" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run pdflatex" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview over all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" clean: -rm -rf _build/* @@ -112,6 +113,11 @@ latexpdf: make -C _build/latex all-pdf @echo "pdflatex finished; the PDF files are in _build/latex." +gettext: + $(SPHINXBUILD) -b gettext $(ALLSPHINXOPTS) _build/locale + @echo + @echo "Build finished. The message catalogs are in _build/locale." + changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes @echo diff --git a/doc/builders.rst b/doc/builders.rst index 80203e759..4f95b6147 100644 --- a/doc/builders.rst +++ b/doc/builders.rst @@ -220,6 +220,18 @@ Note that a direct PDF builder using ReportLab is available in `rst2pdf .. versionadded:: 0.5 +.. module:: sphinx.builders.intl +.. class:: MessageCatalogBuilder + + This builder produces a message catalog file for each reST file or + subdirectory. + + See the documentation on :ref:`internationalization ` for further reference. + + Its name is ``gettext``. + + .. versionadded:: 1.XXX + .. module:: sphinx.builders.changes .. class:: ChangesBuilder diff --git a/doc/contents.rst b/doc/contents.rst index 079f93f26..c8c0988bc 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -14,6 +14,7 @@ Sphinx documentation contents domains builders config + intl theming templating extensions diff --git a/doc/intl.rst b/doc/intl.rst new file mode 100644 index 000000000..8722177de --- /dev/null +++ b/doc/intl.rst @@ -0,0 +1,11 @@ +.. _intl: + +Internationalization +==================== + +.. versionadded:: 1.XXX + +Complementary to translations provided for internal messages such as navigation +bars Sphinx provides mechanisms facilitating *document* translations in itself. +It relies on the existing configuration values :confval:`language` and +:confval:`locale_dirs`. diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index e345d570f..cf7f39557 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -329,4 +329,5 @@ BUILTIN_BUILDERS = { 'man': ('manpage', 'ManualPageBuilder'), 'changes': ('changes', 'ChangesBuilder'), 'linkcheck': ('linkcheck', 'CheckExternalLinksBuilder'), + 'gettext': ('intl', 'MessageCatalogBuilder'), } diff --git a/sphinx/builders/intl.py b/sphinx/builders/intl.py new file mode 100644 index 000000000..0af5b19a2 --- /dev/null +++ b/sphinx/builders/intl.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +""" + sphinx.builders.intl + ~~~~~~~~~~~~~~~~~~~~ + + The MessageCatalogBuilder class. + + :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from collections import defaultdict +from datetime import datetime +from os import path +from codecs import open + +from docutils import nodes + +from sphinx.builders import Builder +from sphinx.builders.versioning import VersioningBuilderMixin +from sphinx.util.nodes import extract_messages +from sphinx.util.osutil import SEP, copyfile +from sphinx.util.console import darkgreen + +POHEADER = ur""" +# SOME DESCRIPTIVE TITLE. +# Copyright (C) %(copyright)s +# This file is distributed under the same license as the %(project)s package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: %(version)s\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: %(ctime)s\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +"""[1:] + +class I18NBuilder(Builder, VersioningBuilderMixin): + name = 'i18n' + + def init(self): + Builder.init(self) + VersioningBuilderMixin.init(self) + self.catalogs = defaultdict(dict) + + def get_target_uri(self, docname, typ=None): + return '' + + def get_outdated_docs(self): + return self.env.found_docs + + def prepare_writing(self, docnames): + return + + def write_doc(self, docname, doctree): + catalog = self.catalogs[docname.split(SEP, 1)[0]] + + self.handle_versioning(docname, doctree, nodes.TextElement) + + for node, msg in extract_messages(doctree): + catalog.setdefault(node.uid, msg) + + def finish(self): + Builder.finish(self) + VersioningBuilderMixin.finish(self) + +class MessageCatalogBuilder(I18NBuilder): + """ + Builds gettext-style message catalogs (.pot files). + """ + name = 'gettext' + + def finish(self): + I18NBuilder.finish(self) + data = dict( + version = self.config.version, + copyright = self.config.copyright, + project = self.config.project, + # XXX should supply tz + ctime = datetime.now().strftime('%Y-%m-%d %H:%M%z'), + ) + for section, messages in self.status_iterator( + self.catalogs.iteritems(), "writing message catalogs... ", + lambda (section, _):darkgreen(section), len(self.catalogs)): + + pofp = path.join(self.outdir, section + '.pot') + pofile = open(pofp, 'w', encoding='utf-8') + try: + pofile.write(POHEADER % data) + for uid, message in messages.iteritems(): + # message contains *one* line of text ready for translation + message = message.replace(u'\\', ur'\\'). \ + replace(u'"', ur'\"') + pomsg = u'#%s\nmsgid "%s"\nmsgstr ""\n\n' % (uid, message) + pofile.write(pomsg) + finally: + pofile.close() diff --git a/sphinx/environment.py b/sphinx/environment.py index a051d6361..4809158d3 100644 --- a/sphinx/environment.py +++ b/sphinx/environment.py @@ -17,6 +17,7 @@ import types import codecs import imghdr import string +import posixpath import cPickle as pickle from os import path from glob import glob @@ -25,9 +26,9 @@ from itertools import izip, groupby from docutils import nodes from docutils.io import FileInput, NullOutput from docutils.core import Publisher -from docutils.utils import Reporter, relative_path +from docutils.utils import Reporter, relative_path, new_document from docutils.readers import standalone -from docutils.parsers.rst import roles, directives +from docutils.parsers.rst import roles, directives, Parser as RSTParser from docutils.parsers.rst.languages import en as english from docutils.parsers.rst.directives.html import MetaBody from docutils.writers import UnfilteredWriter @@ -37,12 +38,12 @@ from docutils.transforms.parts import ContentsFilter from sphinx import addnodes from sphinx.util import url_re, get_matching_docs, docname_join, \ FilenameUniqDict -from sphinx.util.nodes import clean_astext, make_refnode +from sphinx.util.nodes import clean_astext, make_refnode, extract_messages from sphinx.util.osutil import movefile, SEP, ustrftime from sphinx.util.matching import compile_matchers from sphinx.util.pycompat import all, class_types from sphinx.errors import SphinxError, ExtensionError -from sphinx.locale import _ +from sphinx.locale import _, init as init_locale fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() @@ -183,13 +184,50 @@ class CitationReferences(Transform): refnode += nodes.Text('[' + cittext + ']') citnode.parent.replace(citnode, refnode) +class Locale(Transform): + """ + Replace translatable nodes with their translated doctree. + """ + default_priority = 0 + def apply(self): + env = self.document.settings.env + settings, source = self.document.settings, self.document['source'] + # XXX check if this is reliable + assert source.startswith(env.srcdir) + docname = posixpath.splitext(source[len(env.srcdir):].lstrip('/'))[0] + section = docname.split(SEP, 1)[0] + + # fetch translations + dirs = [path.join(env.srcdir, x) + for x in env.config.locale_dirs] + catalog, empty = init_locale(dirs, env.config.language, section) + if not empty: + return + + parser = RSTParser() + + for node, msg in extract_messages(self.document): + ctx = node.parent + patch = new_document(source, settings) + msgstr = catalog.gettext(msg) + #XXX add marker to untranslated parts + if not msgstr or msgstr == msg: # as-of-yet untranslated + continue + parser.parse(msgstr, patch) + patch = patch[0] + assert isinstance(patch, nodes.paragraph) + for child in patch.children: # update leaves + child.parent = node + node.children = patch.children + + class SphinxStandaloneReader(standalone.Reader): """ Add our own transforms. """ - transforms = [CitationReferences, DefaultSubstitutions, MoveModuleTargets, - HandleCodeBlocks, SortIds] + transforms = [Locale, CitationReferences, DefaultSubstitutions, + MoveModuleTargets, HandleCodeBlocks, SortIds] def get_transforms(self): return standalone.Reader.get_transforms(self) + self.transforms @@ -754,6 +792,7 @@ class BuildEnvironment: if node['level'] < filterlevel: node.parent.remove(node) + def process_dependencies(self, docname, doctree): """ Process docutils-generated dependency info. diff --git a/sphinx/ext/viewcode.py b/sphinx/ext/viewcode.py index db04ac794..b9bb9d77e 100644 --- a/sphinx/ext/viewcode.py +++ b/sphinx/ext/viewcode.py @@ -51,7 +51,7 @@ def doctree_read(app, doctree): for signode in objnode: if not isinstance(signode, addnodes.desc_signature): continue - modname = signode['module'] + modname = signode.get('module') if not modname: continue fullname = signode['fullname'] diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 02958457b..2d3ab0269 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -178,22 +178,32 @@ pairindextypes = { 'builtin': l_('built-in function'), } -translator = None +translators = {} if sys.version_info >= (3, 0): def _(message): - return translator.gettext(message) + return translators['sphinx'].gettext(message) else: def _(message): - return translator.ugettext(message) + return translators['sphinx'].ugettext(message) -def init(locale_dirs, language): - global translator +def init(locale_dirs, language, catalog='sphinx'): + """ + Look for message catalogs in `locale_dirs` and *ensure* that there is at + least a NullTranslations catalog set in `translators`. If called multiple + times or several ``.mo`` files are found their contents are merged + together (thus making `init` reentrable). + """ + global translators + translator = translators.get(catalog) + # ignore previously failed attempts to find message catalogs + if isinstance(translator, gettext.NullTranslations): + translator = None # the None entry is the system's default locale path has_translation = True for dir_ in locale_dirs: try: - trans = gettext.translation('sphinx', localedir=dir_, + trans = gettext.translation(catalog, localedir=dir_, languages=[language]) if translator is None: translator = trans @@ -202,7 +212,11 @@ def init(locale_dirs, language): except Exception: # Language couldn't be found in the specified path pass + # guarantee translations[catalog] exists if translator is None: translator = gettext.NullTranslations() has_translation = False + translators[catalog] = translator + if hasattr(translator, 'ugettext'): + translator.gettext = translator.ugettext return translator, has_translation diff --git a/sphinx/quickstart.py b/sphinx/quickstart.py index 7e38a742a..fdac4cbe8 100644 --- a/sphinx/quickstart.py +++ b/sphinx/quickstart.py @@ -344,7 +344,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) \ $(SPHINXOPTS) %(rsrcdir)s .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp \ -epub latex latexpdf text man changes linkcheck doctest +epub latex latexpdf text man changes linkcheck doctest gettext help: \t@echo "Please use \\`make ' where is one of" @@ -361,6 +361,7 @@ help: \t@echo " latexpdf to make LaTeX files and run them through pdflatex" \t@echo " text to make text files" \t@echo " man to make manual pages" +\t@echo " gettext to make PO message catalogs" \t@echo " changes to make an overview of all changed/added/deprecated items" \t@echo " linkcheck to check all external links for integrity" \t@echo " doctest to run all doctests embedded in the documentation \ @@ -447,6 +448,11 @@ man: \t@echo \t@echo "Build finished. The manual pages are in $(BUILDDIR)/man." +gettext: +\t$(SPHINXBUILD) -b gettext $(ALLSPHINXOPTS) $(BUILDDIR)/locale +\t@echo +\t@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + changes: \t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes \t@echo @@ -495,6 +501,7 @@ if "%%1" == "help" ( \techo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter \techo. text to make text files \techo. man to make manual pages +\techo. gettext to make PO message catalogs \techo. changes to make an overview over all changed/added/deprecated items \techo. linkcheck to check all external links for integrity \techo. doctest to run all doctests embedded in the documentation if enabled @@ -596,6 +603,13 @@ if "%%1" == "man" ( \tgoto end ) +if "%%1" == "gettext" ( +\t%%SPHINXBUILD%% -b gettext %%ALLSPHINXOPTS%% %%BUILDDIR%%/locale +\techo. +\techo.Build finished. The message catalogs are in %%BUILDDIR%%/locale. +\tgoto end +) + if "%%1" == "changes" ( \t%%SPHINXBUILD%% -b changes %%ALLSPHINXOPTS%% %%BUILDDIR%%/changes \techo. diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py index aab8f0142..c2f96f078 100644 --- a/sphinx/util/nodes.py +++ b/sphinx/util/nodes.py @@ -22,6 +22,20 @@ explicit_title_re = re.compile(r'^(.+?)\s*(?$', re.DOTALL) caption_ref_re = explicit_title_re # b/w compat alias +def extract_messages(doctree): + """Extract translatable messages from a document tree.""" + for node in doctree.traverse(nodes.TextElement): + if isinstance(node, (nodes.Invisible, nodes.Inline)): + continue + # orphan + if isinstance(node, nodes.field_name) and node.children[0] == 'orphan': + continue + msg = node.rawsource.replace('\n', ' ').strip() + # XXX nodes rendering empty are likely a bug in sphinx.addnodes + if msg: + yield node, msg + + def nested_parse_with_titles(state, content, node): # hack around title style bookkeeping surrounding_title_styles = state.memo.title_styles diff --git a/tests/root/bom.po b/tests/root/bom.po new file mode 100644 index 000000000..c6025eb1e --- /dev/null +++ b/tests/root/bom.po @@ -0,0 +1,12 @@ +#, fuzzy +msgid "" +msgstr "" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "File with UTF-8 BOM" +msgstr "Datei mit UTF-8" + +msgid "This file has a UTF-8 \"BOM\"." +msgstr "This file has umlauts: äöü." diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py new file mode 100644 index 000000000..581c1cb85 --- /dev/null +++ b/tests/test_build_gettext.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +""" + test_build_gettext + ~~~~~~~~~~~~~~~~ + + Test the build process with gettext builder with the test root. + + :copyright: Copyright 2010 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import gettext +import os +from subprocess import Popen, PIPE + +from util import * + + +def teardown_module(): + (test_root / '_build').rmtree(True) + + +@with_app(buildername='gettext') +def test_build(app): + app.builder.build(['extapi', 'subdir/includes']) + # documents end up in a message catalog + assert (app.outdir / 'extapi.pot').isfile() + # ..and are grouped into sections + assert (app.outdir / 'subdir.pot').isfile() + +@with_app(buildername='gettext') +def test_gettext(app): + app.builder.build(['markup']) + + (app.outdir / 'en' / 'LC_MESSAGES').makedirs() + cwd = os.getcwd() + os.chdir(app.outdir) + try: + try: + p = Popen(['msginit', '--no-translator', '-i', 'markup.pot'], + stdout=PIPE, stderr=PIPE) + except OSError: + return # most likely msginit was not found + else: + stdout, stderr = p.communicate() + if p.returncode != 0: + print stdout + print stderr + assert False, 'msginit exited with return code %s' % \ + p.returncode + assert (app.outdir / 'en_US.po').isfile(), 'msginit failed' + try: + p = Popen(['msgfmt', 'en_US.po', '-o', + os.path.join('en', 'LC_MESSAGES', 'test_root.mo')], + stdout=PIPE, stderr=PIPE) + except OSError: + return # most likely msgfmt was not found + else: + stdout, stderr = p.communicate() + if p.returncode != 0: + print stdout + print stderr + assert False, 'msgfmt exited with return code %s' % \ + p.returncode + assert (app.outdir / 'en' / 'LC_MESSAGES' / 'test_root.mo').isfile(), \ + 'msgfmt failed' + finally: + os.chdir(cwd) + + _ = gettext.translation('test_root', app.outdir, languages=['en']).gettext + assert _("Testing various markup") == u"Testing various markup" + +@with_app(buildername='gettext') +def test_all(app): + app.builder.build_all() + +@with_app(buildername='text', + confoverrides={'language': 'xx', 'locale_dirs': ['.']}) +def test_patch(app): + app.builder.build(['bom']) + result = (app.outdir / 'bom.txt').text(encoding='utf-8') + expect = (u"\nDatei mit UTF-8" + u"\n***************\n" # underline matches new translation + u"\nThis file has umlauts: äöü.\n") + assert result == expect + +def setup_patch(): + (test_root / 'xx' / 'LC_MESSAGES').makedirs() + try: + p = Popen(['msgfmt', test_root / 'bom.po', '-o', + test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo'], + stdout=PIPE, stderr=PIPE) + except OSError: + return # most likely msgfmt was not found + else: + stdout, stderr = p.communicate() + if p.returncode != 0: + print stdout + print stderr + assert False, 'msgfmt exited with return code %s' % p.returncode + assert (test_root / 'xx' / 'LC_MESSAGES' / 'bom.mo').isfile(), \ + 'msgfmt failed' + +def teardown_patch(): + (test_root / 'xx').rmtree() +test_patch.setup = setup_patch +test_patch.teardown = teardown_patch