Merged birkenfeld/sphinx into default

CHANGES

@@ -13,8 +13,8 @@ Incompatible changes
   should appear literally (i.e. no smart quotes) in strong font.  Custom writers
   will have to be adapted to handle this node.
 
-New features
-------------
+Features added
+--------------
 
 * Add support for Python 3.4.
 * Added ``sphinx.ext.napoleon`` extension for NumPy and Google style docstring
@@ -40,6 +40,11 @@ New features
 * #1434: Provide non-minified JS files for jquery.js and underscore.js to
   clarify the source of the minified files.
 * PR#252, #1291: Windows color console support. Thanks to meu31.
+* PR#255: When generating latex references, also insert latex target/anchor
+  for the ids defined on the node. Thanks to Olivier Heurtier.
+* PR#229: Allow registration of other translators. Thanks to Russell Sim.
+* Add app.set_translator() API to register or override a Docutils translator
+  class like :confval:`html_translator_class`.
 
 Bugs fixed
 ----------
@@ -154,6 +159,10 @@ Bugs fixed
   attributes. Thanks to David Ham.
 * PR#159: Add coverage targets to quickstart generated Makefile and make.bat.
   Thanks to Matthias Troffaes.
+* #1251: When specifying toctree :numbered: option and :tocdepth: metadata,
+  sub section number that is larger depth than `:tocdepth:` is shrinked.
+* PR#260: Encode underscore in citation labels for latex export. Thanks to
+  Lennart Fricke.
 
 Documentation
 -------------

doc/config.rst

@@ -706,6 +706,8 @@ that use Sphinx's HTMLWriter class.
    used to translate document trees to HTML.  Default is ``None`` (use the
    builtin translator).
 
+   .. seealso::  :meth:`~sphinx.application.Sphinx.set_translator`
+
 .. confval:: html_show_copyright
 
    If true, "(C) Copyright ..." is shown in the HTML footer. Default is

doc/extdev/appapi.rst

@@ -82,16 +82,31 @@ package.
 
    Register an event called *name*.  This is needed to be able to emit it.
 
+.. method:: Sphinx.set_translator(name, translator_class)
+               
+   Register or override a Docutils translator class. This is used to register
+   a custom output translator or to replace a builtin translator.
+   This allows extensions to use custom translator and define custom
+   nodes for the translator (see :meth:`add_node`).
+
+   This is a API version of :confval:`html_translator_class` for all other
+   builders. Note that if :confval:`html_translator_class` is specified and
+   this API is called for html related builders, API overriding takes
+   precedence.
+
+   .. versionadded:: 1.3
+
 .. method:: Sphinx.add_node(node, **kwds)
 
    Register a Docutils node class.  This is necessary for Docutils internals.
    It may also be used in the future to validate nodes in the parsed documents.
 
    Node visitor functions for the Sphinx HTML, LaTeX, text and manpage writers
-   can be given as keyword arguments: the keyword must be one or more of
-   ``'html'``, ``'latex'``, ``'text'``, ``'man'``, ``'texinfo'``, the value a
-   2-tuple of ``(visit, depart)`` methods.  ``depart`` can be ``None`` if the
-   ``visit`` function raises :exc:`docutils.nodes.SkipNode`.  Example:
+   can be given as keyword arguments: the keyword should be one or more of
+   ``'html'``, ``'latex'``, ``'text'``, ``'man'``, ``'texinfo'`` or any other
+   supported translators, the value a 2-tuple of ``(visit, depart)`` methods.
+   ``depart`` can be ``None`` if the ``visit`` function raises
+   :exc:`docutils.nodes.SkipNode`.  Example:
 
    .. code-block:: python
 

setup.py

@@ -46,7 +46,7 @@ if sys.version_info < (2, 6) or (3, 0) <= sys.version_info < (3, 3):
     sys.exit(1)
 
 requires = [
-    'six', 'Jinja2>=2.3', 'Pygments>=1.2', 'docutils>=0.10', 'snowballstemmer>=1.1'
+    'six>=1.4', 'Jinja2>=2.3', 'Pygments>=1.2', 'docutils>=0.10', 'snowballstemmer>=1.1'
 ]
 
 if sys.platform == 'win32':

sphinx/application.py

@@ -99,6 +99,7 @@ class Sphinx(object):
         self.warningiserror = warningiserror
 
         self._events = events.copy()
+        self._translators = {}
 
         # say hello to the world
         self.info(bold('Running Sphinx v%s' % sphinx.__version__))
@@ -449,6 +450,10 @@ class Sphinx(object):
             raise ExtensionError('Event %r already present' % name)
         self._events[name] = ''
 
+    def set_translator(self, name, translator_class):
+        self.info(bold('A Translator for the %s builder is changed.' % name))
+        self._translators[name] = translator_class
+
     def add_node(self, node, **kwds):
         self.debug('[app] adding node: %r', (node, kwds))
         nodes._add_node_class_names([node.__name__])
@@ -458,7 +463,10 @@ class Sphinx(object):
             except ValueError:
                 raise ExtensionError('Value for key %r must be a '
                                      '(visit, depart) function tuple' % key)
-            if key == 'html':
+            translator = self._translators.get(key)
+            if translator is not None:
+                pass
+            elif key == 'html':
                 from sphinx.writers.html import HTMLTranslator as translator
             elif key == 'latex':
                 from sphinx.writers.latex import LaTeXTranslator as translator

sphinx/builders/__init__.py

@@ -65,6 +65,9 @@ class Builder(object):
         # images that need to be copied over (source -> dest)
         self.images = {}
 
+        # load default translator class
+        self.translator_class = app._translators.get(self.name)
+
         self.init()
 
     # helper methods

sphinx/builders/html.py

@@ -152,7 +152,9 @@ class StandaloneHTMLBuilder(Builder):
                                           self.config.trim_doctest_flags)
 
     def init_translator_class(self):
-        if self.config.html_translator_class:
+        if self.translator_class is not None:
+            pass
+        elif self.config.html_translator_class:
             self.translator_class = self.app.import_object(
                 self.config.html_translator_class,
                 'html_translator_class setting')

sphinx/builders/websupport.py

@@ -46,7 +46,8 @@ class WebSupportBuilder(PickleHTMLBuilder):
         self.storage = storage
 
     def init_translator_class(self):
-        self.translator_class = WebSupportTranslator
+        if self.translator_class is None:
+            self.translator_class = WebSupportTranslator
 
     def prepare_writing(self, docnames):
         PickleHTMLBuilder.prepare_writing(self, docnames)

sphinx/environment.py

@@ -493,14 +493,14 @@ class BuildEnvironment:
 
     def warn_and_replace(self, error):
         """Custom decoding error handler that warns and replaces."""
-        linestart = error.object.rfind('\n', 0, error.start)
-        lineend = error.object.find('\n', error.start)
+        linestart = error.object.rfind(b'\n', 0, error.start)
+        lineend = error.object.find(b'\n', error.start)
         if lineend == -1: lineend = len(error.object)
-        lineno = error.object.count('\n', 0, error.start) + 1
+        lineno = error.object.count(b'\n', 0, error.start) + 1
         self.warn(self.docname, 'undecodable source characters, '
                   'replacing with "?": %r' %
-                  (error.object[linestart+1:error.start] + '>>>' +
-                   error.object[error.start:error.end] + '<<<' +
+                  (error.object[linestart+1:error.start] + b'>>>' +
+                   error.object[error.start:error.end] + b'<<<' +
                    error.object[error.end:lineend]), lineno)
         return (u'?', error.end)
 
@@ -590,12 +590,13 @@ class BuildEnvironment:
             def __init__(self_, *args, **kwds):
                 # don't call sys.exit() on IOErrors
                 kwds['handle_io_errors'] = False
+                kwds['error_handler'] = 'sphinx'  # py3: handle error on open.
                 FileInput.__init__(self_, *args, **kwds)
 
             def decode(self_, data):
-                if isinstance(data, text_type):
+                if isinstance(data, text_type):  # py3: `data` already decoded.
                     return data
-                return data.decode(self_.encoding, 'sphinx')
+                return data.decode(self_.encoding, 'sphinx')  # py2: decoding
 
             def read(self_):
                 data = FileInput.read(self_)
@@ -849,6 +850,14 @@ class BuildEnvironment:
             else:
                 name, body = node
                 md[name.astext()] = body.astext()
+        for name, value in md.items():
+            if name in ('tocdepth',):
+                try:
+                    value = int(value)
+                except ValueError:
+                    value = 0
+                md[name] = value
+
         del doctree[0]
 
     def process_refonly_bullet_lists(self, docname, doctree):
@@ -968,11 +977,6 @@ class BuildEnvironment:
         """Build a TOC from the doctree and store it in the inventory."""
         numentries = [0] # nonlocal again...
 
-        try:
-            maxdepth = int(self.metadata[docname].get('tocdepth', 0))
-        except ValueError:
-            maxdepth = 0
-
         def traverse_in_section(node, cls):
             """Like traverse(), but stay within the same section."""
             result = []
@@ -1025,8 +1029,7 @@ class BuildEnvironment:
                 para = addnodes.compact_paragraph('', '', reference)
                 item = nodes.list_item('', para)
                 sub_item = build_toc(sectionnode, depth + 1)
-                if maxdepth == 0 or depth < maxdepth:
-                    item += sub_item
+                item += sub_item
                 entries.append(item)
             if entries:
                 return nodes.bullet_list('', *entries)
@@ -1242,6 +1245,8 @@ class BuildEnvironment:
                             continue
                         refdoc = ref
                         toc = self.tocs[ref].deepcopy()
+                        maxdepth = self.metadata[ref].get('tocdepth', 0)
+                        _toctree_prune(toc, 2, maxdepth)
                         self.process_only_nodes(toc, builder, ref)
                         if title and toc.children and len(toc.children) == 1:
                             child = toc.children[0]

sphinx/roles.py

@@ -264,10 +264,12 @@ def abbr_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     text = utils.unescape(text)
     m = _abbr_re.search(text)
     if m is None:
-        return [addnodes.abbreviation(text, text)], []
+        return [addnodes.abbreviation(text, text, **options)], []
     abbr = text[:m.start()].strip()
     expl = m.group(1)
-    return [addnodes.abbreviation(abbr, abbr, explanation=expl)], []
+    options = options.copy()
+    options['explanation'] = expl
+    return [addnodes.abbreviation(abbr, abbr, **options)], []
 
 
 def index_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):

sphinx/writers/latex.py

@@ -89,9 +89,11 @@ class LaTeXWriter(writers.Writer):
     def __init__(self, builder):
         writers.Writer.__init__(self)
         self.builder = builder
+        self.translator_class = (
+            self.builder.translator_class or LaTeXTranslator)
 
     def translate(self):
-        visitor = LaTeXTranslator(self.document, self.builder)
+        visitor = self.translator_class(self.document, self.builder)
         self.document.walkabout(visitor)
         self.output = visitor.astext()
 
@@ -382,7 +384,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
                 target = self.hypertarget(bi[2] + ':' + bi[3],
                                           withdoc=False)
                 self.body.append(u'\\bibitem[%s]{%s}{%s %s}\n' %
-                    (bi[0], self.idescape(bi[0]), target, bi[1]))
+                    (self.encode(bi[0]), self.idescape(bi[0]), target, bi[1]))
             self.body.append(u'\\end{thebibliography}\n')
             self.bibitems = []
 
@@ -1166,6 +1168,8 @@ class LaTeXTranslator(nodes.NodeVisitor):
         raise nodes.SkipNode
 
     def visit_reference(self, node):
+        for id in node.get('ids'):
+            self.body += self.hypertarget(id, anchor=True)
         uri = node.get('refuri', '')
         if not uri and node.get('refid'):
             uri = '%' + self.curfilestack[-1] + '#' + node['refid']

sphinx/writers/manpage.py

@@ -26,9 +26,11 @@ class ManualPageWriter(Writer):
     def __init__(self, builder):
         Writer.__init__(self)
         self.builder = builder
+        self.translator_class = (
+            self.builder.translator_class or ManualPageTranslator)
 
     def translate(self):
-        visitor = ManualPageTranslator(self.builder, self.document)
+        visitor = self.translator_class(self.builder, self.document)
         self.visitor = visitor
         self.document.walkabout(visitor)
         self.output = visitor.astext()

sphinx/writers/texinfo.py

@@ -120,9 +120,12 @@ class TexinfoWriter(writers.Writer):
     def __init__(self, builder):
         writers.Writer.__init__(self)
         self.builder = builder
+        self.translator_class = (
+            self.builder.translator_class or TexinfoTranslator)
 
     def translate(self):
-        self.visitor = visitor = TexinfoTranslator(self.document, self.builder)
+        self.visitor = visitor = self.translator_class(
+            self.document, self.builder)
         self.document.walkabout(visitor)
         visitor.finish()
         for attr in self.visitor_attributes:

sphinx/writers/text.py

@@ -140,9 +140,10 @@ class TextWriter(writers.Writer):
     def __init__(self, builder):
         writers.Writer.__init__(self)
         self.builder = builder
+        self.translator_class = self.builder.translator_class or TextTranslator
 
     def translate(self):
-        visitor = TextTranslator(self.document, self.builder)
+        visitor = self.translator_class(self.document, self.builder)
         self.document.walkabout(visitor)
         self.output = visitor.body
 

sphinx/writers/xml.py

@@ -18,6 +18,8 @@ class XMLWriter(BaseXMLWriter):
     def __init__(self, builder):
         BaseXMLWriter.__init__(self)
         self.builder = builder
+        if self.builder.translator_class:
+            self.translator_class = self.builder.translator_class
 
     def translate(self, *args, **kwargs):
         self.document.settings.newlines = \

tests/root/contents.txt

@@ -43,3 +43,4 @@ References
 ==========
 
 .. [Ref1] Reference target.
+.. [Ref_1] Reference target 2.

tests/root/markup.txt

@@ -240,6 +240,7 @@ Misc stuff
 Stuff [#]_
 
 Reference lookup: [Ref1]_ (defined in another file).
+Reference lookup underscore: [Ref_1]_
 
 .. seealso:: something, something else, something more
 

tests/root/metadata.txt

@@ -32,6 +32,10 @@
     language, containing examples of all basic reStructuredText
     constructs and many advanced constructs.
 
+:nocomments:
+:orphan:
+:tocdepth: 1
+
 .. meta::
    :keywords: reStructuredText, demonstration, demo, parser
    :description lang=en: A demonstration of the reStructuredText

tests/roots/test-api-set-translator/conf.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+## set this by test
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+from sphinx.writers.html import HTMLTranslator
+from sphinx.writers.latex import LaTeXTranslator
+from sphinx.writers.manpage import ManualPageTranslator
+from sphinx.writers.texinfo import TexinfoTranslator
+from sphinx.writers.text import TextTranslator
+from sphinx.writers.websupport import WebSupportTranslator
+from docutils.writers.docutils_xml import XMLTranslator
+
+
+project = 'test'
+master_doc = 'index'
+
+
+class ConfHTMLTranslator(HTMLTranslator):
+    pass
+
+
+class ConfDirHTMLTranslator(HTMLTranslator):
+    pass
+
+
+class ConfSingleHTMLTranslator(HTMLTranslator):
+    pass
+
+
+class ConfPickleTranslator(HTMLTranslator):
+    pass
+
+
+class ConfJsonTranslator(HTMLTranslator):
+    pass
+
+
+class ConfLaTeXTranslator(LaTeXTranslator):
+    pass
+
+
+class ConfManualPageTranslator(ManualPageTranslator):
+    pass
+
+
+class ConfTexinfoTranslator(TexinfoTranslator):
+    pass
+
+
+class ConfTextTranslator(TextTranslator):
+    pass
+
+
+class ConfWebSupportTranslator(WebSupportTranslator):
+    pass
+
+
+class ConfXMLTranslator(XMLTranslator):
+    pass
+
+
+class ConfPseudoXMLTranslator(XMLTranslator):
+    pass
+
+
+def setup(app):
+    app.set_translator('html', ConfHTMLTranslator)
+    app.set_translator('dirhtml', ConfDirHTMLTranslator)
+    app.set_translator('singlehtml', ConfSingleHTMLTranslator)
+    app.set_translator('pickle', ConfPickleTranslator)
+    app.set_translator('json', ConfJsonTranslator)
+    app.set_translator('latex', ConfLaTeXTranslator)
+    app.set_translator('man', ConfManualPageTranslator)
+    app.set_translator('texinfo', ConfTexinfoTranslator)
+    app.set_translator('text', ConfTextTranslator)
+    app.set_translator('websupport', ConfWebSupportTranslator)
+    app.set_translator('xml', ConfXMLTranslator)
+    app.set_translator('pseudoxml', ConfPseudoXMLTranslator)

tests/roots/test-api-set-translator/index.rst

@@ -0,0 +1,3 @@
+=======================
+Test API set_translator
+=======================

tests/roots/test-api-set-translator/nonext/conf.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath('.')))
+
+project = 'test'
+master_doc = 'index'

tests/roots/test-api-set-translator/translator.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from sphinx.writers.html import HTMLTranslator
+
+class ExtHTMLTranslator(HTMLTranslator):
+    pass

tests/roots/test-intl/admonitions.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.2\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-07-03 12:00\n"
+"POT-Creation-Date: 2013-07-03 12:00+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/definition_terms.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-01-01 05:00\n"
+"POT-Creation-Date: 2013-01-01 05:00+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/docfields.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: Sphinx <Tests> 0.6\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-12-16 14:11\n"
+"POT-Creation-Date: 2012-12-16 14:11+0000\n"
 "PO-Revision-Date: 2012-12-18 06:14+0900\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/external_links.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-11-22 08:28\n"
+"POT-Creation-Date: 2012-11-22 08:28+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/figure_caption.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-01-04 7:00\n"
+"POT-Creation-Date: 2013-01-04 07:00+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/footnote.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-11-22 08:28\n"
+"POT-Creation-Date: 2012-11-22 08:28+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/glossary_terms.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-01-29 14:10\n"
+"POT-Creation-Date: 2013-01-29 14:10+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/glossary_terms_inconsistency.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-01-29 14:10\n"
+"POT-Creation-Date: 2013-01-29 14:10+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/index_entries.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: foo foo\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-01-05 18:10\n"
+"POT-Creation-Date: 2013-01-05 18:10+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/literalblock.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-11-22 08:28\n"
+"POT-Creation-Date: 2012-11-22 08:28+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/refs_inconsistency.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-12-05 08:28\n"
+"POT-Creation-Date: 2012-12-05 08:28+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/role_xref.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-02-04 14:00\n"
+"POT-Creation-Date: 2013-02-04 14:00+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/rubric.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-11-12 7:00\n"
+"POT-Creation-Date: 2013-11-12 07:00+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/seealso.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: Sphinx <Tests> 0.6\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-12-16 06:06\n"
+"POT-Creation-Date: 2012-12-16 06:06+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/sphinx.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-11-22 08:28\n"
+"POT-Creation-Date: 2012-11-22 08:28+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/versionchange.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: sphinx 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2012-12-15 03:17\n"
+"POT-Creation-Date: 2012-12-15 03:17+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-intl/warnings.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: Sphinx <Tests> 0.6\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2013-02-04 13:06\n"
+"POT-Creation-Date: 2013-02-04 13:06+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

tests/roots/test-tocdepth/bar.rst

@@ -0,0 +1,28 @@
+:tocdepth: 2
+
+===
+Bar
+===
+
+should be 2
+
+Bar A
+=====
+
+should be 2.1
+
+Bar A1
+------
+
+should be 2.1.1
+
+Bar B
+=====
+
+should be 2.2
+
+Bar B1
+------
+
+should be 2.2.1
+

tests/roots/test-tocdepth/conf.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+master_doc = 'index'

tests/roots/test-tocdepth/foo.rst

@@ -0,0 +1,26 @@
+===
+Foo
+===
+
+should be 1
+
+Foo A
+=====
+
+should be 1.1
+
+Foo A1
+------
+
+should be 1.1.1
+
+Foo B
+=====
+
+should be 1.2
+
+Foo B1
+------
+
+should be 1.2.1
+

tests/roots/test-tocdepth/index.rst

@@ -0,0 +1,8 @@
+test-tocdepth
+=============
+
+.. toctree::
+   :numbered:
+
+   foo
+   bar

tests/test_api_translator.py

@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+"""
+    test_api_translator
+    ~~~~~~~~~~~~~~~~~~~
+
+    Test the Sphinx API for translator.
+
+    :copyright: Copyright 2007-2014 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import sys
+
+from nose.tools import with_setup
+
+from util import with_app, test_roots
+
+
+def setup_module():
+    sys.path.insert(0, test_roots / 'test-api-set-translator')
+
+
+def teardown_module():
+    sys.path.remove(test_roots / 'test-api-set-translator')
+
+
+def teardown_websupport():
+    (test_roots / 'test-api-set-translator' / 'generated').rmtree(True)
+    (test_roots / 'test-api-set-translator' / 'websupport').rmtree(True)
+
+
+@with_app(
+    buildername='html',
+    srcdir=(test_roots / 'test-api-set-translator'),
+    confdir=(test_roots / 'test-api-set-translator' / 'nonext'),
+)
+def test_html_translator(app):
+    # no set_translator(), no html_translator_class
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'SmartyPantsHTMLTranslator'
+
+
+@with_app(
+    buildername='html',
+    srcdir=(test_roots / 'test-api-set-translator'),
+    confdir=(test_roots / 'test-api-set-translator' / 'nonext'),
+    confoverrides={
+        'html_translator_class': 'translator.ExtHTMLTranslator'},
+)
+def test_html_with_html_translator_class(app):
+    # no set_translator(), but html_translator_class
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ExtHTMLTranslator'
+
+
+@with_app(
+    buildername='html',
+    srcdir=(test_roots / 'test-api-set-translator'),
+    confdir=(test_roots / 'test-api-set-translator' / 'nonext'),
+    confoverrides={'html_use_smartypants': False},
+)
+def test_html_with_smartypants(app):
+    # no set_translator(), html_use_smartypants=False
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'HTMLTranslator'
+
+
+@with_app(
+    buildername='html',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_html_with_set_translator_for_html_(app):
+    # use set_translator(), no html_translator_class
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfHTMLTranslator'
+
+
+@with_app(
+    buildername='html',
+    srcdir=(test_roots / 'test-api-set-translator'),
+    confoverrides={'html_translator_class': 'ext.ExtHTMLTranslator'},
+)
+def test_html_with_set_translator_for_html_and_html_translator_class(app):
+    # use set_translator() and html_translator_class.
+    # set_translator() is given priority over html_translator_clas.
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfHTMLTranslator'
+
+
+## this test break test_websupport.test_comments test. why?
+# @with_app(
+#     buildername='dirhtml',
+#     srcdir=(test_roots / 'test-api-set-translator'),
+# )
+# def test_dirhtml_set_translator_for_dirhtml(app):
+#     translator_class = app.builder.translator_class
+#     assert translator_class
+#     assert translator_class.__name__ == 'ConfDirHTMLTranslator'
+
+
+@with_app(
+    buildername='singlehtml',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_singlehtml_set_translator_for_singlehtml(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfSingleHTMLTranslator'
+
+
+@with_app(
+    buildername='pickle',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_pickle_set_translator_for_pickle(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfPickleTranslator'
+
+
+@with_app(
+    buildername='json',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_json_set_translator_for_json(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfJsonTranslator'
+
+
+@with_app(
+    buildername='latex',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_html_with_set_translator_for_latex(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfLaTeXTranslator'
+
+
+@with_app(
+    buildername='man',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_html_with_set_translator_for_man(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfManualPageTranslator'
+
+
+@with_app(
+    buildername='texinfo',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_html_with_set_translator_for_texinfo(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfTexinfoTranslator'
+
+
+@with_app(
+    buildername='text',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_html_with_set_translator_for_text(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfTextTranslator'
+
+
+@with_setup(teardown=teardown_websupport)
+@with_app(
+    buildername='websupport',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_html_with_set_translator_for_websupport(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfWebSupportTranslator'
+
+
+@with_app(
+    buildername='xml',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_html_with_set_translator_for_xml(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfXMLTranslator'
+
+
+@with_app(
+    buildername='pseudoxml',
+    srcdir=(test_roots / 'test-api-set-translator'),
+)
+def test_html_with_set_translator_for_pseudoxml(app):
+    translator_class = app.builder.translator_class
+    assert translator_class
+    assert translator_class.__name__ == 'ConfPseudoXMLTranslator'

tests/test_build_html.py

@@ -21,7 +21,7 @@ except ImportError:
     pygments = None
 
 from sphinx import __version__
-from util import test_root, remove_unicode_literals, gen_with_app, with_app
+from util import test_root, test_roots, remove_unicode_literals, gen_with_app, with_app
 from etree13 import ElementTree as ET
 
 
@@ -296,7 +296,7 @@ class NslessParser(ET.XMLParser):
             return name
 
 
-def check_xpath(etree, fname, path, check):
+def check_xpath(etree, fname, path, check, be_found=True):
     nodes = list(etree.findall(path))
     assert nodes != [], ('did not find any node matching xpath '
                          '%r in file %s' % (path, fname))
@@ -308,7 +308,7 @@ def check_xpath(etree, fname, path, check):
     else:
         rex = re.compile(check)
         for node in nodes:
-            if node.text and rex.search(node.text):
+            if node.text and (bool(rex.search(node.text)) ^ (not be_found)):
                 break
         else:
             assert False, ('%r not found in any node matching '
@@ -371,3 +371,44 @@ def test_html_with_globaltoc_and_hidden_toctree(app):
             '\n   :hidden:'
             '\n')
     app.builder.build_all()
+
+
+@gen_with_app(buildername='html', srcdir=(test_roots / 'test-tocdepth'))
+def test_tocdepth(app):
+    # issue #1251
+    app.builder.build_all()
+
+    expects = {
+        'index.html': [
+            (".//li[@class='toctree-l3']/a", '1.1.1. Foo A1', True),
+            (".//li[@class='toctree-l3']/a", '1.2.1. Foo B1', True),
+            (".//li[@class='toctree-l3']/a", '2.1.1. Bar A1', False),
+            (".//li[@class='toctree-l3']/a", '2.2.1. Bar B1', False),
+            ],
+        'foo.html': [
+            (".//h1", '1. Foo', True),
+            (".//h2", '1.1. Foo A', True),
+            (".//h3", '1.1.1. Foo A1', True),
+            (".//h2", '1.2. Foo B', True),
+            (".//h3", '1.2.1. Foo B1', True),
+            ],
+        'bar.html': [
+            (".//h1", '2. Bar', True),
+            (".//h2", '2.1. Bar A', True),
+            (".//h3", '2.1.1. Bar A1', True),
+            (".//h2", '2.2. Bar B', True),
+            (".//h3", '2.2.1. Bar B1', True),
+        ],
+    }
+
+    for fname, paths in iteritems(expects):
+        parser = NslessParser()
+        parser.entity.update(html_entities.entitydefs)
+        fp = open(os.path.join(app.outdir, fname), 'rb')
+        try:
+            etree = ET.parse(fp, parser)
+        finally:
+            fp.close()
+
+        for xpath, check, be_found in paths:
+            yield check_xpath, etree, fname, xpath, check, be_found

tests/test_env.py

@@ -10,7 +10,7 @@
 """
 from six import PY3
 
-from util import TestApp, remove_unicode_literals, path
+from util import TestApp, remove_unicode_literals, path, with_app
 
 from sphinx.builders.html import StandaloneHTMLBuilder
 from sphinx.builders.latex import LaTeXBuilder
@@ -95,6 +95,20 @@ def test_second_update():
     assert 'autodoc' not in env.all_docs
     assert 'autodoc' not in env.found_docs
 
+
+@with_app(srcdir='(empty)')
+def test_undecodable_source_reading_emit_warnings(app):
+    # issue #1524
+    warnings[:] = []
+    app.env.set_warnfunc(lambda *args: warnings.append(args))
+    (app.srcdir / 'contents.rst').write_bytes(b'1\xbb2')
+    _, _, it = app.env.update(app.config, app.srcdir, app.doctreedir, app)
+    list(it)  # the generator does all the work
+    assert warning_emitted(
+        'contents', 'undecodable source characters, replacing with "?":'
+    )
+
+
 def test_object_inventory():
     refs = env.domaindata['py']['objects']
 

tests/test_metadata.py

@@ -61,6 +61,9 @@ def test_docinfo():
         'date': u'2006-05-21',
         'organization': u'humankind',
         'revision': u'4564',
+        'tocdepth': 1,
+        'orphan': u'',
+        'nocomments': u'',
     }
     # I like this way of comparing dicts - easier to see the error.
     for key in exampledocinfo: