merge stable into default

CHANGES

@@ -45,6 +45,24 @@ Bugs fixed
 * #933: Do not crash if an ``:option:`` value is malformed (contains spaces
   but no option name).
 
+* #908: On Python 3, handle error messages from LaTeX correctly in the pngmath
+  extension.
+
+* #943: In autosummary, recognize "first sentences" to pull from the docstring
+  if they contain uppercase letters.
+
+* #923: Take the entire LaTeX document into account when caching
+  pngmath-generated images.  This rebuilds them correctly when
+  ``pngmath_latex_preamble`` changes.
+
+* #901: Emit a warning when using docutils' new "math" markup without a Sphinx
+  math extension active.
+
+* #845: In code blocks, when the selected lexer fails, display line numbers
+  nevertheless if configured.
+
+* #929: Support parsed-literal blocks in LaTeX output correctly.
+
 Documentation
 -------------
 

doc/domains.rst

@@ -382,8 +382,8 @@ a matching identifier is found:
 
 .. rst:role:: py:const
 
-   Reference a "defined" constant.  This may be a C-language ``#define`` or a
+   Reference a "defined" constant.  This may be a Python variable that is not
-   Python variable that is not intended to be changed.
+   intended to be changed.
 
 .. rst:role:: py:class
 
@@ -471,8 +471,8 @@ The C domain (name **c**) is suited for documentation of C API.
 
    Describes a "simple" C macro.  Simple macros are macros which are used for
    code expansion, but which do not take arguments so cannot be described as
-   functions.  This is not to be used for simple constant definitions.  Examples
+   functions.  This is a simple C-language ``#define``.  Examples of its use in
-   of its use in the Python documentation include :c:macro:`PyObject_HEAD` and
+   the Python documentation include :c:macro:`PyObject_HEAD` and
    :c:macro:`Py_BEGIN_ALLOW_THREADS`.
 
 .. rst:directive:: .. c:type:: name
@@ -712,24 +712,24 @@ The JavaScript domain (name **js**) provides the following directives:
       .. js:function:: $.getJSON(href, callback[, errback])
 
          :param string href: An URI to the location of the resource.
-         :param callback: Get's called with the object.
+         :param callback: Gets called with the object.
          :param errback:
-             Get's called in case the request fails. And a lot of other
+             Gets called in case the request fails. And a lot of other
-             text so we need multiple lines
+             text so we need multiple lines.
          :throws SomeError: For whatever reason in that case.
-         :returns: Something
+         :returns: Something.
 
    This is rendered as:
 
       .. js:function:: $.getJSON(href, callback[, errback])
 
         :param string href: An URI to the location of the resource.
-        :param callback: Get's called with the object.
+        :param callback: Gets called with the object.
         :param errback:
-            Get's called in case the request fails. And a lot of other
+            Gets called in case the request fails. And a lot of other
             text so we need multiple lines.
         :throws SomeError: For whatever reason in that case.
-        :returns: Something
+        :returns: Something.
 
 .. rst:directive:: .. js:class:: name
 

doc/markup/code.rst

@@ -62,8 +62,8 @@ installed) and handled in a smart way:
   * ``c``
   * ... and any other lexer name that Pygments supports.
 
-* If highlighting with the selected language fails, the block is not highlighted
+* If highlighting with the selected language fails (i.e. Pygments emits an
-  in any way.
+  "Error" token), the block is not highlighted in any way.
 
 Line numbers
 ^^^^^^^^^^^^

sphinx/builders/text.py

@@ -54,6 +54,7 @@ class TextBuilder(Builder):
         self.writer = TextWriter(self)
 
     def write_doc(self, docname, doctree):
+        self.current_docname = docname
         destination = StringOutput(encoding='utf-8')
         self.writer.write(doctree, destination)
         outfilename = path.join(self.outdir, os_path(docname) + self.out_suffix)

sphinx/ext/autosummary/__init__.py

@@ -199,7 +199,6 @@ class Autosummary(Directive):
         nodes = self.get_table(items)
 
         if 'toctree' in self.options:
-            suffix = env.config.source_suffix
             dirname = posixpath.dirname(env.docname)
 
             tree_prefix = self.options['toctree'].strip()
@@ -276,7 +275,7 @@ class Autosummary(Directive):
 
             while doc and not doc[0].strip():
                 doc.pop(0)
-            m = re.search(r"^([A-Z][^A-Z]*?\.\s)", " ".join(doc).strip())
+            m = re.search(r"^([A-Z].*?\.)(?:\s|$)", " ".join(doc).strip())
             if m:
                 summary = m.group(1).strip()
             elif doc:

sphinx/ext/mathbase.py

@@ -84,7 +84,7 @@ class MathDirective(Directive):
 
 
 def latex_visit_math(self, node):
-    self.body.append('$' + node['latex'] + '$')
+    self.body.append('\\(' + node['latex'] + '\\)')
     raise nodes.SkipNode
 
 def latex_visit_displaymath(self, node):

sphinx/ext/pngmath.py

@@ -26,7 +26,7 @@ from docutils import nodes
 from sphinx.errors import SphinxError
 from sphinx.util.png import read_png_depth, write_png_depth
 from sphinx.util.osutil import ensuredir, ENOENT
-from sphinx.util.pycompat import b
+from sphinx.util.pycompat import b, sys_encoding
 from sphinx.ext.mathbase import setup_math as mathbase_setup, wrap_displaymath
 
 class MathExtError(SphinxError):
@@ -34,9 +34,9 @@ class MathExtError(SphinxError):
 
     def __init__(self, msg, stderr=None, stdout=None):
         if stderr:
-            msg += '\n[stderr]\n' + stderr
+            msg += '\n[stderr]\n' + stderr.decode(sys_encoding, 'replace')
         if stdout:
-            msg += '\n[stdout]\n' + stdout
+            msg += '\n[stdout]\n' + stdout.decode(sys_encoding, 'replace')
         SphinxError.__init__(self, msg)
 
 
@@ -82,8 +82,10 @@ def render_math(self, math):
     may not fail since that indicates a problem in the math source.
     """
     use_preview = self.builder.config.pngmath_use_preview
+    latex = DOC_HEAD + self.builder.config.pngmath_latex_preamble
+    latex += (use_preview and DOC_BODY_PREVIEW or DOC_BODY) % math
 
-    shasum = "%s.png" % sha(math.encode('utf-8')).hexdigest()
+    shasum = "%s.png" % sha(latex.encode('utf-8')).hexdigest()
     relfn = posixpath.join(self.builder.imgpath, 'math', shasum)
     outfn = path.join(self.builder.outdir, '_images', 'math', shasum)
     if path.isfile(outfn):
@@ -95,9 +97,6 @@ def render_math(self, math):
        hasattr(self.builder, '_mathpng_warned_dvipng'):
         return None, None
 
-    latex = DOC_HEAD + self.builder.config.pngmath_latex_preamble
-    latex += (use_preview and DOC_BODY_PREVIEW or DOC_BODY) % math
-
     # use only one tempdir per build -- the use of a directory is cleaner
     # than using temporary files, since we can clean up everything at once
     # just removing the whole directory (see cleanup_tempdir)
@@ -192,11 +191,11 @@ def html_visit_math(self, node):
     try:
         fname, depth = render_math(self, '$'+node['latex']+'$')
     except MathExtError, exc:
-        msg = unicode(str(exc), 'utf-8', 'replace')
+        msg = unicode(exc)
         sm = nodes.system_message(msg, type='WARNING', level=2,
                                   backrefs=[], source=node['latex'])
         sm.walkabout(self)
-        self.builder.warn('display latex %r: ' % node['latex'] + str(exc))
+        self.builder.warn('display latex %r: ' % node['latex'] + msg)
         raise nodes.SkipNode
     if fname is None:
         # something failed -- use text-only as a bad substitute

sphinx/highlighting.py

@@ -175,7 +175,7 @@ class PygmentsBridge(object):
                 if self.try_parse(source):
                     lexer = lexers['python']
                 else:
-                    return self.unhighlighted(source)
+                    lexer = lexers['none']
             else:
                 lexer = lexers['python']
         elif lang in ('python3', 'py3') and source.startswith('>>>'):
@@ -185,7 +185,7 @@ class PygmentsBridge(object):
             try:
                 lexer = guess_lexer(source)
             except Exception:
-                return self.unhighlighted(source)
+                lexer = lexers['none']
         else:
             if lang in lexers:
                 lexer = lexers[lang]
@@ -195,7 +195,7 @@ class PygmentsBridge(object):
                 except ClassNotFound:
                     if warn:
                         warn('Pygments lexer name %r is not known' % lang)
-                        return self.unhighlighted(source)
+                        lexer = lexers['none']
                     else:
                         raise
                 else:
@@ -207,19 +207,19 @@ class PygmentsBridge(object):
             source = doctest.doctestopt_re.sub('', source)
 
         # highlight via Pygments
+        formatter = self.get_formatter(**kwargs)
         try:
-            formatter = self.get_formatter(**kwargs)
             hlsource = highlight(source, lexer, formatter)
-            if self.dest == 'html':
-                return hlsource
-            else:
-                if not isinstance(hlsource, unicode):  # Py2 / Pygments < 1.6
-                    hlsource = hlsource.decode()
-                return hlsource.translate(tex_hl_escape_map_new)
         except ErrorToken:
             # this is most probably not the selected language,
             # so let it pass unhighlighted
-            return self.unhighlighted(source)
+            hlsource = highlight(source, lexers['none'], formatter)
+        if self.dest == 'html':
+            return hlsource
+        else:
+            if not isinstance(hlsource, unicode):  # Py2 / Pygments < 1.6
+                hlsource = hlsource.decode()
+            return hlsource.translate(tex_hl_escape_map_new)
 
     def get_stylesheet(self):
         if not pygments:

sphinx/quickstart.py

@@ -348,7 +348,7 @@ epub_copyright = u'%(copyright_str)s'
 #epub_post_files = []
 
 # A list of files that should not be packed into the epub file.
-#epub_exclude_files = []
+epub_exclude_files = ['search.html']
 
 # The depth of the table of contents in toc.ncx.
 #epub_tocdepth = 3
@@ -986,7 +986,7 @@ Enter the root path for documentation.'''
 You have two options for placing the build directory for Sphinx output.
 Either, you use a directory "_build" within the root path, or you separate
 "source" and "build" directories within the root path.'''
-        do_prompt(d, 'sep', 'Separate source and build directories (y/N)', 'n',
+        do_prompt(d, 'sep', 'Separate source and build directories (y/n)', 'n',
                   boolean)
 
     if 'dot' not in d:
@@ -1042,50 +1042,50 @@ document is a custom template, you can also set this to another filename.'''
     if 'epub' not in d:
         print '''
 Sphinx can also add configuration for epub output:'''
-        do_prompt(d, 'epub', 'Do you want to use the epub builder (y/N)',
+        do_prompt(d, 'epub', 'Do you want to use the epub builder (y/n)',
                   'n', boolean)
 
     if 'ext_autodoc' not in d:
         print '''
 Please indicate if you want to use one of the following Sphinx extensions:'''
         do_prompt(d, 'ext_autodoc', 'autodoc: automatically insert docstrings '
-                  'from modules (y/N)', 'n', boolean)
+                  'from modules (y/n)', 'n', boolean)
     if 'ext_doctest' not in d:
         do_prompt(d, 'ext_doctest', 'doctest: automatically test code snippets '
-                  'in doctest blocks (y/N)', 'n', boolean)
+                  'in doctest blocks (y/n)', 'n', boolean)
     if 'ext_intersphinx' not in d:
         do_prompt(d, 'ext_intersphinx', 'intersphinx: link between Sphinx '
-                  'documentation of different projects (y/N)', 'n', boolean)
+                  'documentation of different projects (y/n)', 'n', boolean)
     if 'ext_todo' not in d:
         do_prompt(d, 'ext_todo', 'todo: write "todo" entries '
-                  'that can be shown or hidden on build (y/N)', 'n', boolean)
+                  'that can be shown or hidden on build (y/n)', 'n', boolean)
     if 'ext_coverage' not in d:
         do_prompt(d, 'ext_coverage', 'coverage: checks for documentation '
-                  'coverage (y/N)', 'n', boolean)
+                  'coverage (y/n)', 'n', boolean)
     if 'ext_pngmath' not in d:
         do_prompt(d, 'ext_pngmath', 'pngmath: include math, rendered '
-                  'as PNG images (y/N)', 'n', boolean)
+                  'as PNG images (y/n)', 'n', boolean)
     if 'ext_mathjax' not in d:
         do_prompt(d, 'ext_mathjax', 'mathjax: include math, rendered in the '
-                  'browser by MathJax (y/N)', 'n', boolean)
+                  'browser by MathJax (y/n)', 'n', boolean)
     if d['ext_pngmath'] and d['ext_mathjax']:
         print '''Note: pngmath and mathjax cannot be enabled at the same time.
 pngmath has been deselected.'''
     if 'ext_ifconfig' not in d:
         do_prompt(d, 'ext_ifconfig', 'ifconfig: conditional inclusion of '
-                  'content based on config values (y/N)', 'n', boolean)
+                  'content based on config values (y/n)', 'n', boolean)
     if 'ext_viewcode' not in d:
         do_prompt(d, 'ext_viewcode', 'viewcode: include links to the source '
-                  'code of documented Python objects (y/N)', 'n', boolean)
+                  'code of documented Python objects (y/n)', 'n', boolean)
 
     if 'makefile' not in d:
         print '''
 A Makefile and a Windows command file can be generated for you so that you
 only have to run e.g. `make html' instead of invoking sphinx-build
 directly.'''
-        do_prompt(d, 'makefile', 'Create Makefile? (Y/n)', 'y', boolean)
+        do_prompt(d, 'makefile', 'Create Makefile? (y/n)', 'y', boolean)
     if 'batchfile' not in d:
-        do_prompt(d, 'batchfile', 'Create Windows command file? (Y/n)',
+        do_prompt(d, 'batchfile', 'Create Windows command file? (y/n)',
                   'y', boolean)
     print
 

sphinx/texinputs/sphinx.sty

@@ -30,6 +30,8 @@
 \RequirePackage{wrapfig}
 % Separate paragraphs by space by default.
 \RequirePackage{parskip}
+% For parsed-literal blocks.
+\RequirePackage{alltt}
 
 % Redefine these colors to your liking in the preamble.
 \definecolor{TitleColor}{rgb}{0.126,0.263,0.361}

sphinx/util/pycompat.py

@@ -30,6 +30,9 @@ if sys.version_info >= (3, 0):
     # safely encode a string for printing to the terminal
     def terminal_safe(s):
         return s.encode('ascii', 'backslashreplace').decode('ascii')
+    # some kind of default system encoding; should be used with a lenient
+    # error handler
+    sys_encoding = sys.getdefaultencoding()
     # support for running 2to3 over config files
     def convert_with_2to3(filepath):
         from lib2to3.refactor import RefactoringTool, get_fixers_from_package
@@ -62,6 +65,10 @@ else:
     # safely encode a string for printing to the terminal
     def terminal_safe(s):
         return s.encode('ascii', 'backslashreplace')
+    # some kind of default system encoding; should be used with a lenient
+    # error handler
+    import locale
+    sys_encoding = locale.getpreferredencoding()
 
 
 def execfile_(filepath, _globals):

sphinx/writers/html.py

@@ -551,6 +551,13 @@ class HTMLTranslator(BaseTranslator):
             node['classes'].append('field-odd')
         self.body.append(self.starttag(node, 'tr', '', CLASS='field'))
 
+    def visit_math(self, node, math_env=''):
+        self.builder.warn('using "math" markup without a Sphinx math extension '
+                          'active, please use one of the math extensions '
+                          'described at http://sphinx-doc.org/ext/math.html',
+                          (self.builder.current_docname, node.line))
+        raise nodes.SkipNode
+
     def unknown_visit(self, node):
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
 

sphinx/writers/latex.py

@@ -264,7 +264,6 @@ class LaTeXTranslator(nodes.NodeVisitor):
         self.next_figure_ids = set()
         self.next_table_ids = set()
         # flags
-        self.verbatim = None
         self.in_title = 0
         self.in_production_list = 0
         self.in_footnote = 0
@@ -1318,36 +1317,40 @@ class LaTeXTranslator(nodes.NodeVisitor):
             raise UnsupportedError('%s:%s: literal blocks in footnotes are '
                                    'not supported by LaTeX' %
                                    (self.curfilestack[-1], node.line))
-        self.verbatim = ''
+        if node.rawsource != node.astext():
+            # most probably a parsed-literal block -- don't highlight
+            self.body.append('\\begin{alltt}\n')
+        else:
+            code = node.astext().rstrip('\n')
+            lang = self.hlsettingstack[-1][0]
+            linenos = code.count('\n') >= self.hlsettingstack[-1][1] - 1
+            highlight_args = node.get('highlight_args', {})
+            if 'language' in node:
+                # code-block directives
+                lang = node['language']
+                highlight_args['force'] = True
+            if 'linenos' in node:
+                linenos = node['linenos']
+            def warner(msg):
+                self.builder.warn(msg, (self.curfilestack[-1], node.line))
+            hlcode = self.highlighter.highlight_block(code, lang, warn=warner,
+                    linenos=linenos, **highlight_args)
+            # workaround for Unicode issue
+            hlcode = hlcode.replace(u'€', u'@texteuro[]')
+            # must use original Verbatim environment and "tabular" environment
+            if self.table:
+                hlcode = hlcode.replace('\\begin{Verbatim}',
+                                        '\\begin{OriginalVerbatim}')
+                self.table.has_problematic = True
+                self.table.has_verbatim = True
+            # get consistent trailer
+            hlcode = hlcode.rstrip()[:-14] # strip \end{Verbatim}
+            hlcode = hlcode.rstrip() + '\n'
+            self.body.append('\n' + hlcode + '\\end{%sVerbatim}\n' %
+                             (self.table and 'Original' or ''))
+            raise nodes.SkipNode
     def depart_literal_block(self, node):
-        code = self.verbatim.rstrip('\n')
+        self.body.append('\n\\end{alltt}\n')
-        lang = self.hlsettingstack[-1][0]
-        linenos = code.count('\n') >= self.hlsettingstack[-1][1] - 1
-        highlight_args = node.get('highlight_args', {})
-        if 'language' in node:
-            # code-block directives
-            lang = node['language']
-            highlight_args['force'] = True
-        if 'linenos' in node:
-            linenos = node['linenos']
-        def warner(msg):
-            self.builder.warn(msg, (self.curfilestack[-1], node.line))
-        hlcode = self.highlighter.highlight_block(code, lang, warn=warner,
-                linenos=linenos, **highlight_args)
-        # workaround for Unicode issue
-        hlcode = hlcode.replace(u'€', u'@texteuro[]')
-        # must use original Verbatim environment and "tabular" environment
-        if self.table:
-            hlcode = hlcode.replace('\\begin{Verbatim}',
-                                    '\\begin{OriginalVerbatim}')
-            self.table.has_problematic = True
-            self.table.has_verbatim = True
-        # get consistent trailer
-        hlcode = hlcode.rstrip()[:-14] # strip \end{Verbatim}
-        hlcode = hlcode.rstrip() + '\n'
-        self.body.append('\n' + hlcode + '\\end{%sVerbatim}\n' %
-                         (self.table and 'Original' or ''))
-        self.verbatim = None
     visit_doctest_block = visit_literal_block
     depart_doctest_block = depart_literal_block
 
@@ -1510,13 +1513,10 @@ class LaTeXTranslator(nodes.NodeVisitor):
         return self.encode(text).replace('\\textasciitilde{}', '~')
 
     def visit_Text(self, node):
-        if self.verbatim is not None:
+        text = self.encode(node.astext())
-            self.verbatim += node.astext()
+        if not self.no_contractions:
-        else:
+            text = educate_quotes_latex(text)
-            text = self.encode(node.astext())
+        self.body.append(text)
-            if not self.no_contractions:
-                text = educate_quotes_latex(text)
-            self.body.append(text)
     def depart_Text(self, node):
         pass
 
@@ -1532,5 +1532,14 @@ class LaTeXTranslator(nodes.NodeVisitor):
     def depart_system_message(self, node):
         self.body.append('\n')
 
+    def visit_math(self, node):
+        self.builder.warn('using "math" markup without a Sphinx math extension '
+                          'active, please use one of the math extensions '
+                          'described at http://sphinx-doc.org/ext/math.html',
+                          (self.curfilestack[-1], node.line))
+        raise nodes.SkipNode
+
+    visit_math_block = visit_math
+
     def unknown_visit(self, node):
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)

sphinx/writers/manpage.py

@@ -342,5 +342,13 @@ class ManualPageTranslator(BaseTranslator):
     def depart_inline(self, node):
         pass
 
+    def visit_math(self, node):
+        self.builder.warn('using "math" markup without a Sphinx math extension '
+                          'active, please use one of the math extensions '
+                          'described at http://sphinx-doc.org/ext/math.html')
+        raise nodes.SkipNode
+
+    visit_math_block = visit_math
+
     def unknown_visit(self, node):
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)

sphinx/writers/texinfo.py

@@ -1393,3 +1393,11 @@ class TexinfoTranslator(nodes.NodeVisitor):
         pass
     def depart_pending_xref(self, node):
         pass
+
+    def visit_math(self, node):
+        self.builder.warn('using "math" markup without a Sphinx math extension '
+                          'active, please use one of the math extensions '
+                          'described at http://sphinx-doc.org/ext/math.html')
+        raise nodes.SkipNode
+
+    visit_math_block = visit_math

sphinx/writers/text.py

@@ -152,6 +152,7 @@ class TextTranslator(nodes.NodeVisitor):
 
     def __init__(self, document, builder):
         nodes.NodeVisitor.__init__(self, document)
+        self.builder = builder
 
         newlines = builder.config.text_newlines
         if newlines == 'windows':
@@ -838,5 +839,14 @@ class TextTranslator(nodes.NodeVisitor):
             self.body.append(node.astext())
         raise nodes.SkipNode
 
+    def visit_math(self, node):
+        self.builder.warn('using "math" markup without a Sphinx math extension '
+                          'active, please use one of the math extensions '
+                          'described at http://sphinx-doc.org/ext/math.html',
+                          (self.builder.current_docname, node.line))
+        raise nodes.SkipNode
+
+    visit_math_block = visit_math
+
     def unknown_visit(self, node):
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)

tests/root/markup.txt

@@ -79,6 +79,10 @@ Body directives
 
    b
 
+.. parsed-literal::
+
+   with some *markup* inside
+
 
 .. _admonition-section:
 

tests/test_build_html.py

@@ -42,7 +42,7 @@ http://www.python.org/logo.png
 reading included file u'.*?wrongenc.inc' seems to be wrong, try giving an \
 :encoding: option\\n?
 %(root)s/includes.txt:4: WARNING: download file not readable: .*?nonexisting.png
-%(root)s/markup.txt:142: WARNING: Malformed :option: u'Python c option', does \
+%(root)s/markup.txt:\\d+: WARNING: Malformed :option: u'Python c option', does \
 not contain option marker - or -- or /
 %(root)s/objects.txt:\\d*: WARNING: using old C markup; please migrate to \
 new-style markup \(e.g. c:function instead of cfunction\), see \

tests/test_markup.py

@@ -132,7 +132,7 @@ def test_latex_escaping():
     # in verbatim code fragments
     yield (verify, u'::\n\n @Γ\\∞${}', None,
            u'\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n'
-           u'@\\(\\Gamma\\)\\PYGZbs{}\\(\\infty\\)\\$\\PYGZob{}\\PYGZcb{}\n'
+           u'@\\(\\Gamma\\)\\PYGZbs{}\\(\\infty\\)\\PYGZdl{}\\PYGZob{}\\PYGZcb{}\n'
            u'\\end{Verbatim}')
     # in URIs
     yield (verify_re, u'`test <http://example.com/~me/>`_', None,