Merge with birkenfeld/sphinx

CHANGES

@@ -19,6 +19,37 @@ Release 1.1.3 (in development)
 * PR#36: Make the "bibliography to TOC" fix in LaTeX output specific to
   the document class.
 
+* #695: When the highlight language "python" is specified explicitly,
+  do not try to parse the code to recognize non-Python snippets.
+
+* #859: Fix exception under certain circumstances when not finding
+  appropriate objects to link to.
+
+* #860: Do not crash when encountering invalid doctest examples, just
+  emit a warning.
+
+* #864: Fix crash with some settings of :confval:`modindex_common_prefix`.
+
+* #862: Fix handling of ``-D`` and ``-A`` options on Python 3.
+
+* #851: Recognize and warn about circular toctrees, instead of running
+  into recursion errors.
+
+* #853: Restore compatibility with docutils trunk.
+
+* #852: Fix HtmlHelp index entry links again.
+
+* #854: Fix inheritance_diagram raising attribute errors on builtins.
+
+* #832: Fix crashes when putting comments or lone terms in a glossary.
+
+* #834, #818: Fix HTML help language/encoding mapping for all Sphinx
+  supported languages.
+
+* #844: Fix crashes when dealing with Unicode output in doctest extension.
+
+* #831: Provide ``--project`` flag in setup_command as advertised.
+
 
 Release 1.1.2 (Nov 1, 2011) -- 1.1.1 is a silly version number anyway!
 ======================================================================

doc/config.rst

@@ -277,7 +277,7 @@ Project information
      the format given in :confval:`today_fmt`.
 
    The default is no :confval:`today` and a :confval:`today_fmt` of ``'%B %d,
-   %Y'`` (or, if translation is enabled with :confval:`language`, am equivalent
+   %Y'`` (or, if translation is enabled with :confval:`language`, an equivalent
    %format for the selected locale).
 
 .. confval:: highlight_language
@@ -791,6 +791,8 @@ the `Dublin Core metadata <http://dublincore.org/>`_.
    theme.  These are theme-specific.  For the options understood by the builtin
    themes, see :ref:`this section <builtin-themes>`.
 
+   .. versionadded:: 1.2
+
 .. confval:: epub_title
 
    The title of the document.  It defaults to the :confval:`html_title` option
@@ -912,6 +914,8 @@ the `Dublin Core metadata <http://dublincore.org/>`_.
    to use this option.  The default value is ``False`` because the automatic
    conversion may lose information.
 
+   .. versionadded:: 1.2
+
 .. confval:: epub_max_image_width
 
    This option specifies the maximum width of images.  If it is set to a value
@@ -920,6 +924,8 @@ the `Dublin Core metadata <http://dublincore.org/>`_.
    value is ``0``.  You need the Python Image Library (PIL) installed to use
    this option.
 
+   .. versionadded:: 1.2
+
 
 .. _latex-options:
 

doc/ext/math.rst

@@ -24,6 +24,10 @@ The input language for mathematics is LaTeX markup.  This is the de-facto
 standard for plain-text math notation and has the added advantage that no
 further translation is necessary when building LaTeX output.
 
+Keep in mind that when you put math markup in **Python docstrings** read by
+:mod:`autodoc <sphinx.ext.autodoc>`, you either have to double all backslashes,
+or use Python raw strings (``r"raw"``).
+
 :mod:`.mathbase` defines these new markup elements:
 
 .. rst:role:: math

sphinx/builders/changes.py

@@ -11,7 +11,6 @@
 
 import codecs
 from os import path
-from cgi import escape
 
 from sphinx import package_dir
 from sphinx.util import copy_static_entry
@@ -20,6 +19,7 @@ from sphinx.theming import Theme
 from sphinx.builders import Builder
 from sphinx.util.osutil import ensuredir, os_path
 from sphinx.util.console import bold
+from sphinx.util.pycompat import htmlescape
 
 
 class ChangesBuilder(Builder):
@@ -115,7 +115,7 @@ class ChangesBuilder(Builder):
                   '.. deprecated:: %s' % version]
 
         def hl(no, line):
-            line = '<a name="L%s"> </a>' % no + escape(line)
+            line = '<a name="L%s"> </a>' % no + htmlescape(line)
             for x in hltext:
                 if x in line:
                     line = '<span class="hl">%s</span>' % line
@@ -125,7 +125,10 @@ class ChangesBuilder(Builder):
         self.info(bold('copying source files...'))
         for docname in self.env.all_docs:
             f = codecs.open(self.env.doc2path(docname), 'r', 'latin1')
+            try:
                 lines = f.readlines()
+            finally:
+                f.close()
             targetfn = path.join(self.outdir, 'rst', os_path(docname)) + '.html'
             ensuredir(path.dirname(targetfn))
             f = codecs.open(targetfn, 'w', 'latin1')
@@ -148,7 +151,7 @@ class ChangesBuilder(Builder):
                           self.outdir, self)
 
     def hl(self, text, version):
-        text = escape(text)
+        text = htmlescape(text)
         for directive in ['versionchanged', 'versionadded', 'deprecated']:
             text = text.replace('.. %s:: %s' % (directive, version),
                                 '<b>.. %s:: %s</b>' % (directive, version))

sphinx/builders/htmlhelp.py

@@ -11,7 +11,6 @@
 """
 
 import os
-import cgi
 import codecs
 from os import path
 
@@ -19,6 +18,7 @@ from docutils import nodes
 
 from sphinx import addnodes
 from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.util.pycompat import htmlescape
 
 
 # Project file (*.hhp) template.  'outname' is the file basename (like
@@ -124,20 +124,31 @@ was  will  with
 # See http://msdn.microsoft.com/en-us/library/ms930130.aspx for more.
 chm_locales = {
     # lang:   LCID,  encoding
-    'cs':    (0x405, 'iso8859_2'),
+    'ca':    (0x403, 'cp1252'),
-    'de':    (0x407, 'iso8859_1'),
+    'cs':    (0x405, 'cp1250'),
-    'en':    (0x409, 'iso8859_1'),
+    'da':    (0x406, 'cp1252'),
-    'es':    (0x40a, 'iso8859_1'),
+    'de':    (0x407, 'cp1252'),
-    'fi':    (0x40b, 'iso8859_1'),
+    'en':    (0x409, 'cp1252'),
-    'fr':    (0x40c, 'iso8859_1'),
+    'es':    (0x40a, 'cp1252'),
-    'it':    (0x410, 'iso8859_1'),
+    'et':    (0x425, 'cp1257'),
+    'fa':    (0x429, 'cp1256'),
+    'fi':    (0x40b, 'cp1252'),
+    'fr':    (0x40c, 'cp1252'),
+    'hr':    (0x41a, 'cp1250'),
+    'hu':    (0x40e, 'cp1250'),
+    'it':    (0x410, 'cp1252'),
     'ja':    (0x411, 'cp932'),
+    'ko':    (0x412, 'cp949'),
+    'lt':    (0x427, 'cp1257'),
     'lv':    (0x426, 'cp1257'),
-    'nl':    (0x413, 'iso8859_1'),
+    'nl':    (0x413, 'cp1252'),
-    'pl':    (0x415, 'iso8859_2'),
+    'pl':    (0x415, 'cp1250'),
-    'pt_BR': (0x416, 'iso8859_1'),
+    'pt_BR': (0x416, 'cp1252'),
     'ru':    (0x419, 'cp1251'),
-    'sl':    (0x424, 'iso8859_2'),
+    'sk':    (0x41b, 'cp1250'),
+    'sl':    (0x424, 'cp1250'),
+    'sv':    (0x41d, 'cp1252'),
+    'tr':    (0x41f, 'cp1254'),
     'uk_UA': (0x422, 'cp1251'),
     'zh_CN': (0x804, 'cp936'),
     'zh_TW': (0x404, 'cp950'),
@@ -230,7 +241,7 @@ class HTMLHelpBuilder(StandaloneHTMLBuilder):
                         write_toc(subnode, ullevel)
                 elif isinstance(node, nodes.reference):
                     link = node['refuri']
-                    title = cgi.escape(node.astext()).replace('"','"')
+                    title = htmlescape(node.astext()).replace('"','"')
                     f.write(object_sitemap % (title, link))
                 elif isinstance(node, nodes.bullet_list):
                     if ullevel != 0:
@@ -259,20 +270,20 @@ class HTMLHelpBuilder(StandaloneHTMLBuilder):
             def write_index(title, refs, subitems):
                 def write_param(name, value):
                     item = '    <param name="%s" value="%s">\n' % \
-                        (name, value[1])
+                        (name, value)
                     f.write(item)
-                title = cgi.escape(title)
+                title = htmlescape(title)
                 f.write('<LI> <OBJECT type="text/sitemap">\n')
                 write_param('Keyword', title)
                 if len(refs) == 0:
                     write_param('See Also', title)
                 elif len(refs) == 1:
-                    write_param('Local', refs[0])
+                    write_param('Local', refs[0][1])
                 else:
                     for i, ref in enumerate(refs):
                         # XXX: better title?
-                        write_param('Name', '[%d] %s' % (i, ref))
+                        write_param('Name', '[%d] %s' % (i, ref[1]))
-                        write_param('Local', ref)
+                        write_param('Local', ref[1])
                 f.write('</OBJECT>\n')
                 if subitems:
                     f.write('<UL> ')

sphinx/builders/qthelp.py

@@ -14,12 +14,12 @@ import re
 import codecs
 import posixpath
 from os import path
-from cgi import escape
 
 from docutils import nodes
 
 from sphinx import addnodes
 from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.util.pycompat import htmlescape
 
 
 _idpattern = re.compile(
@@ -164,7 +164,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
                        fn.endswith('.html'):
                     filename = path.join(root, fn)[olen:]
                     projectfiles.append(file_template %
-                                        {'filename': escape(filename)})
+                                        {'filename': htmlescape(filename)})
         projectfiles = '\n'.join(projectfiles)
 
         # it seems that the "namespace" may not contain non-alphanumeric
@@ -179,12 +179,12 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
         f = codecs.open(path.join(outdir, outname+'.qhp'), 'w', 'utf-8')
         try:
             f.write(project_template % {
-                'outname': escape(outname),
+                'outname': htmlescape(outname),
-                'title': escape(self.config.html_title),
+                'title': htmlescape(self.config.html_title),
-                'version': escape(self.config.version),
+                'version': htmlescape(self.config.version),
-                'project': escape(self.config.project),
+                'project': htmlescape(self.config.project),
-                'namespace': escape(nspace),
+                'namespace': htmlescape(nspace),
-                'masterdoc': escape(self.config.master_doc),
+                'masterdoc': htmlescape(self.config.master_doc),
                 'sections': sections,
                 'keywords': keywords,
                 'files': projectfiles})
@@ -199,10 +199,10 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
         f = codecs.open(path.join(outdir, outname+'.qhcp'), 'w', 'utf-8')
         try:
             f.write(collection_template % {
-                'outname': escape(outname),
+                'outname': htmlescape(outname),
-                'title': escape(self.config.html_short_title),
+                'title': htmlescape(self.config.html_short_title),
-                'homepage': escape(homepage),
+                'homepage': htmlescape(homepage),
-                'startpage': escape(startpage)})
+                'startpage': htmlescape(startpage)})
         finally:
             f.close()
 
@@ -224,7 +224,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
         if self.isdocnode(node):
             refnode = node.children[0][0]
             link = refnode['refuri']
-            title = escape(refnode.astext()).replace('"','"')
+            title = htmlescape(refnode.astext()).replace('"','"')
             item = '<section title="%(title)s" ref="%(ref)s">' % {
                                                                 'title': title,
                                                                 'ref': link}
@@ -237,7 +237,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
                 parts.extend(self.write_toc(subnode, indentlevel))
         elif isinstance(node, nodes.reference):
             link = node['refuri']
-            title = escape(node.astext()).replace('"','&quot;')
+            title = htmlescape(node.astext()).replace('"','&quot;')
             item = section_template % {'title': title, 'ref': link}
             item = u' ' * 4 * indentlevel + item
             parts.append(item.encode('ascii', 'xmlcharrefreplace'))
@@ -274,7 +274,7 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
     def build_keywords(self, title, refs, subitems):
         keywords = []
 
-        title = escape(title)
+        title = htmlescape(title)
 #        if len(refs) == 0: # XXX
 #            write_param('See Also', title)
         if len(refs) == 1:

sphinx/cmdline.py

@@ -22,7 +22,7 @@ from sphinx.errors import SphinxError
 from sphinx.application import Sphinx
 from sphinx.util import Tee, format_exception_cut_frames, save_traceback
 from sphinx.util.console import red, nocolor, color_terminal
-from sphinx.util.pycompat import terminal_safe
+from sphinx.util.pycompat import terminal_safe, bytes
 
 
 def usage(argv, msg=None):
@@ -67,7 +67,8 @@ def main(argv):
         allopts = set(opt[0] for opt in opts)
         srcdir = confdir = path.abspath(args[0])
         if not path.isdir(srcdir):
-            print >>sys.stderr, 'Error: Cannot find source directory.'
+            print >>sys.stderr, 'Error: Cannot find source directory `%s\'.' % (
+                                srcdir,)
             return 1
         if not path.isfile(path.join(srcdir, 'conf.py')) and \
                '-c' not in allopts and '-C' not in allopts:
@@ -137,7 +138,7 @@ def main(argv):
             try:
                 val = int(val)
             except ValueError:
-                if likely_encoding:
+                if likely_encoding and isinstance(val, bytes):
                     try:
                         val = val.decode(likely_encoding)
                     except UnicodeError:
@@ -153,7 +154,7 @@ def main(argv):
             try:
                 val = int(val)
             except ValueError:
-                if likely_encoding:
+                if likely_encoding and isinstance(val, bytes):
                     try:
                         val = val.decode(likely_encoding)
                     except UnicodeError:

sphinx/domains/python.py

@@ -525,6 +525,7 @@ class PythonModuleIndex(Index):
                 # it's a submodule
                 if prev_modname == package:
                     # first submodule - make parent a group head
+                    if entries:
                         entries[-1][1] = 1
                 elif not prev_modname.startswith(package):
                     # submodule without parent in list, add dummy entry
@@ -625,6 +626,7 @@ class PythonDomain(Domain):
         newname = None
         if searchmode == 1:
             objtypes = self.objtypes_for_role(type)
+            if objtypes is not None:
                 if modname and classname:
                     fullname = modname + '.' + classname + '.' + name
                     if fullname in objects and objects[fullname][1] in objtypes:

sphinx/domains/std.py

@@ -245,6 +245,9 @@ class Glossary(Directive):
                 continue
             # unindented line -> a term
             if line and not line[0].isspace():
+                # enable comments
+                if line.startswith('.. '):
+                    continue
                 # first term of definition
                 if in_definition:
                     if not was_empty:
@@ -315,6 +318,7 @@ class Glossary(Directive):
             term += system_messages
 
             defnode = nodes.definition()
+            if definition:
                 self.state.nested_parse(definition, definition.items[0][1], defnode)
 
             items.append((termtexts,

sphinx/environment.py

@@ -1313,12 +1313,14 @@ class BuildEnvironment:
                             subnode['iscurrent'] = True
                             subnode = subnode.parent
 
-        def _entries_from_toctree(toctreenode, separate=False, subtree=False):
+        def _entries_from_toctree(toctreenode, parents,
+                                  separate=False, subtree=False):
             """Return TOC entries for a toctree node."""
             refs = [(e[0], str(e[1])) for e in toctreenode['entries']]
             entries = []
             for (title, ref) in refs:
                 try:
+                    refdoc = None
                     if url_re.match(ref):
                         reference = nodes.reference('', '', internal=False,
                                                     refuri=ref, anchorname='',
@@ -1341,6 +1343,12 @@ class BuildEnvironment:
                         # don't show subitems
                         toc = nodes.bullet_list('', item)
                     else:
+                        if ref in parents:
+                            self.warn(ref, 'circular toctree references '
+                                      'detected, ignoring: %s <- %s' %
+                                      (ref, ' <- '.join(parents)))
+                            continue
+                        refdoc = ref
                         toc = self.tocs[ref].deepcopy()
                         self.process_only_nodes(toc, builder, ref)
                         if title and toc.children and len(toc.children) == 1:
@@ -1376,7 +1384,8 @@ class BuildEnvironment:
                         if not (toctreenode.get('hidden', False)
                                 and not includehidden):
                             i = toctreenode.parent.index(toctreenode) + 1
-                            for item in _entries_from_toctree(toctreenode,
+                            for item in _entries_from_toctree(
+                                    toctreenode, [refdoc] + parents,
                                     subtree=True):
                                 toctreenode.parent.insert(i, item)
                                 i += 1
@@ -1398,7 +1407,7 @@ class BuildEnvironment:
         # NOTE: previously, this was separate=True, but that leads to artificial
         # separation when two or more toctree entries form a logical unit, so
         # separating mode is no longer used -- it's kept here for history's sake
-        tocentries = _entries_from_toctree(toctree, separate=False)
+        tocentries = _entries_from_toctree(toctree, [], separate=False)
         if not tocentries:
             return None
 
@@ -1686,7 +1695,11 @@ class BuildEnvironment:
     def collect_relations(self):
         relations = {}
         getinc = self.toctree_includes.get
-        def collect(parents, docname, previous, next):
+        def collect(parents, parents_set, docname, previous, next):
+            # circular relationship?
+            if docname in parents_set:
+                # we will warn about this in resolve_toctree()
+                return
             includes = getinc(docname)
             # previous
             if not previous:
@@ -1723,9 +1736,10 @@ class BuildEnvironment:
                 for subindex, args in enumerate(izip(includes,
                                                      [None] + includes,
                                                      includes[1:] + [None])):
-                    collect([(docname, subindex)] + parents, *args)
+                    collect([(docname, subindex)] + parents,
+                            parents_set.union([docname]), *args)
             relations[docname] = [parents[0][0], previous, next]
-        collect([(None, 0)], self.config.master_doc, None, None)
+        collect([(None, 0)], set(), self.config.master_doc, None, None)
         return relations
 
     def check_consistency(self):

sphinx/ext/doctest.py

@@ -23,9 +23,11 @@ from docutils import nodes
 from docutils.parsers.rst import directives
 
 from sphinx.builders import Builder
+from sphinx.util import force_decode
 from sphinx.util.nodes import set_source_info
 from sphinx.util.compat import Directive
 from sphinx.util.console import bold
+from sphinx.util.pycompat import bytes
 
 blankline_re = re.compile(r'^\s*<BLANKLINE>', re.MULTILINE)
 doctestopt_re = re.compile(r'#\s*doctest:.+$', re.MULTILINE)
@@ -231,6 +233,8 @@ Results of doctest builder run on %s
         self.info(text, nonl=True)
         if self.app.quiet:
             self.warn(text)
+        if isinstance(text, bytes):
+            text = force_decode(text, None)
         self.outfile.write(text)
 
     def get_target_uri(self, docname, typ=None):
@@ -375,8 +379,13 @@ Doctest summary
         for code in group.tests:
             if len(code) == 1:
                 # ordinary doctests (code/output interleaved)
+                try:
                     test = parser.get_doctest(code[0].code, {}, group.name,
                                               filename, code[0].lineno)
+                except Exception:
+                    self.warn('ignoring invalid doctest code: %r' % code[0].code,
+                              '%s:%s' % (filename, code[0].lineno))
+                    continue
                 if not test.examples:
                     continue
                 for example in test.examples:

sphinx/ext/graphviz.py

@@ -298,13 +298,15 @@ def texinfo_visit_graphviz(self, node):
 def text_visit_graphviz(self, node):
     if 'alt' in node.attributes:
         self.add_text(_('[graph: %s]') % node['alt'])
+    else:
         self.add_text(_('[graph]'))
     raise nodes.SkipNode
 
 
 def man_visit_graphviz(self, node):
     if 'alt' in node.attributes:
-        self.body.append(_('[graph: %s]') % node['alt'] + '\n')
+        self.body.append(_('[graph: %s]') % node['alt'])
+    else:
         self.body.append(_('[graph]'))
     raise nodes.SkipNode
 

sphinx/ext/inheritance_diagram.py

@@ -39,6 +39,7 @@ r"""
 import re
 import sys
 import inspect
+import __builtin__
 try:
     from hashlib import md5
 except ImportError:
@@ -142,7 +143,7 @@ class InheritanceGraph(object):
         displayed node names.
         """
         all_classes = {}
-        builtins = __builtins__.values()
+        builtins = vars(__builtin__).values()
 
         def recurse(cls):
             if not show_builtins and cls in builtins:

sphinx/highlighting.py

@@ -10,7 +10,6 @@
 """
 
 import sys
-import cgi
 import re
 import textwrap
 
@@ -20,6 +19,7 @@ except ImportError:
     # parser is not available on Jython
     parser = None
 
+from sphinx.util.pycompat import htmlescape
 from sphinx.util.texescape import tex_hl_escape_map_new
 from sphinx.ext import doctest
 
@@ -105,7 +105,7 @@ class PygmentsBridge(object):
 
     def unhighlighted(self, source):
         if self.dest == 'html':
-            return '<pre>' + cgi.escape(source) + '</pre>\n'
+            return '<pre>' + htmlescape(source) + '</pre>\n'
         else:
             # first, escape highlighting characters like Pygments does
             source = source.translate(escape_hl_chars)
@@ -153,7 +153,7 @@ class PygmentsBridge(object):
         else:
             return True
 
-    def highlight_block(self, source, lang, warn=None, **kwargs):
+    def highlight_block(self, source, lang, warn=None, force=False, **kwargs):
         if not isinstance(source, unicode):
             source = source.decode()
         if not pygments:
@@ -164,12 +164,14 @@ class PygmentsBridge(object):
             if source.startswith('>>>'):
                 # interactive session
                 lexer = lexers['pycon']
-            else:
+            elif not force:
                 # maybe Python -- try parsing it
                 if self.try_parse(source):
                     lexer = lexers['python']
                 else:
                     return self.unhighlighted(source)
+            else:
+                lexer = lexers['python']
         elif lang in ('python3', 'py3') and source.startswith('>>>'):
             # for py3, recognize interactive sessions, but do not try parsing...
             lexer = lexers['pycon3']

sphinx/setup_command.py

@@ -81,6 +81,7 @@ class BuildDoc(Command):
         self.fresh_env = self.all_files = False
         self.source_dir = self.build_dir = None
         self.builder = 'html'
+        self.project = ''
         self.version = ''
         self.release = ''
         self.today = ''
@@ -125,6 +126,8 @@ class BuildDoc(Command):
         else:
             status_stream = sys.stdout
         confoverrides = {}
+        if self.project:
+             confoverrides['project'] = self.project        
         if self.version:
              confoverrides['version'] = self.version
         if self.release:

sphinx/util/inspect.py

@@ -16,6 +16,7 @@ import sys
 inspect = __import__('inspect')
 
 from sphinx.util import force_decode
+from sphinx.util.pycompat import bytes
 
 
 if sys.version_info >= (2, 5):
@@ -89,4 +90,6 @@ def safe_repr(object):
         s = repr(object)
     except Exception:
         raise ValueError
+    if isinstance(s, bytes):
         return force_decode(s, None).replace('\n', ' ')
+    return s.replace('\n', ' ')

sphinx/util/nodes.py

@@ -179,8 +179,12 @@ def set_source_info(directive, node):
         directive.state_machine.get_source_and_line(directive.lineno)
 
 def set_role_source_info(inliner, lineno, node):
+    try:
         node.source, node.line = \
             inliner.reporter.locator(lineno)
+    except AttributeError:
+        # docutils 0.9+
+        node.source, node.line = inliner.reporter.get_source_and_line(lineno)
 
 # monkey-patch Node.__contains__ to get consistent "in" operator behavior
 # across docutils versions

sphinx/util/pycompat.py

@@ -64,6 +64,11 @@ else:
         return s.encode('ascii', 'backslashreplace')
 
 
+try:
+    from html import escape as htmlescape
+except ImportError:
+    from cgi import escape as htmlescape
+
 # ------------------------------------------------------------------------------
 # Missing builtins and itertools in Python < 2.6
 

sphinx/websupport/__init__.py

@@ -9,7 +9,6 @@
     :license: BSD, see LICENSE for details.
 """
 
-import cgi
 import sys
 import cPickle as pickle
 import posixpath
@@ -22,6 +21,7 @@ from docutils.core import publish_parts
 from sphinx.application import Sphinx
 from sphinx.util.osutil import ensuredir
 from sphinx.util.jsonimpl import dumps as dump_json
+from sphinx.util.pycompat import htmlescape
 from sphinx.websupport import errors
 from sphinx.websupport.search import BaseSearch, SEARCH_ADAPTERS
 from sphinx.websupport.storage import StorageBackend
@@ -452,5 +452,5 @@ class WebSupport(object):
             ret = publish_parts(text, writer_name='html',
                                 settings_overrides=settings)['fragment']
         except Exception:
-            ret = cgi.escape(text)
+            ret = htmlescape(text)
         return ret

sphinx/websupport/storage/differ.py

@@ -10,9 +10,10 @@
 """
 
 import re
-from cgi import escape
 from difflib import Differ
 
+from sphinx.util.pycompat import htmlescape
+
 
 class CombinedHtmlDiff(object):
     """Create an HTML representation of the differences between two pieces
@@ -21,7 +22,7 @@ class CombinedHtmlDiff(object):
     highlight_regex = re.compile(r'([\+\-\^]+)')
 
     def __init__(self, source, proposal):
-        proposal = escape(proposal)
+        proposal = htmlescape(proposal)
 
         differ = Differ()
         self.diff = list(differ.compare(source.splitlines(1),

sphinx/writers/html.py

@@ -65,6 +65,8 @@ class HTMLTranslator(BaseTranslator):
             self.permalink_text = self.permalink_text and u'\u00B6' or ''
         self.permalink_text = self.encode(self.permalink_text)
         self.secnumber_suffix = builder.config.html_secnumber_suffix
+        self.param_separator = ''
+        self._table_row_index = 0
 
     def visit_start_of_file(self, node):
         # only occurs in the single-file builder
@@ -233,12 +235,13 @@ class HTMLTranslator(BaseTranslator):
         lang = self.highlightlang
         linenos = node.rawsource.count('\n') >= \
                   self.highlightlinenothreshold - 1
+        highlight_args = node.get('highlight_args', {})
         if node.has_key('language'):
             # code-block directives
             lang = node['language']
+            highlight_args['force'] = True
         if node.has_key('linenos'):
             linenos = node['linenos']
-        highlight_args = node.get('highlight_args', {})
         def warner(msg):
             self.builder.warn(msg, (self.builder.current_docname, node.line))
         highlighted = self.highlighter.highlight_block(

sphinx/writers/latex.py

@@ -1299,12 +1299,13 @@ class LaTeXTranslator(nodes.NodeVisitor):
         code = self.verbatim.rstrip('\n')
         lang = self.hlsettingstack[-1][0]
         linenos = code.count('\n') >= self.hlsettingstack[-1][1] - 1
+        highlight_args = node.get('highlight_args', {})
         if 'language' in node:
             # code-block directives
             lang = node['language']
+            highlight_args['force'] = True
         if 'linenos' in node:
             linenos = node['linenos']
-        highlight_args = node.get('highlight_args', {})
         def warner(msg):
             self.builder.warn(msg, (self.curfilestack[-1], node.line))
         hlcode = self.highlighter.highlight_block(code, lang, warn=warner,

tests/test_build_html.py

@@ -258,7 +258,7 @@ if pygments:
             r'def'),
         (".//div[@class='inc-tab3 highlight-text']//pre",
             r'-| |-'),
-        (".//div[@class='inc-tab8 highlight-python']//pre",
+        (".//div[@class='inc-tab8 highlight-python']//pre/span",
             r'-|      |-'),
     ])
     HTML_XPATH['subdir/includes.html'].extend([
@@ -328,7 +328,11 @@ def test_html(app):
     for fname, paths in HTML_XPATH.iteritems():
         parser = NslessParser()
         parser.entity.update(htmlentitydefs.entitydefs)
-        etree = ET.parse(os.path.join(app.outdir, fname), parser)
+        fp = open(os.path.join(app.outdir, fname))
+        try:
+            etree = ET.parse(fp, parser)
+        finally:
+            fp.close()
         for path, check in paths:
             yield check_xpath, etree, fname, path, check