Add experimental HTML5 writer

CHANGES

@@ -77,6 +77,8 @@ Features added
 * #3476: setuptools: Support multiple builders
 * latex: merged cells in LaTeX tables allow code-blocks, lists, blockquotes...
   as do normal cells (refs: #3435)
+* HTML buildre uses experimental HTML5 writer if ``html_experimental_html5_builder`` is True
+  and docutils 0.13 and newer is installed.
 
 Bugs fixed
 ----------

doc/config.rst

@@ -1088,6 +1088,11 @@ that use Sphinx's HTMLWriter class.
 
    Output file base name for HTML help builder.  Default is ``'pydoc'``.
 
+.. confval:: html_experimental_html5_writer
+
+   Output is processed with HTML5 writer.  This feature needs docutils 0.13 or newer.  Default is ``False``.
+
+   .. versionadded:: 1.6
 
 .. _applehelp-options:
 

sphinx/application.py

@@ -46,6 +46,7 @@ from sphinx.util import status_iterator, old_status_iterator, display_chunk
 from sphinx.util.tags import Tags
 from sphinx.util.osutil import ENOENT
 from sphinx.util.console import bold, darkgreen  # type: ignore
+from sphinx.util.docutils import is_html5_writer_available
 from sphinx.util.i18n import find_catalog_source_files
 
 if False:
@@ -623,24 +624,32 @@ class Sphinx(object):
                 raise ExtensionError('Value for key %r must be a '
                                      '(visit, depart) function tuple' % key)
             translator = self._translators.get(key)
+            translators = []
             if translator is not None:
-                pass
+                translators.append(translator)
             elif key == 'html':
-                from sphinx.writers.html import HTMLTranslator as translator  # type: ignore
+                from sphinx.writers.html import HTMLTranslator
+                translators.append(HTMLTranslator)
+                if is_html5_writer_available():
+                    from sphinx.writers.html5 import HTML5Translator
+                    translators.append(HTML5Translator)
             elif key == 'latex':
-                from sphinx.writers.latex import LaTeXTranslator as translator  # type: ignore
+                from sphinx.writers.latex import LaTeXTranslator
+                translators.append(LaTeXTranslator)
             elif key == 'text':
-                from sphinx.writers.text import TextTranslator as translator  # type: ignore
+                from sphinx.writers.text import TextTranslator
+                translators.append(TextTranslator)
             elif key == 'man':
-                from sphinx.writers.manpage import ManualPageTranslator as translator  # type: ignore  # NOQA
+                from sphinx.writers.manpage import ManualPageTranslator
+                translators.append(ManualPageTranslator)
             elif key == 'texinfo':
-                from sphinx.writers.texinfo import TexinfoTranslator as translator  # type: ignore  # NOQA
-            else:
-                # ignore invalid keys for compatibility
-                continue
-            setattr(translator, 'visit_' + node.__name__, visit)
-            if depart:
-                setattr(translator, 'depart_' + node.__name__, depart)
+                from sphinx.writers.texinfo import TexinfoTranslator
+                translators.append(TexinfoTranslator)
+
+            for translator in translators:
+                setattr(translator, 'visit_' + node.__name__, visit)
+                if depart:
+                    setattr(translator, 'depart_' + node.__name__, depart)
 
     def add_enumerable_node(self, node, figtype, title_getter=None, **kwds):
         # type: (nodes.Node, unicode, Callable, Any) -> None

sphinx/builders/html.py

@@ -34,6 +34,7 @@ from sphinx.util.i18n import format_date
 from sphinx.util.osutil import SEP, os_path, relative_uri, ensuredir, \
     movefile, copyfile
 from sphinx.util.nodes import inline_all_toctrees
+from sphinx.util.docutils import is_html5_writer_available, __version_info__
 from sphinx.util.fileutil import copy_asset
 from sphinx.util.matching import patmatch, Matcher, DOTFILES
 from sphinx.config import string_classes
@@ -55,6 +56,13 @@ if False:
     from sphinx.domains import Domain, Index  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
+# Experimental HTML5 Writer
+if is_html5_writer_available():
+    from sphinx.writers.html5 import HTML5Translator, SmartyPantsHTML5Translator
+    html5_ready = True
+else:
+    html5_ready = False
+
 #: the filename for the inventory of objects
 INVENTORY_FILENAME = 'objects.inv'
 #: the filename for the "last build" file (for serializing builders)
@@ -145,6 +153,12 @@ class StandaloneHTMLBuilder(Builder):
                 self.script_files.append('_static/translations.js')
         self.use_index = self.get_builder_config('use_index', 'html')
 
+        if self.config.html_experimental_html5_writer and not html5_ready:
+            self.app.warn(' '.join((
+                'html_experimental_html5_writer is set, but current version is old.',
+                'Docutils\' version should be or newer than 0.13, but %s.',
+            )) % '.'.join(map(str, __version_info__)))
+
     def _get_translations_js(self):
         # type: () -> unicode
         candidates = [path.join(dir, self.config.language,
@@ -188,10 +202,16 @@ class StandaloneHTMLBuilder(Builder):
     def init_translator_class(self):
         # type: () -> None
         if self.translator_class is None:
-            if self.config.html_use_smartypants:
-                self.translator_class = SmartyPantsHTMLTranslator
+            if self.config.html_experimental_html5_writer and html5_ready:
+                if self.config.html_use_smartypants:
+                    self.translator_class = SmartyPantsHTML5Translator
+                else:
+                    self.translator_class = HTML5Translator
             else:
-                self.translator_class = HTMLTranslator
+                if self.config.html_use_smartypants:
+                    self.translator_class = SmartyPantsHTMLTranslator
+                else:
+                    self.translator_class = HTMLTranslator
 
     def get_outdated_docs(self):
         # type: () -> Iterator[unicode]
@@ -374,6 +394,7 @@ class StandaloneHTMLBuilder(Builder):
             parents = [],
             logo = logo,
             favicon = favicon,
+            html5_doctype = self.config.html_experimental_html5_writer and html5_ready,
         )  # type: Dict[unicode, Any]
         if self.theme:
             self.globalcontext.update(
@@ -1320,6 +1341,7 @@ def setup(app):
     app.add_config_value('html_search_options', {}, 'html')
     app.add_config_value('html_search_scorer', '', None)
     app.add_config_value('html_scaled_image_link', True, 'html')
+    app.add_config_value('html_experimental_html5_writer', False, 'html')
 
     return {
         'version': 'builtin',

sphinx/themes/basic/layout.html

@@ -7,10 +7,12 @@
     :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 #}
-{%- block doctype -%}
+{%- block doctype -%}{%- if html5_doctype %}
+<!DOCTYPE html>
+{%- else %}
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-{%- endblock %}
+{%- endif %}{%- endblock %}
 {%- set reldelim1 = reldelim1 is not defined and ' &#187;' or reldelim1 %}
 {%- set reldelim2 = reldelim2 is not defined and ' |' or reldelim2 %}
 {%- set render_sidebar = (not embedded) and (not theme_nosidebar|tobool) and

sphinx/util/docutils.py

@@ -150,3 +150,8 @@ class LoggingReporter(Reporter):
     def set_conditions(self, category, report_level, halt_level, debug=False):
         # type: (unicode, int, int, bool) -> None
         Reporter.set_conditions(self, category, report_level, halt_level, debug=debug)
+
+
+def is_html5_writer_available():
+    # type: () -> bool
+    return __version_info__ > (0, 13, 0)

sphinx/writers/html5.py

@@ -0,0 +1,929 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.writers.html5
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Experimental docutils writers for HTML5 handling Sphinx' custom nodes.
+
+    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys
+import posixpath
+import os
+
+from six import string_types
+from docutils import nodes
+from docutils.writers.html5_polyglot import HTMLTranslator as BaseTranslator
+
+from sphinx import addnodes
+from sphinx.locale import admonitionlabels, _
+from sphinx.util import logging
+from sphinx.util.images import get_image_size
+from sphinx.util.smartypants import sphinx_smarty_pants
+
+if False:
+    # For type annotation
+    from typing import Any  # NOQA
+    from sphinx.builders.html import StandaloneHTMLBuilder  # NOQA
+
+
+logger = logging.getLogger(__name__)
+
+# A good overview of the purpose behind these classes can be found here:
+# http://www.arnebrodowski.de/blog/write-your-own-restructuredtext-writer.html
+
+
+class HTML5Translator(BaseTranslator):
+    """
+    Our custom HTML translator.
+    """
+
+    def __init__(self, builder, *args, **kwds):
+        # type: (StandaloneHTMLBuilder, Any, Any) -> None
+        BaseTranslator.__init__(self, *args, **kwds)
+        self.highlighter = builder.highlighter
+        self.no_smarty = 0
+        self.builder = builder
+        self.highlightlang = self.highlightlang_base = \
+            builder.config.highlight_language
+        self.highlightopts = builder.config.highlight_options
+        self.highlightlinenothreshold = sys.maxsize
+        self.docnames = [builder.current_docname]  # for singlehtml builder
+        self.protect_literal_text = 0
+        self.permalink_text = builder.config.html_add_permalinks
+        # support backwards-compatible setting to a bool
+        if not isinstance(self.permalink_text, string_types):
+            self.permalink_text = self.permalink_text and u'\u00B6' or ''
+        self.permalink_text = self.encode(self.permalink_text)
+        self.secnumber_suffix = builder.config.html_secnumber_suffix
+        self.param_separator = ''
+        self.optional_param_level = 0
+        self._table_row_index = 0
+        self.required_params_left = 0
+
+    def visit_start_of_file(self, node):
+        # type: (nodes.Node) -> None
+        # only occurs in the single-file builder
+        self.docnames.append(node['docname'])
+        self.body.append('<span id="document-%s"></span>' % node['docname'])
+
+    def depart_start_of_file(self, node):
+        # type: (nodes.Node) -> None
+        self.docnames.pop()
+
+    def visit_desc(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'dl', CLASS=node['objtype']))
+
+    def depart_desc(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</dl>\n\n')
+
+    def visit_desc_signature(self, node):
+        # type: (nodes.Node) -> None
+        # the id is set automatically
+        self.body.append(self.starttag(node, 'dt'))
+        # anchor for per-desc interactive data
+        if node.parent['objtype'] != 'describe' \
+           and node['ids'] and node['first']:
+            self.body.append('<!--[%s]-->' % node['ids'][0])
+
+    def depart_desc_signature(self, node):
+        # type: (nodes.Node) -> None
+        if not node.get('is_multiline'):
+            self.add_permalink_ref(node, _('Permalink to this definition'))
+        self.body.append('</dt>\n')
+
+    def visit_desc_signature_line(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def depart_desc_signature_line(self, node):
+        # type: (nodes.Node) -> None
+        if node.get('add_permalink'):
+            # the permalink info is on the parent desc_signature node
+            self.add_permalink_ref(node.parent, _('Permalink to this definition'))
+        self.body.append('<br />')
+
+    def visit_desc_addname(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'code', '', CLASS='descclassname'))
+
+    def depart_desc_addname(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</code>')
+
+    def visit_desc_type(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def depart_desc_type(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_desc_returns(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(' &#x2192; ')
+
+    def depart_desc_returns(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_desc_name(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'code', '', CLASS='descname'))
+
+    def depart_desc_name(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</code>')
+
+    def visit_desc_parameterlist(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('<span class="sig-paren">(</span>')
+        self.first_param = 1
+        self.optional_param_level = 0
+        # How many required parameters are left.
+        self.required_params_left = sum([isinstance(c, addnodes.desc_parameter)
+                                         for c in node.children])
+        self.param_separator = node.child_text_separator
+
+    def depart_desc_parameterlist(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('<span class="sig-paren">)</span>')
+
+    # If required parameters are still to come, then put the comma after
+    # the parameter.  Otherwise, put the comma before.  This ensures that
+    # signatures like the following render correctly (see issue #1001):
+    #
+    #     foo([a, ]b, c[, d])
+    #
+    def visit_desc_parameter(self, node):
+        # type: (nodes.Node) -> None
+        if self.first_param:
+            self.first_param = 0
+        elif not self.required_params_left:
+            self.body.append(self.param_separator)
+        if self.optional_param_level == 0:
+            self.required_params_left -= 1
+        if not node.hasattr('noemph'):
+            self.body.append('<em>')
+
+    def depart_desc_parameter(self, node):
+        # type: (nodes.Node) -> None
+        if not node.hasattr('noemph'):
+            self.body.append('</em>')
+        if self.required_params_left:
+            self.body.append(self.param_separator)
+
+    def visit_desc_optional(self, node):
+        # type: (nodes.Node) -> None
+        self.optional_param_level += 1
+        self.body.append('<span class="optional">[</span>')
+
+    def depart_desc_optional(self, node):
+        # type: (nodes.Node) -> None
+        self.optional_param_level -= 1
+        self.body.append('<span class="optional">]</span>')
+
+    def visit_desc_annotation(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'em', '', CLASS='property'))
+
+    def depart_desc_annotation(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</em>')
+
+    def visit_desc_content(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'dd', ''))
+
+    def depart_desc_content(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</dd>')
+
+    def visit_versionmodified(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'div', CLASS=node['type']))
+
+    def depart_versionmodified(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</div>\n')
+
+    # overwritten
+    def visit_reference(self, node):
+        # type: (nodes.Node) -> None
+        atts = {'class': 'reference'}
+        if node.get('internal') or 'refuri' not in node:
+            atts['class'] += ' internal'
+        else:
+            atts['class'] += ' external'
+        if 'refuri' in node:
+            atts['href'] = node['refuri'] or '#'
+            if self.settings.cloak_email_addresses and \
+               atts['href'].startswith('mailto:'):
+                atts['href'] = self.cloak_mailto(atts['href'])
+                self.in_mailto = 1
+        else:
+            assert 'refid' in node, \
+                   'References must have "refuri" or "refid" attribute.'
+            atts['href'] = '#' + node['refid']
+        if not isinstance(node.parent, nodes.TextElement):
+            assert len(node) == 1 and isinstance(node[0], nodes.image)
+            atts['class'] += ' image-reference'
+        if 'reftitle' in node:
+            atts['title'] = node['reftitle']
+        if 'target' in node:
+            atts['target'] = node['target']
+        self.body.append(self.starttag(node, 'a', '', **atts))
+
+        if node.get('secnumber'):
+            self.body.append(('%s' + self.secnumber_suffix) %
+                             '.'.join(map(str, node['secnumber'])))
+
+    def visit_number_reference(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_reference(node)
+
+    def depart_number_reference(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_reference(node)
+
+    # overwritten -- we don't want source comments to show up in the HTML
+    def visit_comment(self, node):
+        # type: (nodes.Node) -> None
+        raise nodes.SkipNode
+
+    # overwritten
+    def visit_admonition(self, node, name=''):
+        # type: (nodes.Node, unicode) -> None
+        self.body.append(self.starttag(
+            node, 'div', CLASS=('admonition ' + name)))
+        if name:
+            node.insert(0, nodes.title(name, admonitionlabels[name]))
+
+    def visit_seealso(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'seealso')
+
+    def depart_seealso(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition(node)
+
+    def add_secnumber(self, node):
+        # type: (nodes.Node) -> None
+        if node.get('secnumber'):
+            self.body.append('.'.join(map(str, node['secnumber'])) +
+                             self.secnumber_suffix)
+        elif isinstance(node.parent, nodes.section):
+            if self.builder.name == 'singlehtml':
+                docname = self.docnames[-1]
+                anchorname = "%s/#%s" % (docname, node.parent['ids'][0])
+                if anchorname not in self.builder.secnumbers:
+                    anchorname = "%s/" % docname  # try first heading which has no anchor
+            else:
+                anchorname = '#' + node.parent['ids'][0]
+                if anchorname not in self.builder.secnumbers:
+                    anchorname = ''  # try first heading which has no anchor
+            if self.builder.secnumbers.get(anchorname):
+                numbers = self.builder.secnumbers[anchorname]
+                self.body.append('.'.join(map(str, numbers)) +
+                                 self.secnumber_suffix)
+
+    def add_fignumber(self, node):
+        # type: (nodes.Node) -> None
+        def append_fignumber(figtype, figure_id):
+            # type: (unicode, unicode) -> None
+            if self.builder.name == 'singlehtml':
+                key = u"%s/%s" % (self.docnames[-1], figtype)
+            else:
+                key = figtype
+
+            if figure_id in self.builder.fignumbers.get(key, {}):
+                self.body.append('<span class="caption-number">')
+                prefix = self.builder.config.numfig_format.get(figtype)
+                if prefix is None:
+                    msg = 'numfig_format is not defined for %s' % figtype
+                    logger.warning(msg)
+                else:
+                    numbers = self.builder.fignumbers[key][figure_id]
+                    self.body.append(prefix % '.'.join(map(str, numbers)) + ' ')
+                    self.body.append('</span>')
+
+        figtype = self.builder.env.domains['std'].get_figtype(node)  # type: ignore
+        if figtype:
+            if len(node['ids']) == 0:
+                msg = 'Any IDs not assigned for %s node' % node.tagname
+                logger.warning(msg, location=node)
+            else:
+                append_fignumber(figtype, node['ids'][0])
+
+    def add_permalink_ref(self, node, title):
+        # type: (nodes.Node, unicode) -> None
+        if node['ids'] and self.permalink_text and self.builder.add_permalinks:
+            format = u'<a class="headerlink" href="#%s" title="%s">%s</a>'
+            self.body.append(format % (node['ids'][0], title, self.permalink_text))
+
+    def generate_targets_for_listing(self, node):
+        # type: (nodes.Node) -> None
+        """Generate hyperlink targets for listings.
+
+        Original visit_bullet_list(), visit_definition_list() and visit_enumerated_list()
+        generates hyperlink targets inside listing tags (<ul>, <ol> and <dl>) if multiple
+        IDs are assigned to listings.  That is invalid DOM structure.
+        (This is a bug of docutils <= 0.12)
+
+        This exports hyperlink targets before listings to make valid DOM structure.
+        """
+        for id in node['ids'][1:]:
+            self.body.append('<span id="%s"></span>' % id)
+            node['ids'].remove(id)
+
+    # overwritten
+    def visit_bullet_list(self, node):
+        # type: (nodes.Node) -> None
+        if len(node) == 1 and node[0].tagname == 'toctree':
+            # avoid emitting empty <ul></ul>
+            raise nodes.SkipNode
+        self.generate_targets_for_listing(node)
+        BaseTranslator.visit_bullet_list(self, node)
+
+    # overwritten
+    def visit_enumerated_list(self, node):
+        # type: (nodes.Node) -> None
+        self.generate_targets_for_listing(node)
+        BaseTranslator.visit_enumerated_list(self, node)
+
+    # overwritten
+    def visit_title(self, node):
+        # type: (nodes.Node) -> None
+        BaseTranslator.visit_title(self, node)
+        self.add_secnumber(node)
+        self.add_fignumber(node.parent)
+        if isinstance(node.parent, nodes.table):
+            self.body.append('<span class="caption-text">')
+
+    def depart_title(self, node):
+        # type: (nodes.Node) -> None
+        close_tag = self.context[-1]
+        if (self.permalink_text and self.builder.add_permalinks and
+           node.parent.hasattr('ids') and node.parent['ids']):
+            # add permalink anchor
+            if close_tag.startswith('</h'):
+                self.add_permalink_ref(node.parent, _('Permalink to this headline'))
+            elif close_tag.startswith('</a></h'):
+                self.body.append(u'</a><a class="headerlink" href="#%s" ' %
+                                 node.parent['ids'][0] +
+                                 u'title="%s">%s' % (
+                                     _('Permalink to this headline'),
+                                     self.permalink_text))
+            elif isinstance(node.parent, nodes.table):
+                self.body.append('</span>')
+                self.add_permalink_ref(node.parent, _('Permalink to this table'))
+        elif isinstance(node.parent, nodes.table):
+            self.body.append('</span>')
+
+        BaseTranslator.depart_title(self, node)
+
+    # overwritten
+    def visit_literal_block(self, node):
+        # type: (nodes.Node) -> None
+        if node.rawsource != node.astext():
+            # most probably a parsed-literal block -- don't highlight
+            return BaseTranslator.visit_literal_block(self, node)
+        lang = self.highlightlang
+        linenos = node.rawsource.count('\n') >= \
+            self.highlightlinenothreshold - 1
+        highlight_args = node.get('highlight_args', {})
+        if 'language' in node:
+            # code-block directives
+            lang = node['language']
+            highlight_args['force'] = True
+        if 'linenos' in node:
+            linenos = node['linenos']
+        if lang is self.highlightlang_base:
+            # only pass highlighter options for original language
+            opts = self.highlightopts
+        else:
+            opts = {}
+
+        highlighted = self.highlighter.highlight_block(
+            node.rawsource, lang, opts=opts, linenos=linenos,
+            location=(self.builder.current_docname, node.line), **highlight_args
+        )
+        starttag = self.starttag(node, 'div', suffix='',
+                                 CLASS='highlight-%s' % lang)
+        self.body.append(starttag + highlighted + '</div>\n')
+        raise nodes.SkipNode
+
+    def visit_caption(self, node):
+        # type: (nodes.Node) -> None
+        if isinstance(node.parent, nodes.container) and node.parent.get('literal_block'):
+            self.body.append('<div class="code-block-caption">')
+        else:
+            BaseTranslator.visit_caption(self, node)
+        self.add_fignumber(node.parent)
+        self.body.append(self.starttag(node, 'span', '', CLASS='caption-text'))
+
+    def depart_caption(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</span>')
+
+        # append permalink if available
+        if isinstance(node.parent, nodes.container) and node.parent.get('literal_block'):
+            self.add_permalink_ref(node.parent, _('Permalink to this code'))
+        elif isinstance(node.parent, nodes.figure):
+            image_nodes = node.parent.traverse(nodes.image)
+            target_node = image_nodes and image_nodes[0] or node.parent
+            self.add_permalink_ref(target_node, _('Permalink to this image'))
+        elif node.parent.get('toctree'):
+            self.add_permalink_ref(node.parent.parent, _('Permalink to this toctree'))
+
+        if isinstance(node.parent, nodes.container) and node.parent.get('literal_block'):
+            self.body.append('</div>\n')
+        else:
+            BaseTranslator.depart_caption(self, node)
+
+    def visit_doctest_block(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_literal_block(node)
+
+    # overwritten to add the <div> (for XHTML compliance)
+    def visit_block_quote(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'blockquote') + '<div>')
+
+    def depart_block_quote(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</div></blockquote>\n')
+
+    # overwritten
+    def visit_literal(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'code', '',
+                                       CLASS='docutils literal'))
+        self.protect_literal_text += 1
+
+    def depart_literal(self, node):
+        # type: (nodes.Node) -> None
+        self.protect_literal_text -= 1
+        self.body.append('</code>')
+
+    def visit_productionlist(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'pre'))
+        names = []
+        for production in node:
+            names.append(production['tokenname'])
+        maxlen = max(len(name) for name in names)
+        lastname = None
+        for production in node:
+            if production['tokenname']:
+                lastname = production['tokenname'].ljust(maxlen)
+                self.body.append(self.starttag(production, 'strong', ''))
+                self.body.append(lastname + '</strong> ::= ')
+            elif lastname is not None:
+                self.body.append('%s     ' % (' ' * len(lastname)))
+            production.walkabout(self)
+            self.body.append('\n')
+        self.body.append('</pre>\n')
+        raise nodes.SkipNode
+
+    def depart_productionlist(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_production(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def depart_production(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_centered(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.starttag(node, 'p', CLASS="centered") +
+                         '<strong>')
+
+    def depart_centered(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</strong></p>')
+
+    # overwritten
+    def should_be_compact_paragraph(self, node):
+        # type: (nodes.Node) -> bool
+        """Determine if the <p> tags around paragraph can be omitted."""
+        if isinstance(node.parent, addnodes.desc_content):
+            # Never compact desc_content items.
+            return False
+        if isinstance(node.parent, addnodes.versionmodified):
+            # Never compact versionmodified nodes.
+            return False
+        return BaseTranslator.should_be_compact_paragraph(self, node)
+
+    def visit_compact_paragraph(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def depart_compact_paragraph(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_highlightlang(self, node):
+        # type: (nodes.Node) -> None
+        self.highlightlang = node['lang']
+        self.highlightlinenothreshold = node['linenothreshold']
+
+    def depart_highlightlang(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_download_reference(self, node):
+        # type: (nodes.Node) -> None
+        if self.builder.download_support and node.hasattr('filename'):
+            self.body.append(
+                '<a class="reference download internal" href="%s" download="">' %
+                posixpath.join(self.builder.dlpath, node['filename']))
+            self.context.append('</a>')
+        else:
+            self.context.append('')
+
+    def depart_download_reference(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append(self.context.pop())
+
+    # overwritten
+    def visit_image(self, node):
+        # type: (nodes.Node) -> None
+        olduri = node['uri']
+        # rewrite the URI if the environment knows about it
+        if olduri in self.builder.images:
+            node['uri'] = posixpath.join(self.builder.imgpath,
+                                         self.builder.images[olduri])
+
+        uri = node['uri']
+        if uri.lower().endswith(('svg', 'svgz')):
+            atts = {'src': uri}
+            if 'width' in node:
+                atts['width'] = node['width']
+            if 'height' in node:
+                atts['height'] = node['height']
+            atts['alt'] = node.get('alt', uri)
+            if 'align' in node:
+                self.body.append('<div align="%s" class="align-%s">' %
+                                 (node['align'], node['align']))
+                self.context.append('</div>\n')
+            else:
+                self.context.append('')
+            self.body.append(self.emptytag(node, 'img', '', **atts))
+            return
+
+        if 'scale' in node:
+            # Try to figure out image height and width.  Docutils does that too,
+            # but it tries the final file name, which does not necessarily exist
+            # yet at the time the HTML file is written.
+            if not ('width' in node and 'height' in node):
+                size = get_image_size(os.path.join(self.builder.srcdir, olduri))
+                if size is None:
+                    logger.warning('Could not obtain image size. :scale: option is ignored.',
+                                   location=node)
+                else:
+                    if 'width' not in node:
+                        node['width'] = str(size[0])
+                    if 'height' not in node:
+                        node['height'] = str(size[1])
+        BaseTranslator.visit_image(self, node)
+
+    # overwritten
+    def depart_image(self, node):
+        # type: (nodes.Node) -> None
+        if node['uri'].lower().endswith(('svg', 'svgz')):
+            self.body.append(self.context.pop())
+        else:
+            BaseTranslator.depart_image(self, node)
+
+    def visit_toctree(self, node):
+        # type: (nodes.Node) -> None
+        # this only happens when formatting a toc from env.tocs -- in this
+        # case we don't want to include the subtree
+        raise nodes.SkipNode
+
+    def visit_index(self, node):
+        # type: (nodes.Node) -> None
+        raise nodes.SkipNode
+
+    def visit_tabular_col_spec(self, node):
+        # type: (nodes.Node) -> None
+        raise nodes.SkipNode
+
+    def visit_glossary(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def depart_glossary(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_acks(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def depart_acks(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_hlist(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('<table class="hlist"><tr>')
+
+    def depart_hlist(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</tr></table>\n')
+
+    def visit_hlistcol(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('<td>')
+
+    def depart_hlistcol(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</td>')
+
+    def bulk_text_processor(self, text):
+        # type: (unicode) -> unicode
+        return text
+
+    # overwritten
+    def visit_Text(self, node):
+        # type: (nodes.Node) -> None
+        text = node.astext()
+        encoded = self.encode(text)
+        if self.protect_literal_text:
+            # moved here from base class's visit_literal to support
+            # more formatting in literal nodes
+            for token in self.words_and_spaces.findall(encoded):
+                if token.strip():
+                    # protect literal text from line wrapping
+                    self.body.append('<span class="pre">%s</span>' % token)
+                elif token in ' \n':
+                    # allow breaks at whitespace
+                    self.body.append(token)
+                else:
+                    # protect runs of multiple spaces; the last one can wrap
+                    self.body.append('&#160;' * (len(token) - 1) + ' ')
+        else:
+            if self.in_mailto and self.settings.cloak_email_addresses:
+                encoded = self.cloak_email(encoded)
+            else:
+                encoded = self.bulk_text_processor(encoded)
+            self.body.append(encoded)
+
+    def visit_note(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'note')
+
+    def depart_note(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition(node)
+
+    def visit_warning(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'warning')
+
+    def depart_warning(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition(node)
+
+    def visit_attention(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'attention')
+
+    def depart_attention(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition()
+
+    def visit_caution(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'caution')
+
+    def depart_caution(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition()
+
+    def visit_danger(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'danger')
+
+    def depart_danger(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition()
+
+    def visit_error(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'error')
+
+    def depart_error(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition()
+
+    def visit_hint(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'hint')
+
+    def depart_hint(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition()
+
+    def visit_important(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'important')
+
+    def depart_important(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition()
+
+    def visit_tip(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_admonition(node, 'tip')
+
+    def depart_tip(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_admonition()
+
+    # these are only handled specially in the SmartyPantsHTML5Translator
+    def visit_literal_emphasis(self, node):
+        # type: (nodes.Node) -> None
+        return self.visit_emphasis(node)
+
+    def depart_literal_emphasis(self, node):
+        # type: (nodes.Node) -> None
+        return self.depart_emphasis(node)
+
+    def visit_literal_strong(self, node):
+        # type: (nodes.Node) -> None
+        return self.visit_strong(node)
+
+    def depart_literal_strong(self, node):
+        # type: (nodes.Node) -> None
+        return self.depart_strong(node)
+
+    def visit_abbreviation(self, node):
+        # type: (nodes.Node) -> None
+        attrs = {}
+        if node.hasattr('explanation'):
+            attrs['title'] = node['explanation']
+        self.body.append(self.starttag(node, 'abbr', '', **attrs))
+
+    def depart_abbreviation(self, node):
+        # type: (nodes.Node) -> None
+        self.body.append('</abbr>')
+
+    def visit_manpage(self, node):
+        # type: (nodes.Node) -> None
+        self.visit_literal_emphasis(node)
+
+    def depart_manpage(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_literal_emphasis(node)
+
+    # overwritten to add even/odd classes
+
+    def visit_table(self, node):
+        # type: (nodes.Node) -> None
+        self._table_row_index = 0
+
+        classes = [cls.strip(u' \t\n')
+                   for cls in self.settings.table_style.split(',')]
+        classes.insert(0, "docutils")  # compat
+        if 'align' in node:
+            classes.append('align-%s' % node['align'])
+        tag = self.starttag(node, 'table', CLASS=' '.join(classes))
+        self.body.append(tag)
+
+    def visit_row(self, node):
+        # type: (nodes.Node) -> None
+        self._table_row_index += 1
+        if self._table_row_index % 2 == 0:
+            node['classes'].append('row-even')
+        else:
+            node['classes'].append('row-odd')
+        self.body.append(self.starttag(node, 'tr', ''))
+        node.column = 0
+
+    def visit_field_list(self, node):
+        # type: (nodes.Node) -> None
+        self._fieldlist_row_index = 0
+        return BaseTranslator.visit_field_list(self, node)
+
+    def visit_field(self, node):
+        # type: (nodes.Node) -> None
+        self._fieldlist_row_index += 1
+        if self._fieldlist_row_index % 2 == 0:
+            node['classes'].append('field-even')
+        else:
+            node['classes'].append('field-odd')
+        return node
+
+    def visit_math(self, node, math_env=''):
+        # type: (nodes.Node, unicode) -> None
+        logger.warning('using "math" markup without a Sphinx math extension '
+                       'active, please use one of the math extensions '
+                       'described at http://sphinx-doc.org/ext/math.html',
+                       location=(self.builder.current_docname, node.line))
+        raise nodes.SkipNode
+
+    def unknown_visit(self, node):
+        # type: (nodes.Node) -> None
+        raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
+
+
+class SmartyPantsHTML5Translator(HTML5Translator):
+    """
+    Handle ordinary text via smartypants, converting quotes and dashes
+    to the correct entities.
+    """
+
+    def __init__(self, *args, **kwds):
+        # type: (Any, Any) -> None
+        self.no_smarty = 0
+        HTML5Translator.__init__(self, *args, **kwds)
+
+    def visit_literal(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty += 1
+        try:
+            # this raises SkipNode
+            HTML5Translator.visit_literal(self, node)
+        finally:
+            self.no_smarty -= 1
+
+    def visit_literal_block(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty += 1
+        try:
+            HTML5Translator.visit_literal_block(self, node)
+        except nodes.SkipNode:
+            # HTML5Translator raises SkipNode for simple literal blocks,
+            # but not for parsed literal blocks
+            self.no_smarty -= 1
+            raise
+
+    def depart_literal_block(self, node):
+        # type: (nodes.Node) -> None
+        HTML5Translator.depart_literal_block(self, node)
+        self.no_smarty -= 1
+
+    def visit_literal_emphasis(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty += 1
+        self.visit_emphasis(node)
+
+    def depart_literal_emphasis(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_emphasis(node)
+        self.no_smarty -= 1
+
+    def visit_literal_strong(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty += 1
+        self.visit_strong(node)
+
+    def depart_literal_strong(self, node):
+        # type: (nodes.Node) -> None
+        self.depart_strong(node)
+        self.no_smarty -= 1
+
+    def visit_desc_signature(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty += 1
+        HTML5Translator.visit_desc_signature(self, node)
+
+    def depart_desc_signature(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty -= 1
+        HTML5Translator.depart_desc_signature(self, node)
+
+    def visit_productionlist(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty += 1
+        try:
+            HTML5Translator.visit_productionlist(self, node)
+        finally:
+            self.no_smarty -= 1
+
+    def visit_option(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty += 1
+        HTML5Translator.visit_option(self, node)
+
+    def depart_option(self, node):
+        # type: (nodes.Node) -> None
+        self.no_smarty -= 1
+        HTML5Translator.depart_option(self, node)
+
+    def bulk_text_processor(self, text):
+        # type: (unicode) -> unicode
+        if self.no_smarty <= 0:
+            return sphinx_smarty_pants(text)
+        return text

tests/test_build_html.py

@@ -109,9 +109,9 @@ def check_xpath(etree, fname, path, check, be_found=True):
                 # Since pygments-2.1.1, empty <span> tag is inserted at top of
                 # highlighting block
                 if len(node) == 1 and node[0].tag == 'span' and node[0].text is None:
-                    return node[0].tail
-                else:
-                    return ''
+                    if node[0].tail is not None:
+                        return node[0].tail
+                return ''
 
         rex = re.compile(check)
         if be_found:

tests/test_build_html5.py

@@ -0,0 +1,330 @@
+# -*- coding: utf-8 -*-
+"""
+    test_build_html5
+    ~~~~~~~~~~~~~~~~
+
+    Test the HTML5 writer and check output against XPath.
+
+    This code is digest to reduce test running time.
+    Complete test code is here:
+
+    https://github.com/sphinx-doc/sphinx/pull/2805/files
+
+    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import re
+from itertools import cycle, chain
+import xml.etree.cElementTree as ElementTree
+
+from six import PY3
+import pytest
+from html5lib import getTreeBuilder, HTMLParser
+
+from sphinx import __display_version__
+from sphinx.util.docutils import is_html5_writer_available
+
+from util import remove_unicode_literals, strip_escseq, skip_unless
+from test_build_html import flat_dict, tail_check, check_xpath
+
+TREE_BUILDER = getTreeBuilder('etree', implementation=ElementTree)
+HTML_PARSER = HTMLParser(TREE_BUILDER, namespaceHTMLElements=False)
+
+
+etree_cache = {}
+
+@skip_unless(is_html5_writer_available())
+@pytest.fixture(scope='module')
+def cached_etree_parse():
+    def parse(fname):
+        if fname in etree_cache:
+            return etree_cache[fname]
+        with (fname).open('rb') as fp:
+            etree = HTML_PARSER.parse(fp)
+            etree_cache.clear()
+            etree_cache[fname] = etree
+            return etree
+    yield parse
+    etree_cache.clear()
+
+
+@skip_unless(is_html5_writer_available())
+@pytest.mark.parametrize("fname,expect", flat_dict({
+    'images.html': [
+        (".//img[@src='_images/img.png']", ''),
+        (".//img[@src='_images/img1.png']", ''),
+        (".//img[@src='_images/simg.png']", ''),
+        (".//img[@src='_images/svgimg.svg']", ''),
+        (".//a[@href='_sources/images.txt']", ''),
+    ],
+    'subdir/images.html': [
+        (".//img[@src='../_images/img1.png']", ''),
+        (".//img[@src='../_images/rimg.png']", ''),
+    ],
+    'subdir/includes.html': [
+        (".//a[@href='../_downloads/img.png']", ''),
+        (".//img[@src='../_images/img.png']", ''),
+        (".//p", 'This is an include file.'),
+        (".//pre/span", 'line 1'),
+        (".//pre/span", 'line 2'),
+    ],
+    'includes.html': [
+        (".//pre", u'Max Strauß'),
+        (".//a[@href='_downloads/img.png']", ''),
+        (".//a[@href='_downloads/img1.png']", ''),
+        (".//pre/span", u'"quotes"'),
+        (".//pre/span", u"'included'"),
+        (".//pre/span[@class='s2']", u'üöä'),
+        (".//div[@class='inc-pyobj1 highlight-text']//pre",
+         r'^class Foo:\n    pass\n\s*$'),
+        (".//div[@class='inc-pyobj2 highlight-text']//pre",
+         r'^    def baz\(\):\n        pass\n\s*$'),
+        (".//div[@class='inc-lines highlight-text']//pre",
+         r'^class Foo:\n    pass\nclass Bar:\n$'),
+        (".//div[@class='inc-startend highlight-text']//pre",
+         u'^foo = "Including Unicode characters: üöä"\\n$'),
+        (".//div[@class='inc-preappend highlight-text']//pre",
+         r'(?m)^START CODE$'),
+        (".//div[@class='inc-pyobj-dedent highlight-python']//span",
+         r'def'),
+        (".//div[@class='inc-tab3 highlight-text']//pre",
+         r'-| |-'),
+        (".//div[@class='inc-tab8 highlight-python']//pre/span",
+         r'-|      |-'),
+    ],
+    'autodoc.html': [
+        (".//dt[@id='test_autodoc.Class']", ''),
+        (".//dt[@id='test_autodoc.function']/em", r'\*\*kwds'),
+        (".//dd/p", r'Return spam\.'),
+    ],
+    'extapi.html': [
+        (".//strong", 'from function: Foo'),
+        (".//strong", 'from class: Bar'),
+    ],
+    'markup.html': [
+        (".//title", 'set by title directive'),
+        (".//p/em", 'Section author: Georg Brandl'),
+        (".//p/em", 'Module author: Georg Brandl'),
+        # created by the meta directive
+        (".//meta[@name='author'][@content='Me']", ''),
+        (".//meta[@name='keywords'][@content='docs, sphinx']", ''),
+        # a label created by ``.. _label:``
+        (".//div[@id='label']", ''),
+        # code with standard code blocks
+        (".//pre", '^some code$'),
+        # an option list
+        (".//span[@class='option']", '--help'),
+        # admonitions
+        (".//p[@class='admonition-title']", 'My Admonition'),
+        (".//div[@class='admonition note']/p", 'Note text.'),
+        (".//div[@class='admonition warning']/p", 'Warning text.'),
+        # inline markup
+        (".//li/p/strong", r'^command\\n$'),
+        (".//li/p/strong", r'^program\\n$'),
+        (".//li/p/em", r'^dfn\\n$'),
+        (".//li/p/code/span[@class='pre']", r'^kbd\\n$'),
+        (".//li/p/span", u'File \N{TRIANGULAR BULLET} Close'),
+        (".//li/p/code/span[@class='pre']", '^a/$'),
+        (".//li/p/code/em/span[@class='pre']", '^varpart$'),
+        (".//li/p/code/em/span[@class='pre']", '^i$'),
+        (".//a[@href='https://www.python.org/dev/peps/pep-0008']"
+         "[@class='pep reference external']/strong", 'PEP 8'),
+        (".//a[@href='https://www.python.org/dev/peps/pep-0008']"
+         "[@class='pep reference external']/strong",
+         'Python Enhancement Proposal #8'),
+        (".//a[@href='https://tools.ietf.org/html/rfc1.html']"
+         "[@class='rfc reference external']/strong", 'RFC 1'),
+        (".//a[@href='https://tools.ietf.org/html/rfc1.html']"
+         "[@class='rfc reference external']/strong", 'Request for Comments #1'),
+        (".//a[@href='objects.html#envvar-HOME']"
+         "[@class='reference internal']/code/span[@class='pre']", 'HOME'),
+        (".//a[@href='#with']"
+         "[@class='reference internal']/code/span[@class='pre']", '^with$'),
+        (".//a[@href='#grammar-token-try_stmt']"
+         "[@class='reference internal']/code/span", '^statement$'),
+        (".//a[@href='#some-label'][@class='reference internal']/span", '^here$'),
+        (".//a[@href='#some-label'][@class='reference internal']/span", '^there$'),
+        (".//a[@href='subdir/includes.html']"
+         "[@class='reference internal']/span", 'Including in subdir'),
+        (".//a[@href='objects.html#cmdoption-python-c']"
+         "[@class='reference internal']/code/span[@class='pre']", '-c'),
+        # abbreviations
+        (".//abbr[@title='abbreviation']", '^abbr$'),
+        # version stuff
+        (".//div[@class='versionadded']/p/span", 'New in version 0.6: '),
+        (".//div[@class='versionadded']/p/span",
+         tail_check('First paragraph of versionadded')),
+        (".//div[@class='versionchanged']/p/span",
+         tail_check('First paragraph of versionchanged')),
+        (".//div[@class='versionchanged']/p",
+         'Second paragraph of versionchanged'),
+        # footnote reference
+        (".//a[@class='footnote-reference brackets']", r'1'),
+        # created by reference lookup
+        (".//a[@href='contents.html#ref1']", ''),
+        # ``seealso`` directive
+        (".//div/p[@class='admonition-title']", 'See also'),
+        # a ``hlist`` directive
+        (".//table[@class='hlist']/tbody/tr/td/ul/li/p", '^This$'),
+        # a ``centered`` directive
+        (".//p[@class='centered']/strong", 'LICENSE'),
+        # a glossary
+        (".//dl/dt[@id='term-boson']", 'boson'),
+        # a production list
+        (".//pre/strong", 'try_stmt'),
+        (".//pre/a[@href='#grammar-token-try1_stmt']/code/span", 'try1_stmt'),
+        # tests for ``only`` directive
+        (".//p", 'A global substitution.'),
+        (".//p", 'In HTML.'),
+        (".//p", 'In both.'),
+        (".//p", 'Always present'),
+        # tests for ``any`` role
+        (".//a[@href='#with']/span", 'headings'),
+        (".//a[@href='objects.html#func_without_body']/code/span", 'objects'),
+    ],
+    'objects.html': [
+        (".//dt[@id='mod.Cls.meth1']", ''),
+        (".//dt[@id='errmod.Error']", ''),
+        (".//dt/code", r'long\(parameter,\s* list\)'),
+        (".//dt/code", 'another one'),
+        (".//a[@href='#mod.Cls'][@class='reference internal']", ''),
+        (".//dl[@class='userdesc']", ''),
+        (".//dt[@id='userdesc-myobj']", ''),
+        (".//a[@href='#userdesc-myobj'][@class='reference internal']", ''),
+        # docfields
+        (".//a[@class='reference internal'][@href='#TimeInt']/em", 'TimeInt'),
+        (".//a[@class='reference internal'][@href='#Time']", 'Time'),
+        (".//a[@class='reference internal'][@href='#errmod.Error']/strong", 'Error'),
+        # C references
+        (".//span[@class='pre']", 'CFunction()'),
+        (".//a[@href='#c.Sphinx_DoSomething']", ''),
+        (".//a[@href='#c.SphinxStruct.member']", ''),
+        (".//a[@href='#c.SPHINX_USE_PYTHON']", ''),
+        (".//a[@href='#c.SphinxType']", ''),
+        (".//a[@href='#c.sphinx_global']", ''),
+        # test global TOC created by toctree()
+        (".//ul[@class='current']/li[@class='toctree-l1 current']/a[@href='#']",
+         'Testing object descriptions'),
+        (".//li[@class='toctree-l1']/a[@href='markup.html']",
+         'Testing various markup'),
+        # test unknown field names
+        (".//dt[@class='field-odd']", 'Field_name'),
+        (".//dt[@class='field-even']", 'Field_name all lower'),
+        (".//dt[@class='field-odd']", 'FIELD_NAME'),
+        (".//dt[@class='field-even']", 'FIELD_NAME ALL CAPS'),
+        (".//dt[@class='field-odd']", 'Field_Name'),
+        (".//dt[@class='field-even']", 'Field_Name All Word Caps'),
+        (".//dt[@class='field-odd']", 'Field_name'),
+        (".//dt[@class='field-even']", 'Field_name First word cap'),
+        (".//dt[@class='field-odd']", 'FIELd_name'),
+        (".//dt[@class='field-even']", 'FIELd_name PARTial caps'),
+        # custom sidebar
+        (".//h4", 'Custom sidebar'),
+        # docfields
+        (".//dd[@class='field-odd']/p/strong", '^moo$'),
+        (".//dd[@class='field-odd']/p/strong", tail_check(r'\(Moo\) .* Moo')),
+        (".//dd[@class='field-odd']/ul/li/p/strong", '^hour$'),
+        (".//dd[@class='field-odd']/ul/li/p/em", '^DuplicateType$'),
+        (".//dd[@class='field-odd']/ul/li/p/em", tail_check(r'.* Some parameter')),
+        # others
+        (".//a[@class='reference internal'][@href='#cmdoption-perl-arg-p']/code/span",
+         'perl'),
+        (".//a[@class='reference internal'][@href='#cmdoption-perl-arg-p']/code/span",
+         '\\+p'),
+        (".//a[@class='reference internal'][@href='#cmdoption-perl-objc']/code/span",
+         '--ObjC\\+\\+'),
+        (".//a[@class='reference internal'][@href='#cmdoption-perl-plugin-option']/code/span",
+         '--plugin.option'),
+        (".//a[@class='reference internal'][@href='#cmdoption-perl-arg-create-auth-token']"
+         "/code/span",
+         'create-auth-token'),
+        (".//a[@class='reference internal'][@href='#cmdoption-perl-arg-arg']/code/span",
+         'arg'),
+        (".//a[@class='reference internal'][@href='#cmdoption-hg-arg-commit']/code/span",
+         'hg'),
+        (".//a[@class='reference internal'][@href='#cmdoption-hg-arg-commit']/code/span",
+         'commit'),
+        (".//a[@class='reference internal'][@href='#cmdoption-git-commit-p']/code/span",
+         'git'),
+        (".//a[@class='reference internal'][@href='#cmdoption-git-commit-p']/code/span",
+         'commit'),
+        (".//a[@class='reference internal'][@href='#cmdoption-git-commit-p']/code/span",
+         '-p'),
+    ],
+    'contents.html': [
+        (".//meta[@name='hc'][@content='hcval']", ''),
+        (".//meta[@name='hc_co'][@content='hcval_co']", ''),
+        (".//meta[@name='testopt'][@content='testoverride']", ''),
+        (".//dt[@class='label']/span[@class='brackets']", r'Ref1'),
+        (".//dt[@class='label']", ''),
+        (".//li[@class='toctree-l1']/a", 'Testing various markup'),
+        (".//li[@class='toctree-l2']/a", 'Inline markup'),
+        (".//title", 'Sphinx <Tests>'),
+        (".//div[@class='footer']", 'Georg Brandl & Team'),
+        (".//a[@href='http://python.org/']"
+         "[@class='reference external']", ''),
+        (".//li/p/a[@href='genindex.html']/span", 'Index'),
+        (".//li/p/a[@href='py-modindex.html']/span", 'Module Index'),
+        (".//li/p/a[@href='search.html']/span", 'Search Page'),
+        # custom sidebar only for contents
+        (".//h4", 'Contents sidebar'),
+        # custom JavaScript
+        (".//script[@src='file://moo.js']", ''),
+        # URL in contents
+        (".//a[@class='reference external'][@href='http://sphinx-doc.org/']",
+         'http://sphinx-doc.org/'),
+        (".//a[@class='reference external'][@href='http://sphinx-doc.org/latest/']",
+         'Latest reference'),
+        # Indirect hyperlink targets across files
+        (".//a[@href='markup.html#some-label'][@class='reference internal']/span",
+         '^indirect hyperref$'),
+    ],
+    'bom.html': [
+        (".//title", " File with UTF-8 BOM"),
+    ],
+    'extensions.html': [
+        (".//a[@href='http://python.org/dev/']", "http://python.org/dev/"),
+        (".//a[@href='http://bugs.python.org/issue1000']", "issue 1000"),
+        (".//a[@href='http://bugs.python.org/issue1042']", "explicit caption"),
+    ],
+    '_static/statictmpl.html': [
+        (".//project", 'Sphinx <Tests>'),
+    ],
+    'genindex.html': [
+        # index entries
+        (".//a/strong", "Main"),
+        (".//a/strong", "[1]"),
+        (".//a/strong", "Other"),
+        (".//a", "entry"),
+        (".//li/a", "double"),
+    ],
+    'footnote.html': [
+        (".//a[@class='footnote-reference brackets'][@href='#id8'][@id='id1']", r"1"),
+        (".//a[@class='footnote-reference brackets'][@href='#id9'][@id='id2']", r"2"),
+        (".//a[@class='footnote-reference brackets'][@href='#foo'][@id='id3']", r"3"),
+        (".//a[@class='reference internal'][@href='#bar'][@id='id4']", r"\[bar\]"),
+        (".//a[@class='footnote-reference brackets'][@href='#id10'][@id='id5']", r"4"),
+        (".//a[@class='footnote-reference brackets'][@href='#id11'][@id='id6']", r"5"),
+        (".//a[@class='fn-backref'][@href='#id1']", r"1"),
+        (".//a[@class='fn-backref'][@href='#id2']", r"2"),
+        (".//a[@class='fn-backref'][@href='#id3']", r"3"),
+        (".//a[@class='fn-backref'][@href='#id4']", r"bar"),
+        (".//a[@class='fn-backref'][@href='#id5']", r"4"),
+        (".//a[@class='fn-backref'][@href='#id6']", r"5"),
+        (".//a[@class='fn-backref'][@href='#id7']", r"6"),
+    ],
+    'otherext.html': [
+        (".//h1", "Generated section"),
+        (".//a[@href='_sources/otherext.foo.txt']", ''),
+    ]
+}))
+@pytest.mark.sphinx('html', tags=['testtag'], confoverrides={
+    'html_context.hckey_co': 'hcval_co',
+    'html_experimental_html5_writer': True})
+@pytest.mark.test_params(shared_result='test_build_html5_output')
+def test_html5_output(app, cached_etree_parse, fname, expect):
+    app.build()
+    print(app.outdir / fname)
+    check_xpath(cached_etree_parse(app.outdir / fname), fname, *expect)