diff --git a/CHANGES b/CHANGES
index 7ff4978ef..70a81d93f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -33,12 +33,15 @@ Dependencies
        - texlive-fonts-extra
        - Cyrillic letters (in non-Cyrillic documents)
 
-  These extra package are not required by default. The first two are needed if
+  These extra package are not required by default.  The first two are needed if
   the :confval:`latex_elements`.\ ``'fontenc'`` key has been modify to declare
-  the use of the ``LGR`` (Greek) and/or ``T2A`` (Cyrillic) font encoding. Even
+  the use of the ``LGR`` (Greek) and/or ``T2A`` (Cyrillic) font encoding.  Even
   then, the last four are font packages arising in the default value for
   :confval:`latex_elements`.\ ``'fontpkg'``, and may be replaced by other font
   packages providing ``LGR`` and/or ``T2A`` support.
+* LaTeX builder with :confval:`latex_engine` set to ``'xelatex'`` or to
+  ``'lualatex'`` requires (by default) the ``Computer Modern Unicode`` fonts,
+  which in Ubuntu xenial are in ``texlive-fonts-extra``.
 
 Incompatible changes
 --------------------
diff --git a/doc/usage/builders/index.rst b/doc/usage/builders/index.rst
index 3f16c84e5..aa4aba76a 100644
--- a/doc/usage/builders/index.rst
+++ b/doc/usage/builders/index.rst
@@ -167,7 +167,8 @@ The builder's "name" must be given to the **-b** command-line option of
    * ``texlive-fonts-extra``, ``texlive-lang-greek`` (if needed to
      support Greek or Cyrillic letters in non-cyrillic document)
    * ``latexmk`` (for ``make latexpdf`` on GNU/Linux and MacOS X)
-   * ``texlive-luatex``, ``texlive-xetex`` (see :confval:`latex_engine`)
+   * ``texlive-luatex``, ``texlive-xetex``, ``texlive-fonts-extra``
+      (if :confval:`latex_engine` is set to  ``'xelatex'`` or ``'lualatex'``)
 
    The testing of Sphinx LaTeX is done on Ubuntu xenial with the above mentioned
    packages, which are from a TeXLive 2015 snapshot dated March 2016.
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index b77f6234a..aa80d0776 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -158,9 +158,18 @@ ADDITIONAL_SETTINGS = {
         'polyglossia':  '\\usepackage{polyglossia}',
         'babel':        '',
         'fontenc':      '\\usepackage{fontspec}',
-        'fontpkg':     ('\\setmainfont{CMU Serif}\n'
-                        '\\setsansfont{CMU Sans Serif}\n'
-                        '\\setmonofont{CMU Typewriter Text}'),
+        'fontpkg':     ('\\setmainfont{cmunrm.otf}[\n'
+                        '  BoldFont = cmunbx.otf,\n'
+                        '  ItalicFont = cmunti.otf,\n'
+                        '  BoldItalicFont = cmunbi.otf]\n'
+                        '\\setsansfont{cmunss.otf}[\n'
+                        '  BoldFont = cmunsx.otf,\n'
+                        '  ItalicFont = cmunsi.otf,\n'
+                        '  BoldItalicFont = cmunso.otf]\n'
+                        '\\setmonofont{cmuntt.otf}[\n'
+                        '  BoldFont = cmuntb.otf,\n'
+                        '  ItalicFont = cmunit.otf,\n'
+                        '  BoldItalicFont = cmuntx.otf]'),
         'textgreek':    '',
         'utf8extra':   ('\\catcode`^^^^00a0\\active\\protected\\def^^^^00a0'
                         '{\\leavevmode\\nobreak\\ }'),