Fix #777: (part II) LaTeX output "too deeply nested"

The standard classes have a severe cap on maximal nesting of list-like
environments (a total of six levels, and four for each of enumerate and
itemize). This commit defines a new key ``'maxlistdepth'``. _Only_ if it
is set (i.e. non-empty) will sphinx.sty do some hack to lift the LaTeX
limitations and extend the maximal list depth to the desired value.

doc/config.rst

@@ -1742,6 +1742,24 @@ These options influence LaTeX output. See further :doc:`latex`.
 
    * Keys that don't need be overridden unless in special cases are:
 
+     ``'maxlistdepth'``
+        For example, setting it to ``'10'`` (as a string) will allow up to 10
+        nested levels of list or quote environments, either numbered or not.
+        The default is to leave this key empty. LaTeX then allows only a
+        maximum of 6 nested levels, and among them at most 4 for numbered and
+        4 for itemized lists.
+
+        .. attention::
+
+           - This setting may likely be incompatible with LaTeX packages
+             modifying lists, or special document classes.
+
+           - The LaTeX package "enumitem" provides its own commands for
+             extending the maximal list depth. If this package is added to the
+             preamble, its commands should be used and ``'maxlistdepth'``
+             left to an empty string.
+
+        .. versionadded:: 1.5
      ``'inputenc'``
         "inputenc" package inclusion, defaults to
         ``'\\usepackage[utf8]{inputenc}'`` when using pdflatex.

sphinx/templates/latex/content.tex_t

@@ -1,6 +1,6 @@
 %% Generated by Sphinx.
 \def\sphinxdocclass{<%= docclass %>}
-<%= keepoldnames %>
+<%= passoptionstosphinx %>
 \documentclass[<%= papersize %>,<%= pointsize %><%= classoptions %>]{<%= wrapperclass %>}
 \ifdefined\pdfpxdimen
    \let\sphinxpxdimen\pdfpxdimen\else\newdimen\sphinxpxdimen

sphinx/texinputs/sphinx.sty

@@ -7,9 +7,22 @@
 
 \NeedsTeXFormat{LaTeX2e}[1995/12/01]
 \ProvidesPackage{sphinx}[2016/10/12 v1.5 LaTeX package (Sphinx markup)]
+
+% Handle package options
 \newif\ifsphinxKeepOldNames \sphinxKeepOldNamestrue
 \DeclareOption{dontkeepoldnames}{\sphinxKeepOldNamesfalse}
-\DeclareOption*{\PackageWarning{sphinx}{Unknown option `\CurrentOption'}}
+\newcommand*\sphinxMaxListDepth{0}
+\def\spx@tempa {0}
+\long\def\spx@tempb #1maxlistdepth=#2\UnDeFiNed #3\@nil
+    {\def\spx@tempa {#2}}
+\DeclareOption*{%
+  \expandafter\spx@tempb\CurrentOption\UnDeFiNed maxlistdepth=0\UnDeFiNed\@nil
+  \ifnum\spx@tempa=\z@
+    \PackageWarning{sphinx}{Unknown option `\CurrentOption'}%
+  \else
+    \PackageInfo{sphinx}{Maximal list depth will be set at \spx@tempa}%
+    \let\sphinxMaxListDepth \spx@tempa
+  \fi }
 \ProcessOptions\relax
 
 % this is the \ltx@ifundefined of ltxcmds.sty, which is loaded by
@@ -1005,3 +1018,66 @@
 \providecommand*{\sphinxtableofcontents}{\tableofcontents}
 \providecommand*{\sphinxthebibliography}{\thebibliography}
 \providecommand*{\sphinxtheindex}{\theindex}
+
+% remove LaTeX's cap on nesting depth. This is a hack, intrinsically
+% fragile, but it works with the standard classes. It is executed only
+% if  'maxlistdepth' key from  latex_elements is used.
+
+\def\spx@toodeep@hack{\fi\iffalse}
+
+\ifnum\sphinxMaxListDepth=\z@\expandafter\@gobbletwo\fi
+\AtBeginDocument{%
+  \let\spx@toodeepORI\@toodeep
+  \def\@toodeep{%
+    \ifnum\@listdepth>\sphinxMaxListDepth\relax
+      \expandafter\spx@toodeepORI
+    \else
+      \expandafter\spx@toodeep@hack
+    \fi}%
+% define all missing \@list... macros
+  \count@\@ne
+  \loop
+     \spx@ifundefined{@list\romannumeral\the\count@}
+       {\iffalse}{\iftrue\advance\count@\@ne}%
+  \repeat
+  \loop
+     \ifnum\count@>\sphinxMaxListDepth\relax\else
+       \expandafter\let
+         \csname @list\romannumeral\the\count@\expandafter\endcsname
+         \csname @list\romannumeral\the\numexpr\count@-\@ne\endcsname
+     \advance\count@\@ne
+  \repeat
+% define all missing enum... counters and \labelenum... macros and \p@enum..
+  \count@\@ne
+  \loop
+     \spx@ifundefined{c@enum\romannumeral\the\count@}
+       {\iffalse}{\iftrue\advance\count@\@ne}%
+  \repeat
+  \loop
+     \ifnum\count@>\sphinxMaxListDepth\relax\else
+       \newcounter{enum\romannumeral\the\count@}%
+       \expandafter\def
+         \csname labelenum\romannumeral\the\count@\expandafter\endcsname
+         \expandafter
+         {\csname theenum\romannumeral\the\numexpr\count@\endcsname.}%
+       \expandafter\def
+         \csname p@enum\romannumeral\the\count@\expandafter\endcsname
+         \expandafter
+         {\csname p@enum\romannumeral\the\numexpr\count@-\@ne\expandafter
+          \endcsname\csname theenum\romannumeral\the\numexpr\count@-\@ne\endcsname.}%
+     \advance\count@\@ne
+  \repeat
+% define all missing labelitem... macros
+  \count@\@ne
+  \loop
+     \spx@ifundefined{labelitem\romannumeral\the\count@}
+     {\iffalse}{\iftrue\advance\count@\@ne}%
+  \repeat
+  \loop
+     \ifnum\count@>\sphinxMaxListDepth\relax\else
+       \expandafter\let
+         \csname labelitem\romannumeral\the\count@\expandafter\endcsname
+         \csname labelitem\romannumeral\the\numexpr\count@-\@ne\endcsname
+     \advance\count@\@ne
+  \repeat
+  }

sphinx/writers/latex.py

@@ -51,6 +51,7 @@ DEFAULT_SETTINGS = {
     'pxunit':          '49336sp',
     'classoptions':    '',
     'extraclassoptions': '',
+    'maxlistdepth':    '',
     'passoptionstopackages': '',
     'geometry':        '\\usepackage[margin=1in,marginparwidth=0.5in]'
                        '{geometry}',
@@ -374,12 +375,9 @@ class LaTeXTranslator(nodes.NodeVisitor):
             'releasename':  _('Release'),
             'indexname':    _('Index'),
         })
-        # set-up boolean for sphinx.sty
+        sphinxpkgoptions = ''
-        if builder.config.latex_keep_old_macro_names:
+        if not builder.config.latex_keep_old_macro_names:
-            self.elements['keepoldnames'] = ''
+            sphinxpkgoptions = 'dontkeepoldnames'
-        else:
-            self.elements['keepoldnames'] = ('\\PassOptionsToPackage'
-                                             '{dontkeepoldnames}{sphinx}')
         if document.settings.docclass == 'howto':
             docclass = builder.config.latex_docclass.get('howto', 'article')
         else:
@@ -453,6 +451,11 @@ class LaTeXTranslator(nodes.NodeVisitor):
         # allow the user to override them all
         self.check_latex_elements()
         self.elements.update(builder.config.latex_elements)
+        if self.elements['maxlistdepth']:
+            sphinxpkgoptions += ',maxlistdepth=%s' % self.elements['maxlistdepth']
+        if sphinxpkgoptions:
+            self.elements['passoptionstosphinx'] = \
+                '\\PassOptionsToPackage{%s}{sphinx}' % sphinxpkgoptions
         if self.elements['extraclassoptions']:
             self.elements['classoptions'] += ',' + \
                                              self.elements['extraclassoptions']