Add `content_offset parameter to nested_parse_with_titles` (#11147)

Previously, ``nested_parse_with_titles`` always passed ``0`` as the input
offset when invoking ``nested_parse``.  When parsing the content of a
directive, as is a common use case for ``nested_parse_with_titles``,
this leads to incorrect source file/line number information, as it
does not take into account the directive's ``content_offset``, which is
always non-zero.

This issue affects *all* object descriptions due to GH-10887.  It also
affects the ``sphinx.ext.ifconfig`` extension.

The ``py:module`` and ``js:module`` directives employed a workaround for
this issue, by wrapping the calls to ``nested_parse_with_title`` with
``switch_source_input``.  That worked, but was more complicated (and
likely less efficient) than necessary.

This commit adds an optional ``content_offset`` parameter to
``nested_parse_with_titles``, and fixes callers to pass the appropriate
content offset when needed.

This commit eliminates the now-unnecessary calls to
``switch_source_input`` and instead specifies the correct ``content_offset``.

CHANGES

@@ -23,6 +23,8 @@ Bugs fixed
 
 * #11079: LaTeX: figures with align attribute may disappear and strangely impact
   following lists
+* #11147: Fix source file/line number info in object description content and in
+  other uses of ``nested_parse_with_titles``.  Patch by Jeremy Maitin-Shepard.
 
 Testing
 --------

sphinx/directives/__init__.py

@@ -262,7 +262,7 @@ class ObjectDescription(SphinxDirective, Generic[T]):
             # needed for association of version{added,changed} directives
             self.env.temp_data['object'] = self.names[0]
         self.before_content()
-        nested_parse_with_titles(self.state, self.content, contentnode)
+        nested_parse_with_titles(self.state, self.content, contentnode, self.content_offset)
         self.transform_content(contentnode)
         self.env.app.emit('object-description-transform',
                           self.domain, self.objtype, contentnode)

sphinx/domains/javascript.py

@@ -20,7 +20,7 @@ from sphinx.locale import _, __
 from sphinx.roles import XRefRole
 from sphinx.util import logging
 from sphinx.util.docfields import Field, GroupedField, TypedField
-from sphinx.util.docutils import SphinxDirective, switch_source_input
+from sphinx.util.docutils import SphinxDirective
 from sphinx.util.nodes import make_id, make_refnode, nested_parse_with_titles
 from sphinx.util.typing import OptionSpec
 
@@ -297,10 +297,9 @@ class JSModule(SphinxDirective):
         noindex = 'noindex' in self.options
 
         content_node: Element = nodes.section()
-        with switch_source_input(self.state, self.content):
-            # necessary so that the child nodes get the right source/line set
-            content_node.document = self.state.document
-            nested_parse_with_titles(self.state, self.content, content_node)
+        # necessary so that the child nodes get the right source/line set
+        content_node.document = self.state.document
+        nested_parse_with_titles(self.state, self.content, content_node, self.content_offset)
 
         ret: list[Node] = []
         if not noindex:

sphinx/domains/python.py

@@ -26,7 +26,7 @@ from sphinx.locale import _, __
 from sphinx.roles import XRefRole
 from sphinx.util import logging
 from sphinx.util.docfields import Field, GroupedField, TypedField
-from sphinx.util.docutils import SphinxDirective, switch_source_input
+from sphinx.util.docutils import SphinxDirective
 from sphinx.util.inspect import signature_from_str
 from sphinx.util.nodes import (
     find_pending_xref_condition,
@@ -1033,10 +1033,9 @@ class PyModule(SphinxDirective):
         self.env.ref_context['py:module'] = modname
 
         content_node: Element = nodes.section()
-        with switch_source_input(self.state, self.content):
-            # necessary so that the child nodes get the right source/line set
-            content_node.document = self.state.document
-            nested_parse_with_titles(self.state, self.content, content_node)
+        # necessary so that the child nodes get the right source/line set
+        content_node.document = self.state.document
+        nested_parse_with_titles(self.state, self.content, content_node, self.content_offset)
 
         ret: list[Node] = []
         if not noindex:

sphinx/ext/autodoc/__init__.py

@@ -313,7 +313,7 @@ class Documenter:
     #: order if autodoc_member_order is set to 'groupwise'
     member_order = 0
     #: true if the generated content may contain titles
-    titles_allowed = False
+    titles_allowed = True
 
     option_spec: OptionSpec = {
         'noindex': bool_option
@@ -956,7 +956,6 @@ class ModuleDocumenter(Documenter):
     """
     objtype = 'module'
     content_indent = ''
-    titles_allowed = True
     _extra_indent = '   '
 
     option_spec: OptionSpec = {

sphinx/ext/ifconfig.py

@@ -45,7 +45,7 @@ class IfConfig(SphinxDirective):
         node.document = self.state.document
         self.set_source_info(node)
         node['expr'] = self.arguments[0]
-        nested_parse_with_titles(self.state, self.content, node)
+        nested_parse_with_titles(self.state, self.content, node, self.content_offset)
         return [node]
 
 

sphinx/util/nodes.py

@@ -311,7 +311,8 @@ def traverse_translatable_index(
         yield node, entries
 
 
-def nested_parse_with_titles(state: Any, content: StringList, node: Node) -> str:
+def nested_parse_with_titles(state: Any, content: StringList, node: Node,
+                             content_offset: int = 0) -> str:
     """Version of state.nested_parse() that allows titles and does not require
     titles to have the same decoration as the calling document.
 
@@ -324,7 +325,7 @@ def nested_parse_with_titles(state: Any, content: StringList, node: Node) -> str
     state.memo.title_styles = []
     state.memo.section_level = 0
     try:
-        return state.nested_parse(content, 0, node, match_titles=1)
+        return state.nested_parse(content, content_offset, node, match_titles=1)
     finally:
         state.memo.title_styles = surrounding_title_styles
         state.memo.section_level = surrounding_section_level

tests/test_directive_object_description.py

@@ -1,10 +1,12 @@
 """Test object description directives."""
 
+import docutils.utils
 import pytest
 from docutils import nodes
 
 from sphinx import addnodes
 from sphinx.io import create_publisher
+from sphinx.testing import restructuredtext
 from sphinx.util.docutils import sphinx_domains
 
 
@@ -43,3 +45,15 @@ def test_object_description_sections(app):
     assert doctree[1][1][0][0][0] == 'Overview'
     assert isinstance(doctree[1][1][0][1], nodes.paragraph)
     assert doctree[1][1][0][1][0] == 'Lorem ipsum dolar sit amet'
+
+
+def test_object_description_content_line_number(app):
+    text = (".. py:function:: foo(bar)\n" +
+            "\n" +
+            "   Some link here: :ref:`abc`\n")
+    doc = restructuredtext.parse(app, text)
+    xrefs = list(doc.findall(condition=addnodes.pending_xref))
+    assert len(xrefs) == 1
+    source, line = docutils.utils.get_source_line(xrefs[0])
+    assert 'index.rst' in source
+    assert line == 3

tests/test_domain_js.py

@@ -2,6 +2,7 @@
 
 from unittest.mock import Mock
 
+import docutils.utils
 import pytest
 from docutils import nodes
 
@@ -229,3 +230,15 @@ def test_noindexentry(app):
     assert_node(doctree, (addnodes.index, desc, addnodes.index, desc))
     assert_node(doctree[0], addnodes.index, entries=[('single', 'f() (built-in function)', 'f', '', None)])
     assert_node(doctree[2], addnodes.index, entries=[])
+
+
+def test_module_content_line_number(app):
+    text = (".. js:module:: foo\n" +
+            "\n" +
+            "   Some link here: :ref:`abc`\n")
+    doc = restructuredtext.parse(app, text)
+    xrefs = list(doc.findall(condition=addnodes.pending_xref))
+    assert len(xrefs) == 1
+    source, line = docutils.utils.get_source_line(xrefs[0])
+    assert 'index.rst' in source
+    assert line == 3

tests/test_domain_py.py

@@ -1458,3 +1458,15 @@ def test_signature_line_number(app, include_options):
     source, line = docutils.utils.get_source_line(xrefs[0])
     assert 'index.rst' in source
     assert line == 1
+
+
+def test_module_content_line_number(app):
+    text = (".. py:module:: foo\n" +
+            "\n" +
+            "   Some link here: :ref:`abc`\n")
+    doc = restructuredtext.parse(app, text)
+    xrefs = list(doc.findall(condition=addnodes.pending_xref))
+    assert len(xrefs) == 1
+    source, line = docutils.utils.get_source_line(xrefs[0])
+    assert 'index.rst' in source
+    assert line == 3

tests/test_ext_ifconfig.py

@@ -1,7 +1,11 @@
 """Test sphinx.ext.ifconfig extension."""
 
+import docutils.utils
 import pytest
 
+from sphinx import addnodes
+from sphinx.testing import restructuredtext
+
 
 @pytest.mark.sphinx('text', testroot='ext-ifconfig')
 def test_ifconfig(app, status, warning):
@@ -9,3 +13,16 @@ def test_ifconfig(app, status, warning):
     result = (app.outdir / 'index.txt').read_text(encoding='utf8')
     assert 'spam' in result
     assert 'ham' not in result
+
+
+def test_ifconfig_content_line_number(app):
+    app.setup_extension("sphinx.ext.ifconfig")
+    text = (".. ifconfig:: confval1\n" +
+            "\n" +
+            "   Some link here: :ref:`abc`\n")
+    doc = restructuredtext.parse(app, text)
+    xrefs = list(doc.findall(condition=addnodes.pending_xref))
+    assert len(xrefs) == 1
+    source, line = docutils.utils.get_source_line(xrefs[0])
+    assert 'index.rst' in source
+    assert line == 3