Merge branch '4.0.x' into 4.x

CHANGES

@@ -109,6 +109,12 @@ Deprecated
 Features added
 --------------
 
+* C, add C23 keywords ``_Decimal32``, ``_Decimal64``, and ``_Decimal128``.
+* #9354: C, add :confval:`c_extra_keywords` to allow user-defined keywords
+  during parsing.
+* Revert the removal of ``sphinx.util:force_decode()`` to become some 3rd party
+  extensions available again during 5.0
+
 Bugs fixed
 ----------
 
@@ -117,6 +123,9 @@ Bugs fixed
 * #9313: LaTeX: complex table with merged cells broken since 4.0
 * #9305: LaTeX: backslash may cause Improper discretionary list pdf build error
   with Japanese engines
+* #9354: C, remove special macro names from the keyword list.
+  See also :confval:`c_extra_keywords`.
+* #9322: KeyError is raised on PropagateDescDomain transform
 
 Testing
 --------

doc/extdev/deprecated.rst

@@ -1036,7 +1036,7 @@ The following is a list of deprecated interfaces.
 
    * - ``sphinx.util.force_decode()``
      - 2.0
-     - 4.0
+     - 5.0
      - N/A
 
    * - ``sphinx.util.get_matching_docs()``

doc/usage/configuration.rst

@@ -2689,6 +2689,14 @@ Options for the C domain
 
    .. versionadded:: 3.0
 
+.. confval:: c_extra_keywords
+
+  A list of identifiers to be recognized as keywords by the C parser.
+  It defaults to ``['alignas', 'alignof', 'bool', 'complex', 'imaginary',
+  'noreturn', 'static_assert', 'thread_local']``.
+
+  .. versionadded:: 4.0.3
+
 .. confval:: c_allow_pre_v3
 
    A boolean (default ``False``) controlling whether to parse and try to

sphinx/domains/c.py

@@ -55,10 +55,15 @@ _keywords = [
     'else', 'enum', 'extern', 'float', 'for', 'goto', 'if', 'inline', 'int', 'long',
     'register', 'restrict', 'return', 'short', 'signed', 'sizeof', 'static', 'struct',
     'switch', 'typedef', 'union', 'unsigned', 'void', 'volatile', 'while',
-    '_Alignas', 'alignas', '_Alignof', 'alignof', '_Atomic', '_Bool', 'bool',
-    '_Complex', 'complex', '_Generic', '_Imaginary', 'imaginary',
-    '_Noreturn', 'noreturn', '_Static_assert', 'static_assert',
-    '_Thread_local', 'thread_local',
+    '_Alignas', '_Alignof', '_Atomic', '_Bool', '_Complex',
+    '_Decimal32', '_Decimal64', '_Decimal128',
+    '_Generic', '_Imaginary', '_Noreturn', '_Static_assert', '_Thread_local',
+]
+# These are only keyword'y when the corresponding headers are included.
+# They are used as default value for c_extra_keywords.
+_macroKeywords = [
+    'alignas', 'alignof', 'bool', 'complex', 'imaginary', 'noreturn', 'static_assert',
+    'thread_local',
 ]
 
 # these are ordered by preceedence
@@ -2536,6 +2541,12 @@ class DefinitionParser(BaseParser):
             if identifier in _keywords:
                 self.fail("Expected identifier in nested name, "
                           "got keyword: %s" % identifier)
+            if self.matched_text in self.config.c_extra_keywords:
+                msg = "Expected identifier, got user-defined keyword: %s." \
+                      + " Remove it from c_extra_keywords to allow it as identifier.\n" \
+                      + "Currently c_extra_keywords is %s."
+                self.fail(msg % (self.matched_text,
+                                 str(self.config.c_extra_keywords)))
             ident = ASTIdentifier(identifier)
             names.append(ident)
 
@@ -2712,6 +2723,12 @@ class DefinitionParser(BaseParser):
                 if self.matched_text in _keywords:
                     self.fail("Expected identifier, "
                               "got keyword: %s" % self.matched_text)
+                if self.matched_text in self.config.c_extra_keywords:
+                    msg = "Expected identifier, got user-defined keyword: %s." \
+                          + " Remove it from c_extra_keywords to allow it as identifier.\n" \
+                          + "Currently c_extra_keywords is %s."
+                    self.fail(msg % (self.matched_text,
+                                     str(self.config.c_extra_keywords)))
                 identifier = ASTIdentifier(self.matched_text)
                 declId = ASTNestedName([identifier], rooted=False)
             else:
@@ -3878,6 +3895,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_domain(CDomain)
     app.add_config_value("c_id_attributes", [], 'env')
     app.add_config_value("c_paren_attributes", [], 'env')
+    app.add_config_value("c_extra_keywords", _macroKeywords, 'env')
     app.add_post_transform(AliasTransform)
 
     app.add_config_value("c_allow_pre_v3", False, 'env')

sphinx/transforms/post_transforms/__init__.py

@@ -263,7 +263,8 @@ class PropagateDescDomain(SphinxPostTransform):
 
     def run(self, **kwargs: Any) -> None:
         for node in self.document.traverse(addnodes.desc_signature):
-            node['classes'].append(node.parent['domain'])
+            if node.parent.get('domain'):
+                node['classes'].append(node.parent['domain'])
 
 
 def setup(app: Sphinx) -> Dict[str, Any]:

sphinx/util/__init__.py

@@ -337,6 +337,23 @@ def parselinenos(spec: str, total: int) -> List[int]:
     return items
 
 
+def force_decode(string: str, encoding: str) -> str:
+    """Forcibly get a unicode string out of a bytestring."""
+    warnings.warn('force_decode() is deprecated.',
+                  RemovedInSphinx50Warning, stacklevel=2)
+    if isinstance(string, bytes):
+        try:
+            if encoding:
+                string = string.decode(encoding)
+            else:
+                # try decoding with utf-8, should only work for real UTF-8
+                string = string.decode()
+        except UnicodeError:
+            # last resort -- can't fail
+            string = string.decode('latin1')
+    return string
+
+
 def rpartition(s: str, t: str) -> Tuple[str, str]:
     """Similar to str.rpartition from 2.5, but doesn't return the separator."""
     warnings.warn('rpartition() is now deprecated.', RemovedInSphinx50Warning, stacklevel=2)

tests/test_domain_c.py

@@ -15,16 +15,20 @@ import pytest
 
 from sphinx import addnodes
 from sphinx.addnodes import desc
-from sphinx.domains.c import DefinitionError, DefinitionParser, Symbol, _id_prefix, _max_id
+from sphinx.domains.c import (DefinitionError, DefinitionParser, Symbol, _id_prefix,
+                              _macroKeywords, _max_id)
 from sphinx.ext.intersphinx import load_mappings, normalize_intersphinx_mapping
 from sphinx.testing import restructuredtext
 from sphinx.testing.util import assert_node
 
 
+class Config:
+    c_id_attributes = ["id_attr", 'LIGHTGBM_C_EXPORT']
+    c_paren_attributes = ["paren_attr"]
+    c_extra_keywords = _macroKeywords
+
+
 def parse(name, string):
-    class Config:
-        c_id_attributes = ["id_attr", 'LIGHTGBM_C_EXPORT']
-        c_paren_attributes = ["paren_attr"]
     parser = DefinitionParser(string, location=None, config=Config())
     parser.allowFallbackExpressionParsing = False
     ast = parser.parse_declaration(name, name)
@@ -114,9 +118,6 @@ def check(name, input, idDict, output=None, key=None, asTextOutput=None):
 
 def test_domain_c_ast_expressions():
     def exprCheck(expr, output=None):
-        class Config:
-            c_id_attributes = ["id_attr"]
-            c_paren_attributes = ["paren_attr"]
         parser = DefinitionParser(expr, location=None, config=Config())
         parser.allowFallbackExpressionParsing = False
         ast = parser.parse_expression()
@@ -527,6 +528,16 @@ def test_domain_c_ast_attributes():
     check('function', 'LIGHTGBM_C_EXPORT int LGBM_BoosterFree(int handle)',
           {1: 'LGBM_BoosterFree'})
 
+
+def test_extra_keywords():
+    with pytest.raises(DefinitionError,
+                       match='Expected identifier, got user-defined keyword: complex.'):
+        parse('function', 'void f(int complex)')
+    with pytest.raises(DefinitionError,
+                       match='Expected identifier, got user-defined keyword: complex.'):
+        parse('function', 'void complex(void)')
+
+
 # def test_print():
 #     # used for getting all the ids out for checking
 #     for a in ids: