Merge pull request #7380 from jakobandersen/cpp_comma_operator

C++, comma operator, pack expansion, error messages

CHANGES

@@ -20,6 +20,8 @@ Bugs fixed
 * #7370: autosummary: raises UnboundLocalError when unknown module given
 * #7367: C++, alternate operator spellings are now supported.
 * C, alternate operator spellings are now supported.
+* #7368: C++, comma operator in expressions, pack expansion in template
+  argument lists, and more comprehensive error messages in some cases.
 
 Testing
 --------

sphinx/domains/cpp.py

@@ -1505,8 +1505,38 @@ class ASTBinOpExpr(ASTExpression):
             self.exprs[i].describe_signature(signode, mode, env, symbol)
 
 
+class ASTBracedInitList(ASTBase):
+    def __init__(self, exprs: List[Union[ASTExpression, "ASTBracedInitList"]],
+                 trailingComma: bool) -> None:
+        self.exprs = exprs
+        self.trailingComma = trailingComma
+
+    def get_id(self, version: int) -> str:
+        return "il%sE" % ''.join(e.get_id(version) for e in self.exprs)
+
+    def _stringify(self, transform: StringifyTransform) -> str:
+        exprs = [transform(e) for e in self.exprs]
+        trailingComma = ',' if self.trailingComma else ''
+        return '{%s%s}' % (', '.join(exprs), trailingComma)
+
+    def describe_signature(self, signode: TextElement, mode: str,
+                           env: "BuildEnvironment", symbol: "Symbol") -> None:
+        verify_description_mode(mode)
+        signode.append(nodes.Text('{'))
+        first = True
+        for e in self.exprs:
+            if not first:
+                signode.append(nodes.Text(', '))
+            else:
+                first = False
+            e.describe_signature(signode, mode, env, symbol)
+        if self.trailingComma:
+            signode.append(nodes.Text(','))
+        signode.append(nodes.Text('}'))
+
+
 class ASTAssignmentExpr(ASTExpression):
-    def __init__(self, exprs: List[ASTExpression], ops: List[str]):
+    def __init__(self, exprs: List[Union[ASTExpression, ASTBracedInitList]], ops: List[str]):
         assert len(exprs) > 0
         assert len(exprs) == len(ops) + 1
         self.exprs = exprs
@@ -1540,6 +1570,31 @@ class ASTAssignmentExpr(ASTExpression):
             self.exprs[i].describe_signature(signode, mode, env, symbol)
 
 
+class ASTCommaExpr(ASTExpression):
+    def __init__(self, exprs: List[ASTExpression]):
+        assert len(exprs) > 0
+        self.exprs = exprs
+
+    def _stringify(self, transform: StringifyTransform) -> str:
+        return ', '.join(transform(e) for e in self.exprs)
+
+    def get_id(self, version: int) -> str:
+        id_ = _id_operator_v2[',']
+        res = []
+        for i in range(len(self.exprs) - 1):
+            res.append(id_)
+            res.append(self.exprs[i].get_id(version))
+        res.append(self.exprs[-1].get_id(version))
+        return ''.join(res)
+
+    def describe_signature(self, signode: TextElement, mode: str,
+                           env: "BuildEnvironment", symbol: "Symbol") -> None:
+        self.exprs[0].describe_signature(signode, mode, env, symbol)
+        for i in range(1, len(self.exprs)):
+            signode.append(nodes.Text(', '))
+            self.exprs[i].describe_signature(signode, mode, env, symbol)
+
+
 class ASTFallbackExpr(ASTExpression):
     def __init__(self, expr: str):
         self.expr = expr
@@ -1658,9 +1713,11 @@ class ASTTemplateArgConstant(ASTBase):
 
 
 class ASTTemplateArgs(ASTBase):
-    def __init__(self, args: List[Union["ASTType", ASTTemplateArgConstant]]) -> None:
+    def __init__(self, args: List[Union["ASTType", ASTTemplateArgConstant]],
+                 packExpansion: bool) -> None:
         assert args is not None
         self.args = args
+        self.packExpansion = packExpansion
 
     def get_id(self, version: int) -> str:
         if version == 1:
@@ -1672,13 +1729,21 @@ class ASTTemplateArgs(ASTBase):
 
         res = []
         res.append('I')
-        for a in self.args:
-            res.append(a.get_id(version))
+        if len(self.args) > 0:
+            for a in self.args[:-1]:
+                res.append(a.get_id(version))
+            if self.packExpansion:
+                res.append('J')
+            res.append(self.args[-1].get_id(version))
+            if self.packExpansion:
+                res.append('E')
         res.append('E')
         return ''.join(res)
 
     def _stringify(self, transform: StringifyTransform) -> str:
         res = ', '.join(transform(a) for a in self.args)
+        if self.packExpansion:
+            res += '...'
         return '<' + res + '>'
 
     def describe_signature(self, signode: TextElement, mode: str,
@@ -1691,6 +1756,8 @@ class ASTTemplateArgs(ASTBase):
                 signode += nodes.Text(', ')
             first = False
             a.describe_signature(signode, 'markType', env, symbol=symbol)
+        if self.packExpansion:
+            signode += nodes.Text('...')
         signode += nodes.Text('>')
 
 
@@ -2641,7 +2708,7 @@ class ASTDeclaratorParen(ASTDeclarator):
 ##############################################################################################
 
 class ASTPackExpansionExpr(ASTExpression):
-    def __init__(self, expr: ASTExpression):
+    def __init__(self, expr: Union[ASTExpression, ASTBracedInitList]):
         self.expr = expr
 
     def _stringify(self, transform: StringifyTransform) -> str:
@@ -2658,7 +2725,7 @@ class ASTPackExpansionExpr(ASTExpression):
 
 
 class ASTParenExprList(ASTBase):
-    def __init__(self, exprs: List[ASTExpression]) -> None:
+    def __init__(self, exprs: List[Union[ASTExpression, ASTBracedInitList]]) -> None:
         self.exprs = exprs
 
     def get_id(self, version: int) -> str:
@@ -2682,35 +2749,6 @@ class ASTParenExprList(ASTBase):
         signode.append(nodes.Text(')'))
 
 
-class ASTBracedInitList(ASTBase):
-    def __init__(self, exprs: List[ASTExpression], trailingComma: bool) -> None:
-        self.exprs = exprs
-        self.trailingComma = trailingComma
-
-    def get_id(self, version: int) -> str:
-        return "il%sE" % ''.join(e.get_id(version) for e in self.exprs)
-
-    def _stringify(self, transform: StringifyTransform) -> str:
-        exprs = [transform(e) for e in self.exprs]
-        trailingComma = ',' if self.trailingComma else ''
-        return '{%s%s}' % (', '.join(exprs), trailingComma)
-
-    def describe_signature(self, signode: TextElement, mode: str,
-                           env: "BuildEnvironment", symbol: "Symbol") -> None:
-        verify_description_mode(mode)
-        signode.append(nodes.Text('{'))
-        first = True
-        for e in self.exprs:
-            if not first:
-                signode.append(nodes.Text(', '))
-            else:
-                first = False
-            e.describe_signature(signode, mode, env, symbol)
-        if self.trailingComma:
-            signode.append(nodes.Text(','))
-        signode.append(nodes.Text('}'))
-
-
 class ASTInitializer(ASTBase):
     def __init__(self, value: Union[ASTExpression, ASTBracedInitList],
                  hasAssign: bool = True) -> None:
@@ -4755,7 +4793,7 @@ class DefinitionParser(BaseParser):
             self.pos = pos
             # fall back to a paren expression
             try:
-                res = self._parse_expression(inTemplate=False)
+                res = self._parse_expression()
                 self.skip_ws()
                 if not self.skip_string(')'):
                     self.fail("Expected ')' in end of parenthesized expression.")
@@ -4803,7 +4841,9 @@ class DefinitionParser(BaseParser):
         return None
 
     def _parse_initializer_list(self, name: str, open: str, close: str
-                                ) -> Tuple[List[ASTExpression], bool]:
+                                ) -> Tuple[List[Union[ASTExpression,
+                                                      ASTBracedInitList]],
+                                           bool]:
         # Parse open and close with the actual initializer-list inbetween
         # -> initializer-clause '...'[opt]
         #  | initializer-list ',' initializer-clause '...'[opt]
@@ -4813,11 +4853,11 @@ class DefinitionParser(BaseParser):
         if self.skip_string(close):
             return [], False
 
-        exprs = []  # type: List[ASTExpression]
+        exprs = []  # type: List[Union[ASTExpression, ASTBracedInitList]]
         trailingComma = False
         while True:
             self.skip_ws()
-            expr = self._parse_expression(inTemplate=False)
+            expr = self._parse_initializer_clause()
             self.skip_ws()
             if self.skip_string('...'):
                 exprs.append(ASTPackExpansionExpr(expr))
@@ -4847,6 +4887,12 @@ class DefinitionParser(BaseParser):
             return None
         return ASTParenExprList(exprs)
 
+    def _parse_initializer_clause(self) -> Union[ASTExpression, ASTBracedInitList]:
+        bracedInitList = self._parse_braced_init_list()
+        if bracedInitList is not None:
+            return bracedInitList
+        return self._parse_assignment_expression(inTemplate=False)
+
     def _parse_braced_init_list(self) -> ASTBracedInitList:
         # -> '{' initializer-list ','[opt] '}'
         #  | '{' '}'
@@ -4906,7 +4952,7 @@ class DefinitionParser(BaseParser):
                 self.fail("Expected '(' in '%s'." % cast)
 
             def parser():
-                return self._parse_expression(inTemplate=False)
+                return self._parse_expression()
             expr = self._parse_expression_fallback([')'], parser)
             self.skip_ws()
             if not self.skip_string(")"):
@@ -4927,7 +4973,7 @@ class DefinitionParser(BaseParser):
                 try:
 
                     def parser():
-                        return self._parse_expression(inTemplate=False)
+                        return self._parse_expression()
                     expr = self._parse_expression_fallback([')'], parser)
                     prefix = ASTTypeId(expr, isType=False)
                     if not self.skip_string(')'):
@@ -4974,7 +5020,7 @@ class DefinitionParser(BaseParser):
             self.skip_ws()
             if prefixType in ['expr', 'cast', 'typeid']:
                 if self.skip_string_and_ws('['):
-                    expr = self._parse_expression(inTemplate=False)
+                    expr = self._parse_expression()
                     self.skip_ws()
                     if not self.skip_string(']'):
                         self.fail("Expected ']' in end of postfix expression.")
@@ -5065,7 +5111,7 @@ class DefinitionParser(BaseParser):
         if self.skip_word_and_ws('noexcept'):
             if not self.skip_string_and_ws('('):
                 self.fail("Expecting '(' after 'noexcept'.")
-            expr = self._parse_expression(inTemplate=False)
+            expr = self._parse_expression()
             self.skip_ws()
             if not self.skip_string(')'):
                 self.fail("Expecting ')' to end 'noexcept'.")
@@ -5198,7 +5244,7 @@ class DefinitionParser(BaseParser):
         #     logical-or-expression
         #   | logical-or-expression "?" expression ":" assignment-expression
         #   | logical-or-expression assignment-operator initializer-clause
-        exprs = []
+        exprs = []  # type: List[Union[ASTExpression, ASTBracedInitList]]
         ops = []
         orExpr = self._parse_logical_or_expression(inTemplate=inTemplate)
         exprs.append(orExpr)
@@ -5213,7 +5259,7 @@ class DefinitionParser(BaseParser):
                 else:
                     if not self.skip_string(op):
                         continue
-                expr = self._parse_logical_or_expression(False)
+                expr = self._parse_initializer_clause()
                 exprs.append(expr)
                 ops.append(op)
                 oneMore = True
@@ -5230,11 +5276,19 @@ class DefinitionParser(BaseParser):
         # TODO: use _parse_conditional_expression_tail
         return orExpr
 
-    def _parse_expression(self, inTemplate: bool) -> ASTExpression:
+    def _parse_expression(self) -> ASTExpression:
         # -> assignment-expression
         #  | expression "," assignment-expresion
-        # TODO: actually parse the second production
-        return self._parse_assignment_expression(inTemplate=inTemplate)
+        exprs = [self._parse_assignment_expression(inTemplate=False)]
+        while True:
+            self.skip_ws()
+            if not self.skip_string(','):
+                break
+            exprs.append(self._parse_assignment_expression(inTemplate=False))
+        if len(exprs) == 1:
+            return exprs[0]
+        else:
+            return ASTCommaExpr(exprs)
 
     def _parse_expression_fallback(self, end: List[str],
                                    parser: Callable[[], ASTExpression],
@@ -5313,13 +5367,21 @@ class DefinitionParser(BaseParser):
         return ASTOperatorType(type)
 
     def _parse_template_argument_list(self) -> ASTTemplateArgs:
+        # template-argument-list: (but we include the < and > here
+        #    template-argument ...[opt]
+        #    template-argument-list, template-argument ...[opt]
+        # template-argument:
+        #    constant-expression
+        #    type-id
+        #    id-expression
         self.skip_ws()
         if not self.skip_string_and_ws('<'):
             return None
         if self.skip_string('>'):
-            return ASTTemplateArgs([])
+            return ASTTemplateArgs([], False)
         prevErrors = []
         templateArgs = []  # type: List[Union[ASTType, ASTTemplateArgConstant]]
+        packExpansion = False
         while 1:
             pos = self.pos
             parsedComma = False
@@ -5327,31 +5389,35 @@ class DefinitionParser(BaseParser):
             try:
                 type = self._parse_type(named=False)
                 self.skip_ws()
-                if self.skip_string('>'):
+                if self.skip_string_and_ws('...'):
+                    packExpansion = True
+                    parsedEnd = True
+                    if not self.skip_string('>'):
+                        self.fail('Expected ">" after "..." in template argument list.')
+                elif self.skip_string('>'):
                     parsedEnd = True
                 elif self.skip_string(','):
                     parsedComma = True
                 else:
-                    self.fail('Expected ">" or "," in template argument list.')
+                    self.fail('Expected "...>", ">" or "," in template argument list.')
                 templateArgs.append(type)
             except DefinitionError as e:
                 prevErrors.append((e, "If type argument"))
                 self.pos = pos
                 try:
-                    # actually here we shouldn't use the fallback parser (hence allow=False),
-                    # because if actually took the < in an expression, then we _will_ fail,
-                    # which is handled elsewhere. E.g., :cpp:expr:`A <= 0`.
-                    def parser():
-                        return self._parse_constant_expression(inTemplate=True)
-                    value = self._parse_expression_fallback(
-                        [',', '>'], parser, allow=False)
+                    value = self._parse_constant_expression(inTemplate=True)
                     self.skip_ws()
-                    if self.skip_string('>'):
+                    if self.skip_string_and_ws('...'):
+                        packExpansion = True
+                        parsedEnd = True
+                        if not self.skip_string('>'):
+                            self.fail('Expected ">" after "..." in template argument list.')
+                    elif self.skip_string('>'):
                         parsedEnd = True
                     elif self.skip_string(','):
                         parsedComma = True
                     else:
-                        self.fail('Expected ">" or "," in template argument list.')
+                        self.fail('Expected "...>", ">" or "," in template argument list.')
                     templateArgs.append(ASTTemplateArgConstant(value))
                 except DefinitionError as e:
                     self.pos = pos
@@ -5361,7 +5427,9 @@ class DefinitionParser(BaseParser):
             if parsedEnd:
                 assert not parsedComma
                 break
-        return ASTTemplateArgs(templateArgs)
+            else:
+                assert not packExpansion
+        return ASTTemplateArgs(templateArgs, packExpansion)
 
     def _parse_nested_name(self, memberPointer: bool = False) -> ASTNestedName:
         names = []  # type: List[ASTNestedNameElement]
@@ -5451,7 +5519,7 @@ class DefinitionParser(BaseParser):
                 if not self.skip_string(')'):
                     self.fail("Expected ')' after 'decltype(auto'.")
                 return ASTTrailingTypeSpecDecltypeAuto()
-            expr = self._parse_expression(inTemplate=False)
+            expr = self._parse_expression()
             self.skip_ws()
             if not self.skip_string(')'):
                 self.fail("Expected ')' after 'decltype(<expr>'.")
@@ -5464,7 +5532,6 @@ class DefinitionParser(BaseParser):
             if self.skip_word_and_ws(k):
                 prefix = k
                 break
-
         nestedName = self._parse_nested_name()
         return ASTTrailingTypeSpecName(prefix, nestedName)
 
@@ -5696,7 +5763,7 @@ class DefinitionParser(BaseParser):
                     continue
 
                 def parser():
-                    return self._parse_expression(inTemplate=False)
+                    return self._parse_expression()
                 value = self._parse_expression_fallback([']'], parser)
                 if not self.skip_string(']'):
                     self.fail("Expected ']' in end of array operator.")
@@ -5758,32 +5825,6 @@ class DefinitionParser(BaseParser):
         if typed and self.skip_string("..."):
             next = self._parse_declarator(named, paramMode, False)
             return ASTDeclaratorParamPack(next=next)
-        if typed:  # pointer to member
-            pos = self.pos
-            try:
-                name = self._parse_nested_name(memberPointer=True)
-                self.skip_ws()
-                if not self.skip_string('*'):
-                    self.fail("Expected '*' in pointer to member declarator.")
-                self.skip_ws()
-            except DefinitionError as e:
-                self.pos = pos
-                prevErrors.append((e, "If pointer to member declarator"))
-            else:
-                volatile = False
-                const = False
-                while 1:
-                    if not volatile:
-                        volatile = self.skip_word_and_ws('volatile')
-                        if volatile:
-                            continue
-                    if not const:
-                        const = self.skip_word_and_ws('const')
-                        if const:
-                            continue
-                    break
-                next = self._parse_declarator(named, paramMode, typed)
-                return ASTDeclaratorMemPtr(name, const, volatile, next=next)
         if typed and self.current_char == '(':  # note: peeking, not skipping
             if paramMode == "operatorCast":
                 # TODO: we should be able to parse cast operators which return
@@ -5820,9 +5861,40 @@ class DefinitionParser(BaseParser):
                     prevErrors.append((exNoPtrParen, "If parenthesis in noptr-declarator"))
                     header = "Error in declarator"
                     raise self._make_multi_error(prevErrors, header)
+        if typed:  # pointer to member
+            pos = self.pos
+            try:
+                name = self._parse_nested_name(memberPointer=True)
+                self.skip_ws()
+                if not self.skip_string('*'):
+                    self.fail("Expected '*' in pointer to member declarator.")
+                self.skip_ws()
+            except DefinitionError as e:
+                self.pos = pos
+                prevErrors.append((e, "If pointer to member declarator"))
+            else:
+                volatile = False
+                const = False
+                while 1:
+                    if not volatile:
+                        volatile = self.skip_word_and_ws('volatile')
+                        if volatile:
+                            continue
+                    if not const:
+                        const = self.skip_word_and_ws('const')
+                        if const:
+                            continue
+                    break
+                next = self._parse_declarator(named, paramMode, typed)
+                return ASTDeclaratorMemPtr(name, const, volatile, next=next)
         pos = self.pos
         try:
-            return self._parse_declarator_name_suffix(named, paramMode, typed)
+            res = self._parse_declarator_name_suffix(named, paramMode, typed)
+            # this is a heuristic for error messages, for when there is a < after a
+            # nested name, but it was not a successful template argument list
+            if self.current_char == '<':
+                self.otherErrors.append(self._make_multi_error(prevErrors, ""))
+            return res
         except DefinitionError as e:
             self.pos = pos
             prevErrors.append((e, "If declarator-id"))
@@ -5891,7 +5963,6 @@ class DefinitionParser(BaseParser):
                 raise Exception('Internal error, unknown outer "%s".' % outer)
             if outer != 'operatorCast':
                 assert named
-
         if outer in ('type', 'function'):
             # We allow type objects to just be a name.
             # Some functions don't have normal return types: constructors,
@@ -6085,8 +6156,8 @@ class DefinitionParser(BaseParser):
         self.skip_ws()
         if not self.skip_string("<"):
             self.fail("Expected '<' after 'template'")
+        prevErrors = []
         while 1:
-            prevErrors = []
             self.skip_ws()
             if self.skip_word('template'):
                 # declare a tenplate template parameter
@@ -6139,6 +6210,7 @@ class DefinitionParser(BaseParser):
             if self.skip_string('>'):
                 return ASTTemplateParams(templateParams)
             elif self.skip_string(','):
+                prevErrors = []
                 continue
             else:
                 header = "Error in template parameter list."
@@ -6358,7 +6430,7 @@ class DefinitionParser(BaseParser):
     def parse_expression(self) -> Union[ASTExpression, ASTType]:
         pos = self.pos
         try:
-            expr = self._parse_expression(False)
+            expr = self._parse_expression()
             self.skip_ws()
             self.assert_end()
             return expr

tests/test_domain_cpp.py

@@ -274,6 +274,9 @@ def test_expressions():
     exprCheck('a xor_eq 5', 'eO1aL5E')
     exprCheck('a |= 5', 'oR1aL5E')
     exprCheck('a or_eq 5', 'oR1aL5E')
+    exprCheck('a = {1, 2, 3}', 'aS1ailL1EL2EL3EE')
+    # comma operator
+    exprCheck('a, 5', 'cm1aL5E')
 
     # Additional tests
     # a < expression that starts with something that could be a template
@@ -626,6 +629,12 @@ def test_class_definitions():
           {2: 'I0E7has_varI1TNSt6void_tIDTadN1T3varEEEEE'})
 
 
+    check('class', 'template<typename ...Ts> T<int (*)(Ts)...>',
+          {2: 'IDpE1TIJPFi2TsEEE'})
+    check('class', 'template<int... Is> T<(Is)...>',
+          {2: 'I_DpiE1TIJX(Is)EEE', 3: 'I_DpiE1TIJX2IsEEE'})
+
+
 def test_union_definitions():
     check('union', 'A', {2: "1A"})