Merge pull request #7471 from jakobandersen/c_attributes

C, parse attributes
This commit is contained in:
Jakob Lykke Andersen 2020-04-13 13:54:40 +02:00 committed by GitHub
commit e65fb93d20
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 265 additions and 269 deletions

View File

@ -13,6 +13,9 @@ Deprecated
Features added
--------------
* C, parse attributes and add :confval:`c_id_attributes`
and :confval:`c_paren_attributes` to support user-defined attributes.
Bugs fixed
----------

View File

@ -2472,6 +2472,30 @@ Options for the XML builder
match any sequence of characters *including* slashes.
.. _c-config:
Options for the C domain
------------------------
.. confval:: c_id_attributes
A list of strings that the parser additionally should accept as attributes.
This can for example be used when attributes have been ``#define`` d for
portability.
.. versionadded:: 3.0
.. confval:: c_paren_attributes
A list of strings that the parser additionally should accept as attributes
with one argument. That is, if ``my_align_as`` is in the list, then
``my_align_as(X)`` is parsed as an attribute for all strings ``X`` that have
balanced braces (``()``, ``[]``, and ``{}``). This can for example be used
when attributes have been ``#define`` d for portability.
.. versionadded:: 3.0
.. _cpp-config:
Options for the C++ domain

View File

@ -706,6 +706,12 @@ Inline Expressions and Types
.. versionadded:: 3.0
Configuration Variables
~~~~~~~~~~~~~~~~~~~~~~~
See :ref:`c-config`.
.. _cpp-domain:
The C++ Domain

View File

@ -1990,6 +1990,14 @@ class DefinitionParser(BaseParser):
def language(self) -> str:
return 'C'
@property
def id_attributes(self):
return self.config.c_id_attributes
@property
def paren_attributes(self):
return self.config.c_paren_attributes
def _parse_string(self) -> str:
if self.current_char != '"':
return None
@ -2009,66 +2017,6 @@ class DefinitionParser(BaseParser):
self.pos += 1
return self.definition[startPos:self.pos]
def _parse_attribute(self) -> Any:
return None
# self.skip_ws()
# # try C++11 style
# startPos = self.pos
# if self.skip_string_and_ws('['):
# if not self.skip_string('['):
# self.pos = startPos
# else:
# # TODO: actually implement the correct grammar
# arg = self._parse_balanced_token_seq(end=[']'])
# if not self.skip_string_and_ws(']'):
# self.fail("Expected ']' in end of attribute.")
# if not self.skip_string_and_ws(']'):
# self.fail("Expected ']' in end of attribute after [[...]")
# return ASTCPPAttribute(arg)
#
# # try GNU style
# if self.skip_word_and_ws('__attribute__'):
# if not self.skip_string_and_ws('('):
# self.fail("Expected '(' after '__attribute__'.")
# if not self.skip_string_and_ws('('):
# self.fail("Expected '(' after '__attribute__('.")
# attrs = []
# while 1:
# if self.match(identifier_re):
# name = self.matched_text
# self.skip_ws()
# if self.skip_string_and_ws('('):
# self.fail('Parameterized GNU style attribute not yet supported.')
# attrs.append(ASTGnuAttribute(name, None))
# # TODO: parse arguments for the attribute
# if self.skip_string_and_ws(','):
# continue
# elif self.skip_string_and_ws(')'):
# break
# else:
# self.fail("Expected identifier, ')', or ',' in __attribute__.")
# if not self.skip_string_and_ws(')'):
# self.fail("Expected ')' after '__attribute__((...)'")
# return ASTGnuAttributeList(attrs)
#
# # try the simple id attributes defined by the user
# for id in self.config.cpp_id_attributes:
# if self.skip_word_and_ws(id):
# return ASTIdAttribute(id)
#
# # try the paren attributes defined by the user
# for id in self.config.cpp_paren_attributes:
# if not self.skip_string_and_ws(id):
# continue
# if not self.skip_string('('):
# self.fail("Expected '(' after user-defined paren-attribute.")
# arg = self._parse_balanced_token_seq(end=[')'])
# if not self.skip_string(')'):
# self.fail("Expected ')' to end user-defined paren-attribute.")
# return ASTParenAttribute(id, arg)
return None
def _parse_literal(self) -> ASTLiteral:
# -> integer-literal
# | character-literal
@ -3081,7 +3029,7 @@ class CObject(ObjectDescription):
def handle_signature(self, sig: str, signode: TextElement) -> ASTDeclaration:
parentSymbol = self.env.temp_data['c:parent_symbol'] # type: Symbol
parser = DefinitionParser(sig, location=signode)
parser = DefinitionParser(sig, location=signode, config=self.env.config)
try:
ast = self.parse_definition(parser)
parser.assert_end()
@ -3214,7 +3162,8 @@ class CExprRole(SphinxRole):
def run(self) -> Tuple[List[Node], List[system_message]]:
text = self.text.replace('\n', ' ')
parser = DefinitionParser(text, location=self.get_source_info())
parser = DefinitionParser(text, location=self.get_source_info(),
config=self.env.config)
# attempt to mimic XRefRole classes, except that...
classes = ['xref', 'c', self.class_type]
try:
@ -3344,7 +3293,7 @@ class CDomain(Domain):
def _resolve_xref_inner(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
typ: str, target: str, node: pending_xref,
contnode: Element) -> Tuple[Element, str]:
parser = DefinitionParser(target, location=node)
parser = DefinitionParser(target, location=node, config=env.config)
try:
name = parser.parse_xref_object()
except DefinitionError as e:
@ -3401,6 +3350,8 @@ class CDomain(Domain):
def setup(app: Sphinx) -> Dict[str, Any]:
app.add_domain(CDomain)
app.add_config_value("c_id_attributes", [], 'env')
app.add_config_value("c_paren_attributes", [], 'env')
return {
'version': 'builtin',

View File

@ -21,7 +21,6 @@ from sphinx import addnodes
from sphinx.addnodes import desc_signature, pending_xref
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.config import Config
from sphinx.directives import ObjectDescription
from sphinx.domains import Domain, ObjType
from sphinx.environment import BuildEnvironment
@ -32,7 +31,7 @@ from sphinx.transforms import SphinxTransform
from sphinx.transforms.post_transforms import ReferencesResolver
from sphinx.util import logging
from sphinx.util.cfamily import (
NoOldIdError, ASTBaseBase, verify_description_mode, StringifyTransform,
NoOldIdError, ASTBaseBase, ASTAttribute, verify_description_mode, StringifyTransform,
BaseParser, DefinitionError, UnsupportedMultiCharacterCharLiteral,
identifier_re, anon_identifier_re, integer_literal_re, octal_literal_re,
hex_literal_re, binary_literal_re, float_literal_re,
@ -769,89 +768,6 @@ class ASTNestedName(ASTBase):
raise Exception('Unknown description mode: %s' % mode)
################################################################################
# Attributes
################################################################################
class ASTAttribute(ASTBase):
def describe_signature(self, signode: TextElement) -> None:
raise NotImplementedError(repr(self))
class ASTCPPAttribute(ASTAttribute):
def __init__(self, arg: str) -> None:
self.arg = arg
def _stringify(self, transform: StringifyTransform) -> str:
return "[[" + self.arg + "]]"
def describe_signature(self, signode: TextElement) -> None:
txt = str(self)
signode.append(nodes.Text(txt, txt))
class ASTGnuAttribute(ASTBase):
def __init__(self, name: str, args: Any) -> None:
self.name = name
self.args = args
def _stringify(self, transform: StringifyTransform) -> str:
res = [self.name]
if self.args:
res.append('(')
res.append(transform(self.args))
res.append(')')
return ''.join(res)
class ASTGnuAttributeList(ASTAttribute):
def __init__(self, attrs: List[ASTGnuAttribute]) -> None:
self.attrs = attrs
def _stringify(self, transform: StringifyTransform) -> str:
res = ['__attribute__((']
first = True
for attr in self.attrs:
if not first:
res.append(', ')
first = False
res.append(transform(attr))
res.append('))')
return ''.join(res)
def describe_signature(self, signode: TextElement) -> None:
txt = str(self)
signode.append(nodes.Text(txt, txt))
class ASTIdAttribute(ASTAttribute):
"""For simple attributes defined by the user."""
def __init__(self, id: str) -> None:
self.id = id
def _stringify(self, transform: StringifyTransform) -> str:
return self.id
def describe_signature(self, signode: TextElement) -> None:
signode.append(nodes.Text(self.id, self.id))
class ASTParenAttribute(ASTAttribute):
"""For paren attributes defined by the user."""
def __init__(self, id: str, arg: str) -> None:
self.id = id
self.arg = arg
def _stringify(self, transform: StringifyTransform) -> str:
return self.id + '(' + self.arg + ')'
def describe_signature(self, signode: TextElement) -> None:
txt = str(self)
signode.append(nodes.Text(txt, txt))
################################################################################
# Expressions
################################################################################
@ -4667,16 +4583,18 @@ class DefinitionParser(BaseParser):
_prefix_keys = ('class', 'struct', 'enum', 'union', 'typename')
def __init__(self, definition: str, *,
location: Union[nodes.Node, Tuple[str, int]],
config: "Config") -> None:
super().__init__(definition, location=location)
self.config = config
@property
def language(self) -> str:
return 'C++'
@property
def id_attributes(self):
return self.config.cpp_id_attributes
@property
def paren_attributes(self):
return self.config.cpp_paren_attributes
def _parse_string(self) -> str:
if self.current_char != '"':
return None
@ -4696,85 +4614,6 @@ class DefinitionParser(BaseParser):
self.pos += 1
return self.definition[startPos:self.pos]
def _parse_balanced_token_seq(self, end: List[str]) -> str:
# TODO: add handling of string literals and similar
brackets = {'(': ')', '[': ']', '{': '}'}
startPos = self.pos
symbols = [] # type: List[str]
while not self.eof:
if len(symbols) == 0 and self.current_char in end:
break
if self.current_char in brackets.keys():
symbols.append(brackets[self.current_char])
elif len(symbols) > 0 and self.current_char == symbols[-1]:
symbols.pop()
elif self.current_char in ")]}":
self.fail("Unexpected '%s' in balanced-token-seq." % self.current_char)
self.pos += 1
if self.eof:
self.fail("Could not find end of balanced-token-seq starting at %d."
% startPos)
return self.definition[startPos:self.pos]
def _parse_attribute(self) -> ASTAttribute:
self.skip_ws()
# try C++11 style
startPos = self.pos
if self.skip_string_and_ws('['):
if not self.skip_string('['):
self.pos = startPos
else:
# TODO: actually implement the correct grammar
arg = self._parse_balanced_token_seq(end=[']'])
if not self.skip_string_and_ws(']'):
self.fail("Expected ']' in end of attribute.")
if not self.skip_string_and_ws(']'):
self.fail("Expected ']' in end of attribute after [[...]")
return ASTCPPAttribute(arg)
# try GNU style
if self.skip_word_and_ws('__attribute__'):
if not self.skip_string_and_ws('('):
self.fail("Expected '(' after '__attribute__'.")
if not self.skip_string_and_ws('('):
self.fail("Expected '(' after '__attribute__('.")
attrs = []
while 1:
if self.match(identifier_re):
name = self.matched_text
self.skip_ws()
if self.skip_string_and_ws('('):
self.fail('Parameterized GNU style attribute not yet supported.')
attrs.append(ASTGnuAttribute(name, None))
# TODO: parse arguments for the attribute
if self.skip_string_and_ws(','):
continue
elif self.skip_string_and_ws(')'):
break
else:
self.fail("Expected identifier, ')', or ',' in __attribute__.")
if not self.skip_string_and_ws(')'):
self.fail("Expected ')' after '__attribute__((...)'")
return ASTGnuAttributeList(attrs)
# try the simple id attributes defined by the user
for id in self.config.cpp_id_attributes:
if self.skip_word_and_ws(id):
return ASTIdAttribute(id)
# try the paren attributes defined by the user
for id in self.config.cpp_paren_attributes:
if not self.skip_string_and_ws(id):
continue
if not self.skip_string('('):
self.fail("Expected '(' after user-defined paren-attribute.")
arg = self._parse_balanced_token_seq(end=[')'])
if not self.skip_string(')'):
self.fail("Expected ')' to end user-defined paren-attribute.")
return ASTParenAttribute(id, arg)
return None
def _parse_literal(self) -> ASTLiteral:
# -> integer-literal
# | character-literal
@ -7200,8 +7039,7 @@ class CPPDomain(Domain):
# add parens again for those that could be functions
if typ == 'any' or typ == 'func':
target += '()'
parser = DefinitionParser(target, location=node,
config=env.config)
parser = DefinitionParser(target, location=node, config=env.config)
try:
ast, isShorthand = parser.parse_xref_object()
except DefinitionError as e:

View File

@ -16,7 +16,9 @@ from typing import (
)
from docutils import nodes
from docutils.nodes import TextElement
from sphinx.config import Config
from sphinx.deprecation import RemovedInSphinx40Warning
from sphinx.util import logging
@ -112,6 +114,92 @@ class ASTBaseBase:
return '<%s>' % self.__class__.__name__
################################################################################
# Attributes
################################################################################
class ASTAttribute(ASTBaseBase):
def describe_signature(self, signode: TextElement) -> None:
raise NotImplementedError(repr(self))
class ASTCPPAttribute(ASTAttribute):
def __init__(self, arg: str) -> None:
self.arg = arg
def _stringify(self, transform: StringifyTransform) -> str:
return "[[" + self.arg + "]]"
def describe_signature(self, signode: TextElement) -> None:
txt = str(self)
signode.append(nodes.Text(txt, txt))
class ASTGnuAttribute(ASTBaseBase):
def __init__(self, name: str, args: Any) -> None:
self.name = name
self.args = args
def _stringify(self, transform: StringifyTransform) -> str:
res = [self.name]
if self.args:
res.append('(')
res.append(transform(self.args))
res.append(')')
return ''.join(res)
class ASTGnuAttributeList(ASTAttribute):
def __init__(self, attrs: List[ASTGnuAttribute]) -> None:
self.attrs = attrs
def _stringify(self, transform: StringifyTransform) -> str:
res = ['__attribute__((']
first = True
for attr in self.attrs:
if not first:
res.append(', ')
first = False
res.append(transform(attr))
res.append('))')
return ''.join(res)
def describe_signature(self, signode: TextElement) -> None:
txt = str(self)
signode.append(nodes.Text(txt, txt))
class ASTIdAttribute(ASTAttribute):
"""For simple attributes defined by the user."""
def __init__(self, id: str) -> None:
self.id = id
def _stringify(self, transform: StringifyTransform) -> str:
return self.id
def describe_signature(self, signode: TextElement) -> None:
signode.append(nodes.Text(self.id, self.id))
class ASTParenAttribute(ASTAttribute):
"""For paren attributes defined by the user."""
def __init__(self, id: str, arg: str) -> None:
self.id = id
self.arg = arg
def _stringify(self, transform: StringifyTransform) -> str:
return self.id + '(' + self.arg + ')'
def describe_signature(self, signode: TextElement) -> None:
txt = str(self)
signode.append(nodes.Text(txt, txt))
################################################################################
class UnsupportedMultiCharacterCharLiteral(Exception):
@property
def decoded(self) -> str:
@ -132,9 +220,11 @@ class DefinitionError(Exception):
class BaseParser:
def __init__(self, definition: str, *,
location: Union[nodes.Node, Tuple[str, int]]) -> None:
location: Union[nodes.Node, Tuple[str, int]],
config: "Config") -> None:
self.definition = definition.strip()
self.location = location # for warnings
self.config = config
self.pos = 0
self.end = len(self.definition)
@ -252,3 +342,92 @@ class BaseParser:
self.skip_ws()
if not self.eof:
self.fail('Expected end of definition.')
################################################################################
@property
def id_attributes(self):
raise NotImplementedError
@property
def paren_attributes(self):
raise NotImplementedError
def _parse_balanced_token_seq(self, end: List[str]) -> str:
# TODO: add handling of string literals and similar
brackets = {'(': ')', '[': ']', '{': '}'}
startPos = self.pos
symbols = [] # type: List[str]
while not self.eof:
if len(symbols) == 0 and self.current_char in end:
break
if self.current_char in brackets.keys():
symbols.append(brackets[self.current_char])
elif len(symbols) > 0 and self.current_char == symbols[-1]:
symbols.pop()
elif self.current_char in ")]}":
self.fail("Unexpected '%s' in balanced-token-seq." % self.current_char)
self.pos += 1
if self.eof:
self.fail("Could not find end of balanced-token-seq starting at %d."
% startPos)
return self.definition[startPos:self.pos]
def _parse_attribute(self) -> ASTAttribute:
self.skip_ws()
# try C++11 style
startPos = self.pos
if self.skip_string_and_ws('['):
if not self.skip_string('['):
self.pos = startPos
else:
# TODO: actually implement the correct grammar
arg = self._parse_balanced_token_seq(end=[']'])
if not self.skip_string_and_ws(']'):
self.fail("Expected ']' in end of attribute.")
if not self.skip_string_and_ws(']'):
self.fail("Expected ']' in end of attribute after [[...]")
return ASTCPPAttribute(arg)
# try GNU style
if self.skip_word_and_ws('__attribute__'):
if not self.skip_string_and_ws('('):
self.fail("Expected '(' after '__attribute__'.")
if not self.skip_string_and_ws('('):
self.fail("Expected '(' after '__attribute__('.")
attrs = []
while 1:
if self.match(identifier_re):
name = self.matched_text
self.skip_ws()
if self.skip_string_and_ws('('):
self.fail('Parameterized GNU style attribute not yet supported.')
attrs.append(ASTGnuAttribute(name, None))
# TODO: parse arguments for the attribute
if self.skip_string_and_ws(','):
continue
elif self.skip_string_and_ws(')'):
break
else:
self.fail("Expected identifier, ')', or ',' in __attribute__.")
if not self.skip_string_and_ws(')'):
self.fail("Expected ')' after '__attribute__((...)'")
return ASTGnuAttributeList(attrs)
# try the simple id attributes defined by the user
for id in self.id_attributes:
if self.skip_word_and_ws(id):
return ASTIdAttribute(id)
# try the paren attributes defined by the user
for id in self.paren_attributes:
if not self.skip_string_and_ws(id):
continue
if not self.skip_string('('):
self.fail("Expected '(' after user-defined paren-attribute.")
arg = self._parse_balanced_token_seq(end=[')'])
if not self.skip_string(')'):
self.fail("Expected ')' to end user-defined paren-attribute.")
return ASTParenAttribute(id, arg)
return None

View File

@ -7,28 +7,20 @@
:copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
import pytest
from docutils import nodes
import sphinx.domains.c as cDomain
from sphinx import addnodes
from sphinx.addnodes import (
desc, desc_addname, desc_annotation, desc_content, desc_name, desc_optional,
desc_parameter, desc_parameterlist, desc_returns, desc_signature, desc_type,
pending_xref
)
from sphinx.domains.c import DefinitionParser, DefinitionError
from sphinx.domains.c import _max_id, _id_prefix, Symbol
from sphinx.testing import restructuredtext
from sphinx.testing.util import assert_node
from sphinx.util import docutils
def parse(name, string):
parser = DefinitionParser(string, location=None)
class Config:
c_id_attributes = ["id_attr", 'LIGHTGBM_C_EXPORT']
c_paren_attributes = ["paren_attr"]
parser = DefinitionParser(string, location=None, config=Config())
parser.allowFallbackExpressionParsing = False
ast = parser.parse_declaration(name, name)
parser.assert_end()
@ -87,7 +79,10 @@ def check(name, input, idDict, output=None):
def test_expressions():
def exprCheck(expr, output=None):
parser = DefinitionParser(expr, location=None)
class Config:
c_id_attributes = ["id_attr"]
c_paren_attributes = ["paren_attr"]
parser = DefinitionParser(expr, location=None, config=Config())
parser.allowFallbackExpressionParsing = False
ast = parser.parse_expression()
parser.assert_end()
@ -404,24 +399,23 @@ def test_initializers():
def test_attributes():
return # TODO
# style: C++
check('member', '[[]] int f', {1: 'f__i', 2: '1f'})
check('member', '[ [ ] ] int f', {1: 'f__i', 2: '1f'},
check('member', '[[]] int f', {1: 'f'})
check('member', '[ [ ] ] int f', {1: 'f'},
# this will fail when the proper grammar is implemented
output='[[ ]] int f')
check('member', '[[a]] int f', {1: 'f__i', 2: '1f'})
check('member', '[[a]] int f', {1: 'f'})
# style: GNU
check('member', '__attribute__(()) int f', {1: 'f__i', 2: '1f'})
check('member', '__attribute__((a)) int f', {1: 'f__i', 2: '1f'})
check('member', '__attribute__((a, b)) int f', {1: 'f__i', 2: '1f'})
check('member', '__attribute__(()) int f', {1: 'f'})
check('member', '__attribute__((a)) int f', {1: 'f'})
check('member', '__attribute__((a, b)) int f', {1: 'f'})
# style: user-defined id
check('member', 'id_attr int f', {1: 'f__i', 2: '1f'})
check('member', 'id_attr int f', {1: 'f'})
# style: user-defined paren
check('member', 'paren_attr() int f', {1: 'f__i', 2: '1f'})
check('member', 'paren_attr(a) int f', {1: 'f__i', 2: '1f'})
check('member', 'paren_attr("") int f', {1: 'f__i', 2: '1f'})
check('member', 'paren_attr(()[{}][]{}) int f', {1: 'f__i', 2: '1f'})
check('member', 'paren_attr() int f', {1: 'f'})
check('member', 'paren_attr(a) int f', {1: 'f'})
check('member', 'paren_attr("") int f',{1: 'f'})
check('member', 'paren_attr(()[{}][]{}) int f', {1: 'f'})
with pytest.raises(DefinitionError):
parse('member', 'paren_attr(() int f')
with pytest.raises(DefinitionError):
@ -437,18 +431,20 @@ def test_attributes():
# position: decl specs
check('function', 'static inline __attribute__(()) void f()',
{1: 'f', 2: '1fv'},
{1: 'f'},
output='__attribute__(()) static inline void f()')
check('function', '[[attr1]] [[attr2]] void f()',
{1: 'f', 2: '1fv'},
{1: 'f'},
output='[[attr1]] [[attr2]] void f()')
# position: declarator
check('member', 'int *[[attr]] i', {1: 'i__iP', 2: '1i'})
check('member', 'int *const [[attr]] volatile i', {1: 'i__iPVC', 2: '1i'},
check('member', 'int *[[attr]] i', {1: 'i'})
check('member', 'int *const [[attr]] volatile i', {1: 'i'},
output='int *[[attr]] volatile const i')
check('member', 'int &[[attr]] i', {1: 'i__iR', 2: '1i'})
check('member', 'int *[[attr]] *i', {1: 'i__iPP', 2: '1i'})
check('member', 'int *[[attr]] *i', {1: 'i'})
# issue michaeljones/breathe#500
check('function', 'LIGHTGBM_C_EXPORT int LGBM_BoosterFree(int handle)',
{1: 'LGBM_BoosterFree'})
# def test_print():
# # used for getting all the ids out for checking

View File

@ -23,8 +23,7 @@ def parse(name, string):
class Config:
cpp_id_attributes = ["id_attr"]
cpp_paren_attributes = ["paren_attr"]
parser = DefinitionParser(string, location=None,
config=Config())
parser = DefinitionParser(string, location=None, config=Config())
parser.allowFallbackExpressionParsing = False
ast = parser.parse_declaration(name, name)
parser.assert_end()