From 0f49e30c51b5cc5055cda5b4b294c2dd9d1df573 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Thu, 12 Mar 2020 16:59:30 +0100 Subject: [PATCH 01/12] C, initial rewrite --- doc/usage/restructuredtext/domains.rst | 45 +- sphinx/domains/c.py | 2256 ++++++++++++++++++++++-- sphinx/domains/cpp.py | 268 +-- sphinx/util/cfamily.py | 209 +++ tests/roots/test-domain-c/conf.py | 1 + tests/roots/test-domain-c/index.rst | 17 + tests/roots/test-root/objects.txt | 6 +- tests/test_build_changes.py | 2 +- tests/test_build_html.py | 10 +- tests/test_domain_c.py | 536 +++++- 10 files changed, 2881 insertions(+), 469 deletions(-) create mode 100644 sphinx/util/cfamily.py create mode 100644 tests/roots/test-domain-c/conf.py create mode 100644 tests/roots/test-domain-c/index.rst diff --git a/doc/usage/restructuredtext/domains.rst b/doc/usage/restructuredtext/domains.rst index 9c69fe88f..6b51f1135 100644 --- a/doc/usage/restructuredtext/domains.rst +++ b/doc/usage/restructuredtext/domains.rst @@ -556,43 +556,29 @@ The C domain (name **c**) is suited for documentation of C API. Describes a C function. The signature should be given as in C, e.g.:: - .. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - - This is also used to describe function-like preprocessor macros. The names - of the arguments should be given so they may be used in the description. + .. c:function:: PyObject *PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) Note that you don't have to backslash-escape asterisks in the signature, as it is not parsed by the reST inliner. .. rst:directive:: .. c:member:: declaration + .. c:var:: declaration - Describes a C struct member. Example signature:: + Describes a C struct member or variable. Example signature:: - .. c:member:: PyObject* PyTypeObject.tp_bases - - The text of the description should include the range of values allowed, how - the value should be interpreted, and whether the value can be changed. - References to structure members in text should use the ``member`` role. + .. c:member:: PyObject *PyTypeObject.tp_bases .. rst:directive:: .. c:macro:: name + .. c:macro:: name(arg list) - Describes a "simple" C macro. Simple macros are macros which are used for - code expansion, but which do not take arguments so cannot be described as - functions. This is a simple C-language ``#define``. Examples of its use in - the Python documentation include :c:macro:`PyObject_HEAD` and - :c:macro:`Py_BEGIN_ALLOW_THREADS`. + Describes a C macro, i.e., a C-language ``#define``, without the replacement + text. -.. rst:directive:: .. c:type:: name +.. rst:directive:: .. c:type:: typedef-like declaration + .. c:type:: name - Describes a C type (whether defined by a typedef or struct). The signature - should just be the type name. - -.. rst:directive:: .. c:var:: declaration - - Describes a global C variable. The signature should include the type, such - as:: - - .. c:var:: PyObject* PyClass_Type + Describes a C type, either as a typedef, or the alias for an unspecified + type. .. _c-roles: @@ -607,21 +593,18 @@ are defined in the documentation: Reference a C-language function. Should include trailing parentheses. .. rst:role:: c:member + c:data - Reference a C-language member of a struct. + Reference a C-language member of a struct or variable. .. rst:role:: c:macro - Reference a "simple" C macro, as defined above. + Reference a simple C macro, as defined above. .. rst:role:: c:type Reference a C-language type. -.. rst:role:: c:data - - Reference a C-language variable. - .. _cpp-domain: diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 22d6ea82f..021e9d8ba 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -10,11 +10,15 @@ import re import string -from typing import Any, Dict, Iterator, List, Tuple +import warnings +from copy import deepcopy +from typing import ( + Any, Callable, Dict, Iterator, List, Match, Pattern, Tuple, Type, TypeVar, Union +) from typing import cast from docutils import nodes -from docutils.nodes import Element +from docutils.nodes import Element, Node, TextElement from sphinx import addnodes from sphinx.addnodes import pending_xref, desc_signature @@ -26,36 +30,1915 @@ from sphinx.environment import BuildEnvironment from sphinx.locale import _, __ from sphinx.roles import XRefRole from sphinx.util import logging +from sphinx.util.cfamily import ( + NoOldIdError, ASTBase, verify_description_mode, StringifyTransform, + BaseParser, DefinitionError, identifier_re, anon_identifier_re +) from sphinx.util.docfields import Field, TypedField -from sphinx.util.nodes import make_id, make_refnode - +from sphinx.util.nodes import make_refnode logger = logging.getLogger(__name__) -# RE to split at word boundaries -wsplit_re = re.compile(r'(\W+)') +# https://en.cppreference.com/w/c/keyword +_keywords = [ + 'auto', 'break', 'case', 'char', 'const', 'continue', 'default', 'do', 'double', + 'else', 'enum', 'extern', 'float', 'for', 'goto', 'if', 'inline', 'int', 'long', + 'register', 'restrict', 'return', 'short', 'signed', 'sizeof', 'static', 'struct', + 'switch', 'typedef', 'union', 'unsigned', 'void', 'volatile', 'while', + '_Alignas', 'alignas', '_Alignof', 'alignof', '_Atomic', '_Bool', 'bool', + '_Complex', 'complex', '_Generic', '_Imaginary', 'imaginary', + '_Noreturn', 'noreturn', '_Static_assert', 'static_assert', + '_Thread_local', 'thread_local', +] -# REs for C signatures -c_sig_re = re.compile( - r'''^([^(]*?) # return type - ([\w:.]+) \s* # thing name (colon allowed for C++) - (?: \((.*)\) )? # optionally arguments - (\s+const)? $ # const specifier - ''', re.VERBOSE) -c_funcptr_sig_re = re.compile( - r'''^([^(]+?) # return type - (\( [^()]+ \)) \s* # name in parentheses - \( (.*) \) # arguments - (\s+const)? $ # const specifier - ''', re.VERBOSE) -c_funcptr_arg_sig_re = re.compile( - r'''^\s*([^(,]+?) # return type - \( ([^()]+) \) \s* # name in parentheses - \( (.*) \) # arguments - (\s+const)? # const specifier - \s*(?=$|,) # end with comma or end of string - ''', re.VERBOSE) -c_funcptr_name_re = re.compile(r'^\(\s*\*\s*(.*?)\s*\)$') +_max_id = 1 +_id_prefix = [None, 'c.', 'Cv2.'] +# Ids are used in lookup keys which are used across pickled files, +# so when _max_id changes, make sure to update the ENV_VERSION. + +_string_re = re.compile(r"[LuU8]?('([^'\\]*(?:\\.[^'\\]*)*)'" + r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) + + +class _DuplicateSymbolError(Exception): + def __init__(self, symbol: "Symbol", declaration: Any) -> None: + assert symbol + assert declaration + self.symbol = symbol + self.declaration = declaration + + def __str__(self) -> str: + return "Internal C duplicate symbol error:\n%s" % self.symbol.dump(0) + + +############################################################################################## + +class ASTFallbackExpr(ASTBase): + def __init__(self, expr): + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return self.expr + + def get_id(self, version: int) -> str: + return str(self.expr) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode += nodes.Text(self.expr) + + +################################################################################ +# The Rest +################################################################################ + +class ASTIdentifier(ASTBase): + def __init__(self, identifier: str) -> None: + assert identifier is not None + assert len(identifier) != 0 + self.identifier = identifier + + def is_anon(self) -> bool: + return self.identifier[0] == '@' + + def get_id(self, version: int) -> str: + assert False + if self.is_anon() and version < 3: + raise NoOldIdError() + if version == 1: + if self.identifier == 'size_t': + return 's' + else: + return self.identifier + if self.identifier == "std": + return 'St' + elif self.identifier[0] == "~": + # a destructor, just use an arbitrary version of dtors + return 'D0' + else: + if self.is_anon(): + return 'Ut%d_%s' % (len(self.identifier) - 1, self.identifier[1:]) + else: + return str(len(self.identifier)) + self.identifier + + # and this is where we finally make a difference between __str__ and the display string + + def __str__(self) -> str: + return self.identifier + + def get_display_string(self) -> str: + return "[anonymous]" if self.is_anon() else self.identifier + + def describe_signature(self, signode: Any, mode: str, env: "BuildEnvironment", + prefix: str, symbol: "Symbol") -> None: + verify_description_mode(mode) + if mode == 'markType': + targetText = prefix + self.identifier + pnode = addnodes.pending_xref('', refdomain='c', + reftype='identifier', + reftarget=targetText, modname=None, + classname=None) + # key = symbol.get_lookup_key() + # pnode['c:parent_key'] = key + if self.is_anon(): + pnode += nodes.strong(text="[anonymous]") + else: + pnode += nodes.Text(self.identifier) + signode += pnode + elif mode == 'lastIsName': + if self.is_anon(): + signode += nodes.strong(text="[anonymous]") + else: + signode += addnodes.desc_name(self.identifier, self.identifier) + elif mode == 'noneIsName': + if self.is_anon(): + signode += nodes.strong(text="[anonymous]") + else: + signode += nodes.Text(self.identifier) + else: + raise Exception('Unknown description mode: %s' % mode) + + +class ASTNestedName(ASTBase): + def __init__(self, names: List[ASTIdentifier], rooted: bool) -> None: + assert len(names) > 0 + self.names = names + self.rooted = rooted + + @property + def name(self) -> "ASTNestedName": + return self + + def get_id(self, version: int) -> str: + return '.'.join(str(n) for n in self.names) + + def _stringify(self, transform: StringifyTransform) -> str: + res = '.'.join(transform(n) for n in self.names) + if self.rooted: + return '.' + res + else: + return res + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + # just print the name part, with template args, not template params + if mode == 'noneIsName': + signode += nodes.Text(str(self)) + elif mode == 'param': + name = str(self) + signode += nodes.emphasis(name, name) + elif mode == 'markType' or mode == 'lastIsName' or mode == 'markName': + # Each element should be a pending xref targeting the complete + # prefix. + prefix = '' + first = True + names = self.names[:-1] if mode == 'lastIsName' else self.names + # If lastIsName, then wrap all of the prefix in a desc_addname, + # else append directly to signode. + # TODO: also for C? + # NOTE: Breathe relies on the prefix being in the desc_addname node, + # so it can remove it in inner declarations. + dest = signode + if mode == 'lastIsName': + dest = addnodes.desc_addname() # type: ignore + for i in range(len(names)): + ident = names[i] + if not first: + dest += nodes.Text('.') + prefix += '.' + first = False + txt_ident = str(ident) + if txt_ident != '': + ident.describe_signature(dest, 'markType', env, prefix, symbol) + prefix += txt_ident + if mode == 'lastIsName': + if len(self.names) > 1: + dest += addnodes.desc_addname('.', '.') + signode += dest + self.names[-1].describe_signature(signode, mode, env, '', symbol) + else: + raise Exception('Unknown description mode: %s' % mode) + + +class ASTTrailingTypeSpecFundamental(ASTBase): + def __init__(self, name: str) -> None: + self.name = name + + def _stringify(self, transform: StringifyTransform) -> str: + return self.name + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode += nodes.Text(str(self.name)) + + +class ASTTrailingTypeSpecName(ASTBase): + def __init__(self, prefix: str, nestedName: Any) -> None: + self.prefix = prefix + self.nestedName = nestedName + + @property + def name(self) -> ASTNestedName: + return self.nestedName + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + if self.prefix: + res.append(self.prefix) + res.append(' ') + res.append(transform(self.nestedName)) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + if self.prefix: + signode += addnodes.desc_annotation(self.prefix, self.prefix) + signode += nodes.Text(' ') + self.nestedName.describe_signature(signode, mode, env, symbol=symbol) + + +class ASTFunctionParameter(ASTBase): + def __init__(self, arg: Any, ellipsis: bool = False) -> None: + self.arg = arg + self.ellipsis = ellipsis + + def _stringify(self, transform: StringifyTransform) -> str: + if self.ellipsis: + return '...' + else: + return transform(self.arg) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + if self.ellipsis: + signode += nodes.Text('...') + else: + self.arg.describe_signature(signode, mode, env, symbol=symbol) + + +class ASTParameters(ASTBase): + def __init__(self, args: List[Any]) -> None: + self.args = args + + @property + def function_params(self) -> Any: + return self.args + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + res.append('(') + first = True + for a in self.args: + if not first: + res.append(', ') + first = False + res.append(str(a)) + res.append(')') + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + paramlist = addnodes.desc_parameterlist() + for arg in self.args: + param = addnodes.desc_parameter('', '', noemph=True) + if mode == 'lastIsName': # i.e., outer-function params + arg.describe_signature(param, 'param', env, symbol=symbol) + else: + arg.describe_signature(param, 'markType', env, symbol=symbol) + paramlist += param + signode += paramlist + + +class ASTDeclSpecsSimple(ASTBase): + def __init__(self, storage: str, threadLocal: str, inline: bool, + restrict: bool, volatile: bool, const: bool, attrs: List[Any]) -> None: + self.storage = storage + self.threadLocal = threadLocal + self.inline = inline + self.restrict = restrict + self.volatile = volatile + self.const = const + self.attrs = attrs + + def mergeWith(self, other: "ASTDeclSpecsSimple") -> "ASTDeclSpecsSimple": + if not other: + return self + return ASTDeclSpecsSimple(self.storage or other.storage, + self.threadLocal or other.threadLocal, + self.inline or other.inline, + self.volatile or other.volatile, + self.const or other.const, + self.restrict or other.restrict, + self.attrs + other.attrs) + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] # type: List[str] + res.extend(transform(attr) for attr in self.attrs) + if self.storage: + res.append(self.storage) + if self.threadLocal: + res.append(self.threadLocal) + if self.inline: + res.append('inline') + if self.restrict: + res.append('restrict') + if self.volatile: + res.append('volatile') + if self.const: + res.append('const') + return ' '.join(res) + + def describe_signature(self, modifiers: List[Node]) -> None: + def _add(modifiers: List[Node], text: str) -> None: + if len(modifiers) > 0: + modifiers.append(nodes.Text(' ')) + # TODO: should probably do + # modifiers.append(addnodes.desc_annotation(text, text)) + # but for now emulate the old output: + modifiers.append(nodes.Text(text)) + + for attr in self.attrs: + if len(modifiers) > 0: + modifiers.append(nodes.Text(' ')) + modifiers.append(attr.describe_signature(modifiers)) + if self.storage: + _add(modifiers, self.storage) + if self.threadLocal: + _add(modifiers, self.threadLocal) + if self.inline: + _add(modifiers, 'inline') + if self.restrict: + _add(modifiers, 'restrict') + if self.volatile: + _add(modifiers, 'volatile') + if self.const: + _add(modifiers, 'const') + + +class ASTDeclSpecs(ASTBase): + def __init__(self, outer: Any, leftSpecs: ASTDeclSpecsSimple, + rightSpecs: ASTDeclSpecsSimple, trailing: Any) -> None: + # leftSpecs and rightSpecs are used for output + # allSpecs are used for id generation TODO: remove? + self.outer = outer + self.leftSpecs = leftSpecs + self.rightSpecs = rightSpecs + self.allSpecs = self.leftSpecs.mergeWith(self.rightSpecs) + self.trailingTypeSpec = trailing + + @property + def name(self) -> ASTNestedName: + return self.trailingTypeSpec.name + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] # type: List[str] + l = transform(self.leftSpecs) + if len(l) > 0: + res.append(l) + if self.trailingTypeSpec: + if len(res) > 0: + res.append(" ") + res.append(transform(self.trailingTypeSpec)) + r = str(self.rightSpecs) + if len(r) > 0: + if len(res) > 0: + res.append(" ") + res.append(r) + return "".join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + modifiers = [] # type: List[Node] + + def _add(modifiers: List[Node], text: str) -> None: + if len(modifiers) > 0: + modifiers.append(nodes.Text(' ')) + modifiers.append(addnodes.desc_annotation(text, text)) + + self.leftSpecs.describe_signature(modifiers) + + for m in modifiers: + signode += m + if self.trailingTypeSpec: + if len(modifiers) > 0: + signode += nodes.Text(' ') + self.trailingTypeSpec.describe_signature(signode, mode, env, + symbol=symbol) + modifiers = [] + self.rightSpecs.describe_signature(modifiers) + if len(modifiers) > 0: + signode += nodes.Text(' ') + for m in modifiers: + signode += m + + +class ASTArray(ASTBase): + def __init__(self, size): + self.size = size + + def _stringify(self, transform: StringifyTransform) -> str: + if self.size: + return '[' + transform(self.size) + ']' + else: + return '[]' + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + signode.append(nodes.Text("[")) + if self.size: + self.size.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text("]")) + + +class ASTDeclaratorPtr(ASTBase): + def __init__(self, next: Any, restrict: bool, volatile: bool, const: bool, + attrs: Any) -> None: + assert next + self.next = next + self.restrict = restrict + self.volatile = volatile + self.const = const + self.attrs = attrs + + @property + def name(self) -> ASTNestedName: + return self.next.name + + @property + def function_params(self) -> Any: + return self.next.function_params + + def require_space_after_declSpecs(self) -> bool: + return True + + def _stringify(self, transform: StringifyTransform) -> str: + res = ['*'] + for a in self.attrs: + res.append(transform(a)) + if len(self.attrs) > 0 and (self.restrict or self.volatile or self.const): + res.append(' ') + if self.restrict: + res.append('restrict') + if self.volatile: + if self.restrict: + res.append(' ') + res.append('volatile') + if self.const: + if self.restrict or self.volatile: + res.append(' ') + res.append('const') + if self.const or self.volatile or self.restrict or len(self.attrs) > 0: + if self.next.require_space_after_declSpecs(): + res.append(' ') + res.append(transform(self.next)) + return ''.join(res) + + def is_function_type(self) -> bool: + assert False + return self.next.is_function_type() + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + signode += nodes.Text("*") + for a in self.attrs: + a.describe_signature(signode) + if len(self.attrs) > 0 and (self.restrict or self.volatile or self.const): + signode += nodes.Text(' ') + + def _add_anno(signode: desc_signature, text: str) -> None: + signode += addnodes.desc_annotation(text, text) + + if self.restrict: + _add_anno(signode, 'restrict') + if self.volatile: + if self.restrict: + signode += nodes.Text(' ') + _add_anno(signode, 'volatile') + if self.const: + if self.restrict or self.volatile: + signode += nodes.Text(' ') + _add_anno(signode, 'const') + if self.const or self.volatile or self.restrict or len(self.attrs) > 0: + if self.next.require_space_after_declSpecs(): + signode += nodes.Text(' ') + self.next.describe_signature(signode, mode, env, symbol) + + +class ASTDeclaratorParen(ASTBase): + def __init__(self, inner: Any, next: Any) -> None: + assert inner + assert next + self.inner = inner + self.next = next + # TODO: we assume the name and params are in inner + + @property + def name(self) -> ASTNestedName: + return self.inner.name + + @property + def function_params(self) -> Any: + return self.inner.function_params + + def require_space_after_declSpecs(self) -> bool: + return True + + def _stringify(self, transform: StringifyTransform) -> str: + res = ['('] + res.append(transform(self.inner)) + res.append(')') + res.append(transform(self.next)) + return ''.join(res) + + def is_function_type(self) -> bool: + return self.inner.is_function_type() + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + signode += nodes.Text('(') + self.inner.describe_signature(signode, mode, env, symbol) + signode += nodes.Text(')') + self.next.describe_signature(signode, "noneIsName", env, symbol) + + +class ASTDeclaratorNameParam(ASTBase): + def __init__(self, declId: Any, arrayOps: List[Any], param: Any, prefix: str) -> None: + self.declId = declId + self.arrayOps = arrayOps + self.param = param + self.prefix = prefix + + @property + def name(self) -> ASTNestedName: + return self.declId + + @property + def function_params(self) -> Any: + return self.param.function_params + + # ------------------------------------------------------------------------ + + def require_space_after_declSpecs(self) -> bool: + return self.declId is not None + + def is_function_type(self) -> bool: + return self.param is not None + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + if self.prefix is not None: + res.append(self.prefix) + res.append(' ') + assert self.declId + if self.declId: + res.append(transform(self.declId)) + for op in self.arrayOps: + res.append(transform(op)) + if self.param: + res.append(transform(self.param)) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + if self.prefix: + signode += addnodes.desc_annotation(self.prefix, self.prefix) + signode += nodes.Text(' ') + if self.declId: + self.declId.describe_signature(signode, mode, env, symbol) + for op in self.arrayOps: + op.describe_signature(signode, mode, env, symbol) + if self.param: + self.param.describe_signature(signode, mode, env, symbol) + + +class ASTDeclaratorNameBitField(ASTBase): + def __init__(self, declId, size): + self.declId = declId + self.size = size + + @property + def name(self) -> ASTNestedName: + return self.declId + + # ------------------------------------------------------------------------ + + def require_space_after_declSpecs(self) -> bool: + return self.declId is not None + + def is_function_type(self) -> bool: + return False + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + if self.declId: + res.append(transform(self.declId)) + res.append(" : ") + # TODO: when size is properly parsed + # res.append(transform(self.size)) + res.append(self.size) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + if self.declId: + self.declId.describe_signature(signode, mode, env, symbol) + signode += nodes.Text(' : ', ' : ') + # TODO: when size is properly parsed + # self.size.describe_signature(signode, mode, env, symbol) + signode += nodes.Text(self.size, self.size) + + +class ASTInitializer(ASTBase): + def __init__(self, value: Any, hasAssign: bool = True) -> None: + self.value = value + self.hasAssign = hasAssign + + def _stringify(self, transform: StringifyTransform) -> str: + val = transform(self.value) + if self.hasAssign: + return ' = ' + val + else: + return val + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + if self.hasAssign: + signode.append(nodes.Text(' = ')) + self.value.describe_signature(signode, 'markType', env, symbol) + + +class ASTType(ASTBase): + def __init__(self, declSpecs: Any, decl: Any) -> None: + assert declSpecs + assert decl + self.declSpecs = declSpecs + self.decl = decl + + @property + def name(self) -> ASTNestedName: + return self.decl.name + + @property + def function_params(self) -> Any: + return self.decl.function_params + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + declSpecs = transform(self.declSpecs) + res.append(declSpecs) + if self.decl.require_space_after_declSpecs() and len(declSpecs) > 0: + res.append(' ') + res.append(transform(self.decl)) + return ''.join(res) + + def get_type_declaration_prefix(self) -> str: + assert False + if self.declSpecs.trailingTypeSpec: + return 'typedef' + else: + return 'type' + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + self.declSpecs.describe_signature(signode, 'markType', env, symbol) + if (self.decl.require_space_after_declSpecs() and + len(str(self.declSpecs)) > 0): + signode += nodes.Text(' ') + # for parameters that don't really declare new names we get 'markType', + # this should not be propagated, but be 'noneIsName'. + if mode == 'markType': + mode = 'noneIsName' + self.decl.describe_signature(signode, mode, env, symbol) + + +class ASTTypeWithInit(ASTBase): + def __init__(self, type: Any, init: Any) -> None: + self.type = type + self.init = init + + @property + def name(self) -> ASTNestedName: + return self.type.name + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + res.append(transform(self.type)) + if self.init: + res.append(transform(self.init)) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + self.type.describe_signature(signode, mode, env, symbol) + if self.init: + self.init.describe_signature(signode, mode, env, symbol) + + +class ASTMacro(ASTBase): + def __init__(self, ident: ASTNestedName, args: List[ASTFunctionParameter]) -> None: + self.ident = ident + self.args = args + + @property + def name(self) -> ASTNestedName: + return self.ident + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + res.append(transform(self.ident)) + if self.args is not None: + res.append('(') + first = True + for arg in self.args: + if not first: + res.append(', ') + first = False + res.append(transform(arg)) + res.append(')') + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + self.ident.describe_signature(signode, mode, env, symbol) + if self.args is None: + return + paramlist = addnodes.desc_parameterlist() + for arg in self.args: + param = addnodes.desc_parameter('', '', noemph=True) + arg.describe_signature(param, 'param', env, symbol=symbol) + paramlist += param + signode += paramlist + + +class ASTDeclaration(ASTBase): + def __init__(self, objectType: str, directiveType: str, declaration: Any) -> None: + self.objectType = objectType + self.directiveType = directiveType + self.declaration = declaration + + self.symbol = None # type: Symbol + # set by CPPObject._add_enumerator_to_parent + # self.enumeratorScopedSymbol = None # type: Symbol + + @property + def name(self) -> ASTNestedName: + return self.declaration.name + + @property + def function_params(self) -> Any: + if self.objectType != 'function': + return None + return self.declaration.function_params + + def get_id(self, version: int, prefixed: bool = True) -> str: + id_ = self.symbol.get_full_nested_name().get_id(version) + if prefixed: + return _id_prefix[version] + id_ + else: + return id_ + + def get_newest_id(self) -> str: + return self.get_id(_max_id, True) + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.declaration) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", options: Dict) -> None: + verify_description_mode(mode) + assert self.symbol + self.declaration.describe_signature(signode, mode, env, self.symbol) + + +class SymbolLookupResult: + def __init__(self, symbols: Iterator["Symbol"], parentSymbol: "Symbol", + ident: ASTIdentifier) -> None: + self.symbols = symbols + self.parentSymbol = parentSymbol + self.ident = ident + + +class LookupKey: + def __init__(self, data: List[Tuple[ASTIdentifier, str]]) -> None: + self.data = data + + +class Symbol: + debug_indent = 0 + debug_indent_string = " " + debug_lookup = False + debug_show_tree = False + + @staticmethod + def debug_print(*args: Any) -> None: + print(Symbol.debug_indent_string * Symbol.debug_indent, end="") + print(*args) + + def _assert_invariants(self) -> None: + if not self.parent: + # parent == None means global scope, so declaration means a parent + assert not self.declaration + assert not self.docname + else: + if self.declaration: + assert self.docname + + def __setattr__(self, key, value): + if key == "children": + assert False + else: + return super().__setattr__(key, value) + + def __init__(self, parent: "Symbol", ident: ASTIdentifier, + declaration: ASTDeclaration, docname: str) -> None: + self.parent = parent + # declarations in a single directive are linked together + self.siblingAbove = None # type: Symbol + self.siblingBelow = None # type: Symbol + self.ident = ident + self.declaration = declaration + self.docname = docname + self.isRedeclaration = False + self._assert_invariants() + + # Remember to modify Symbol.remove if modifications to the parent change. + self._children = [] # type: List[Symbol] + self._anonChildren = [] # type: List[Symbol] + # note: _children includes _anonChildren + if self.parent: + self.parent._children.append(self) + if self.declaration: + self.declaration.symbol = self + + # Do symbol addition after self._children has been initialised. + self._add_function_params() + + def _fill_empty(self, declaration: ASTDeclaration, docname: str) -> None: + self._assert_invariants() + assert not self.declaration + assert not self.docname + assert declaration + assert docname + self.declaration = declaration + self.declaration.symbol = self + self.docname = docname + self._assert_invariants() + # and symbol addition should be done as well + self._add_function_params() + + def _add_function_params(self) -> None: + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("_add_function_params:") + # Note: we may be called from _fill_empty, so the symbols we want + # to add may actually already be present (as empty symbols). + + # add symbols for function parameters, if any + if self.declaration is not None and self.declaration.function_params is not None: + for p in self.declaration.function_params: + if p.arg is None: + continue + nn = p.arg.name + if nn is None: + continue + # (comparing to the template params: we have checked that we are a declaration) + decl = ASTDeclaration('functionParam', None, p) + assert not nn.rooted + assert len(nn.names) == 1 + self._add_symbols(nn, decl, self.docname) + if Symbol.debug_lookup: + Symbol.debug_indent -= 1 + + def remove(self) -> None: + if self.parent is None: + return + assert self in self.parent._children + self.parent._children.remove(self) + self.parent = None + + def clear_doc(self, docname: str) -> None: + newChildren = [] # type: List[Symbol] + for sChild in self._children: + sChild.clear_doc(docname) + if sChild.declaration and sChild.docname == docname: + sChild.declaration = None + sChild.docname = None + if sChild.siblingAbove is not None: + sChild.siblingAbove.siblingBelow = sChild.siblingBelow + if sChild.siblingBelow is not None: + sChild.siblingBelow.siblingAbove = sChild.siblingAbove + sChild.siblingAbove = None + sChild.siblingBelow = None + newChildren.append(sChild) + self._children = newChildren + + def get_all_symbols(self) -> Iterator[Any]: + yield self + for sChild in self._children: + for s in sChild.get_all_symbols(): + yield s + + @property + def children_recurse_anon(self): + for c in self._children: + yield c + if not c.ident.is_anon(): + continue + yield from c.children_recurse_anon + + def get_lookup_key(self) -> "LookupKey": + # The pickle files for the environment and for each document are distinct. + # The environment has all the symbols, but the documents has xrefs that + # must know their scope. A lookup key is essentially a specification of + # how to find a specific symbol. + symbols = [] + s = self + while s.parent: + symbols.append(s) + s = s.parent + symbols.reverse() + key = [] + for s in symbols: + if s.declaration is not None: + # TODO: do we need the ID? + key.append((s.ident, s.declaration.get_newest_id())) + else: + key.append((s.ident, None)) + return LookupKey(key) + + def get_full_nested_name(self) -> ASTNestedName: + symbols = [] + s = self + while s.parent: + symbols.append(s) + s = s.parent + symbols.reverse() + names = [] + for s in symbols: + names.append(s.ident) + return ASTNestedName(names, rooted=False) + + def _find_first_named_symbol(self, ident: ASTIdentifier, + matchSelf: bool, recurseInAnon: bool) -> "Symbol": + # TODO: further simplification from C++ to C + if Symbol.debug_lookup: + Symbol.debug_print("_find_first_named_symbol ->") + res = self._find_named_symbols(ident, matchSelf, recurseInAnon, + searchInSiblings=False) + try: + return next(res) + except StopIteration: + return None + + def _find_named_symbols(self, ident: ASTIdentifier, + matchSelf: bool, recurseInAnon: bool, + searchInSiblings: bool) -> Iterator["Symbol"]: + # TODO: further simplification from C++ to C + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("_find_named_symbols:") + Symbol.debug_indent += 1 + Symbol.debug_print("self:") + print(self.to_string(Symbol.debug_indent + 1), end="") + Symbol.debug_print("ident: ", ident) + Symbol.debug_print("matchSelf: ", matchSelf) + Symbol.debug_print("recurseInAnon: ", recurseInAnon) + Symbol.debug_print("searchInSiblings: ", searchInSiblings) + + def candidates(): + s = self + if Symbol.debug_lookup: + Symbol.debug_print("searching in self:") + print(s.to_string(Symbol.debug_indent + 1), end="") + while True: + if matchSelf: + yield s + if recurseInAnon: + yield from s.children_recurse_anon + else: + yield from s._children + + if s.siblingAbove is None: + break + s = s.siblingAbove + if Symbol.debug_lookup: + Symbol.debug_print("searching in sibling:") + print(s.to_string(Symbol.debug_indent + 1), end="") + + for s in candidates(): + if Symbol.debug_lookup: + Symbol.debug_print("candidate:") + print(s.to_string(Symbol.debug_indent + 1), end="") + if s.ident == ident: + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("matches") + Symbol.debug_indent -= 3 + yield s + if Symbol.debug_lookup: + Symbol.debug_indent += 2 + if Symbol.debug_lookup: + Symbol.debug_indent -= 2 + + def _symbol_lookup(self, nestedName: ASTNestedName, + onMissingQualifiedSymbol: Callable[["Symbol", ASTIdentifier, Any], "Symbol"], # NOQA + ancestorLookupType: str, matchSelf: bool, + recurseInAnon: bool, searchInSiblings: bool) -> SymbolLookupResult: + # TODO: further simplification from C++ to C + # ancestorLookupType: if not None, specifies the target type of the lookup + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("_symbol_lookup:") + Symbol.debug_indent += 1 + Symbol.debug_print("self:") + print(self.to_string(Symbol.debug_indent + 1), end="") + Symbol.debug_print("nestedName: ", nestedName) + Symbol.debug_print("ancestorLookupType:", ancestorLookupType) + Symbol.debug_print("matchSelf: ", matchSelf) + Symbol.debug_print("recurseInAnon: ", recurseInAnon) + Symbol.debug_print("searchInSiblings: ", searchInSiblings) + + names = nestedName.names + + # find the right starting point for lookup + parentSymbol = self + if nestedName.rooted: + while parentSymbol.parent: + parentSymbol = parentSymbol.parent + if ancestorLookupType is not None: + # walk up until we find the first identifier + firstName = names[0] + while parentSymbol.parent: + if parentSymbol.find_identifier(firstName.ident, + matchSelf=matchSelf, + recurseInAnon=recurseInAnon, + searchInSiblings=searchInSiblings): + # if we are in the scope of a constructor but wants to + # reference the class we need to walk one extra up + if (len(names) == 1 and ancestorLookupType == 'class' and matchSelf and + parentSymbol.parent and + parentSymbol.parent.ident == firstName.ident): + pass + else: + break + parentSymbol = parentSymbol.parent + + if Symbol.debug_lookup: + Symbol.debug_print("starting point:") + print(parentSymbol.to_string(Symbol.debug_indent + 1), end="") + + # and now the actual lookup + for ident in names[:-1]: + symbol = parentSymbol._find_first_named_symbol( + ident, matchSelf=matchSelf, recurseInAnon=recurseInAnon) + if symbol is None: + symbol = onMissingQualifiedSymbol(parentSymbol, ident) + if symbol is None: + if Symbol.debug_lookup: + Symbol.debug_indent -= 2 + return None + # We have now matched part of a nested name, and need to match more + # so even if we should matchSelf before, we definitely shouldn't + # even more. (see also issue #2666) + matchSelf = False + parentSymbol = symbol + + if Symbol.debug_lookup: + Symbol.debug_print("handle last name from:") + print(parentSymbol.to_string(Symbol.debug_indent + 1), end="") + + # handle the last name + ident = names[-1] + + symbols = parentSymbol._find_named_symbols( + ident, matchSelf=matchSelf, + recurseInAnon=recurseInAnon, + searchInSiblings=searchInSiblings) + if Symbol.debug_lookup: + symbols = list(symbols) # type: ignore + Symbol.debug_indent -= 2 + return SymbolLookupResult(symbols, parentSymbol, ident) + + def _add_symbols(self, nestedName: ASTNestedName, + declaration: ASTDeclaration, docname: str) -> "Symbol": + # TODO: further simplification from C++ to C + # Used for adding a whole path of symbols, where the last may or may not + # be an actual declaration. + + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("_add_symbols:") + Symbol.debug_indent += 1 + Symbol.debug_print("nn: ", nestedName) + Symbol.debug_print("decl: ", declaration) + Symbol.debug_print("doc: ", docname) + + def onMissingQualifiedSymbol(parentSymbol: "Symbol", ident: ASTIdentifier) -> "Symbol": + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("_add_symbols, onMissingQualifiedSymbol:") + Symbol.debug_indent += 1 + Symbol.debug_print("ident: ", ident) + Symbol.debug_indent -= 2 + return Symbol(parent=parentSymbol, ident=ident, + declaration=None, docname=None) + + lookupResult = self._symbol_lookup(nestedName, + onMissingQualifiedSymbol, + ancestorLookupType=None, + matchSelf=False, + recurseInAnon=True, + searchInSiblings=False) + assert lookupResult is not None # we create symbols all the way, so that can't happen + symbols = list(lookupResult.symbols) + if len(symbols) == 0: + if Symbol.debug_lookup: + Symbol.debug_print("_add_symbols, result, no symbol:") + Symbol.debug_indent += 1 + Symbol.debug_print("ident: ", lookupResult.ident) + Symbol.debug_print("declaration: ", declaration) + Symbol.debug_print("docname: ", docname) + Symbol.debug_indent -= 1 + symbol = Symbol(parent=lookupResult.parentSymbol, + ident=lookupResult.ident, + declaration=declaration, + docname=docname) + if Symbol.debug_lookup: + Symbol.debug_indent -= 2 + return symbol + + if Symbol.debug_lookup: + Symbol.debug_print("_add_symbols, result, symbols:") + Symbol.debug_indent += 1 + Symbol.debug_print("number symbols:", len(symbols)) + Symbol.debug_indent -= 1 + + if not declaration: + if Symbol.debug_lookup: + Symbol.debug_print("no delcaration") + Symbol.debug_indent -= 2 + # good, just a scope creation + # TODO: what if we have more than one symbol? + return symbols[0] + + noDecl = [] + withDecl = [] + dupDecl = [] + for s in symbols: + if s.declaration is None: + noDecl.append(s) + elif s.isRedeclaration: + dupDecl.append(s) + else: + withDecl.append(s) + if Symbol.debug_lookup: + Symbol.debug_print("#noDecl: ", len(noDecl)) + Symbol.debug_print("#withDecl:", len(withDecl)) + Symbol.debug_print("#dupDecl: ", len(dupDecl)) + # With partial builds we may start with a large symbol tree stripped of declarations. + # Essentially any combination of noDecl, withDecl, and dupDecls seems possible. + # TODO: make partial builds fully work. What should happen when the primary symbol gets + # deleted, and other duplicates exist? The full document should probably be rebuild. + + # First check if one of those with a declaration matches. + # If it's a function, we need to compare IDs, + # otherwise there should be only one symbol with a declaration. + def makeCandSymbol(): + if Symbol.debug_lookup: + Symbol.debug_print("begin: creating candidate symbol") + symbol = Symbol(parent=lookupResult.parentSymbol, + ident=lookupResult.ident, + declaration=declaration, + docname=docname) + if Symbol.debug_lookup: + Symbol.debug_print("end: creating candidate symbol") + return symbol + if len(withDecl) == 0: + candSymbol = None + else: + candSymbol = makeCandSymbol() + + def handleDuplicateDeclaration(symbol, candSymbol): + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("redeclaration") + Symbol.debug_indent -= 1 + Symbol.debug_indent -= 2 + # Redeclaration of the same symbol. + # Let the new one be there, but raise an error to the client + # so it can use the real symbol as subscope. + # This will probably result in a duplicate id warning. + candSymbol.isRedeclaration = True + raise _DuplicateSymbolError(symbol, declaration) + + if declaration.objectType != "function": + assert len(withDecl) <= 1 + handleDuplicateDeclaration(withDecl[0], candSymbol) + # (not reachable) + + # a function, so compare IDs + candId = declaration.get_newest_id() + if Symbol.debug_lookup: + Symbol.debug_print("candId:", candId) + for symbol in withDecl: + oldId = symbol.declaration.get_newest_id() + if Symbol.debug_lookup: + Symbol.debug_print("oldId: ", oldId) + if candId == oldId: + handleDuplicateDeclaration(symbol, candSymbol) + # (not reachable) + # no candidate symbol found with matching ID + # if there is an empty symbol, fill that one + if len(noDecl) == 0: + if Symbol.debug_lookup: + Symbol.debug_print("no match, no empty, candSybmol is not None?:", candSymbol is not None) # NOQA + Symbol.debug_indent -= 2 + if candSymbol is not None: + return candSymbol + else: + return makeCandSymbol() + else: + if Symbol.debug_lookup: + Symbol.debug_print("no match, but fill an empty declaration, candSybmol is not None?:", candSymbol is not None) # NOQA + Symbol.debug_indent -= 2 + if candSymbol is not None: + candSymbol.remove() + # assert len(noDecl) == 1 + # TODO: enable assertion when we at some point find out how to do cleanup + # for now, just take the first one, it should work fine ... right? + symbol = noDecl[0] + # If someone first opened the scope, and then later + # declares it, e.g, + # .. namespace:: Test + # .. namespace:: nullptr + # .. class:: Test + symbol._fill_empty(declaration, docname) + return symbol + + def merge_with(self, other: "Symbol", docnames: List[str], + env: "BuildEnvironment") -> None: + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("merge_with:") + assert other is not None + for otherChild in other._children: + ourChild = self._find_first_named_symbol( + ident=otherChild.ident, matchSelf=False, + recurseInAnon=False) + if ourChild is None: + # TODO: hmm, should we prune by docnames? + self._children.append(otherChild) + otherChild.parent = self + otherChild._assert_invariants() + continue + if otherChild.declaration and otherChild.docname in docnames: + if not ourChild.declaration: + ourChild._fill_empty(otherChild.declaration, otherChild.docname) + elif ourChild.docname != otherChild.docname: + name = str(ourChild.declaration) + msg = __("Duplicate declaration, also defined in '%s'.\n" + "Declaration is '%s'.") + msg = msg % (ourChild.docname, name) + logger.warning(msg, location=otherChild.docname) + else: + # Both have declarations, and in the same docname. + # This can apparently happen, it should be safe to + # just ignore it, right? + pass + ourChild.merge_with(otherChild, docnames, env) + if Symbol.debug_lookup: + Symbol.debug_indent -= 1 + + def add_name(self, nestedName: ASTNestedName) -> "Symbol": + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("add_name:") + res = self._add_symbols(nestedName, declaration=None, docname=None) + if Symbol.debug_lookup: + Symbol.debug_indent -= 1 + return res + + def add_declaration(self, declaration: ASTDeclaration, docname: str) -> "Symbol": + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("add_declaration:") + assert declaration + assert docname + nestedName = declaration.name + res = self._add_symbols(nestedName, declaration, docname) + if Symbol.debug_lookup: + Symbol.debug_indent -= 1 + return res + + def to_string(self, indent: int) -> str: + res = [Symbol.debug_indent_string * indent] + if not self.parent: + res.append('::') + else: + if self.ident: + res.append(str(self.ident)) + else: + res.append(str(self.declaration)) + if self.declaration: + res.append(": ") + if self.isRedeclaration: + res.append('!!duplicate!! ') + res.append(str(self.declaration)) + if self.docname: + res.append('\t(') + res.append(self.docname) + res.append(')') + res.append('\n') + return ''.join(res) + + def dump(self, indent: int) -> str: + res = [self.to_string(indent)] + for c in self._children: + res.append(c.dump(indent + 1)) + return ''.join(res) + + +class DefinitionParser(BaseParser): + # those without signedness and size modifiers + # see https://en.cppreference.com/w/cpp/language/types + _simple_fundemental_types = ( + 'void', '_Bool', 'bool', 'char', 'int', 'float', 'double', + ) + + _prefix_keys = ('struct', 'enum', 'union') + + def _parse_attribute(self) -> Any: + return None + self.skip_ws() + # try C++11 style + startPos = self.pos + if self.skip_string_and_ws('['): + if not self.skip_string('['): + self.pos = startPos + else: + # TODO: actually implement the correct grammar + arg = self._parse_balanced_token_seq(end=[']']) + if not self.skip_string_and_ws(']'): + self.fail("Expected ']' in end of attribute.") + if not self.skip_string_and_ws(']'): + self.fail("Expected ']' in end of attribute after [[...]") + return ASTCPPAttribute(arg) + + # try GNU style + if self.skip_word_and_ws('__attribute__'): + if not self.skip_string_and_ws('('): + self.fail("Expected '(' after '__attribute__'.") + if not self.skip_string_and_ws('('): + self.fail("Expected '(' after '__attribute__('.") + attrs = [] + while 1: + if self.match(identifier_re): + name = self.matched_text + self.skip_ws() + if self.skip_string_and_ws('('): + self.fail('Parameterized GNU style attribute not yet supported.') + attrs.append(ASTGnuAttribute(name, None)) + # TODO: parse arguments for the attribute + if self.skip_string_and_ws(','): + continue + elif self.skip_string_and_ws(')'): + break + else: + self.fail("Expected identifier, ')', or ',' in __attribute__.") + if not self.skip_string_and_ws(')'): + self.fail("Expected ')' after '__attribute__((...)'") + return ASTGnuAttributeList(attrs) + + # try the simple id attributes defined by the user + for id in self.config.cpp_id_attributes: + if self.skip_word_and_ws(id): + return ASTIdAttribute(id) + + # try the paren attributes defined by the user + for id in self.config.cpp_paren_attributes: + if not self.skip_string_and_ws(id): + continue + if not self.skip_string('('): + self.fail("Expected '(' after user-defined paren-attribute.") + arg = self._parse_balanced_token_seq(end=[')']) + if not self.skip_string(')'): + self.fail("Expected ')' to end user-defined paren-attribute.") + return ASTParenAttribute(id, arg) + + return None + + + def _parse_expression_fallback(self, end, parser, allow=True): + # Stupidly "parse" an expression. + # 'end' should be a list of characters which ends the expression. + + # first try to use the provided parser + # TODO: copy-paste and adapt real expression parser + # prevPos = self.pos + # try: + # return parser() + # except DefinitionError as e: + # # some places (e.g., template parameters) we really don't want to use fallback, + # # and for testing we may want to globally disable it + # if not allow or not self.allowFallbackExpressionParsing: + # raise + # self.warn("Parsing of expression failed. Using fallback parser." + # " Error was:\n%s" % e) + # self.pos = prevPos + # and then the fallback scanning + assert end is not None + self.skip_ws() + startPos = self.pos + if self.match(_string_re): + value = self.matched_text + else: + # TODO: add handling of more bracket-like things, and quote handling + brackets = {'(': ')', '{': '}', '[': ']'} + symbols = [] # type: List[str] + while not self.eof: + if (len(symbols) == 0 and self.current_char in end): + break + if self.current_char in brackets.keys(): + symbols.append(brackets[self.current_char]) + elif len(symbols) > 0 and self.current_char == symbols[-1]: + symbols.pop() + self.pos += 1 + if len(end) > 0 and self.eof: + self.fail("Could not find end of expression starting at %d." + % startPos) + value = self.definition[startPos:self.pos].strip() + return ASTFallbackExpr(value.strip()) + + def _parse_nested_name(self) -> ASTNestedName: + names = [] # type: List[Any] + + self.skip_ws() + rooted = False + if self.skip_string('.'): + rooted = True + while 1: + self.skip_ws() + if not self.match(identifier_re): + self.fail("Expected identifier in nested name.") + identifier = self.matched_text + # make sure there isn't a keyword + if identifier in _keywords: + self.fail("Expected identifier in nested name, " + "got keyword: %s" % identifier) + ident = ASTIdentifier(identifier) + names.append(ident) + + self.skip_ws() + if not self.skip_string('.'): + break + return ASTNestedName(names, rooted) + + def _parse_trailing_type_spec(self) -> Any: + # fundemental types + self.skip_ws() + for t in self._simple_fundemental_types: + if self.skip_word(t): + return ASTTrailingTypeSpecFundamental(t) + + # TODO: this could/should be more strict + elements = [] + if self.skip_word_and_ws('signed'): + elements.append('signed') + elif self.skip_word_and_ws('unsigned'): + elements.append('unsigned') + while 1: + if self.skip_word_and_ws('short'): + elements.append('short') + elif self.skip_word_and_ws('long'): + elements.append('long') + else: + break + if self.skip_word_and_ws('char'): + elements.append('char') + elif self.skip_word_and_ws('int'): + elements.append('int') + elif self.skip_word_and_ws('double'): + elements.append('double') + if len(elements) > 0: + return ASTTrailingTypeSpecFundamental(' '.join(elements)) + + # prefixed + prefix = None + self.skip_ws() + for k in self._prefix_keys: + if self.skip_word_and_ws(k): + prefix = k + break + + nestedName = self._parse_nested_name() + return ASTTrailingTypeSpecName(prefix, nestedName) + + def _parse_parameters(self, paramMode: str) -> ASTParameters: + self.skip_ws() + if not self.skip_string('('): + if paramMode == 'function': + self.fail('Expecting "(" in parameters.') + else: + return None + + args = [] + self.skip_ws() + if not self.skip_string(')'): + while 1: + self.skip_ws() + if self.skip_string('...'): + args.append(ASTFunctionParameter(None, True)) + self.skip_ws() + if not self.skip_string(')'): + self.fail('Expected ")" after "..." in parameters.') + break + # note: it seems that function arguments can always be named, + # even in function pointers and similar. + arg = self._parse_type_with_init(outer=None, named='single') + # TODO: parse default parameters # TODO: didn't we just do that? + args.append(ASTFunctionParameter(arg)) + + self.skip_ws() + if self.skip_string(','): + continue + elif self.skip_string(')'): + break + else: + self.fail( + 'Expecting "," or ")" in parameters, ' + 'got "%s".' % self.current_char) + + return ASTParameters(args) + + def _parse_decl_specs_simple(self, outer: str, typed: bool) -> ASTDeclSpecsSimple: + """Just parse the simple ones.""" + storage = None + threadLocal = None + inline = None + restrict = None + volatile = None + const = None + attrs = [] + while 1: # accept any permutation of a subset of some decl-specs + self.skip_ws() + if not storage: + if outer == 'member': + if self.skip_word('auto'): + storage = 'auto' + continue + if self.skip_word('register'): + storage = 'register' + continue + if outer in ('member', 'function'): + if self.skip_word('static'): + storage = 'static' + continue + if self.skip_word('extern'): + storage = 'extern' + continue + if outer == 'member' and not threadLocal: + if self.skip_word('thread_local'): + threadLocal = 'thread_local' + continue + if self.skip_word('_Thread_local'): + threadLocal = '_Thread_local' + continue + if outer == 'function' and not inline: + inline = self.skip_word('inline') + if inline: + continue + + if not restrict and typed: + restrict = self.skip_word('restrict') + if restrict: + continue + if not volatile and typed: + volatile = self.skip_word('volatile') + if volatile: + continue + if not const and typed: + const = self.skip_word('const') + if const: + continue + attr = self._parse_attribute() + if attr: + attrs.append(attr) + continue + break + return ASTDeclSpecsSimple(storage, threadLocal, inline, + restrict, volatile, const, attrs) + + def _parse_decl_specs(self, outer: str, typed: bool = True) -> ASTDeclSpecs: + if outer: + if outer not in ('type', 'member', 'function'): + raise Exception('Internal error, unknown outer "%s".' % outer) + leftSpecs = self._parse_decl_specs_simple(outer, typed) + rightSpecs = None + + if typed: + trailing = self._parse_trailing_type_spec() + rightSpecs = self._parse_decl_specs_simple(outer, typed) + else: + trailing = None + return ASTDeclSpecs(outer, leftSpecs, rightSpecs, trailing) + + def _parse_declarator_name_suffix( + self, named: Union[bool, str], paramMode: str, typed: bool + ) -> Union[ASTDeclaratorNameParam, ASTDeclaratorNameBitField]: + # now we should parse the name, and then suffixes + prefix = None # struct/union/enum for when named but not typed + if named == 'maybe': + pos = self.pos + try: + declId = self._parse_nested_name() + except DefinitionError: + self.pos = pos + declId = None + elif named == 'single': + if self.match(identifier_re): + identifier = ASTIdentifier(self.matched_text) + declId = ASTNestedName([identifier], rooted=False) + else: + declId = None + elif named: + if not typed: + self.skip_ws() + for p in self._prefix_keys: + if self.skip_word(p): + prefix = p + break + declId = self._parse_nested_name() + else: + declId = None + arrayOps = [] + while 1: + self.skip_ws() + if typed and self.skip_string('['): + self.skip_ws() + if self.skip_string(']'): + arrayOps.append(ASTArray(None)) + continue + + def parser(): + return self._parse_expression(inTemplate=False) + + value = self._parse_expression_fallback([']'], parser) + if not self.skip_string(']'): + self.fail("Expected ']' in end of array operator.") + arrayOps.append(ASTArray(value)) + continue + else: + break + param = self._parse_parameters(paramMode) + if param is None and len(arrayOps) == 0: + # perhaps a bit-field + if named and paramMode == 'type' and typed: + self.skip_ws() + if self.skip_string(':'): + # TODO: copy-paste and adapt proper parser + # size = self._parse_constant_expression(inTemplate=False) + size = self.read_rest().strip() + return ASTDeclaratorNameBitField(declId=declId, size=size) + return ASTDeclaratorNameParam(declId=declId, arrayOps=arrayOps, + param=param, prefix=prefix) + + def _parse_declarator(self, named: Union[bool, str], paramMode: str, + typed: bool = True) -> Any: + # 'typed' here means 'parse return type stuff' + if paramMode not in ('type', 'function'): + raise Exception( + "Internal error, unknown paramMode '%s'." % paramMode) + prevErrors = [] + self.skip_ws() + if typed and self.skip_string('*'): + self.skip_ws() + restrict = False + volatile = False + const = False + attrs = [] + while 1: + if not restrict: + restrict = self.skip_word_and_ws('restrict') + if restrict: + continue + if not volatile: + volatile = self.skip_word_and_ws('volatile') + if volatile: + continue + if not const: + const = self.skip_word_and_ws('const') + if const: + continue + attr = self._parse_attribute() + if attr is not None: + attrs.append(attr) + continue + break + next = self._parse_declarator(named, paramMode, typed) + return ASTDeclaratorPtr(next=next, + restrict=restrict, volatile=volatile, const=const, + attrs=attrs) + if typed and self.current_char == '(': # note: peeking, not skipping + # maybe this is the beginning of params,try that first, + # otherwise assume it's noptr->declarator > ( ptr-declarator ) + pos = self.pos + try: + # assume this is params + res = self._parse_declarator_name_suffix(named, paramMode, + typed) + return res + except DefinitionError as exParamQual: + prevErrors.append((exParamQual, "If declId and parameters")) + self.pos = pos + try: + assert self.current_char == '(' + self.skip_string('(') + # TODO: hmm, if there is a name, it must be in inner, right? + # TODO: hmm, if there must be parameters, they must b + # inside, right? + inner = self._parse_declarator(named, paramMode, typed) + if not self.skip_string(')'): + self.fail("Expected ')' in \"( ptr-declarator )\"") + next = self._parse_declarator(named=False, + paramMode="type", + typed=typed) + return ASTDeclaratorParen(inner=inner, next=next) + except DefinitionError as exNoPtrParen: + self.pos = pos + prevErrors.append((exNoPtrParen, "If parenthesis in noptr-declarator")) + header = "Error in declarator" + raise self._make_multi_error(prevErrors, header) + pos = self.pos + try: + return self._parse_declarator_name_suffix(named, paramMode, typed) + except DefinitionError as e: + self.pos = pos + prevErrors.append((e, "If declarator-id")) + header = "Error in declarator or parameters" + raise self._make_multi_error(prevErrors, header) + + def _parse_initializer(self, outer: str = None, allowFallback: bool = True + ) -> ASTInitializer: + self.skip_ws() + if outer == 'member' and False: # TODO + bracedInit = self._parse_braced_init_list() + if bracedInit is not None: + return ASTInitializer(bracedInit, hasAssign=False) + + if not self.skip_string('='): + return None + + # TODO: implement + #bracedInit = self._parse_braced_init_list() + #if bracedInit is not None: + # return ASTInitializer(bracedInit) + + if outer == 'member': + fallbackEnd = [] # type: List[str] + elif outer is None: # function parameter + fallbackEnd = [',', ')'] + else: + self.fail("Internal error, initializer for outer '%s' not " + "implemented." % outer) + + def parser(): + return self._parse_assignment_expression() + value = self._parse_expression_fallback(fallbackEnd, parser, allow=allowFallback) + return ASTInitializer(value) + + def _parse_type(self, named: Union[bool, str], outer: str = None) -> ASTType: + """ + named=False|'maybe'|True: 'maybe' is e.g., for function objects which + doesn't need to name the arguments + """ + if outer: # always named + if outer not in ('type', 'member', 'function'): + raise Exception('Internal error, unknown outer "%s".' % outer) + assert named + + if outer == 'type': + # We allow type objects to just be a name. + prevErrors = [] + startPos = self.pos + # first try without the type + try: + declSpecs = self._parse_decl_specs(outer=outer, typed=False) + decl = self._parse_declarator(named=True, paramMode=outer, + typed=False) + self.assert_end() + except DefinitionError as exUntyped: + desc = "If just a name" + prevErrors.append((exUntyped, desc)) + self.pos = startPos + try: + declSpecs = self._parse_decl_specs(outer=outer) + decl = self._parse_declarator(named=True, paramMode=outer) + except DefinitionError as exTyped: + self.pos = startPos + desc = "If typedef-like declaration" + prevErrors.append((exTyped, desc)) + # Retain the else branch for easier debugging. + # TODO: it would be nice to save the previous stacktrace + # and output it here. + if True: + header = "Type must be either just a name or a " + header += "typedef-like declaration." + raise self._make_multi_error(prevErrors, header) + else: + # For testing purposes. + # do it again to get the proper traceback (how do you + # reliably save a traceback when an exception is + # constructed?) + self.pos = startPos + typed = True + declSpecs = self._parse_decl_specs(outer=outer, typed=typed) + decl = self._parse_declarator(named=True, paramMode=outer, + typed=typed) + elif outer == 'function': + declSpecs = self._parse_decl_specs(outer=outer) + decl = self._parse_declarator(named=True, paramMode=outer) + else: + paramMode = 'type' + if outer == 'member': # i.e., member + named = True + declSpecs = self._parse_decl_specs(outer=outer) + decl = self._parse_declarator(named=named, paramMode=paramMode) + return ASTType(declSpecs, decl) + + def _parse_type_with_init(self, named: Union[bool, str], outer: str) -> Any: + if outer: + assert outer in ('type', 'member', 'function') + type = self._parse_type(outer=outer, named=named) + init = self._parse_initializer(outer=outer) + return ASTTypeWithInit(type, init) + + def _parse_macro(self) -> ASTMacro: + self.skip_ws() + ident = self._parse_nested_name() + if ident is None: + self.fail("Expected identifier in macro definition.") + self.skip_ws() + if not self.skip_string_and_ws('('): + return ASTMacro(ident, None) + if self.skip_string(')'): + return ASTMacro(ident, []) + args = [] + while 1: + self.skip_ws() + if self.skip_string('...'): + args.append(ASTFunctionParameter(None, True)) + self.skip_ws() + if not self.skip_string(')'): + self.fail('Expected ")" after "..." in macro parameters.') + break + if not self.match(identifier_re): + self.fail("Expected identifier in macro parameters.") + nn = ASTNestedName([ASTIdentifier(self.matched_text)], rooted=False) + arg = ASTFunctionParameter(nn) + args.append(arg) + self.skip_ws() + if self.skip_string_and_ws(','): + continue + elif self.skip_string_and_ws(')'): + break + else: + self.fail("Expected identifier, ')', or ',' in macro parameter list.") + return ASTMacro(ident, args) + + def parse_declaration(self, objectType: str, directiveType: str) -> ASTDeclaration: + if objectType not in ('function', 'macro', 'member', 'type'): + raise Exception('Internal error, unknown objectType "%s".' % objectType) + if directiveType not in ('function', 'macro', 'member', 'var', 'type'): + raise Exception('Internal error, unknown directiveType "%s".' % directiveType) + + if objectType == 'function': + declaration = self._parse_type(named=True, outer='function') + elif objectType == 'macro': + declaration = self._parse_macro() + elif objectType == 'member': + declaration = self._parse_type_with_init(named=True, outer='member') + elif objectType == 'type': + declaration = self._parse_type(named=True, outer='type') + else: + assert False + return ASTDeclaration(objectType, directiveType, declaration) + + +def _make_phony_error_name() -> ASTNestedName: + return ASTNestedName([ASTIdentifier("PhonyNameDueToError")], rooted=False) class CObject(ObjectDescription): @@ -73,114 +1956,57 @@ class CObject(ObjectDescription): names=('rtype',)), ] - # These C types aren't described anywhere, so don't try to create - # a cross-reference to them - stopwords = { - 'const', 'void', 'char', 'wchar_t', 'int', 'short', - 'long', 'float', 'double', 'unsigned', 'signed', 'FILE', - 'clock_t', 'time_t', 'ptrdiff_t', 'size_t', 'ssize_t', - 'struct', '_Bool', - } + def warn(self, msg: Union[str, Exception]) -> None: + self.state_machine.reporter.warning(msg, line=self.lineno) - def _parse_type(self, node: Element, ctype: str) -> None: - # add cross-ref nodes for all words - for part in [_f for _f in wsplit_re.split(ctype) if _f]: - tnode = nodes.Text(part, part) - if part[0] in string.ascii_letters + '_' and \ - part not in self.stopwords: - pnode = pending_xref('', refdomain='c', reftype='type', reftarget=part, - modname=None, classname=None) - pnode += tnode - node += pnode - else: - node += tnode - - def _parse_arglist(self, arglist: str) -> Iterator[str]: - while True: - m = c_funcptr_arg_sig_re.match(arglist) - if m: - yield m.group() - arglist = c_funcptr_arg_sig_re.sub('', arglist) - if ',' in arglist: - _, arglist = arglist.split(',', 1) - else: - break - else: - if ',' in arglist: - arg, arglist = arglist.split(',', 1) - yield arg - else: - yield arglist - break - - def handle_signature(self, sig: str, signode: desc_signature) -> str: - """Transform a C signature into RST nodes.""" - # first try the function pointer signature regex, it's more specific - m = c_funcptr_sig_re.match(sig) - if m is None: - m = c_sig_re.match(sig) - if m is None: - raise ValueError('no match') - rettype, name, arglist, const = m.groups() - - desc_type = addnodes.desc_type('', '') - signode += desc_type - self._parse_type(desc_type, rettype) - try: - classname, funcname = name.split('::', 1) - classname += '::' - signode += addnodes.desc_addname(classname, classname) - signode += addnodes.desc_name(funcname, funcname) - # name (the full name) is still both parts - except ValueError: - signode += addnodes.desc_name(name, name) - # clean up parentheses from canonical name - m = c_funcptr_name_re.match(name) - if m: - name = m.group(1) - - typename = self.env.ref_context.get('c:type') - if self.name == 'c:member' and typename: - fullname = typename + '.' + name - else: - fullname = name - - if not arglist: - if self.objtype == 'function' or \ - self.objtype == 'macro' and sig.rstrip().endswith('()'): - # for functions, add an empty parameter list - signode += addnodes.desc_parameterlist() - if const: - signode += addnodes.desc_addname(const, const) - return fullname - - paramlist = addnodes.desc_parameterlist() - arglist = arglist.replace('`', '').replace('\\ ', '') # remove markup - # this messes up function pointer types, but not too badly ;) - for arg in self._parse_arglist(arglist): - arg = arg.strip() - param = addnodes.desc_parameter('', '', noemph=True) + def add_target_and_index(self, ast: ASTDeclaration, sig: str, signode: desc_signature) -> None: + ids = [] + for i in range(1, _max_id + 1): try: - m = c_funcptr_arg_sig_re.match(arg) - if m: - self._parse_type(param, m.group(1) + '(') - param += nodes.emphasis(m.group(2), m.group(2)) - self._parse_type(param, ')(' + m.group(3) + ')') - if m.group(4): - param += addnodes.desc_addname(m.group(4), m.group(4)) - else: - ctype, argname = arg.rsplit(' ', 1) - self._parse_type(param, ctype) - # separate by non-breaking space in the output - param += nodes.emphasis(' ' + argname, '\xa0' + argname) - except ValueError: - # no argument name given, only the type - self._parse_type(param, arg) - paramlist += param - signode += paramlist - if const: - signode += addnodes.desc_addname(const, const) - return fullname + id = ast.get_id(version=i) + ids.append(id) + except NoOldIdError: + assert i < _max_id + # let's keep the newest first + ids = list(reversed(ids)) + newestId = ids[0] + assert newestId # shouldn't be None + + name = ast.symbol.get_full_nested_name().get_display_string().lstrip('.') + if newestId not in self.state.document.ids: + signode['names'].append(name) + # always add the newest id + assert newestId + signode['ids'].append(newestId) + # only add compatibility ids when there are no conflicts + for id in ids[1:]: + if not id: # is None when the element didn't exist in that version + continue + if id not in self.state.document.ids: + signode['ids'].append(id) + + self.state.document.note_explicit_target(signode) + + domain = cast(CDomain, self.env.get_domain('c')) + domain.note_object(name, self.objtype, newestId) + + strippedName = name + # for prefix in self.env.config.cpp_index_common_prefix: + # if name.startswith(prefix): + # strippedName = strippedName[len(prefix):] + # break + indexText = self.get_index_text(strippedName) + self.indexnode['entries'].append(('single', indexText, newestId, '', None)) + + def get_object_type(self) -> str: + if self.objtype in ('function', 'macro'): + return self.objtype + elif self.objtype in ('member', 'var'): + return 'member' + elif self.objtype == 'type': + return 'type' # hmm + else: + assert False def get_index_text(self, name: str) -> str: if self.objtype == 'function': @@ -194,37 +2020,79 @@ class CObject(ObjectDescription): elif self.objtype == 'var': return _('%s (C variable)') % name else: - return '' + assert False - def add_target_and_index(self, name: str, sig: str, signode: desc_signature) -> None: - node_id = make_id(self.env, self.state.document, 'c', name) - signode['ids'].append(node_id) + def parse_definition(self, parser: DefinitionParser) -> ASTDeclaration: + return parser.parse_declaration(self.get_object_type(), self.objtype) - # Assign old styled node_id not to break old hyperlinks (if possible) - # Note: Will be removed in Sphinx-5.0 (RemovedInSphinx50Warning) - old_node_id = self.make_old_id(name) - if old_node_id not in self.state.document.ids and old_node_id not in signode['ids']: - signode['ids'].append(old_node_id) + def describe_signature(self, signode: desc_signature, ast: Any, options: Dict) -> None: + ast.describe_signature(signode, 'lastIsName', self.env, options) - self.state.document.note_explicit_target(signode) + def run(self) -> List[Node]: + env = self.state.document.settings.env # from ObjectDescription.run + if 'c:parent_symbol' not in env.temp_data: + root = env.domaindata['c']['root_symbol'] + env.temp_data['c:parent_symbol'] = root + env.ref_context['c:parent_key'] = root.get_lookup_key() - domain = cast(CDomain, self.env.get_domain('c')) - domain.note_object(name, self.objtype, node_id) + # When multiple declarations are made in the same directive + # they need to know about each other to provide symbol lookup for function parameters. + # We use last_symbol to store the latest added declaration in a directive. + env.temp_data['c:last_symbol'] = None + return super().run() - indextext = self.get_index_text(name) - if indextext: - self.indexnode['entries'].append(('single', indextext, node_id, '', None)) + def handle_signature(self, sig: str, signode: desc_signature) -> str: + parentSymbol = self.env.temp_data['c:parent_symbol'] # type: Symbol + + parser = DefinitionParser(sig, self) + try: + ast = self.parse_definition(parser) + parser.assert_end() + except DefinitionError as e: + self.warn(e) + # It is easier to assume some phony name than handling the error in + # the possibly inner declarations. + name = _make_phony_error_name() + symbol = parentSymbol.add_name(name) + self.env.temp_data['c:last_symbol'] = symbol + raise ValueError + + try: + symbol = parentSymbol.add_declaration(ast, docname=self.env.docname) + # append the new declaration to the sibling list + assert symbol.siblingAbove is None + assert symbol.siblingBelow is None + symbol.siblingAbove = self.env.temp_data['c:last_symbol'] + if symbol.siblingAbove is not None: + assert symbol.siblingAbove.siblingBelow is None + symbol.siblingAbove.siblingBelow = symbol + self.env.temp_data['c:last_symbol'] = symbol + except _DuplicateSymbolError as e: + # Assume we are actually in the old symbol, + # instead of the newly created duplicate. + self.env.temp_data['c:last_symbol'] = e.symbol + self.warn("Duplicate declaration, %s" % sig) + + if ast.objectType == 'enumerator': + self._add_enumerator_to_parent(ast) + + # note: handle_signature may be called multiple time per directive, + # if it has multiple signatures, so don't mess with the original options. + options = dict(self.options) + self.describe_signature(signode, ast, options) + return ast def before_content(self) -> None: - self.typename_set = False - if self.name == 'c:type': - if self.names: - self.env.ref_context['c:type'] = self.names[0] - self.typename_set = True + lastSymbol = self.env.temp_data['c:last_symbol'] # type: Symbol + assert lastSymbol + self.oldParentSymbol = self.env.temp_data['c:parent_symbol'] + self.oldParentKey = self.env.ref_context['c:parent_key'] # type: LookupKey + self.env.temp_data['c:parent_symbol'] = lastSymbol + self.env.ref_context['c:parent_key'] = lastSymbol.get_lookup_key() def after_content(self) -> None: - if self.typename_set: - self.env.ref_context.pop('c:type', None) + self.env.temp_data['c:parent_symbol'] = self.oldParentSymbol + self.env.ref_context['c:parent_key'] = self.oldParentKey def make_old_id(self, name: str) -> str: """Generate old styled node_id for C objects. @@ -256,29 +2124,30 @@ class CDomain(Domain): label = 'C' object_types = { 'function': ObjType(_('function'), 'func'), - 'member': ObjType(_('member'), 'member'), - 'macro': ObjType(_('macro'), 'macro'), - 'type': ObjType(_('type'), 'type'), - 'var': ObjType(_('variable'), 'data'), + 'member': ObjType(_('member'), 'member'), + 'macro': ObjType(_('macro'), 'macro'), + 'type': ObjType(_('type'), 'type'), + 'var': ObjType(_('variable'), 'data'), } directives = { 'function': CObject, - 'member': CObject, - 'macro': CObject, - 'type': CObject, - 'var': CObject, + 'member': CObject, + 'macro': CObject, + 'type': CObject, + 'var': CObject, } roles = { - 'func': CXRefRole(fix_parens=True), + 'func': CXRefRole(fix_parens=True), 'member': CXRefRole(), - 'macro': CXRefRole(), - 'data': CXRefRole(), - 'type': CXRefRole(), + 'macro': CXRefRole(), + 'data': CXRefRole(), + 'type': CXRefRole(), } initial_data = { + 'root_symbol': Symbol(None, None, None, None), 'objects': {}, # fullname -> docname, node_id, objtype - } # type: Dict[str, Dict[str, Tuple[str, str, str]]] + } # type: Dict[str, Dict[str, Tuple[str, Any]]] @property def objects(self) -> Dict[str, Tuple[str, str, str]]: @@ -287,21 +2156,60 @@ class CDomain(Domain): def note_object(self, name: str, objtype: str, node_id: str, location: Any = None) -> None: if name in self.objects: docname = self.objects[name][0] - logger.warning(__('duplicate C object description of %s, ' + logger.warning(__('Duplicate C object description of %s, ' 'other instance in %s, use :noindex: for one of them'), name, docname, location=location) self.objects[name] = (self.env.docname, node_id, objtype) def clear_doc(self, docname: str) -> None: - for fullname, (fn, node_id, _l) in list(self.objects.items()): + if Symbol.debug_show_tree: + print("clear_doc:", docname) + print("\tbefore:") + print(self.data['root_symbol'].dump(1)) + print("\tbefore end") + + rootSymbol = self.data['root_symbol'] + rootSymbol.clear_doc(docname) + + if Symbol.debug_show_tree: + print("\tafter:") + print(self.data['root_symbol'].dump(1)) + print("\tafter end") + print("clear_doc end:", docname) + for fullname, (fn, _id, _l) in list(self.objects.items()): if fn == docname: del self.objects[fullname] + def process_doc(self, env: BuildEnvironment, docname: str, + document: nodes.document) -> None: + if Symbol.debug_show_tree: + print("process_doc:", docname) + print(self.data['root_symbol'].dump(0)) + print("process_doc end:", docname) + def merge_domaindata(self, docnames: List[str], otherdata: Dict) -> None: - # XXX check duplicates - for fullname, (fn, node_id, objtype) in otherdata['objects'].items(): + if Symbol.debug_show_tree: + print("merge_domaindata:") + print("\tself:") + print(self.data['root_symbol'].dump(1)) + print("\tself end") + print("\tother:") + print(otherdata['root_symbol'].dump(1)) + print("\tother end") + print("merge_domaindata end") + + self.data['root_symbol'].merge_with(otherdata['root_symbol'], + docnames, self.env) + ourObjects = self.data['objects'] + for fullname, (fn, id_, objtype) in otherdata['objects'].items(): if fn in docnames: - self.data['objects'][fullname] = (fn, node_id, objtype) + if fullname in ourObjects: + msg = __("Duplicate declaration, also defined in '%s'.\n" + "Name of declaration is '%s'.") + msg = msg % (ourObjects[fullname], fullname) + logger.warning(msg, location=fn) + else: + ourObjects[fullname] = (fn, id_, objtype) def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder, typ: str, target: str, node: pending_xref, contnode: Element @@ -309,7 +2217,7 @@ class CDomain(Domain): # strip pointer asterisk target = target.rstrip(' *') # becase TypedField can generate xrefs - if target in CObject.stopwords: + if target in _keywords: return contnode if target not in self.objects: return None diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py index 0be9f5afd..d23d5be35 100644 --- a/sphinx/domains/cpp.py +++ b/sphinx/domains/cpp.py @@ -10,7 +10,6 @@ import re import warnings -from copy import deepcopy from typing import ( Any, Callable, Dict, Iterator, List, Match, Pattern, Tuple, Type, TypeVar, Union ) @@ -35,13 +34,16 @@ from sphinx.roles import XRefRole from sphinx.transforms import SphinxTransform from sphinx.transforms.post_transforms import ReferencesResolver from sphinx.util import logging +from sphinx.util.cfamily import ( + NoOldIdError, ASTBase, verify_description_mode, StringifyTransform, + BaseParser, DefinitionError, identifier_re, anon_identifier_re +) from sphinx.util.docfields import Field, GroupedField from sphinx.util.docutils import SphinxDirective from sphinx.util.nodes import make_refnode logger = logging.getLogger(__name__) -StringifyTransform = Callable[[Any], str] T = TypeVar('T') """ @@ -325,16 +327,6 @@ _char_literal_re = re.compile(r'''(?x) )' ''') -_anon_identifier_re = re.compile(r'(@[a-zA-Z0-9_])[a-zA-Z0-9_]*\b') -_identifier_re = re.compile(r'''(?x) - ( # This 'extends' _anon_identifier_re with the ordinary identifiers, - # make sure they are in sync. - (~?\b[a-zA-Z_]) # ordinary identifiers - | (@[a-zA-Z0-9_]) # our extension for names of anonymous entities - ) - [a-zA-Z0-9_]*\b -''') -_whitespace_re = re.compile(r'(?u)\s+') _string_re = re.compile(r"[LuU8]?('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) _visibility_re = re.compile(r'\b(public|private|protected)\b') @@ -582,25 +574,6 @@ _id_explicit_cast = { } -class NoOldIdError(Exception): - # Used to avoid implementing unneeded id generation for old id schemes. - @property - def description(self) -> str: - warnings.warn('%s.description is deprecated. ' - 'Coerce the instance to a string instead.' % self.__class__.__name__, - RemovedInSphinx40Warning, stacklevel=2) - return str(self) - - -class DefinitionError(Exception): - @property - def description(self) -> str: - warnings.warn('%s.description is deprecated. ' - 'Coerce the instance to a string instead.' % self.__class__.__name__, - RemovedInSphinx40Warning, stacklevel=2) - return str(self) - - class _DuplicateSymbolError(Exception): def __init__(self, symbol: "Symbol", declaration: Any) -> None: assert symbol @@ -612,42 +585,6 @@ class _DuplicateSymbolError(Exception): return "Internal C++ duplicate symbol error:\n%s" % self.symbol.dump(0) -class ASTBase: - def __eq__(self, other: Any) -> bool: - if type(self) is not type(other): - return False - try: - for key, value in self.__dict__.items(): - if value != getattr(other, key): - return False - except AttributeError: - return False - return True - - __hash__ = None # type: Callable[[], int] - - def clone(self) -> Any: - """Clone a definition expression node.""" - return deepcopy(self) - - def _stringify(self, transform: StringifyTransform) -> str: - raise NotImplementedError(repr(self)) - - def __str__(self) -> str: - return self._stringify(lambda ast: str(ast)) - - def get_display_string(self) -> str: - return self._stringify(lambda ast: ast.get_display_string()) - - def __repr__(self) -> str: - return '<%s>' % self.__class__.__name__ - - -def _verify_description_mode(mode: str) -> None: - if mode not in ('lastIsName', 'noneIsName', 'markType', 'markName', 'param'): - raise Exception("Description mode '%s' is invalid." % mode) - - ################################################################################ # Attributes ################################################################################ @@ -1441,7 +1378,7 @@ class ASTIdentifier(ASTBase): def describe_signature(self, signode: Any, mode: str, env: "BuildEnvironment", prefix: str, templateArgs: str, symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) if mode == 'markType': targetText = prefix + self.identifier + templateArgs pnode = addnodes.pending_xref('', refdomain='cpp', @@ -1847,7 +1784,7 @@ class ASTTemplateDeclarationPrefix(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol", lineSpec: bool) -> None: - _verify_description_mode(mode) + verify_description_mode(mode) for t in self.templates: t.describe_signature(signode, 'lastIsName', env, symbol, lineSpec) @@ -1868,7 +1805,7 @@ class ASTOperator(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", prefix: str, templateArgs: str, symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) identifier = str(self) if mode == 'lastIsName': signode += addnodes.desc_name(identifier, identifier) @@ -1947,7 +1884,7 @@ class ASTTemplateArgConstant(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) self.value.describe_signature(signode, mode, env, symbol) @@ -1977,7 +1914,7 @@ class ASTTemplateArgs(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) signode += nodes.Text('<') first = True for a in self.args: @@ -2072,7 +2009,7 @@ class ASTNestedName(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) # just print the name part, with template args, not template params if mode == 'noneIsName': signode += nodes.Text(str(self)) @@ -2243,7 +2180,7 @@ class ASTFunctionParameter(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) if self.ellipsis: signode += nodes.Text('...') else: @@ -2323,7 +2260,7 @@ class ASTParametersQualifiers(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) paramlist = addnodes.desc_parameterlist() for arg in self.args: param = addnodes.desc_parameter('', '', noemph=True) @@ -2489,7 +2426,7 @@ class ASTDeclSpecs(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) modifiers = [] # type: List[Node] def _add(modifiers: List[Node], text: str) -> None: @@ -2539,7 +2476,7 @@ class ASTArray(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) signode.append(nodes.Text("[")) if self.size: self.size.describe_signature(signode, mode, env, symbol) @@ -2623,7 +2560,7 @@ class ASTDeclaratorPtr(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) signode += nodes.Text("*") for a in self.attrs: a.describe_signature(signode) @@ -2696,7 +2633,7 @@ class ASTDeclaratorRef(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) signode += nodes.Text("&") for a in self.attrs: a.describe_signature(signode) @@ -2749,7 +2686,7 @@ class ASTDeclaratorParamPack(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) signode += nodes.Text("...") if self.next.name: signode += nodes.Text(' ') @@ -2826,7 +2763,7 @@ class ASTDeclaratorMemPtr(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) self.className.describe_signature(signode, mode, env, symbol) signode += nodes.Text('::*') @@ -2896,7 +2833,7 @@ class ASTDeclaratorParen(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) signode += nodes.Text('(') self.inner.describe_signature(signode, mode, env, symbol) signode += nodes.Text(')') @@ -2972,7 +2909,7 @@ class ASTDeclaratorNameParamQual(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) if self.declId: self.declId.describe_signature(signode, mode, env, symbol) for op in self.arrayOps: @@ -3014,7 +2951,7 @@ class ASTDeclaratorNameBitField(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) if self.declId: self.declId.describe_signature(signode, mode, env, symbol) signode.append(nodes.Text(' : ', ' : ')) @@ -3034,7 +2971,7 @@ class ASTParenExprList(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) signode.append(nodes.Text('(')) first = True for e in self.exprs: @@ -3061,7 +2998,7 @@ class ASTBracedInitList(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) signode.append(nodes.Text('{')) first = True for e in self.exprs: @@ -3089,7 +3026,7 @@ class ASTInitializer(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) if self.hasAssign: signode.append(nodes.Text(' = ')) self.value.describe_signature(signode, 'markType', env, symbol) @@ -3184,7 +3121,7 @@ class ASTType(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) self.declSpecs.describe_signature(signode, 'markType', env, symbol) if (self.decl.require_space_after_declSpecs() and len(str(self.declSpecs)) > 0): @@ -3227,7 +3164,7 @@ class ASTTypeWithInit(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) self.type.describe_signature(signode, mode, env, symbol) if self.init: self.init.describe_signature(signode, mode, env, symbol) @@ -3257,7 +3194,7 @@ class ASTTypeUsing(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) self.name.describe_signature(signode, mode, env, symbol=symbol) if self.type: signode += nodes.Text(' = ') @@ -3315,7 +3252,7 @@ class ASTBaseClass(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) if self.visibility is not None: signode += addnodes.desc_annotation(self.visibility, self.visibility) @@ -3354,7 +3291,7 @@ class ASTClass(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) self.name.describe_signature(signode, mode, env, symbol=symbol) if self.final: signode += nodes.Text(' ') @@ -3381,7 +3318,7 @@ class ASTUnion(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) self.name.describe_signature(signode, mode, env, symbol=symbol) @@ -3409,7 +3346,7 @@ class ASTEnum(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) # self.scoped has been done by the CPPEnumObject self.name.describe_signature(signode, mode, env, symbol=symbol) if self.underlyingType: @@ -3437,7 +3374,7 @@ class ASTEnumerator(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: - _verify_description_mode(mode) + verify_description_mode(mode) self.name.describe_signature(signode, mode, env, symbol) if self.init: self.init.describe_signature(signode, 'markType', env, symbol) @@ -3509,7 +3446,7 @@ class ASTDeclaration(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", options: Dict) -> None: - _verify_description_mode(mode) + verify_description_mode(mode) assert self.symbol # The caller of the domain added a desc_signature node. # Always enable multiline: @@ -4497,7 +4434,7 @@ class Symbol: return ''.join(res) -class DefinitionParser: +class DefinitionParser(BaseParser): # those without signedness and size modifiers # see https://en.cppreference.com/w/cpp/language/types _simple_fundemental_types = ( @@ -4508,128 +4445,9 @@ class DefinitionParser: _prefix_keys = ('class', 'struct', 'enum', 'union', 'typename') def __init__(self, definition: Any, warnEnv: Any, config: "Config") -> None: - self.definition = definition.strip() - self.pos = 0 - self.end = len(self.definition) - self.last_match = None # type: Match - self._previous_state = (0, None) # type: Tuple[int, Match] - self.otherErrors = [] # type: List[DefinitionError] - # in our tests the following is set to False to capture bad parsing - self.allowFallbackExpressionParsing = True - - self.warnEnv = warnEnv + super().__init__(definition, warnEnv) self.config = config - def _make_multi_error(self, errors: List[Any], header: str) -> DefinitionError: - if len(errors) == 1: - if len(header) > 0: - return DefinitionError(header + '\n' + str(errors[0][0])) - else: - return DefinitionError(str(errors[0][0])) - result = [header, '\n'] - for e in errors: - if len(e[1]) > 0: - ident = ' ' - result.append(e[1]) - result.append(':\n') - for line in str(e[0]).split('\n'): - if len(line) == 0: - continue - result.append(ident) - result.append(line) - result.append('\n') - else: - result.append(str(e[0])) - return DefinitionError(''.join(result)) - - def status(self, msg: str) -> None: - # for debugging - indicator = '-' * self.pos + '^' - print("%s\n%s\n%s" % (msg, self.definition, indicator)) - - def fail(self, msg: str) -> None: - errors = [] - indicator = '-' * self.pos + '^' - exMain = DefinitionError( - 'Invalid definition: %s [error at %d]\n %s\n %s' % - (msg, self.pos, self.definition, indicator)) - errors.append((exMain, "Main error")) - for err in self.otherErrors: - errors.append((err, "Potential other error")) - self.otherErrors = [] - raise self._make_multi_error(errors, '') - - def warn(self, msg: str) -> None: - if self.warnEnv: - self.warnEnv.warn(msg) - else: - print("Warning: %s" % msg) - - def match(self, regex: Pattern) -> bool: - match = regex.match(self.definition, self.pos) - if match is not None: - self._previous_state = (self.pos, self.last_match) - self.pos = match.end() - self.last_match = match - return True - return False - - def backout(self) -> None: - self.pos, self.last_match = self._previous_state - - def skip_string(self, string: str) -> bool: - strlen = len(string) - if self.definition[self.pos:self.pos + strlen] == string: - self.pos += strlen - return True - return False - - def skip_word(self, word: str) -> bool: - return self.match(re.compile(r'\b%s\b' % re.escape(word))) - - def skip_ws(self) -> bool: - return self.match(_whitespace_re) - - def skip_word_and_ws(self, word: str) -> bool: - if self.skip_word(word): - self.skip_ws() - return True - return False - - def skip_string_and_ws(self, string: str) -> bool: - if self.skip_string(string): - self.skip_ws() - return True - return False - - @property - def eof(self) -> bool: - return self.pos >= self.end - - @property - def current_char(self) -> str: - try: - return self.definition[self.pos] - except IndexError: - return 'EOF' - - @property - def matched_text(self) -> str: - if self.last_match is not None: - return self.last_match.group() - else: - return None - - def read_rest(self) -> str: - rv = self.definition[self.pos:] - self.pos = self.end - return rv - - def assert_end(self) -> None: - self.skip_ws() - if not self.eof: - self.fail('Expected end of definition.') - def _parse_string(self): if self.current_char != '"': return None @@ -4693,7 +4511,7 @@ class DefinitionParser: self.fail("Expected '(' after '__attribute__('.") attrs = [] while 1: - if self.match(_identifier_re): + if self.match(identifier_re): name = self.matched_text self.skip_ws() if self.skip_string_and_ws('('): @@ -5085,7 +4903,7 @@ class DefinitionParser: if self.skip_string_and_ws('...'): if not self.skip_string_and_ws('('): self.fail("Expecting '(' after 'sizeof...'.") - if not self.match(_identifier_re): + if not self.match(identifier_re): self.fail("Expecting identifier for 'sizeof...'.") ident = ASTIdentifier(self.matched_text) self.skip_ws() @@ -5335,7 +5153,7 @@ class DefinitionParser: # user-defined literal? if self.skip_string('""'): self.skip_ws() - if not self.match(_identifier_re): + if not self.match(identifier_re): self.fail("Expected user-defined literal suffix.") identifier = ASTIdentifier(self.matched_text) return ASTOperatorLiteral(identifier) @@ -5411,7 +5229,7 @@ class DefinitionParser: if self.skip_word_and_ws('operator'): identOrOp = self._parse_operator() else: - if not self.match(_identifier_re): + if not self.match(identifier_re): if memberPointer and len(names) > 0: templates.pop() break @@ -5699,7 +5517,7 @@ class DefinitionParser: self.pos = pos declId = None elif named == 'single': - if self.match(_identifier_re): + if self.match(identifier_re): identifier = ASTIdentifier(self.matched_text) nne = ASTNestedNameElement(identifier, None) declId = ASTNestedName([nne], [False], rooted=False) @@ -6129,7 +5947,7 @@ class DefinitionParser: self.skip_ws() parameterPack = self.skip_string('...') self.skip_ws() - if self.match(_identifier_re): + if self.match(identifier_re): identifier = ASTIdentifier(self.matched_text) else: identifier = None @@ -6188,7 +6006,7 @@ class DefinitionParser: self.skip_ws() parameterPack = self.skip_string('...') self.skip_ws() - if not self.match(_identifier_re): + if not self.match(identifier_re): self.fail("Expected identifier in template introduction list.") txt_identifier = self.matched_text # make sure there isn't a keyword @@ -6907,7 +6725,7 @@ class CPPXRefRole(XRefRole): if not has_explicit_title: # major hax: replace anon names via simple string manipulation. # Can this actually fail? - title = _anon_identifier_re.sub("[anonymous]", str(title)) + title = anon_identifier_re.sub("[anonymous]", str(title)) if refnode['reftype'] == 'any': # Assume the removal part of fix_parens for :any: refs. diff --git a/sphinx/util/cfamily.py b/sphinx/util/cfamily.py new file mode 100644 index 000000000..0402732de --- /dev/null +++ b/sphinx/util/cfamily.py @@ -0,0 +1,209 @@ +""" + sphinx.util.cfamily + ~~~~~~~~~~~~~~~~ + + Utility functions common to the C and C++ domains. + + :copyright: Copyright 2020-2020 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +import warnings +from copy import deepcopy +from typing import ( + Any, Callable, Dict, Iterator, List, Match, Pattern, Tuple, Type, TypeVar, Union +) + +from sphinx.deprecation import RemovedInSphinx40Warning + + +StringifyTransform = Callable[[Any], str] + + +_whitespace_re = re.compile(r'(?u)\s+') +anon_identifier_re = re.compile(r'(@[a-zA-Z0-9_])[a-zA-Z0-9_]*\b') +identifier_re = re.compile(r'''(?x) + ( # This 'extends' _anon_identifier_re with the ordinary identifiers, + # make sure they are in sync. + (~?\b[a-zA-Z_]) # ordinary identifiers + | (@[a-zA-Z0-9_]) # our extension for names of anonymous entities + ) + [a-zA-Z0-9_]*\b +''') + + +def verify_description_mode(mode: str) -> None: + if mode not in ('lastIsName', 'noneIsName', 'markType', 'markName', 'param'): + raise Exception("Description mode '%s' is invalid." % mode) + + +class NoOldIdError(Exception): + # Used to avoid implementing unneeded id generation for old id schemes. + @property + def description(self) -> str: + warnings.warn('%s.description is deprecated. ' + 'Coerce the instance to a string instead.' % self.__class__.__name__, + RemovedInSphinx40Warning, stacklevel=2) + return str(self) + + +class ASTBase: + def __eq__(self, other: Any) -> bool: + if type(self) is not type(other): + return False + try: + for key, value in self.__dict__.items(): + if value != getattr(other, key): + return False + except AttributeError: + return False + return True + + __hash__ = None # type: Callable[[], int] + + def clone(self) -> Any: + """Clone a definition expression node.""" + return deepcopy(self) + + def _stringify(self, transform: StringifyTransform) -> str: + raise NotImplementedError(repr(self)) + + def __str__(self) -> str: + return self._stringify(lambda ast: str(ast)) + + def get_display_string(self) -> str: + return self._stringify(lambda ast: ast.get_display_string()) + + def __repr__(self) -> str: + return '<%s>' % self.__class__.__name__ + + +class DefinitionError(Exception): + @property + def description(self) -> str: + warnings.warn('%s.description is deprecated. ' + 'Coerce the instance to a string instead.' % self.__class__.__name__, + RemovedInSphinx40Warning, stacklevel=2) + return str(self) + + +class BaseParser: + def __init__(self, definition: Any, warnEnv: Any) -> None: + self.warnEnv = warnEnv + self.definition = definition.strip() + self.pos = 0 + self.end = len(self.definition) + self.last_match = None # type: Match + self._previous_state = (0, None) # type: Tuple[int, Match] + self.otherErrors = [] # type: List[DefinitionError] + + # in our tests the following is set to False to capture bad parsing + self.allowFallbackExpressionParsing = True + + def _make_multi_error(self, errors: List[Any], header: str) -> DefinitionError: + if len(errors) == 1: + if len(header) > 0: + return DefinitionError(header + '\n' + str(errors[0][0])) + else: + return DefinitionError(str(errors[0][0])) + result = [header, '\n'] + for e in errors: + if len(e[1]) > 0: + ident = ' ' + result.append(e[1]) + result.append(':\n') + for line in str(e[0]).split('\n'): + if len(line) == 0: + continue + result.append(ident) + result.append(line) + result.append('\n') + else: + result.append(str(e[0])) + return DefinitionError(''.join(result)) + + def status(self, msg: str) -> None: + # for debugging + indicator = '-' * self.pos + '^' + print("%s\n%s\n%s" % (msg, self.definition, indicator)) + + def fail(self, msg: str) -> None: + errors = [] + indicator = '-' * self.pos + '^' + exMain = DefinitionError( + 'Invalid definition: %s [error at %d]\n %s\n %s' % + (msg, self.pos, self.definition, indicator)) + errors.append((exMain, "Main error")) + for err in self.otherErrors: + errors.append((err, "Potential other error")) + self.otherErrors = [] + raise self._make_multi_error(errors, '') + + def warn(self, msg: str) -> None: + if self.warnEnv: + self.warnEnv.warn(msg) + else: + print("Warning: %s" % msg) + + def match(self, regex: Pattern) -> bool: + match = regex.match(self.definition, self.pos) + if match is not None: + self._previous_state = (self.pos, self.last_match) + self.pos = match.end() + self.last_match = match + return True + return False + + def skip_string(self, string: str) -> bool: + strlen = len(string) + if self.definition[self.pos:self.pos + strlen] == string: + self.pos += strlen + return True + return False + + def skip_word(self, word: str) -> bool: + return self.match(re.compile(r'\b%s\b' % re.escape(word))) + + def skip_ws(self) -> bool: + return self.match(_whitespace_re) + + def skip_word_and_ws(self, word: str) -> bool: + if self.skip_word(word): + self.skip_ws() + return True + return False + + def skip_string_and_ws(self, string: str) -> bool: + if self.skip_string(string): + self.skip_ws() + return True + return False + + @property + def eof(self) -> bool: + return self.pos >= self.end + + @property + def current_char(self) -> str: + try: + return self.definition[self.pos] + except IndexError: + return 'EOF' + + @property + def matched_text(self) -> str: + if self.last_match is not None: + return self.last_match.group() + else: + return None + + def read_rest(self) -> str: + rv = self.definition[self.pos:] + self.pos = self.end + return rv + + def assert_end(self) -> None: + self.skip_ws() + if not self.eof: + self.fail('Expected end of definition.') \ No newline at end of file diff --git a/tests/roots/test-domain-c/conf.py b/tests/roots/test-domain-c/conf.py new file mode 100644 index 000000000..a45d22e28 --- /dev/null +++ b/tests/roots/test-domain-c/conf.py @@ -0,0 +1 @@ +exclude_patterns = ['_build'] diff --git a/tests/roots/test-domain-c/index.rst b/tests/roots/test-domain-c/index.rst new file mode 100644 index 000000000..e81f9c8e8 --- /dev/null +++ b/tests/roots/test-domain-c/index.rst @@ -0,0 +1,17 @@ +test-domain-c +============= + +directives +---------- + +.. c:function:: int hello(char *name) + +.. c:member:: float Sphinx.version + +.. c:macro:: IS_SPHINX + +.. c:macro:: SPHINX(arg1, arg2) + +.. c:type:: Sphinx + +.. c:var:: int version diff --git a/tests/roots/test-root/objects.txt b/tests/roots/test-root/objects.txt index 7e9bc15a9..f713e076c 100644 --- a/tests/roots/test-root/objects.txt +++ b/tests/roots/test-root/objects.txt @@ -107,15 +107,15 @@ Referring to :func:`nothing <>`. C items ======= -.. c:function:: Sphinx_DoSomething() +.. c:function:: void Sphinx_DoSomething() -.. c:member:: SphinxStruct.member +.. c:member:: int SphinxStruct.member .. c:macro:: SPHINX_USE_PYTHON .. c:type:: SphinxType -.. c:var:: sphinx_global +.. c:var:: int sphinx_global Javascript items diff --git a/tests/test_build_changes.py b/tests/test_build_changes.py index 2e87fe0bf..3aedd03f5 100644 --- a/tests/test_build_changes.py +++ b/tests/test_build_changes.py @@ -28,7 +28,7 @@ def test_build(app): assert path_html in htmltext malloc_html = ( - 'Test_Malloc: changed: Changed in version 0.6:' + 'void *Test_Malloc(size_t n): changed: Changed in version 0.6:' ' Can now be replaced with a different allocator.') assert malloc_html in htmltext diff --git a/tests/test_build_html.py b/tests/test_build_html.py index f6f350152..98eb56250 100644 --- a/tests/test_build_html.py +++ b/tests/test_build_html.py @@ -289,11 +289,11 @@ def test_html4_output(app, status, warning): (".//a[@class='reference internal'][@href='#errmod-error']/strong", 'Error'), # C references (".//span[@class='pre']", 'CFunction()'), - (".//a[@href='#c-sphinx-dosomething']", ''), - (".//a[@href='#c-sphinxstruct-member']", ''), - (".//a[@href='#c-sphinx-use-python']", ''), - (".//a[@href='#c-sphinxtype']", ''), - (".//a[@href='#c-sphinx-global']", ''), + (".//a[@href='#c.Sphinx_DoSomething']", ''), + (".//a[@href='#c.SphinxStruct.member']", ''), + (".//a[@href='#c.SPHINX_USE_PYTHON']", ''), + (".//a[@href='#c.SphinxType']", ''), + (".//a[@href='#c.sphinx_global']", ''), # test global TOC created by toctree() (".//ul[@class='current']/li[@class='toctree-l1 current']/a[@href='#']", 'Testing object descriptions'), diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index 4e5de2287..efbf332b4 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -8,73 +8,549 @@ :license: BSD, see LICENSE for details. """ -from docutils import nodes +import re +import pytest + +from docutils import nodes +import sphinx.domains.c as cDomain from sphinx import addnodes from sphinx.addnodes import ( desc, desc_addname, desc_annotation, desc_content, desc_name, desc_optional, desc_parameter, desc_parameterlist, desc_returns, desc_signature, desc_type, pending_xref ) +from sphinx.domains.c import DefinitionParser, DefinitionError +from sphinx.domains.c import _max_id, _id_prefix, Symbol from sphinx.testing import restructuredtext from sphinx.testing.util import assert_node +from sphinx.util import docutils + + +def parse(name, string): + parser = DefinitionParser(string, None) + parser.allowFallbackExpressionParsing = False + ast = parser.parse_declaration(name, name) + parser.assert_end() + return ast + + +def check(name, input, idDict, output=None): + # first a simple check of the AST + if output is None: + output = input + ast = parse(name, input) + res = str(ast) + if res != output: + print("") + print("Input: ", input) + print("Result: ", res) + print("Expected: ", output) + raise DefinitionError("") + rootSymbol = Symbol(None, None, None, None) + symbol = rootSymbol.add_declaration(ast, docname="TestDoc") + parentNode = addnodes.desc() + signode = addnodes.desc_signature(input, '') + parentNode += signode + ast.describe_signature(signode, 'lastIsName', symbol, options={}) + + idExpected = [None] + for i in range(1, _max_id + 1): + if i in idDict: + idExpected.append(idDict[i]) + else: + idExpected.append(idExpected[i - 1]) + idActual = [None] + for i in range(1, _max_id + 1): + #try: + id = ast.get_id(version=i) + assert id is not None + idActual.append(id[len(_id_prefix[i]):]) + #except NoOldIdError: + # idActual.append(None) + + res = [True] + for i in range(1, _max_id + 1): + res.append(idExpected[i] == idActual[i]) + + if not all(res): + print("input: %s" % input.rjust(20)) + for i in range(1, _max_id + 1): + if res[i]: + continue + print("Error in id version %d." % i) + print("result: %s" % idActual[i]) + print("expected: %s" % idExpected[i]) + #print(rootSymbol.dump(0)) + raise DefinitionError("") + + +def test_expressions(): + return # TODO + def exprCheck(expr, id, id4=None): + ids = 'IE1CIA%s_1aE' + idDict = {2: ids % expr, 3: ids % id} + if id4 is not None: + idDict[4] = ids % id4 + check('class', 'template<> C' % expr, idDict) + # primary + exprCheck('nullptr', 'LDnE') + exprCheck('true', 'L1E') + exprCheck('false', 'L0E') + ints = ['5', '0', '075', '0x0123456789ABCDEF', '0XF', '0b1', '0B1'] + unsignedSuffix = ['', 'u', 'U'] + longSuffix = ['', 'l', 'L', 'll', 'LL'] + for i in ints: + for u in unsignedSuffix: + for l in longSuffix: + expr = i + u + l + exprCheck(expr, 'L' + expr + 'E') + expr = i + l + u + exprCheck(expr, 'L' + expr + 'E') + for suffix in ['', 'f', 'F', 'l', 'L']: + for e in [ + '5e42', '5e+42', '5e-42', + '5.', '5.e42', '5.e+42', '5.e-42', + '.5', '.5e42', '.5e+42', '.5e-42', + '5.0', '5.0e42', '5.0e+42', '5.0e-42']: + expr = e + suffix + exprCheck(expr, 'L' + expr + 'E') + for e in [ + 'ApF', 'Ap+F', 'Ap-F', + 'A.', 'A.pF', 'A.p+F', 'A.p-F', + '.A', '.ApF', '.Ap+F', '.Ap-F', + 'A.B', 'A.BpF', 'A.Bp+F', 'A.Bp-F']: + expr = "0x" + e + suffix + exprCheck(expr, 'L' + expr + 'E') + exprCheck('"abc\\"cba"', 'LA8_KcE') # string + exprCheck('this', 'fpT') + # character literals + for p, t in [('', 'c'), ('u8', 'c'), ('u', 'Ds'), ('U', 'Di'), ('L', 'w')]: + exprCheck(p + "'a'", t + "97") + exprCheck(p + "'\\n'", t + "10") + exprCheck(p + "'\\012'", t + "10") + exprCheck(p + "'\\0'", t + "0") + exprCheck(p + "'\\x0a'", t + "10") + exprCheck(p + "'\\x0A'", t + "10") + exprCheck(p + "'\\u0a42'", t + "2626") + exprCheck(p + "'\\u0A42'", t + "2626") + exprCheck(p + "'\\U0001f34c'", t + "127820") + exprCheck(p + "'\\U0001F34C'", t + "127820") + + # TODO: user-defined lit + exprCheck('(... + Ns)', '(... + Ns)', id4='flpl2Ns') + exprCheck('(Ns + ...)', '(Ns + ...)', id4='frpl2Ns') + exprCheck('(Ns + ... + 0)', '(Ns + ... + 0)', id4='fLpl2NsL0E') + exprCheck('(5)', 'L5E') + exprCheck('C', '1C') + # postfix + exprCheck('A(2)', 'cl1AL2EE') + exprCheck('A[2]', 'ix1AL2E') + exprCheck('a.b.c', 'dtdt1a1b1c') + exprCheck('a->b->c', 'ptpt1a1b1c') + exprCheck('i++', 'pp1i') + exprCheck('i--', 'mm1i') + exprCheck('dynamic_cast(i)++', 'ppdcR1T1i') + exprCheck('static_cast(i)++', 'ppscR1T1i') + exprCheck('reinterpret_cast(i)++', 'pprcR1T1i') + exprCheck('const_cast(i)++', 'ppccR1T1i') + exprCheck('typeid(T).name', 'dtti1T4name') + exprCheck('typeid(a + b).name', 'dttepl1a1b4name') + # unary + exprCheck('++5', 'pp_L5E') + exprCheck('--5', 'mm_L5E') + exprCheck('*5', 'deL5E') + exprCheck('&5', 'adL5E') + exprCheck('+5', 'psL5E') + exprCheck('-5', 'ngL5E') + exprCheck('!5', 'ntL5E') + exprCheck('~5', 'coL5E') + exprCheck('sizeof...(a)', 'sZ1a') + exprCheck('sizeof(T)', 'st1T') + exprCheck('sizeof -42', 'szngL42E') + exprCheck('alignof(T)', 'at1T') + exprCheck('noexcept(-42)', 'nxngL42E') + # new-expression + exprCheck('new int', 'nw_iE') + exprCheck('new volatile int', 'nw_ViE') + exprCheck('new int[42]', 'nw_AL42E_iE') + exprCheck('new int()', 'nw_ipiE') + exprCheck('new int(5, 42)', 'nw_ipiL5EL42EE') + exprCheck('::new int', 'nw_iE') + exprCheck('new int{}', 'nw_iilE') + exprCheck('new int{5, 42}', 'nw_iilL5EL42EE') + # delete-expression + exprCheck('delete p', 'dl1p') + exprCheck('delete [] p', 'da1p') + exprCheck('::delete p', 'dl1p') + exprCheck('::delete [] p', 'da1p') + # cast + exprCheck('(int)2', 'cviL2E') + # binary op + exprCheck('5 || 42', 'ooL5EL42E') + exprCheck('5 && 42', 'aaL5EL42E') + exprCheck('5 | 42', 'orL5EL42E') + exprCheck('5 ^ 42', 'eoL5EL42E') + exprCheck('5 & 42', 'anL5EL42E') + # ['==', '!='] + exprCheck('5 == 42', 'eqL5EL42E') + exprCheck('5 != 42', 'neL5EL42E') + # ['<=', '>=', '<', '>'] + exprCheck('5 <= 42', 'leL5EL42E') + exprCheck('5 >= 42', 'geL5EL42E') + exprCheck('5 < 42', 'ltL5EL42E') + exprCheck('5 > 42', 'gtL5EL42E') + # ['<<', '>>'] + exprCheck('5 << 42', 'lsL5EL42E') + exprCheck('5 >> 42', 'rsL5EL42E') + # ['+', '-'] + exprCheck('5 + 42', 'plL5EL42E') + exprCheck('5 - 42', 'miL5EL42E') + # ['*', '/', '%'] + exprCheck('5 * 42', 'mlL5EL42E') + exprCheck('5 / 42', 'dvL5EL42E') + exprCheck('5 % 42', 'rmL5EL42E') + # ['.*', '->*'] + exprCheck('5 .* 42', 'dsL5EL42E') + exprCheck('5 ->* 42', 'pmL5EL42E') + # conditional + # TODO + # assignment + exprCheck('a = 5', 'aS1aL5E') + exprCheck('a *= 5', 'mL1aL5E') + exprCheck('a /= 5', 'dV1aL5E') + exprCheck('a %= 5', 'rM1aL5E') + exprCheck('a += 5', 'pL1aL5E') + exprCheck('a -= 5', 'mI1aL5E') + exprCheck('a >>= 5', 'rS1aL5E') + exprCheck('a <<= 5', 'lS1aL5E') + exprCheck('a &= 5', 'aN1aL5E') + exprCheck('a ^= 5', 'eO1aL5E') + exprCheck('a |= 5', 'oR1aL5E') + + # Additional tests + # a < expression that starts with something that could be a template + exprCheck('A < 42', 'lt1AL42E') + check('function', 'template<> void f(A &v)', + {2: "IE1fR1AI1BX2EE", 3: "IE1fR1AI1BXL2EEE", 4: "IE1fvR1AI1BXL2EEE"}) + exprCheck('A<1>::value', 'N1AIXL1EEE5valueE') + check('class', "template A", {2: "I_iE1A"}) + check('enumerator', 'A = std::numeric_limits::max()', {2: "1A"}) + + exprCheck('operator()()', 'clclE') + exprCheck('operator()()', 'clclIiEE') + + # pack expansion + exprCheck('a(b(c, 1 + d...)..., e(f..., g))', 'cl1aspcl1b1cspplL1E1dEcl1esp1f1gEE') + + +def test_type_definitions(): + check('type', "T", {1: "T"}) + check('type', "struct T", {1: 'T'}) + check('type', "enum T", {1: 'T'}) + check('type', "union T", {1: 'T'}) + + check('type', "bool *b", {1: 'b'}) + check('type', "bool *const b", {1: 'b'}) + check('type', "bool *volatile const b", {1: 'b'}) + check('type', "bool *volatile const b", {1: 'b'}) + check('type', "bool *volatile const *b", {1: 'b'}) + check('type', "bool b[]", {1: 'b'}) + check('type', "long long int foo", {1: 'foo'}) + # test decl specs on right + check('type', "bool const b", {1: 'b'}) + + # from breathe#267 (named function parameters for function pointers + check('type', 'void (*gpio_callback_t)(struct device *port, uint32_t pin)', + {1: 'gpio_callback_t'}) + + +def test_macro_definitions(): + check('macro', 'M', {1: 'M'}) + check('macro', 'M()', {1: 'M'}) + check('macro', 'M(arg)', {1: 'M'}) + check('macro', 'M(arg1, arg2)', {1: 'M'}) + check('macro', 'M(arg1, arg2, arg3)', {1: 'M'}) + check('macro', 'M(...)', {1: 'M'}) + check('macro', 'M(arg, ...)', {1: 'M'}) + check('macro', 'M(arg1, arg2, ...)', {1: 'M'}) + check('macro', 'M(arg1, arg2, arg3, ...)', {1: 'M'}) + + +def test_member_definitions(): + check('member', 'void a', {1: 'a'}) + check('member', '_Bool a', {1: 'a'}) + check('member', 'bool a', {1: 'a'}) + check('member', 'char a', {1: 'a'}) + check('member', 'int a', {1: 'a'}) + check('member', 'float a', {1: 'a'}) + check('member', 'double a', {1: 'a'}) + + check('member', 'unsigned long a', {1: 'a'}) + + check('member', 'int .a', {1: 'a'}) + + check('member', 'int *a', {1: 'a'}) + check('member', 'int **a', {1: 'a'}) + check('member', 'const int a', {1: 'a'}) + check('member', 'volatile int a', {1: 'a'}) + check('member', 'restrict int a', {1: 'a'}) + check('member', 'volatile const int a', {1: 'a'}) + check('member', 'restrict const int a', {1: 'a'}) + check('member', 'restrict volatile int a', {1: 'a'}) + check('member', 'restrict volatile const int a', {1: 'a'}) + + check('member', 'T t', {1: 't'}) + check('member', 'struct T t', {1: 't'}) + check('member', 'enum T t', {1: 't'}) + check('member', 'union T t', {1: 't'}) + + check('member', 'int a[]', {1: 'a'}) + + check('member', 'int (*p)[]', {1: 'p'}) + + check('member', 'int a[42]', {1: 'a'}) + check('member', 'int a = 42', {1: 'a'}) + check('member', 'T a = {}', {1: 'a'}) + check('member', 'T a = {1}', {1: 'a'}) + check('member', 'T a = {1, 2}', {1: 'a'}) + check('member', 'T a = {1, 2, 3}', {1: 'a'}) + + # test from issue #1539 + check('member', 'CK_UTF8CHAR model[16]', {1: 'model'}) + + check('member', 'auto int a', {1: 'a'}) + check('member', 'register int a', {1: 'a'}) + check('member', 'extern int a', {1: 'a'}) + check('member', 'static int a', {1: 'a'}) + + check('member', 'thread_local int a', {1: 'a'}) + check('member', '_Thread_local int a', {1: 'a'}) + check('member', 'extern thread_local int a', {1: 'a'}) + check('member', 'thread_local extern int a', {1: 'a'}, + 'extern thread_local int a') + check('member', 'static thread_local int a', {1: 'a'}) + check('member', 'thread_local static int a', {1: 'a'}, + 'static thread_local int a') + + check('member', 'int b : 3', {1: 'b'}) + + +def test_function_definitions(): + check('function', 'void f()', {1: 'f'}) + check('function', 'void f(int)', {1: 'f'}) + check('function', 'void f(int i)', {1: 'f'}) + check('function', 'void f(int i, int j)', {1: 'f'}) + check('function', 'void f(...)', {1: 'f'}) + check('function', 'void f(int i, ...)', {1: 'f'}) + check('function', 'void f(struct T)', {1: 'f'}) + check('function', 'void f(struct T t)', {1: 'f'}) + check('function', 'void f(union T)', {1: 'f'}) + check('function', 'void f(union T t)', {1: 'f'}) + check('function', 'void f(enum T)', {1: 'f'}) + check('function', 'void f(enum T t)', {1: 'f'}) + + # test from issue #1539 + check('function', 'void f(A x[])', {1: 'f'}) + + # test from issue #2377 + check('function', 'void (*signal(int sig, void (*func)(int)))(int)', {1: 'signal'}) + + check('function', 'extern void f()', {1: 'f'}) + check('function', 'static void f()', {1: 'f'}) + check('function', 'inline void f()', {1: 'f'}) + + # tests derived from issue #1753 (skip to keep sanity) + check('function', "void f(float *q(double))", {1: 'f'}) + check('function', "void f(float *(*q)(double))", {1: 'f'}) + check('function', "void f(float (*q)(double))", {1: 'f'}) + check('function', "int (*f(double d))(float)", {1: 'f'}) + check('function', "int (*f(bool b))[5]", {1: 'f'}) + check('function', "void f(int *const p)", {1: 'f'}) + check('function', "void f(int *volatile const p)", {1: 'f'}) + + # from breathe#223 + check('function', 'void f(struct E e)', {1: 'f'}) + check('function', 'void f(enum E e)', {1: 'f'}) + check('function', 'void f(union E e)', {1: 'f'}) + + +def test_union_definitions(): + return # TODO + check('union', 'A', {2: "1A"}) + + +def test_enum_definitions(): + return # TODO + check('enum', 'A', {2: "1A"}) + check('enum', 'A : std::underlying_type::type', {2: "1A"}) + check('enum', 'A : unsigned int', {2: "1A"}) + check('enum', 'public A', {2: "1A"}, output='A') + check('enum', 'private A', {2: "1A"}) + + check('enumerator', 'A', {2: "1A"}) + check('enumerator', 'A = std::numeric_limits::max()', {2: "1A"}) + + +def test_anon_definitions(): + return # TODO + check('class', '@a', {3: "Ut1_a"}) + check('union', '@a', {3: "Ut1_a"}) + check('enum', '@a', {3: "Ut1_a"}) + check('class', '@1', {3: "Ut1_1"}) + check('class', '@a::A', {3: "NUt1_a1AE"}) + + +def test_initializers(): + return # TODO + idsMember = {1: 'v__T', 2: '1v'} + idsFunction = {1: 'f__T', 2: '1f1T'} + idsTemplate = {2: 'I_1TE1fv', 4: 'I_1TE1fvv'} + # no init + check('member', 'T v', idsMember) + check('function', 'void f(T v)', idsFunction) + check('function', 'template void f()', idsTemplate) + # with '=', assignment-expression + check('member', 'T v = 42', idsMember) + check('function', 'void f(T v = 42)', idsFunction) + check('function', 'template void f()', idsTemplate) + # with '=', braced-init + check('member', 'T v = {}', idsMember) + check('function', 'void f(T v = {})', idsFunction) + check('function', 'template void f()', idsTemplate) + check('member', 'T v = {42, 42, 42}', idsMember) + check('function', 'void f(T v = {42, 42, 42})', idsFunction) + check('function', 'template void f()', idsTemplate) + check('member', 'T v = {42, 42, 42,}', idsMember) + check('function', 'void f(T v = {42, 42, 42,})', idsFunction) + check('function', 'template void f()', idsTemplate) + check('member', 'T v = {42, 42, args...}', idsMember) + check('function', 'void f(T v = {42, 42, args...})', idsFunction) + check('function', 'template void f()', idsTemplate) + # without '=', braced-init + check('member', 'T v{}', idsMember) + check('member', 'T v{42, 42, 42}', idsMember) + check('member', 'T v{42, 42, 42,}', idsMember) + check('member', 'T v{42, 42, args...}', idsMember) + # other + check('member', 'T v = T{}', idsMember) + + +def test_attributes(): + return # TODO + # style: C++ + check('member', '[[]] int f', {1: 'f__i', 2: '1f'}) + check('member', '[ [ ] ] int f', {1: 'f__i', 2: '1f'}, + # this will fail when the proper grammar is implemented + output='[[ ]] int f') + check('member', '[[a]] int f', {1: 'f__i', 2: '1f'}) + # style: GNU + check('member', '__attribute__(()) int f', {1: 'f__i', 2: '1f'}) + check('member', '__attribute__((a)) int f', {1: 'f__i', 2: '1f'}) + check('member', '__attribute__((a, b)) int f', {1: 'f__i', 2: '1f'}) + # style: user-defined id + check('member', 'id_attr int f', {1: 'f__i', 2: '1f'}) + # style: user-defined paren + check('member', 'paren_attr() int f', {1: 'f__i', 2: '1f'}) + check('member', 'paren_attr(a) int f', {1: 'f__i', 2: '1f'}) + check('member', 'paren_attr("") int f', {1: 'f__i', 2: '1f'}) + check('member', 'paren_attr(()[{}][]{}) int f', {1: 'f__i', 2: '1f'}) + with pytest.raises(DefinitionError): + parse('member', 'paren_attr(() int f') + with pytest.raises(DefinitionError): + parse('member', 'paren_attr([) int f') + with pytest.raises(DefinitionError): + parse('member', 'paren_attr({) int f') + with pytest.raises(DefinitionError): + parse('member', 'paren_attr([)]) int f') + with pytest.raises(DefinitionError): + parse('member', 'paren_attr((])) int f') + with pytest.raises(DefinitionError): + parse('member', 'paren_attr({]}) int f') + + # position: decl specs + check('function', 'static inline __attribute__(()) void f()', + {1: 'f', 2: '1fv'}, + output='__attribute__(()) static inline void f()') + check('function', '[[attr1]] [[attr2]] void f()', + {1: 'f', 2: '1fv'}, + output='[[attr1]] [[attr2]] void f()') + # position: declarator + check('member', 'int *[[attr]] i', {1: 'i__iP', 2: '1i'}) + check('member', 'int *const [[attr]] volatile i', {1: 'i__iPVC', 2: '1i'}, + output='int *[[attr]] volatile const i') + check('member', 'int &[[attr]] i', {1: 'i__iR', 2: '1i'}) + check('member', 'int *[[attr]] *i', {1: 'i__iPP', 2: '1i'}) + + +def test_xref_parsing(): + return # TODO + def check(target): + class Config: + cpp_id_attributes = ["id_attr"] + cpp_paren_attributes = ["paren_attr"] + parser = DefinitionParser(target, None, Config()) + ast, isShorthand = parser.parse_xref_object() + parser.assert_end() + check('f') + check('f()') + check('void f()') + check('T f()') + + +# def test_print(): +# # used for getting all the ids out for checking +# for a in ids: +# print(a) +# raise DefinitionError("") + + +def filter_warnings(warning, file): + lines = warning.getvalue().split("\n"); + res = [l for l in lines if "domain-c" in l and "{}.rst".format(file) in l and + "WARNING: document isn't included in any toctree" not in l] + print("Filtered warnings for file '{}':".format(file)) + for w in res: + print(w) + return res + + +@pytest.mark.sphinx(testroot='domain-c', confoverrides={'nitpicky': True}) +def test_build_domain_c(app, status, warning): + app.builder.build_all() + ws = filter_warnings(warning, "index") + assert len(ws) == 0 def test_cfunction(app): text = (".. c:function:: PyObject* " "PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)") doctree = restructuredtext.parse(app, text) - assert_node(doctree, - (addnodes.index, - [desc, ([desc_signature, ([desc_type, ([pending_xref, "PyObject"], - "* ")], - [desc_name, "PyType_GenericAlloc"], - [desc_parameterlist, (desc_parameter, - desc_parameter)])], - desc_content)])) assert_node(doctree[1], addnodes.desc, desctype="function", domain="c", objtype="function", noindex=False) - assert_node(doctree[1][0][2][0], - [desc_parameter, ([pending_xref, "PyTypeObject"], - [nodes.emphasis, "\xa0*type"])]) - assert_node(doctree[1][0][2][1], - [desc_parameter, ([pending_xref, "Py_ssize_t"], - [nodes.emphasis, "\xa0nitems"])]) domain = app.env.get_domain('c') entry = domain.objects.get('PyType_GenericAlloc') - assert entry == ('index', 'c-pytype-genericalloc', 'function') + assert entry == ('index', 'c.PyType_GenericAlloc', 'function') def test_cmember(app): text = ".. c:member:: PyObject* PyTypeObject.tp_bases" doctree = restructuredtext.parse(app, text) - assert_node(doctree, - (addnodes.index, - [desc, ([desc_signature, ([desc_type, ([pending_xref, "PyObject"], - "* ")], - [desc_name, "PyTypeObject.tp_bases"])], - desc_content)])) assert_node(doctree[1], addnodes.desc, desctype="member", domain="c", objtype="member", noindex=False) domain = app.env.get_domain('c') entry = domain.objects.get('PyTypeObject.tp_bases') - assert entry == ('index', 'c-pytypeobject-tp-bases', 'member') + assert entry == ('index', 'c.PyTypeObject.tp_bases', 'member') def test_cvar(app): text = ".. c:var:: PyObject* PyClass_Type" doctree = restructuredtext.parse(app, text) - assert_node(doctree, - (addnodes.index, - [desc, ([desc_signature, ([desc_type, ([pending_xref, "PyObject"], - "* ")], - [desc_name, "PyClass_Type"])], - desc_content)])) assert_node(doctree[1], addnodes.desc, desctype="var", domain="c", objtype="var", noindex=False) domain = app.env.get_domain('c') entry = domain.objects.get('PyClass_Type') - assert entry == ('index', 'c-pyclass-type', 'var') + assert entry == ('index', 'c.PyClass_Type', 'var') From 815a9c657d235ec3476dd45871adc52034d81aba Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 15 Mar 2020 00:36:46 +0100 Subject: [PATCH 02/12] C, add new directives, fix xref lookup --- doc/usage/restructuredtext/domains.rst | 100 +++- sphinx/domains/c.py | 652 ++++++++++++++++++------- tests/roots/test-domain-c/index.rst | 27 +- tests/test_domain_c.py | 24 +- 4 files changed, 591 insertions(+), 212 deletions(-) diff --git a/doc/usage/restructuredtext/domains.rst b/doc/usage/restructuredtext/domains.rst index 6b51f1135..f399199b6 100644 --- a/doc/usage/restructuredtext/domains.rst +++ b/doc/usage/restructuredtext/domains.rst @@ -552,6 +552,15 @@ The C Domain The C domain (name **c**) is suited for documentation of C API. +.. rst:directive:: .. c:member:: declaration + .. c:var:: declaration + + Describes a C struct member or variable. Example signature:: + + .. c:member:: PyObject *PyTypeObject.tp_bases + + The difference between the two directives is only cosmetic. + .. rst:directive:: .. c:function:: function prototype Describes a C function. The signature should be given as in C, e.g.:: @@ -561,19 +570,39 @@ The C domain (name **c**) is suited for documentation of C API. Note that you don't have to backslash-escape asterisks in the signature, as it is not parsed by the reST inliner. -.. rst:directive:: .. c:member:: declaration - .. c:var:: declaration - - Describes a C struct member or variable. Example signature:: - - .. c:member:: PyObject *PyTypeObject.tp_bases - .. rst:directive:: .. c:macro:: name .. c:macro:: name(arg list) Describes a C macro, i.e., a C-language ``#define``, without the replacement text. + .. versionadded:: 3.0 + The function style variant. + +.. rst:directive:: .. c:struct:: name + + Describes a C struct. + + .. versionadded:: 3.0 + +.. rst:directive:: .. c:union:: name + + Describes a C union. + + .. versionadded:: 3.0 + +.. rst:directive:: .. c:enum:: name + + Describes a C enum. + + .. versionadded:: 3.0 + +.. rst:directive:: .. c:enumerator:: name + + Describes a C enumerator. + + .. versionadded:: 3.0 + .. rst:directive:: .. c:type:: typedef-like declaration .. c:type:: name @@ -588,22 +617,61 @@ Cross-referencing C constructs The following roles create cross-references to C-language constructs if they are defined in the documentation: -.. rst:role:: c:func - - Reference a C-language function. Should include trailing parentheses. - .. rst:role:: c:member c:data + c:var + c:func + c:macro + c:struct + c:union + c:enum + c:enumerator + c:type - Reference a C-language member of a struct or variable. + Reference a C declaration, as defined above. + Note that :rst:role:`c:member`, :rst:role:`c:data`, and + :rst:role:`c:var` are equivalent. -.. rst:role:: c:macro + .. versionadded:: 3.0 + The var, struct, union, enum, and enumerator roles. - Reference a simple C macro, as defined above. -.. rst:role:: c:type +Anonymous Entities +~~~~~~~~~~~~~~~~~~ - Reference a C-language type. +C supports anonymous structs, enums, and unions. +For the sake of documentation they must be given some name that starts with +``@``, e.g., ``@42`` or ``@data``. +These names can also be used in cross-references, +though nested symbols will be found even when omitted. +The ``@...`` name will always be rendered as **[anonymous]** (possibly as a +link). + +Example:: + + .. c:struct:: Data + + .. c:union:: @data + + .. c:var:: int a + + .. c:var:: double b + + Explicit ref: :c:var:`Data.@data.a`. Short-hand ref: :c:var:`Data.a`. + +This will be rendered as: + +.. c:struct:: Data + + .. c:union:: @data + + .. c:var:: int a + + .. c:var:: double b + +Explicit ref: :c:var:`Data.@data.a`. Short-hand ref: :c:var:`Data.a`. + +.. versionadded:: 3.0 .. _cpp-domain: diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 021e9d8ba..ea22824f4 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -9,16 +9,13 @@ """ import re -import string -import warnings -from copy import deepcopy from typing import ( - Any, Callable, Dict, Iterator, List, Match, Pattern, Tuple, Type, TypeVar, Union + Any, Callable, Dict, Iterator, List, Tuple, Union ) from typing import cast from docutils import nodes -from docutils.nodes import Element, Node, TextElement +from docutils.nodes import Element, Node from sphinx import addnodes from sphinx.addnodes import pending_xref, desc_signature @@ -27,6 +24,7 @@ from sphinx.builders import Builder from sphinx.directives import ObjectDescription from sphinx.domains import Domain, ObjType from sphinx.environment import BuildEnvironment +from sphinx.errors import NoUri from sphinx.locale import _, __ from sphinx.roles import XRefRole from sphinx.util import logging @@ -101,26 +99,6 @@ class ASTIdentifier(ASTBase): def is_anon(self) -> bool: return self.identifier[0] == '@' - def get_id(self, version: int) -> str: - assert False - if self.is_anon() and version < 3: - raise NoOldIdError() - if version == 1: - if self.identifier == 'size_t': - return 's' - else: - return self.identifier - if self.identifier == "std": - return 'St' - elif self.identifier[0] == "~": - # a destructor, just use an arbitrary version of dtors - return 'D0' - else: - if self.is_anon(): - return 'Ut%d_%s' % (len(self.identifier) - 1, self.identifier[1:]) - else: - return str(len(self.identifier)) + self.identifier - # and this is where we finally make a difference between __str__ and the display string def __str__(self) -> str: @@ -498,10 +476,6 @@ class ASTDeclaratorPtr(ASTBase): res.append(transform(self.next)) return ''.join(res) - def is_function_type(self) -> bool: - assert False - return self.next.is_function_type() - def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) @@ -556,9 +530,6 @@ class ASTDeclaratorParen(ASTBase): res.append(transform(self.next)) return ''.join(res) - def is_function_type(self) -> bool: - return self.inner.is_function_type() - def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) @@ -569,11 +540,10 @@ class ASTDeclaratorParen(ASTBase): class ASTDeclaratorNameParam(ASTBase): - def __init__(self, declId: Any, arrayOps: List[Any], param: Any, prefix: str) -> None: + def __init__(self, declId: Any, arrayOps: List[Any], param: Any) -> None: self.declId = declId self.arrayOps = arrayOps self.param = param - self.prefix = prefix @property def name(self) -> ASTNestedName: @@ -588,15 +558,8 @@ class ASTDeclaratorNameParam(ASTBase): def require_space_after_declSpecs(self) -> bool: return self.declId is not None - def is_function_type(self) -> bool: - return self.param is not None - def _stringify(self, transform: StringifyTransform) -> str: res = [] - if self.prefix is not None: - res.append(self.prefix) - res.append(' ') - assert self.declId if self.declId: res.append(transform(self.declId)) for op in self.arrayOps: @@ -608,9 +571,6 @@ class ASTDeclaratorNameParam(ASTBase): def describe_signature(self, signode: desc_signature, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) - if self.prefix: - signode += addnodes.desc_annotation(self.prefix, self.prefix) - signode += nodes.Text(' ') if self.declId: self.declId.describe_signature(signode, mode, env, symbol) for op in self.arrayOps: @@ -633,9 +593,6 @@ class ASTDeclaratorNameBitField(ASTBase): def require_space_after_declSpecs(self) -> bool: return self.declId is not None - def is_function_type(self) -> bool: - return False - def _stringify(self, transform: StringifyTransform) -> str: res = [] if self.declId: @@ -702,7 +659,6 @@ class ASTType(ASTBase): return ''.join(res) def get_type_declaration_prefix(self) -> str: - assert False if self.declSpecs.trailingTypeSpec: return 'typedef' else: @@ -783,6 +739,77 @@ class ASTMacro(ASTBase): signode += paramlist +class ASTStruct(ASTBase): + def __init__(self, name: ASTNestedName) -> None: + self.name = name + + def get_id(self, version: int, objectType: str, symbol: "Symbol") -> str: + return symbol.get_full_nested_name().get_id(version) + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.name) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + self.name.describe_signature(signode, mode, env, symbol=symbol) + + +class ASTUnion(ASTBase): + def __init__(self, name: ASTNestedName) -> None: + self.name = name + + def get_id(self, version: int, objectType: str, symbol: "Symbol") -> str: + return symbol.get_full_nested_name().get_id(version) + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.name) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + self.name.describe_signature(signode, mode, env, symbol=symbol) + + +class ASTEnum(ASTBase): + def __init__(self, name: ASTNestedName) -> None: + self.name = name + + def get_id(self, version: int, objectType: str, symbol: "Symbol") -> str: + return symbol.get_full_nested_name().get_id(version) + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.name) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + self.name.describe_signature(signode, mode, env, symbol=symbol) + + +class ASTEnumerator(ASTBase): + def __init__(self, name: Any, init: Any) -> None: + self.name = name + self.init = init + + def get_id(self, version: int, objectType: str, symbol: "Symbol") -> str: + return symbol.get_full_nested_name().get_id(version) + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + res.append(transform(self.name)) + if self.init: + res.append(transform(self.init)) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + self.name.describe_signature(signode, mode, env, symbol) + if self.init: + self.init.describe_signature(signode, 'markType', env, symbol) + + class ASTDeclaration(ASTBase): def __init__(self, objectType: str, directiveType: str, declaration: Any) -> None: self.objectType = objectType @@ -790,8 +817,8 @@ class ASTDeclaration(ASTBase): self.declaration = declaration self.symbol = None # type: Symbol - # set by CPPObject._add_enumerator_to_parent - # self.enumeratorScopedSymbol = None # type: Symbol + # set by CObject._add_enumerator_to_parent + self.enumeratorScopedSymbol = None # type: Symbol @property def name(self) -> ASTNestedName: @@ -804,6 +831,8 @@ class ASTDeclaration(ASTBase): return self.declaration.function_params def get_id(self, version: int, prefixed: bool = True) -> str: + if self.objectType == 'enumerator' and self.enumeratorScopedSymbol: + return self.enumeratorScopedSymbol.declaration.get_id(version, prefixed) id_ = self.symbol.get_full_nested_name().get_id(version) if prefixed: return _id_prefix[version] + id_ @@ -820,7 +849,30 @@ class ASTDeclaration(ASTBase): env: "BuildEnvironment", options: Dict) -> None: verify_description_mode(mode) assert self.symbol - self.declaration.describe_signature(signode, mode, env, self.symbol) + + mainDeclNode = signode + + if self.objectType == 'member': + pass + elif self.objectType == 'function': + pass + elif self.objectType == 'macro': + pass + elif self.objectType == 'struct': + mainDeclNode += addnodes.desc_annotation('struct ', 'struct ') + elif self.objectType == 'union': + mainDeclNode += addnodes.desc_annotation('union ', 'union ') + elif self.objectType == 'enum': + mainDeclNode += addnodes.desc_annotation('enum ', 'enum ') + elif self.objectType == 'enumerator': + mainDeclNode += addnodes.desc_annotation('enumerator ', 'enumerator ') + elif self.objectType == 'type': + prefix = self.declaration.get_type_declaration_prefix() + prefix += ' ' + mainDeclNode += addnodes.desc_annotation(prefix, prefix) + else: + assert False + self.declaration.describe_signature(mainDeclNode, mode, env, self.symbol) class SymbolLookupResult: @@ -835,6 +887,10 @@ class LookupKey: def __init__(self, data: List[Tuple[ASTIdentifier, str]]) -> None: self.data = data + def __str__(self): + return '[{}]'.format(', '.join("({}, {})".format( + ident, id_) for ident, id_ in self.data)) + class Symbol: debug_indent = 0 @@ -1082,7 +1138,7 @@ class Symbol: # walk up until we find the first identifier firstName = names[0] while parentSymbol.parent: - if parentSymbol.find_identifier(firstName.ident, + if parentSymbol.find_identifier(firstName.identifier, matchSelf=matchSelf, recurseInAnon=recurseInAnon, searchInSiblings=searchInSiblings): @@ -1208,6 +1264,7 @@ class Symbol: Symbol.debug_print("#noDecl: ", len(noDecl)) Symbol.debug_print("#withDecl:", len(withDecl)) Symbol.debug_print("#dupDecl: ", len(dupDecl)) + # With partial builds we may start with a large symbol tree stripped of declarations. # Essentially any combination of noDecl, withDecl, and dupDecls seems possible. # TODO: make partial builds fully work. What should happen when the primary symbol gets @@ -1226,6 +1283,7 @@ class Symbol: if Symbol.debug_lookup: Symbol.debug_print("end: creating candidate symbol") return symbol + if len(withDecl) == 0: candSymbol = None else: @@ -1272,7 +1330,9 @@ class Symbol: return makeCandSymbol() else: if Symbol.debug_lookup: - Symbol.debug_print("no match, but fill an empty declaration, candSybmol is not None?:", candSymbol is not None) # NOQA + Symbol.debug_print( + "no match, but fill an empty declaration, candSybmol is not None?:", + candSymbol is not None) # NOQA Symbol.debug_indent -= 2 if candSymbol is not None: candSymbol.remove() @@ -1343,6 +1403,92 @@ class Symbol: Symbol.debug_indent -= 1 return res + def find_identifier(self, ident: ASTIdentifier, + matchSelf: bool, recurseInAnon: bool, searchInSiblings: bool + ) -> "Symbol": + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("find_identifier:") + Symbol.debug_indent += 1 + Symbol.debug_print("ident: ", ident) + Symbol.debug_print("matchSelf: ", matchSelf) + Symbol.debug_print("recurseInAnon: ", recurseInAnon) + Symbol.debug_print("searchInSiblings:", searchInSiblings) + print(self.to_string(Symbol.debug_indent + 1), end="") + Symbol.debug_indent -= 2 + current = self + while current is not None: + if Symbol.debug_lookup: + Symbol.debug_indent += 2 + Symbol.debug_print("trying:") + print(current.to_string(Symbol.debug_indent + 1), end="") + Symbol.debug_indent -= 2 + if matchSelf and current.ident == ident: + return current + children = current.children_recurse_anon if recurseInAnon else current._children + for s in children: + if s.ident == ident: + return s + if not searchInSiblings: + break + current = current.siblingAbove + return None + + def direct_lookup(self, key: "LookupKey") -> "Symbol": + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("direct_lookup:") + Symbol.debug_indent += 1 + s = self + for name, id_ in key.data: + res = None + for cand in s._children: + if cand.ident == name: + res = cand + break + s = res + if Symbol.debug_lookup: + Symbol.debug_print("name: ", name) + Symbol.debug_print("id: ", id_) + if s is not None: + print(s.to_string(Symbol.debug_indent + 1), end="") + else: + Symbol.debug_print("not found") + if s is None: + if Symbol.debug_lookup: + Symbol.debug_indent -= 2 + return None + if Symbol.debug_lookup: + Symbol.debug_indent -= 2 + return s + + def find_declaration(self, nestedName: ASTNestedName, typ: str, + matchSelf: bool, recurseInAnon: bool) -> "Symbol": + # templateShorthand: missing template parameter lists for templates is ok + if Symbol.debug_lookup: + Symbol.debug_indent += 1 + Symbol.debug_print("find_declaration:") + + def onMissingQualifiedSymbol(parentSymbol: "Symbol", + ident: ASTIdentifier) -> "Symbol": + return None + + lookupResult = self._symbol_lookup(nestedName, + onMissingQualifiedSymbol, + ancestorLookupType=typ, + matchSelf=matchSelf, + recurseInAnon=recurseInAnon, + searchInSiblings=False) + if Symbol.debug_lookup: + Symbol.debug_indent -= 1 + if lookupResult is None: + return None + + symbols = list(lookupResult.symbols) + if len(symbols) == 0: + return None + return symbols[0] + def to_string(self, indent: int) -> str: res = [Symbol.debug_indent_string * indent] if not self.parent: @@ -1382,65 +1528,64 @@ class DefinitionParser(BaseParser): def _parse_attribute(self) -> Any: return None - self.skip_ws() - # try C++11 style - startPos = self.pos - if self.skip_string_and_ws('['): - if not self.skip_string('['): - self.pos = startPos - else: - # TODO: actually implement the correct grammar - arg = self._parse_balanced_token_seq(end=[']']) - if not self.skip_string_and_ws(']'): - self.fail("Expected ']' in end of attribute.") - if not self.skip_string_and_ws(']'): - self.fail("Expected ']' in end of attribute after [[...]") - return ASTCPPAttribute(arg) - - # try GNU style - if self.skip_word_and_ws('__attribute__'): - if not self.skip_string_and_ws('('): - self.fail("Expected '(' after '__attribute__'.") - if not self.skip_string_and_ws('('): - self.fail("Expected '(' after '__attribute__('.") - attrs = [] - while 1: - if self.match(identifier_re): - name = self.matched_text - self.skip_ws() - if self.skip_string_and_ws('('): - self.fail('Parameterized GNU style attribute not yet supported.') - attrs.append(ASTGnuAttribute(name, None)) - # TODO: parse arguments for the attribute - if self.skip_string_and_ws(','): - continue - elif self.skip_string_and_ws(')'): - break - else: - self.fail("Expected identifier, ')', or ',' in __attribute__.") - if not self.skip_string_and_ws(')'): - self.fail("Expected ')' after '__attribute__((...)'") - return ASTGnuAttributeList(attrs) - - # try the simple id attributes defined by the user - for id in self.config.cpp_id_attributes: - if self.skip_word_and_ws(id): - return ASTIdAttribute(id) - - # try the paren attributes defined by the user - for id in self.config.cpp_paren_attributes: - if not self.skip_string_and_ws(id): - continue - if not self.skip_string('('): - self.fail("Expected '(' after user-defined paren-attribute.") - arg = self._parse_balanced_token_seq(end=[')']) - if not self.skip_string(')'): - self.fail("Expected ')' to end user-defined paren-attribute.") - return ASTParenAttribute(id, arg) + # self.skip_ws() + # # try C++11 style + # startPos = self.pos + # if self.skip_string_and_ws('['): + # if not self.skip_string('['): + # self.pos = startPos + # else: + # # TODO: actually implement the correct grammar + # arg = self._parse_balanced_token_seq(end=[']']) + # if not self.skip_string_and_ws(']'): + # self.fail("Expected ']' in end of attribute.") + # if not self.skip_string_and_ws(']'): + # self.fail("Expected ']' in end of attribute after [[...]") + # return ASTCPPAttribute(arg) + # + # # try GNU style + # if self.skip_word_and_ws('__attribute__'): + # if not self.skip_string_and_ws('('): + # self.fail("Expected '(' after '__attribute__'.") + # if not self.skip_string_and_ws('('): + # self.fail("Expected '(' after '__attribute__('.") + # attrs = [] + # while 1: + # if self.match(identifier_re): + # name = self.matched_text + # self.skip_ws() + # if self.skip_string_and_ws('('): + # self.fail('Parameterized GNU style attribute not yet supported.') + # attrs.append(ASTGnuAttribute(name, None)) + # # TODO: parse arguments for the attribute + # if self.skip_string_and_ws(','): + # continue + # elif self.skip_string_and_ws(')'): + # break + # else: + # self.fail("Expected identifier, ')', or ',' in __attribute__.") + # if not self.skip_string_and_ws(')'): + # self.fail("Expected ')' after '__attribute__((...)'") + # return ASTGnuAttributeList(attrs) + # + # # try the simple id attributes defined by the user + # for id in self.config.cpp_id_attributes: + # if self.skip_word_and_ws(id): + # return ASTIdAttribute(id) + # + # # try the paren attributes defined by the user + # for id in self.config.cpp_paren_attributes: + # if not self.skip_string_and_ws(id): + # continue + # if not self.skip_string('('): + # self.fail("Expected '(' after user-defined paren-attribute.") + # arg = self._parse_balanced_token_seq(end=[')']) + # if not self.skip_string(')'): + # self.fail("Expected ')' to end user-defined paren-attribute.") + # return ASTParenAttribute(id, arg) return None - def _parse_expression_fallback(self, end, parser, allow=True): # Stupidly "parse" an expression. # 'end' should be a list of characters which ends the expression. @@ -1659,7 +1804,6 @@ class DefinitionParser(BaseParser): self, named: Union[bool, str], paramMode: str, typed: bool ) -> Union[ASTDeclaratorNameParam, ASTDeclaratorNameBitField]: # now we should parse the name, and then suffixes - prefix = None # struct/union/enum for when named but not typed if named == 'maybe': pos = self.pos try: @@ -1674,12 +1818,6 @@ class DefinitionParser(BaseParser): else: declId = None elif named: - if not typed: - self.skip_ws() - for p in self._prefix_keys: - if self.skip_word(p): - prefix = p - break declId = self._parse_nested_name() else: declId = None @@ -1713,7 +1851,7 @@ class DefinitionParser(BaseParser): size = self.read_rest().strip() return ASTDeclaratorNameBitField(declId=declId, size=size) return ASTDeclaratorNameParam(declId=declId, arrayOps=arrayOps, - param=param, prefix=prefix) + param=param) def _parse_declarator(self, named: Union[bool, str], paramMode: str, typed: bool = True) -> Any: @@ -1802,9 +1940,9 @@ class DefinitionParser(BaseParser): return None # TODO: implement - #bracedInit = self._parse_braced_init_list() - #if bracedInit is not None: - # return ASTInitializer(bracedInit) + # bracedInit = self._parse_braced_init_list() + # if bracedInit is not None: + # return ASTInitializer(bracedInit) if outer == 'member': fallbackEnd = [] # type: List[str] @@ -1816,6 +1954,7 @@ class DefinitionParser(BaseParser): def parser(): return self._parse_assignment_expression() + value = self._parse_expression_fallback(fallbackEnd, parser, allow=allowFallback) return ASTInitializer(value) @@ -1918,24 +2057,68 @@ class DefinitionParser(BaseParser): self.fail("Expected identifier, ')', or ',' in macro parameter list.") return ASTMacro(ident, args) + def _parse_struct(self) -> ASTStruct: + name = self._parse_nested_name() + return ASTStruct(name) + + def _parse_union(self) -> ASTUnion: + name = self._parse_nested_name() + return ASTUnion(name) + + def _parse_enum(self) -> ASTEnum: + name = self._parse_nested_name() + return ASTEnum(name) + + def _parse_enumerator(self) -> ASTEnumerator: + name = self._parse_nested_name() + self.skip_ws() + init = None + if self.skip_string('='): + self.skip_ws() + + def parser(): + return self._parse_constant_expression(inTemplate=False) + + initVal = self._parse_expression_fallback([], parser) + init = ASTInitializer(initVal) + return ASTEnumerator(name, init) + def parse_declaration(self, objectType: str, directiveType: str) -> ASTDeclaration: - if objectType not in ('function', 'macro', 'member', 'type'): + if objectType not in ('function', 'member', + 'macro', 'struct', 'union', 'enum', 'enumerator', 'type'): raise Exception('Internal error, unknown objectType "%s".' % objectType) - if directiveType not in ('function', 'macro', 'member', 'var', 'type'): + if directiveType not in ('function', 'member', 'var', + 'macro', 'struct', 'union', 'enum', 'enumerator', 'type'): raise Exception('Internal error, unknown directiveType "%s".' % directiveType) - if objectType == 'function': + if objectType == 'member': + declaration = self._parse_type_with_init(named=True, outer='member') + elif objectType == 'function': declaration = self._parse_type(named=True, outer='function') elif objectType == 'macro': declaration = self._parse_macro() - elif objectType == 'member': - declaration = self._parse_type_with_init(named=True, outer='member') + elif objectType == 'struct': + declaration = self._parse_struct() + elif objectType == 'union': + declaration = self._parse_union() + elif objectType == 'enum': + declaration = self._parse_enum() + elif objectType == 'enumerator': + declaration = self._parse_enumerator() elif objectType == 'type': declaration = self._parse_type(named=True, outer='type') else: assert False return ASTDeclaration(objectType, directiveType, declaration) + def parse_xref_object(self) -> ASTNestedName: + name = self._parse_nested_name() + # if there are '()' left, just skip them + self.skip_ws() + self.skip_string('()') + self.assert_end() + return name + def _make_phony_error_name() -> ASTNestedName: return ASTNestedName([ASTIdentifier("PhonyNameDueToError")], rooted=False) @@ -1959,7 +2142,45 @@ class CObject(ObjectDescription): def warn(self, msg: Union[str, Exception]) -> None: self.state_machine.reporter.warning(msg, line=self.lineno) - def add_target_and_index(self, ast: ASTDeclaration, sig: str, signode: desc_signature) -> None: + def _add_enumerator_to_parent(self, ast: ASTDeclaration) -> None: + assert ast.objectType == 'enumerator' + # find the parent, if it exists && is an enum + # then add the name to the parent scope + symbol = ast.symbol + assert symbol + assert symbol.ident is not None + parentSymbol = symbol.parent + assert parentSymbol + if parentSymbol.parent is None: + # TODO: we could warn, but it is somewhat equivalent to + # enumeratorss, without the enum + return # no parent + parentDecl = parentSymbol.declaration + if parentDecl is None: + # the parent is not explicitly declared + # TODO: we could warn, but? + return + if parentDecl.objectType != 'enum': + # TODO: maybe issue a warning, enumerators in non-enums is weird, + # but it is somewhat equivalent to enumeratorss, without the enum + return + if parentDecl.directiveType != 'enum': + return + + targetSymbol = parentSymbol.parent + s = targetSymbol.find_identifier(symbol.ident, matchSelf=False, recurseInAnon=True, + searchInSiblings=False) + if s is not None: + # something is already declared with that name + return + declClone = symbol.declaration.clone() + declClone.enumeratorScopedSymbol = symbol + Symbol(parent=targetSymbol, ident=symbol.ident, + declaration=declClone, + docname=self.env.docname) + + def add_target_and_index(self, ast: ASTDeclaration, sig: str, + signode: desc_signature) -> None: ids = [] for i in range(1, _max_id + 1): try: @@ -1974,7 +2195,6 @@ class CObject(ObjectDescription): name = ast.symbol.get_full_nested_name().get_display_string().lstrip('.') if newestId not in self.state.document.ids: - signode['names'].append(name) # always add the newest id assert newestId signode['ids'].append(newestId) @@ -1990,40 +2210,22 @@ class CObject(ObjectDescription): domain = cast(CDomain, self.env.get_domain('c')) domain.note_object(name, self.objtype, newestId) - strippedName = name - # for prefix in self.env.config.cpp_index_common_prefix: - # if name.startswith(prefix): - # strippedName = strippedName[len(prefix):] - # break - indexText = self.get_index_text(strippedName) + indexText = self.get_index_text(name) self.indexnode['entries'].append(('single', indexText, newestId, '', None)) - def get_object_type(self) -> str: - if self.objtype in ('function', 'macro'): - return self.objtype - elif self.objtype in ('member', 'var'): - return 'member' - elif self.objtype == 'type': - return 'type' # hmm - else: - assert False + @property + def object_type(self) -> str: + raise NotImplementedError() + + @property + def display_object_type(self) -> str: + return self.object_type def get_index_text(self, name: str) -> str: - if self.objtype == 'function': - return _('%s (C function)') % name - elif self.objtype == 'member': - return _('%s (C member)') % name - elif self.objtype == 'macro': - return _('%s (C macro)') % name - elif self.objtype == 'type': - return _('%s (C type)') % name - elif self.objtype == 'var': - return _('%s (C variable)') % name - else: - assert False + return _('%s (C %s)') % (name, self.display_object_type) def parse_definition(self, parser: DefinitionParser) -> ASTDeclaration: - return parser.parse_declaration(self.get_object_type(), self.objtype) + return parser.parse_declaration(self.object_type, self.objtype) def describe_signature(self, signode: desc_signature, ast: Any, options: Dict) -> None: ast.describe_signature(signode, 'lastIsName', self.env, options) @@ -2103,9 +2305,54 @@ class CObject(ObjectDescription): return 'c.' + name +class CMemberObject(CObject): + object_type = 'member' + + @property + def display_object_type(self) -> str: + # the distinction between var and member is only cosmetic + assert self.objtype in ('member', 'var') + return self.objtype + + +class CFunctionObject(CObject): + object_type = 'function' + + +class CMacroObject(CObject): + object_type = 'macro' + + +class CStructObject(CObject): + object_type = 'struct' + + +class CUnionObject(CObject): + object_type = 'union' + + +class CEnumObject(CObject): + object_type = 'enum' + + +class CEnumeratorObject(CObject): + object_type = 'enumerator' + + +class CTypeObject(CObject): + object_type = 'type' + + class CXRefRole(XRefRole): def process_link(self, env: BuildEnvironment, refnode: Element, has_explicit_title: bool, title: str, target: str) -> Tuple[str, str]: + refnode.attributes.update(env.ref_context) + + if not has_explicit_title: + # major hax: replace anon names via simple string manipulation. + # Can this actually fail? + title = anon_identifier_re.sub("[anonymous]", str(title)) + if not has_explicit_title: target = target.lstrip('~') # only has a meaning for the title # if the first character is a tilde, don't display the module/class @@ -2131,17 +2378,26 @@ class CDomain(Domain): } directives = { - 'function': CObject, - 'member': CObject, - 'macro': CObject, - 'type': CObject, - 'var': CObject, + 'member': CMemberObject, + 'var': CMemberObject, + 'function': CFunctionObject, + 'macro': CMacroObject, + 'struct': CStructObject, + 'union': CUnionObject, + 'enum': CEnumObject, + 'enumerator': CEnumeratorObject, + 'type': CTypeObject, } roles = { - 'func': CXRefRole(fix_parens=True), 'member': CXRefRole(), - 'macro': CXRefRole(), 'data': CXRefRole(), + 'var': CXRefRole(), + 'func': CXRefRole(fix_parens=True), + 'macro': CXRefRole(), + 'struct': CXRefRole(), + 'union': CXRefRole(), + 'enum': CXRefRole(), + 'enumerator': CXRefRole(), 'type': CXRefRole(), } initial_data = { @@ -2211,29 +2467,71 @@ class CDomain(Domain): else: ourObjects[fullname] = (fn, id_, objtype) - def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder, - typ: str, target: str, node: pending_xref, contnode: Element - ) -> Element: + def _resolve_xref_inner(self, env: BuildEnvironment, fromdocname: str, builder: Builder, + typ: str, target: str, node: pending_xref, contnode: Element, + emitWarnings: bool = True) -> Tuple[Element, str]: + class Warner: + def warn(self, msg: str) -> None: + if emitWarnings: + logger.warning(msg, location=node) + + warner = Warner() + # strip pointer asterisk target = target.rstrip(' *') + # becase TypedField can generate xrefs if target in _keywords: - return contnode - if target not in self.objects: - return None - docname, node_id, objtype = self.objects[target] - return make_refnode(builder, fromdocname, docname, node_id, contnode, target) + raise NoUri(target, typ) + + parser = DefinitionParser(target, warner) + try: + name = parser.parse_xref_object() + except DefinitionError as e: + warner.warn('Unparseable C cross-reference: %r\n%s' % (target, e)) + return None, None + parentKey = node.get("c:parent_key", None) # type: LookupKey + rootSymbol = self.data['root_symbol'] + if parentKey: + parentSymbol = rootSymbol.direct_lookup(parentKey) # type: Symbol + if not parentSymbol: + print("Target: ", target) + print("ParentKey: ", parentKey) + print(rootSymbol.dump(1)) + assert parentSymbol # should be there + else: + parentSymbol = rootSymbol + s = parentSymbol.find_declaration(name, typ, + matchSelf=True, recurseInAnon=True) + if s is None or s.declaration is None: + return None, None + + # TODO: check role type vs. object type + + declaration = s.declaration + displayName = name.get_display_string() + docname = s.docname + assert docname + + return make_refnode(builder, fromdocname, docname, + declaration.get_newest_id(), contnode, displayName + ), declaration.objectType + + def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder, + typ: str, target: str, node: pending_xref, contnode: Element, + emitWarnings: bool = True) -> Element: + return self._resolve_xref_inner(env, fromdocname, builder, typ, + target, node, contnode)[0] def resolve_any_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder, target: str, node: pending_xref, contnode: Element ) -> List[Tuple[str, Element]]: - # strip pointer asterisk - target = target.rstrip(' *') - if target not in self.objects: - return [] - docname, node_id, objtype = self.objects[target] - return [('c:' + self.role_for_objtype(objtype), - make_refnode(builder, fromdocname, docname, node_id, contnode, target))] + retnode, objtype = self._resolve_xref_inner(env, fromdocname, builder, + 'any', target, node, contnode, + emitWarnings=False) + if retnode: + return [('c:' + self.role_for_objtype(objtype), retnode)] + return [] def get_objects(self) -> Iterator[Tuple[str, str, str, str, str, int]]: for refname, (docname, node_id, objtype) in list(self.objects.items()): diff --git a/tests/roots/test-domain-c/index.rst b/tests/roots/test-domain-c/index.rst index e81f9c8e8..55e284692 100644 --- a/tests/roots/test-domain-c/index.rst +++ b/tests/roots/test-domain-c/index.rst @@ -7,11 +7,32 @@ directives .. c:function:: int hello(char *name) .. c:member:: float Sphinx.version +.. c:var:: int version .. c:macro:: IS_SPHINX - .. c:macro:: SPHINX(arg1, arg2) -.. c:type:: Sphinx +.. c:struct:: MyStruct +.. c:union:: MyUnion +.. c:enum:: MyEnum -.. c:var:: int version + .. c:enumerator:: MyEnumerator + + :c:enumerator:`MyEnumerator` + + :c:enumerator:`MyEnumerator` + +:c:enumerator:`MyEnumerator` + +.. c:type:: Sphinx +.. c:type:: int SphinxVersionNum + + +.. c:struct:: A + + .. c:union:: @data + + .. c:member:: int a + +- :c:member:`A.@data.a` +- :c:member:`A.a` diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index efbf332b4..1c42ca967 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -246,9 +246,6 @@ def test_expressions(): def test_type_definitions(): check('type', "T", {1: "T"}) - check('type', "struct T", {1: 'T'}) - check('type', "enum T", {1: 'T'}) - check('type', "union T", {1: 'T'}) check('type', "bool *b", {1: 'b'}) check('type', "bool *const b", {1: 'b'}) @@ -301,9 +298,6 @@ def test_member_definitions(): check('member', 'restrict volatile const int a', {1: 'a'}) check('member', 'T t', {1: 't'}) - check('member', 'struct T t', {1: 't'}) - check('member', 'enum T t', {1: 't'}) - check('member', 'union T t', {1: 't'}) check('member', 'int a[]', {1: 'a'}) @@ -376,20 +370,18 @@ def test_function_definitions(): def test_union_definitions(): - return # TODO - check('union', 'A', {2: "1A"}) + check('struct', 'A', {1: 'A'}) + + +def test_union_definitions(): + check('union', 'A', {1: 'A'}) def test_enum_definitions(): - return # TODO - check('enum', 'A', {2: "1A"}) - check('enum', 'A : std::underlying_type::type', {2: "1A"}) - check('enum', 'A : unsigned int', {2: "1A"}) - check('enum', 'public A', {2: "1A"}, output='A') - check('enum', 'private A', {2: "1A"}) + check('enum', 'A', {1: 'A'}) - check('enumerator', 'A', {2: "1A"}) - check('enumerator', 'A = std::numeric_limits::max()', {2: "1A"}) + check('enumerator', 'A', {1: 'A'}) + check('enumerator', 'A = 42', {1: 'A'}) def test_anon_definitions(): From b2ca906830af3f7b3f8f3b6dc29652dac8b9161c Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sat, 14 Mar 2020 18:18:10 +0100 Subject: [PATCH 03/12] C, add expression parsing and expr role --- doc/usage/restructuredtext/domains.rst | 32 + sphinx/domains/c.py | 824 ++++++++++++++++++++++++- sphinx/domains/cpp.py | 51 +- sphinx/util/cfamily.py | 38 ++ tests/roots/test-domain-c/index.rst | 8 + tests/test_domain_c.py | 236 +++---- 6 files changed, 976 insertions(+), 213 deletions(-) diff --git a/doc/usage/restructuredtext/domains.rst b/doc/usage/restructuredtext/domains.rst index f399199b6..7a987be70 100644 --- a/doc/usage/restructuredtext/domains.rst +++ b/doc/usage/restructuredtext/domains.rst @@ -674,6 +674,38 @@ Explicit ref: :c:var:`Data.@data.a`. Short-hand ref: :c:var:`Data.a`. .. versionadded:: 3.0 +Inline Expressions and Types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. rst:role:: c:expr + c:texpr + + Insert a C expression or type either as inline code (``cpp:expr``) + or inline text (``cpp:texpr``). For example:: + + .. c:var:: int a = 42 + + .. c:function:: int f(int i) + + An expression: :c:expr:`a * f(a)` (or as text: :c:texpr:`a * f(a)`). + + A type: :c:expr:`const Data*` + (or as text :c:texpr:`const Data*`). + + will be rendered as follows: + + .. c:var:: int a = 42 + + .. c:function:: int f(int i) + + An expression: :c:expr:`a * f(a)` (or as text: :c:texpr:`a * f(a)`). + + A type: :c:expr:`const Data*` + (or as text :c:texpr:`const Data*`). + + .. versionadded:: 3.0 + + .. _cpp-domain: The C++ Domain diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index ea22824f4..3a65c9b85 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -14,8 +14,9 @@ from typing import ( ) from typing import cast -from docutils import nodes -from docutils.nodes import Element, Node +from docutils import nodes, utils +from docutils.nodes import Element, Node, TextElement, system_message +from docutils.parsers.rst.states import Inliner from sphinx import addnodes from sphinx.addnodes import pending_xref, desc_signature @@ -30,7 +31,10 @@ from sphinx.roles import XRefRole from sphinx.util import logging from sphinx.util.cfamily import ( NoOldIdError, ASTBase, verify_description_mode, StringifyTransform, - BaseParser, DefinitionError, identifier_re, anon_identifier_re + BaseParser, DefinitionError, UnsupportedMultiCharacterCharLiteral, + identifier_re, anon_identifier_re, integer_literal_re, octal_literal_re, + hex_literal_re, binary_literal_re, float_literal_re, + char_literal_re ) from sphinx.util.docfields import Field, TypedField from sphinx.util.nodes import make_refnode @@ -49,6 +53,24 @@ _keywords = [ '_Thread_local', 'thread_local', ] +# these are ordered by preceedence +_expression_bin_ops = [ + ['||'], + ['&&'], + ['|'], + ['^'], + ['&'], + ['==', '!='], + ['<=', '>=', '<', '>'], + ['<<', '>>'], + ['+', '-'], + ['*', '/', '%'], + ['.*', '->*'] +] +_expression_unary_ops = ["++", "--", "*", "&", "+", "-", "!", "~"] +_expression_assignment_ops = ["=", "*=", "/=", "%=", "+=", "-=", + ">>=", "<<=", "&=", "^=", "|="] + _max_id = 1 _id_prefix = [None, 'c.', 'Cv2.'] # Ids are used in lookup keys which are used across pickled files, @@ -69,7 +91,307 @@ class _DuplicateSymbolError(Exception): return "Internal C duplicate symbol error:\n%s" % self.symbol.dump(0) -############################################################################################## +################################################################################ +# Expressions and Literals +################################################################################ + +class ASTBooleanLiteral(ASTBase): + def __init__(self, value: bool) -> None: + self.value = value + + def _stringify(self, transform: StringifyTransform) -> str: + if self.value: + return 'true' + else: + return 'false' + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text(str(self))) + + +class ASTNumberLiteral(ASTBase): + def __init__(self, data: str) -> None: + self.data = data + + def _stringify(self, transform: StringifyTransform) -> str: + return self.data + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + txt = str(self) + signode.append(nodes.Text(txt, txt)) + + +class ASTCharLiteral(ASTBase): + def __init__(self, prefix: str, data: str) -> None: + self.prefix = prefix # may be None when no prefix + self.data = data + decoded = data.encode().decode('unicode-escape') + if len(decoded) == 1: + self.value = ord(decoded) + else: + raise UnsupportedMultiCharacterCharLiteral(decoded) + + def _stringify(self, transform: StringifyTransform) -> str: + if self.prefix is None: + return "'" + self.data + "'" + else: + return self.prefix + "'" + self.data + "'" + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + txt = str(self) + signode.append(nodes.Text(txt, txt)) + + +class ASTStringLiteral(ASTBase): + def __init__(self, data: str) -> None: + self.data = data + + def _stringify(self, transform: StringifyTransform) -> str: + return self.data + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + txt = str(self) + signode.append(nodes.Text(txt, txt)) + + +class ASTParenExpr(ASTBase): + def __init__(self, expr): + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return '(' + transform(self.expr) + ')' + + def get_id(self, version: int) -> str: + return self.expr.get_id(version) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('(', '(')) + self.expr.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')', ')')) + + +class ASTBinOpExpr(ASTBase): + def __init__(self, exprs, ops): + assert len(exprs) > 0 + assert len(exprs) == len(ops) + 1 + self.exprs = exprs + self.ops = ops + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + res.append(transform(self.exprs[0])) + for i in range(1, len(self.exprs)): + res.append(' ') + res.append(self.ops[i - 1]) + res.append(' ') + res.append(transform(self.exprs[i])) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.exprs[0].describe_signature(signode, mode, env, symbol) + for i in range(1, len(self.exprs)): + signode.append(nodes.Text(' ')) + signode.append(nodes.Text(self.ops[i - 1])) + signode.append(nodes.Text(' ')) + self.exprs[i].describe_signature(signode, mode, env, symbol) + + +class ASTAssignmentExpr(ASTBase): + def __init__(self, exprs, ops): + assert len(exprs) > 0 + assert len(exprs) == len(ops) + 1 + self.exprs = exprs + self.ops = ops + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + res.append(transform(self.exprs[0])) + for i in range(1, len(self.exprs)): + res.append(' ') + res.append(self.ops[i - 1]) + res.append(' ') + res.append(transform(self.exprs[i])) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.exprs[0].describe_signature(signode, mode, env, symbol) + for i in range(1, len(self.exprs)): + signode.append(nodes.Text(' ')) + signode.append(nodes.Text(self.ops[i - 1])) + signode.append(nodes.Text(' ')) + self.exprs[i].describe_signature(signode, mode, env, symbol) + + +class ASTCastExpr(ASTBase): + def __init__(self, typ, expr): + self.typ = typ + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + res = ['('] + res.append(transform(self.typ)) + res.append(')') + res.append(transform(self.expr)) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('(')) + self.typ.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')')) + self.expr.describe_signature(signode, mode, env, symbol) + + +class ASTUnaryOpExpr(ASTBase): + def __init__(self, op, expr): + self.op = op + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.op) + transform(self.expr) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text(self.op)) + self.expr.describe_signature(signode, mode, env, symbol) + + +class ASTSizeofType(ASTBase): + def __init__(self, typ): + self.typ = typ + + def _stringify(self, transform: StringifyTransform) -> str: + return "sizeof(" + transform(self.typ) + ")" + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('sizeof(')) + self.typ.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')')) + + +class ASTSizeofExpr(ASTBase): + def __init__(self, expr): + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return "sizeof " + transform(self.expr) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('sizeof ')) + self.expr.describe_signature(signode, mode, env, symbol) + + +class ASTAlignofExpr(ASTBase): + def __init__(self, typ): + self.typ = typ + + def _stringify(self, transform: StringifyTransform) -> str: + return "alignof(" + transform(self.typ) + ")" + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('alignof(')) + self.typ.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')')) + + +class ASTPostfixCallExpr(ASTBase): + def __init__(self, lst: Union["ASTParenExprList", "ASTBracedInitList"]) -> None: + self.lst = lst + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.lst) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.lst.describe_signature(signode, mode, env, symbol) + + +class ASTPostfixArray(ASTBase): + def __init__(self, expr): + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return '[' + transform(self.expr) + ']' + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('[')) + self.expr.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(']')) + + +class ASTPostfixInc(ASTBase): + def _stringify(self, transform: StringifyTransform) -> str: + return '++' + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('++')) + + +class ASTPostfixDec(ASTBase): + def _stringify(self, transform: StringifyTransform) -> str: + return '--' + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('--')) + + +class ASTPostfixMember(ASTBase): + def __init__(self, name): + self.name = name + + def _stringify(self, transform: StringifyTransform) -> str: + return '.' + transform(self.name) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('.')) + self.name.describe_signature(signode, 'noneIsName', env, symbol) + + +class ASTPostfixMemberOfPointer(ASTBase): + def __init__(self, name): + self.name = name + + def _stringify(self, transform: StringifyTransform) -> str: + return '->' + transform(self.name) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('->')) + self.name.describe_signature(signode, 'noneIsName', env, symbol) + + +class ASTPostfixExpr(ASTBase): + def __init__(self, prefix, postFixes): + assert len(postFixes) > 0 + self.prefix = prefix + self.postFixes = postFixes + + def _stringify(self, transform: StringifyTransform) -> str: + res = [transform(self.prefix)] + for p in self.postFixes: + res.append(transform(p)) + return ''.join(res) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.prefix.describe_signature(signode, mode, env, symbol) + for p in self.postFixes: + p.describe_signature(signode, mode, env, symbol) + class ASTFallbackExpr(ASTBase): def __init__(self, expr): @@ -452,7 +774,9 @@ class ASTDeclaratorPtr(ASTBase): return self.next.function_params def require_space_after_declSpecs(self) -> bool: - return True + return self.const or self.volatile or self.restrict or \ + len(self.attrs) > 0 or \ + self.next.require_space_after_declSpecs() def _stringify(self, transform: StringifyTransform) -> str: res = ['*'] @@ -598,9 +922,7 @@ class ASTDeclaratorNameBitField(ASTBase): if self.declId: res.append(transform(self.declId)) res.append(" : ") - # TODO: when size is properly parsed - # res.append(transform(self.size)) - res.append(self.size) + res.append(transform(self.size)) return ''.join(res) def describe_signature(self, signode: desc_signature, mode: str, @@ -609,11 +931,58 @@ class ASTDeclaratorNameBitField(ASTBase): if self.declId: self.declId.describe_signature(signode, mode, env, symbol) signode += nodes.Text(' : ', ' : ') - # TODO: when size is properly parsed - # self.size.describe_signature(signode, mode, env, symbol) + self.size.describe_signature(signode, mode, env, symbol) signode += nodes.Text(self.size, self.size) +class ASTParenExprList(ASTBase): + def __init__(self, exprs: List[Any]) -> None: + self.exprs = exprs + + def _stringify(self, transform: StringifyTransform) -> str: + exprs = [transform(e) for e in self.exprs] + return '(%s)' % ', '.join(exprs) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + signode.append(nodes.Text('(')) + first = True + for e in self.exprs: + if not first: + signode.append(nodes.Text(', ')) + else: + first = False + e.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')')) + + +class ASTBracedInitList(ASTBase): + def __init__(self, exprs: List[Any], trailingComma: bool) -> None: + self.exprs = exprs + self.trailingComma = trailingComma + + def _stringify(self, transform: StringifyTransform) -> str: + exprs = [transform(e) for e in self.exprs] + trailingComma = ',' if self.trailingComma else '' + return '{%s%s}' % (', '.join(exprs), trailingComma) + + def describe_signature(self, signode: desc_signature, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + signode.append(nodes.Text('{')) + first = True + for e in self.exprs: + if not first: + signode.append(nodes.Text(', ')) + else: + first = False + e.describe_signature(signode, mode, env, symbol) + if self.trailingComma: + signode.append(nodes.Text(',')) + signode.append(nodes.Text('}')) + + class ASTInitializer(ASTBase): def __init__(self, value: Any, hasAssign: bool = True) -> None: self.value = value @@ -1526,6 +1895,25 @@ class DefinitionParser(BaseParser): _prefix_keys = ('struct', 'enum', 'union') + def _parse_string(self): + if self.current_char != '"': + return None + startPos = self.pos + self.pos += 1 + escape = False + while True: + if self.eof: + self.fail("Unexpected end during inside string.") + elif self.current_char == '"' and not escape: + self.pos += 1 + break + elif self.current_char == '\\': + escape = True + else: + escape = False + self.pos += 1 + return self.definition[startPos:self.pos] + def _parse_attribute(self) -> Any: return None # self.skip_ws() @@ -1586,23 +1974,340 @@ class DefinitionParser(BaseParser): return None + def _parse_literal(self): + # -> integer-literal + # | character-literal + # | floating-literal + # | string-literal + # | boolean-literal -> "false" | "true" + self.skip_ws() + if self.skip_word('true'): + return ASTBooleanLiteral(True) + if self.skip_word('false'): + return ASTBooleanLiteral(False) + for regex in [float_literal_re, binary_literal_re, hex_literal_re, + integer_literal_re, octal_literal_re]: + pos = self.pos + if self.match(regex): + while self.current_char in 'uUlLfF': + self.pos += 1 + return ASTNumberLiteral(self.definition[pos:self.pos]) + + string = self._parse_string() + if string is not None: + return ASTStringLiteral(string) + + # character-literal + if self.match(char_literal_re): + prefix = self.last_match.group(1) # may be None when no prefix + data = self.last_match.group(2) + try: + return ASTCharLiteral(prefix, data) + except UnicodeDecodeError as e: + self.fail("Can not handle character literal. Internal error was: %s" % e) + except UnsupportedMultiCharacterCharLiteral: + self.fail("Can not handle character literal" + " resulting in multiple decoded characters.") + return None + + def _parse_paren_expression(self): + # "(" expression ")" + if self.current_char != '(': + return None + self.pos += 1 + res = self._parse_expression() + self.skip_ws() + if not self.skip_string(')'): + self.fail("Expected ')' in end of parenthesized expression.") + return ASTParenExpr(res) + + def _parse_primary_expression(self): + # literal + # "(" expression ")" + # id-expression -> we parse this with _parse_nested_name + self.skip_ws() + res = self._parse_literal() + if res is not None: + return res + res = self._parse_paren_expression() + if res is not None: + return res + return self._parse_nested_name() + + def _parse_initializer_list(self, name: str, open: str, close: str + ) -> Tuple[List[Any], bool]: + # Parse open and close with the actual initializer-list inbetween + # -> initializer-clause '...'[opt] + # | initializer-list ',' initializer-clause '...'[opt] + # TODO: designators + self.skip_ws() + if not self.skip_string_and_ws(open): + return None, None + if self.skip_string(close): + return [], False + + exprs = [] + trailingComma = False + while True: + self.skip_ws() + expr = self._parse_expression() + self.skip_ws() + exprs.append(expr) + self.skip_ws() + if self.skip_string(close): + break + if not self.skip_string_and_ws(','): + self.fail("Error in %s, expected ',' or '%s'." % (name, close)) + if self.current_char == close and close == '}': + self.pos += 1 + trailingComma = True + break + return exprs, trailingComma + + def _parse_paren_expression_list(self) -> ASTParenExprList: + # -> '(' expression-list ')' + # though, we relax it to also allow empty parens + # as it's needed in some cases + # + # expression-list + # -> initializer-list + exprs, trailingComma = self._parse_initializer_list("parenthesized expression-list", + '(', ')') + if exprs is None: + return None + return ASTParenExprList(exprs) + + def _parse_braced_init_list(self) -> ASTBracedInitList: + # -> '{' initializer-list ','[opt] '}' + # | '{' '}' + exprs, trailingComma = self._parse_initializer_list("braced-init-list", '{', '}') + if exprs is None: + return None + return ASTBracedInitList(exprs, trailingComma) + + def _parse_postfix_expression(self): + # -> primary + # | postfix "[" expression "]" + # | postfix "[" braced-init-list [opt] "]" + # | postfix "(" expression-list [opt] ")" + # | postfix "." id-expression + # | postfix "->" id-expression + # | postfix "++" + # | postfix "--" + + prefix = self._parse_primary_expression() + + # and now parse postfixes + postFixes = [] # type: List[Any] + while True: + self.skip_ws() + if self.skip_string_and_ws('['): + expr = self._parse_expression() + self.skip_ws() + if not self.skip_string(']'): + self.fail("Expected ']' in end of postfix expression.") + postFixes.append(ASTPostfixArray(expr)) + continue + if self.skip_string('.'): + if self.skip_string('*'): + # don't steal the dot + self.pos -= 2 + elif self.skip_string('..'): + # don't steal the dot + self.pos -= 3 + else: + name = self._parse_nested_name() + postFixes.append(ASTPostfixMember(name)) + continue + if self.skip_string('->'): + if self.skip_string('*'): + # don't steal the arrow + self.pos -= 3 + else: + name = self._parse_nested_name() + postFixes.append(ASTPostfixMemberOfPointer(name)) + continue + if self.skip_string('++'): + postFixes.append(ASTPostfixInc()) + continue + if self.skip_string('--'): + postFixes.append(ASTPostfixDec()) + continue + lst = self._parse_paren_expression_list() + if lst is not None: + postFixes.append(ASTPostfixCallExpr(lst)) + continue + break + if len(postFixes) == 0: + return prefix + else: + return ASTPostfixExpr(prefix, postFixes) + + def _parse_unary_expression(self): + # -> postfix + # | "++" cast + # | "--" cast + # | unary-operator cast -> (* | & | + | - | ! | ~) cast + # The rest: + # | "sizeof" unary + # | "sizeof" "(" type-id ")" + # | "alignof" "(" type-id ")" + self.skip_ws() + for op in _expression_unary_ops: + # TODO: hmm, should we be able to backtrack here? + if self.skip_string(op): + expr = self._parse_cast_expression() + return ASTUnaryOpExpr(op, expr) + if self.skip_word_and_ws('sizeof'): + if self.skip_string_and_ws('('): + typ = self._parse_type(named=False) + self.skip_ws() + if not self.skip_string(')'): + self.fail("Expecting ')' to end 'sizeof'.") + return ASTSizeofType(typ) + expr = self._parse_unary_expression() + return ASTSizeofExpr(expr) + if self.skip_word_and_ws('alignof'): + if not self.skip_string_and_ws('('): + self.fail("Expecting '(' after 'alignof'.") + typ = self._parse_type(named=False) + self.skip_ws() + if not self.skip_string(')'): + self.fail("Expecting ')' to end 'alignof'.") + return ASTAlignofExpr(typ) + return self._parse_postfix_expression() + + def _parse_cast_expression(self): + # -> unary | "(" type-id ")" cast + pos = self.pos + self.skip_ws() + if self.skip_string('('): + try: + typ = self._parse_type(False) + if not self.skip_string(')'): + self.fail("Expected ')' in cast expression.") + expr = self._parse_cast_expression() + return ASTCastExpr(typ, expr) + except DefinitionError as exCast: + self.pos = pos + try: + return self._parse_unary_expression() + except DefinitionError as exUnary: + errs = [] + errs.append((exCast, "If type cast expression")) + errs.append((exUnary, "If unary expression")) + raise self._make_multi_error(errs, "Error in cast expression.") + else: + return self._parse_unary_expression() + + def _parse_logical_or_expression(self): + # logical-or = logical-and || + # logical-and = inclusive-or && + # inclusive-or = exclusive-or | + # exclusive-or = and ^ + # and = equality & + # equality = relational ==, != + # relational = shift <, >, <=, >= + # shift = additive <<, >> + # additive = multiplicative +, - + # multiplicative = pm *, /, % + # pm = cast .*, ->* + def _parse_bin_op_expr(self, opId): + if opId + 1 == len(_expression_bin_ops): + def parser(): + return self._parse_cast_expression() + else: + def parser(): + return _parse_bin_op_expr(self, opId + 1) + exprs = [] + ops = [] + exprs.append(parser()) + while True: + self.skip_ws() + pos = self.pos + oneMore = False + for op in _expression_bin_ops[opId]: + if not self.skip_string(op): + continue + if op == '&' and self.current_char == '&': + # don't split the && 'token' + self.pos -= 1 + # and btw. && has lower precedence, so we are done + break + try: + expr = parser() + exprs.append(expr) + ops.append(op) + oneMore = True + break + except DefinitionError: + self.pos = pos + if not oneMore: + break + return ASTBinOpExpr(exprs, ops) + return _parse_bin_op_expr(self, 0) + + def _parse_conditional_expression_tail(self, orExprHead): + # -> "?" expression ":" assignment-expression + return None + + def _parse_assignment_expression(self): + # -> conditional-expression + # | logical-or-expression assignment-operator initializer-clause + # -> conditional-expression -> + # logical-or-expression + # | logical-or-expression "?" expression ":" assignment-expression + # | logical-or-expression assignment-operator initializer-clause + exprs = [] + ops = [] + orExpr = self._parse_logical_or_expression() + exprs.append(orExpr) + # TODO: handle ternary with _parse_conditional_expression_tail + while True: + oneMore = False + self.skip_ws() + for op in _expression_assignment_ops: + if not self.skip_string(op): + continue + expr = self._parse_logical_or_expression() + exprs.append(expr) + ops.append(op) + oneMore = True + if not oneMore: + break + if len(ops) == 0: + return orExpr + else: + return ASTAssignmentExpr(exprs, ops) + + def _parse_constant_expression(self): + # -> conditional-expression + orExpr = self._parse_logical_or_expression() + # TODO: use _parse_conditional_expression_tail + return orExpr + + def _parse_expression(self): + # -> assignment-expression + # | expression "," assignment-expresion + # TODO: actually parse the second production + return self._parse_assignment_expression() + def _parse_expression_fallback(self, end, parser, allow=True): # Stupidly "parse" an expression. # 'end' should be a list of characters which ends the expression. # first try to use the provided parser - # TODO: copy-paste and adapt real expression parser - # prevPos = self.pos - # try: - # return parser() - # except DefinitionError as e: - # # some places (e.g., template parameters) we really don't want to use fallback, - # # and for testing we may want to globally disable it - # if not allow or not self.allowFallbackExpressionParsing: - # raise - # self.warn("Parsing of expression failed. Using fallback parser." - # " Error was:\n%s" % e) - # self.pos = prevPos + prevPos = self.pos + try: + return parser() + except DefinitionError as e: + # some places (e.g., template parameters) we really don't want to use fallback, + # and for testing we may want to globally disable it + if not allow or not self.allowFallbackExpressionParsing: + raise + self.warn("Parsing of expression failed. Using fallback parser." + " Error was:\n%s" % e) + self.pos = prevPos # and then the fallback scanning assert end is not None self.skip_ws() @@ -1831,7 +2536,7 @@ class DefinitionParser(BaseParser): continue def parser(): - return self._parse_expression(inTemplate=False) + return self._parse_expression() value = self._parse_expression_fallback([']'], parser) if not self.skip_string(']'): @@ -1846,9 +2551,7 @@ class DefinitionParser(BaseParser): if named and paramMode == 'type' and typed: self.skip_ws() if self.skip_string(':'): - # TODO: copy-paste and adapt proper parser - # size = self._parse_constant_expression(inTemplate=False) - size = self.read_rest().strip() + size = self._parse_constant_expression() return ASTDeclaratorNameBitField(declId=declId, size=size) return ASTDeclaratorNameParam(declId=declId, arrayOps=arrayOps, param=param) @@ -1939,10 +2642,9 @@ class DefinitionParser(BaseParser): if not self.skip_string('='): return None - # TODO: implement - # bracedInit = self._parse_braced_init_list() - # if bracedInit is not None: - # return ASTInitializer(bracedInit) + bracedInit = self._parse_braced_init_list() + if bracedInit is not None: + return ASTInitializer(bracedInit) if outer == 'member': fallbackEnd = [] # type: List[str] @@ -2077,7 +2779,7 @@ class DefinitionParser(BaseParser): self.skip_ws() def parser(): - return self._parse_constant_expression(inTemplate=False) + return self._parse_constant_expression() initVal = self._parse_expression_fallback([], parser) init = ASTInitializer(initVal) @@ -2119,6 +2821,26 @@ class DefinitionParser(BaseParser): self.assert_end() return name + def parse_expression(self): + pos = self.pos + try: + expr = self._parse_expression() + self.skip_ws() + self.assert_end() + except DefinitionError as exExpr: + self.pos = pos + try: + expr = self._parse_type(False) + self.skip_ws() + self.assert_end() + except DefinitionError as exType: + header = "Error when parsing (type) expression." + errs = [] + errs.append((exExpr, "If expression")) + errs.append((exType, "If type")) + raise self._make_multi_error(errs, header) + return expr + def _make_phony_error_name() -> ASTNestedName: return ASTNestedName([ASTIdentifier("PhonyNameDueToError")], rooted=False) @@ -2365,6 +3087,44 @@ class CXRefRole(XRefRole): return title, target +class CExprRole: + def __init__(self, asCode: bool) -> None: + if asCode: + # render the expression as inline code + self.class_type = 'c-expr' + self.node_type = nodes.literal # type: Type[TextElement] + else: + # render the expression as inline text + self.class_type = 'c-texpr' + self.node_type = nodes.inline + + def __call__(self, typ: str, rawtext: str, text: str, lineno: int, + inliner: Inliner, options: Dict = {}, content: List[str] = [] + ) -> Tuple[List[Node], List[system_message]]: + class Warner: + def warn(self, msg: str) -> None: + inliner.reporter.warning(msg, line=lineno) + text = utils.unescape(text).replace('\n', ' ') + env = inliner.document.settings.env + parser = DefinitionParser(text, Warner()) + # attempt to mimic XRefRole classes, except that... + classes = ['xref', 'c', self.class_type] + try: + ast = parser.parse_expression() + except DefinitionError as ex: + Warner().warn('Unparseable C expression: %r\n%s' % (text, ex)) + # see below + return [self.node_type(text, text, classes=classes)], [] + parentSymbol = env.temp_data.get('cpp:parent_symbol', None) + if parentSymbol is None: + parentSymbol = env.domaindata['c']['root_symbol'] + # ...most if not all of these classes should really apply to the individual references, + # not the container node + signode = self.node_type(classes=classes) + ast.describe_signature(signode, 'markType', env, parentSymbol) + return [signode], [] + + class CDomain(Domain): """C language domain.""" name = 'c' @@ -2399,6 +3159,8 @@ class CDomain(Domain): 'enum': CXRefRole(), 'enumerator': CXRefRole(), 'type': CXRefRole(), + 'expr': CExprRole(asCode=True), + 'texpr': CExprRole(asCode=False) } initial_data = { 'root_symbol': Symbol(None, None, None, None), diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py index d23d5be35..639ac20b6 100644 --- a/sphinx/domains/cpp.py +++ b/sphinx/domains/cpp.py @@ -36,7 +36,10 @@ from sphinx.transforms.post_transforms import ReferencesResolver from sphinx.util import logging from sphinx.util.cfamily import ( NoOldIdError, ASTBase, verify_description_mode, StringifyTransform, - BaseParser, DefinitionError, identifier_re, anon_identifier_re + BaseParser, DefinitionError, UnsupportedMultiCharacterCharLiteral, + identifier_re, anon_identifier_re, integer_literal_re, octal_literal_re, + hex_literal_re, binary_literal_re, float_literal_re, + char_literal_re ) from sphinx.util.docfields import Field, GroupedField from sphinx.util.docutils import SphinxDirective @@ -296,37 +299,6 @@ T = TypeVar('T') nested-name """ -_integer_literal_re = re.compile(r'[1-9][0-9]*') -_octal_literal_re = re.compile(r'0[0-7]*') -_hex_literal_re = re.compile(r'0[xX][0-9a-fA-F][0-9a-fA-F]*') -_binary_literal_re = re.compile(r'0[bB][01][01]*') -_integer_suffix_re = re.compile(r'') -_float_literal_re = re.compile(r'''(?x) - [+-]?( - # decimal - ([0-9]+[eE][+-]?[0-9]+) - | ([0-9]*\.[0-9]+([eE][+-]?[0-9]+)?) - | ([0-9]+\.([eE][+-]?[0-9]+)?) - # hex - | (0[xX][0-9a-fA-F]+[pP][+-]?[0-9a-fA-F]+) - | (0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+([pP][+-]?[0-9a-fA-F]+)?) - | (0[xX][0-9a-fA-F]+\.([pP][+-]?[0-9a-fA-F]+)?) - ) -''') -_char_literal_re = re.compile(r'''(?x) - ((?:u8)|u|U|L)? - '( - (?:[^\\']) - | (\\( - (?:['"?\\abfnrtv]) - | (?:[0-7]{1,3}) - | (?:x[0-9a-fA-F]{2}) - | (?:u[0-9a-fA-F]{4}) - | (?:U[0-9a-fA-F]{8}) - )) - )' -''') - _string_re = re.compile(r"[LuU8]?('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) _visibility_re = re.compile(r'\b(public|private|protected)\b') @@ -716,15 +688,6 @@ class ASTNumberLiteral(ASTBase): signode.append(nodes.Text(txt, txt)) -class UnsupportedMultiCharacterCharLiteral(Exception): - @property - def decoded(self) -> str: - warnings.warn('%s.decoded is deprecated. ' - 'Coerce the instance to a string instead.' % self.__class__.__name__, - RemovedInSphinx40Warning, stacklevel=2) - return str(self) - - class ASTCharLiteral(ASTBase): def __init__(self, prefix: str, data: str) -> None: self.prefix = prefix # may be None when no prefix @@ -4561,8 +4524,8 @@ class DefinitionParser(BaseParser): return ASTBooleanLiteral(True) if self.skip_word('false'): return ASTBooleanLiteral(False) - for regex in [_float_literal_re, _binary_literal_re, _hex_literal_re, - _integer_literal_re, _octal_literal_re]: + for regex in [float_literal_re, binary_literal_re, hex_literal_re, + integer_literal_re, octal_literal_re]: pos = self.pos if self.match(regex): while self.current_char in 'uUlLfF': @@ -4574,7 +4537,7 @@ class DefinitionParser(BaseParser): return ASTStringLiteral(string) # character-literal - if self.match(_char_literal_re): + if self.match(char_literal_re): prefix = self.last_match.group(1) # may be None when no prefix data = self.last_match.group(2) try: diff --git a/sphinx/util/cfamily.py b/sphinx/util/cfamily.py index 0402732de..87e438851 100644 --- a/sphinx/util/cfamily.py +++ b/sphinx/util/cfamily.py @@ -31,6 +31,35 @@ identifier_re = re.compile(r'''(?x) ) [a-zA-Z0-9_]*\b ''') +integer_literal_re = re.compile(r'[1-9][0-9]*') +octal_literal_re = re.compile(r'0[0-7]*') +hex_literal_re = re.compile(r'0[xX][0-9a-fA-F][0-9a-fA-F]*') +binary_literal_re = re.compile(r'0[bB][01][01]*') +float_literal_re = re.compile(r'''(?x) + [+-]?( + # decimal + ([0-9]+[eE][+-]?[0-9]+) + | ([0-9]*\.[0-9]+([eE][+-]?[0-9]+)?) + | ([0-9]+\.([eE][+-]?[0-9]+)?) + # hex + | (0[xX][0-9a-fA-F]+[pP][+-]?[0-9a-fA-F]+) + | (0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+([pP][+-]?[0-9a-fA-F]+)?) + | (0[xX][0-9a-fA-F]+\.([pP][+-]?[0-9a-fA-F]+)?) + ) +''') +char_literal_re = re.compile(r'''(?x) + ((?:u8)|u|U|L)? + '( + (?:[^\\']) + | (\\( + (?:['"?\\abfnrtv]) + | (?:[0-7]{1,3}) + | (?:x[0-9a-fA-F]{2}) + | (?:u[0-9a-fA-F]{4}) + | (?:U[0-9a-fA-F]{8}) + )) + )' +''') def verify_description_mode(mode: str) -> None: @@ -79,6 +108,15 @@ class ASTBase: return '<%s>' % self.__class__.__name__ +class UnsupportedMultiCharacterCharLiteral(Exception): + @property + def decoded(self) -> str: + warnings.warn('%s.decoded is deprecated. ' + 'Coerce the instance to a string instead.' % self.__class__.__name__, + RemovedInSphinx40Warning, stacklevel=2) + return str(self) + + class DefinitionError(Exception): @property def description(self) -> str: diff --git a/tests/roots/test-domain-c/index.rst b/tests/roots/test-domain-c/index.rst index 55e284692..1265b7967 100644 --- a/tests/roots/test-domain-c/index.rst +++ b/tests/roots/test-domain-c/index.rst @@ -36,3 +36,11 @@ directives - :c:member:`A.@data.a` - :c:member:`A.a` + +- :c:expr:`unsigned int` +- :c:texpr:`unsigned int` + +.. c:var:: A a + +- :c:expr:`a->b` +- :c:texpr:`a->b` diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index 1c42ca967..df3126d38 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -86,17 +86,34 @@ def check(name, input, idDict, output=None): def test_expressions(): - return # TODO - def exprCheck(expr, id, id4=None): - ids = 'IE1CIA%s_1aE' - idDict = {2: ids % expr, 3: ids % id} - if id4 is not None: - idDict[4] = ids % id4 - check('class', 'template<> C' % expr, idDict) + def exprCheck(expr, output=None): + parser = DefinitionParser(expr, None) + parser.allowFallbackExpressionParsing = False + ast = parser.parse_expression() + parser.assert_end() + # first a simple check of the AST + if output is None: + output = expr + res = str(ast) + if res != output: + print("") + print("Input: ", input) + print("Result: ", res) + print("Expected: ", output) + raise DefinitionError("") + # type expressions + exprCheck('int*') + exprCheck('int *const*') + exprCheck('int *volatile*') + exprCheck('int *restrict*') + exprCheck('int *(*)(double)') + exprCheck('const int*') + + # actual expressions + # primary - exprCheck('nullptr', 'LDnE') - exprCheck('true', 'L1E') - exprCheck('false', 'L0E') + exprCheck('true') + exprCheck('false') ints = ['5', '0', '075', '0x0123456789ABCDEF', '0XF', '0b1', '0B1'] unsignedSuffix = ['', 'u', 'U'] longSuffix = ['', 'l', 'L', 'll', 'LL'] @@ -104,9 +121,9 @@ def test_expressions(): for u in unsignedSuffix: for l in longSuffix: expr = i + u + l - exprCheck(expr, 'L' + expr + 'E') + exprCheck(expr) expr = i + l + u - exprCheck(expr, 'L' + expr + 'E') + exprCheck(expr) for suffix in ['', 'f', 'F', 'l', 'L']: for e in [ '5e42', '5e+42', '5e-42', @@ -114,134 +131,90 @@ def test_expressions(): '.5', '.5e42', '.5e+42', '.5e-42', '5.0', '5.0e42', '5.0e+42', '5.0e-42']: expr = e + suffix - exprCheck(expr, 'L' + expr + 'E') + exprCheck(expr) for e in [ 'ApF', 'Ap+F', 'Ap-F', 'A.', 'A.pF', 'A.p+F', 'A.p-F', '.A', '.ApF', '.Ap+F', '.Ap-F', 'A.B', 'A.BpF', 'A.Bp+F', 'A.Bp-F']: expr = "0x" + e + suffix - exprCheck(expr, 'L' + expr + 'E') - exprCheck('"abc\\"cba"', 'LA8_KcE') # string - exprCheck('this', 'fpT') + exprCheck(expr) + exprCheck('"abc\\"cba"') # string # character literals - for p, t in [('', 'c'), ('u8', 'c'), ('u', 'Ds'), ('U', 'Di'), ('L', 'w')]: - exprCheck(p + "'a'", t + "97") - exprCheck(p + "'\\n'", t + "10") - exprCheck(p + "'\\012'", t + "10") - exprCheck(p + "'\\0'", t + "0") - exprCheck(p + "'\\x0a'", t + "10") - exprCheck(p + "'\\x0A'", t + "10") - exprCheck(p + "'\\u0a42'", t + "2626") - exprCheck(p + "'\\u0A42'", t + "2626") - exprCheck(p + "'\\U0001f34c'", t + "127820") - exprCheck(p + "'\\U0001F34C'", t + "127820") + for p in ['', 'u8', 'u', 'U', 'L']: + exprCheck(p + "'a'") + exprCheck(p + "'\\n'") + exprCheck(p + "'\\012'") + exprCheck(p + "'\\0'") + exprCheck(p + "'\\x0a'") + exprCheck(p + "'\\x0A'") + exprCheck(p + "'\\u0a42'") + exprCheck(p + "'\\u0A42'") + exprCheck(p + "'\\U0001f34c'") + exprCheck(p + "'\\U0001F34C'") - # TODO: user-defined lit - exprCheck('(... + Ns)', '(... + Ns)', id4='flpl2Ns') - exprCheck('(Ns + ...)', '(Ns + ...)', id4='frpl2Ns') - exprCheck('(Ns + ... + 0)', '(Ns + ... + 0)', id4='fLpl2NsL0E') - exprCheck('(5)', 'L5E') - exprCheck('C', '1C') + exprCheck('(5)') + exprCheck('C') # postfix - exprCheck('A(2)', 'cl1AL2EE') - exprCheck('A[2]', 'ix1AL2E') - exprCheck('a.b.c', 'dtdt1a1b1c') - exprCheck('a->b->c', 'ptpt1a1b1c') - exprCheck('i++', 'pp1i') - exprCheck('i--', 'mm1i') - exprCheck('dynamic_cast(i)++', 'ppdcR1T1i') - exprCheck('static_cast(i)++', 'ppscR1T1i') - exprCheck('reinterpret_cast(i)++', 'pprcR1T1i') - exprCheck('const_cast(i)++', 'ppccR1T1i') - exprCheck('typeid(T).name', 'dtti1T4name') - exprCheck('typeid(a + b).name', 'dttepl1a1b4name') + exprCheck('A(2)') + exprCheck('A[2]') + exprCheck('a.b.c') + exprCheck('a->b->c') + exprCheck('i++') + exprCheck('i--') # unary - exprCheck('++5', 'pp_L5E') - exprCheck('--5', 'mm_L5E') - exprCheck('*5', 'deL5E') - exprCheck('&5', 'adL5E') - exprCheck('+5', 'psL5E') - exprCheck('-5', 'ngL5E') - exprCheck('!5', 'ntL5E') - exprCheck('~5', 'coL5E') - exprCheck('sizeof...(a)', 'sZ1a') - exprCheck('sizeof(T)', 'st1T') - exprCheck('sizeof -42', 'szngL42E') - exprCheck('alignof(T)', 'at1T') - exprCheck('noexcept(-42)', 'nxngL42E') - # new-expression - exprCheck('new int', 'nw_iE') - exprCheck('new volatile int', 'nw_ViE') - exprCheck('new int[42]', 'nw_AL42E_iE') - exprCheck('new int()', 'nw_ipiE') - exprCheck('new int(5, 42)', 'nw_ipiL5EL42EE') - exprCheck('::new int', 'nw_iE') - exprCheck('new int{}', 'nw_iilE') - exprCheck('new int{5, 42}', 'nw_iilL5EL42EE') - # delete-expression - exprCheck('delete p', 'dl1p') - exprCheck('delete [] p', 'da1p') - exprCheck('::delete p', 'dl1p') - exprCheck('::delete [] p', 'da1p') + exprCheck('++5') + exprCheck('--5') + exprCheck('*5') + exprCheck('&5') + exprCheck('+5') + exprCheck('-5') + exprCheck('!5') + exprCheck('~5') + exprCheck('sizeof(T)') + exprCheck('sizeof -42') + exprCheck('alignof(T)') # cast - exprCheck('(int)2', 'cviL2E') + exprCheck('(int)2') # binary op - exprCheck('5 || 42', 'ooL5EL42E') - exprCheck('5 && 42', 'aaL5EL42E') - exprCheck('5 | 42', 'orL5EL42E') - exprCheck('5 ^ 42', 'eoL5EL42E') - exprCheck('5 & 42', 'anL5EL42E') + exprCheck('5 || 42') + exprCheck('5 && 42') + exprCheck('5 | 42') + exprCheck('5 ^ 42') + exprCheck('5 & 42') # ['==', '!='] - exprCheck('5 == 42', 'eqL5EL42E') - exprCheck('5 != 42', 'neL5EL42E') + exprCheck('5 == 42') + exprCheck('5 != 42') # ['<=', '>=', '<', '>'] - exprCheck('5 <= 42', 'leL5EL42E') - exprCheck('5 >= 42', 'geL5EL42E') - exprCheck('5 < 42', 'ltL5EL42E') - exprCheck('5 > 42', 'gtL5EL42E') + exprCheck('5 <= 42') + exprCheck('5 >= 42') + exprCheck('5 < 42') + exprCheck('5 > 42') # ['<<', '>>'] - exprCheck('5 << 42', 'lsL5EL42E') - exprCheck('5 >> 42', 'rsL5EL42E') + exprCheck('5 << 42') + exprCheck('5 >> 42') # ['+', '-'] - exprCheck('5 + 42', 'plL5EL42E') - exprCheck('5 - 42', 'miL5EL42E') + exprCheck('5 + 42') + exprCheck('5 - 42') # ['*', '/', '%'] - exprCheck('5 * 42', 'mlL5EL42E') - exprCheck('5 / 42', 'dvL5EL42E') - exprCheck('5 % 42', 'rmL5EL42E') + exprCheck('5 * 42') + exprCheck('5 / 42') + exprCheck('5 % 42') # ['.*', '->*'] - exprCheck('5 .* 42', 'dsL5EL42E') - exprCheck('5 ->* 42', 'pmL5EL42E') # conditional # TODO # assignment - exprCheck('a = 5', 'aS1aL5E') - exprCheck('a *= 5', 'mL1aL5E') - exprCheck('a /= 5', 'dV1aL5E') - exprCheck('a %= 5', 'rM1aL5E') - exprCheck('a += 5', 'pL1aL5E') - exprCheck('a -= 5', 'mI1aL5E') - exprCheck('a >>= 5', 'rS1aL5E') - exprCheck('a <<= 5', 'lS1aL5E') - exprCheck('a &= 5', 'aN1aL5E') - exprCheck('a ^= 5', 'eO1aL5E') - exprCheck('a |= 5', 'oR1aL5E') - - # Additional tests - # a < expression that starts with something that could be a template - exprCheck('A < 42', 'lt1AL42E') - check('function', 'template<> void f(A &v)', - {2: "IE1fR1AI1BX2EE", 3: "IE1fR1AI1BXL2EEE", 4: "IE1fvR1AI1BXL2EEE"}) - exprCheck('A<1>::value', 'N1AIXL1EEE5valueE') - check('class', "template A", {2: "I_iE1A"}) - check('enumerator', 'A = std::numeric_limits::max()', {2: "1A"}) - - exprCheck('operator()()', 'clclE') - exprCheck('operator()()', 'clclIiEE') - - # pack expansion - exprCheck('a(b(c, 1 + d...)..., e(f..., g))', 'cl1aspcl1b1cspplL1E1dEcl1esp1f1gEE') + exprCheck('a = 5') + exprCheck('a *= 5') + exprCheck('a /= 5') + exprCheck('a %= 5') + exprCheck('a += 5') + exprCheck('a -= 5') + exprCheck('a >>= 5') + exprCheck('a <<= 5') + exprCheck('a &= 5') + exprCheck('a ^= 5') + exprCheck('a |= 5') def test_type_definitions(): @@ -249,6 +222,9 @@ def test_type_definitions(): check('type', "bool *b", {1: 'b'}) check('type', "bool *const b", {1: 'b'}) + check('type', "bool *const *b", {1: 'b'}) + check('type', "bool *volatile *b", {1: 'b'}) + check('type', "bool *restrict *b", {1: 'b'}) check('type', "bool *volatile const b", {1: 'b'}) check('type', "bool *volatile const b", {1: 'b'}) check('type', "bool *volatile const *b", {1: 'b'}) @@ -394,38 +370,22 @@ def test_anon_definitions(): def test_initializers(): - return # TODO - idsMember = {1: 'v__T', 2: '1v'} - idsFunction = {1: 'f__T', 2: '1f1T'} - idsTemplate = {2: 'I_1TE1fv', 4: 'I_1TE1fvv'} + idsMember = {1: 'v'} + idsFunction = {1: 'f'} # no init check('member', 'T v', idsMember) check('function', 'void f(T v)', idsFunction) - check('function', 'template void f()', idsTemplate) # with '=', assignment-expression check('member', 'T v = 42', idsMember) check('function', 'void f(T v = 42)', idsFunction) - check('function', 'template void f()', idsTemplate) # with '=', braced-init check('member', 'T v = {}', idsMember) check('function', 'void f(T v = {})', idsFunction) - check('function', 'template void f()', idsTemplate) check('member', 'T v = {42, 42, 42}', idsMember) check('function', 'void f(T v = {42, 42, 42})', idsFunction) - check('function', 'template void f()', idsTemplate) check('member', 'T v = {42, 42, 42,}', idsMember) check('function', 'void f(T v = {42, 42, 42,})', idsFunction) - check('function', 'template void f()', idsTemplate) - check('member', 'T v = {42, 42, args...}', idsMember) - check('function', 'void f(T v = {42, 42, args...})', idsFunction) - check('function', 'template void f()', idsTemplate) - # without '=', braced-init - check('member', 'T v{}', idsMember) - check('member', 'T v{42, 42, 42}', idsMember) - check('member', 'T v{42, 42, 42,}', idsMember) - check('member', 'T v{42, 42, args...}', idsMember) - # other - check('member', 'T v = T{}', idsMember) + # TODO: designator-list def test_attributes(): From fdc55201c813799f4314fec7c78cc123ed816960 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 15 Mar 2020 00:08:27 +0100 Subject: [PATCH 04/12] C, add __int64 as a fundamental type --- sphinx/domains/c.py | 19 +++++++++---------- tests/roots/test-domain-c/index.rst | 6 ++++++ tests/test_domain_c.py | 4 ++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 3a65c9b85..a1c96afe5 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -1889,8 +1889,9 @@ class Symbol: class DefinitionParser(BaseParser): # those without signedness and size modifiers # see https://en.cppreference.com/w/cpp/language/types - _simple_fundemental_types = ( + _simple_fundamental_types = ( 'void', '_Bool', 'bool', 'char', 'int', 'float', 'double', + '__int64', ) _prefix_keys = ('struct', 'enum', 'union') @@ -2357,9 +2358,9 @@ class DefinitionParser(BaseParser): return ASTNestedName(names, rooted) def _parse_trailing_type_spec(self) -> Any: - # fundemental types + # fundamental types self.skip_ws() - for t in self._simple_fundemental_types: + for t in self._simple_fundamental_types: if self.skip_word(t): return ASTTrailingTypeSpecFundamental(t) @@ -2382,6 +2383,8 @@ class DefinitionParser(BaseParser): elements.append('int') elif self.skip_word_and_ws('double'): elements.append('double') + elif self.skip_word_and_ws('__int64'): + elements.append('__int64') if len(elements) > 0: return ASTTrailingTypeSpecFundamental(' '.join(elements)) @@ -3205,6 +3208,9 @@ class CDomain(Domain): print(self.data['root_symbol'].dump(0)) print("process_doc end:", docname) + def process_field_xref(self, pnode: pending_xref) -> None: + pnode.attributes.update(self.env.ref_context) + def merge_domaindata(self, docnames: List[str], otherdata: Dict) -> None: if Symbol.debug_show_tree: print("merge_domaindata:") @@ -3239,13 +3245,6 @@ class CDomain(Domain): warner = Warner() - # strip pointer asterisk - target = target.rstrip(' *') - - # becase TypedField can generate xrefs - if target in _keywords: - raise NoUri(target, typ) - parser = DefinitionParser(target, warner) try: name = parser.parse_xref_object() diff --git a/tests/roots/test-domain-c/index.rst b/tests/roots/test-domain-c/index.rst index 1265b7967..22c3bd9e5 100644 --- a/tests/roots/test-domain-c/index.rst +++ b/tests/roots/test-domain-c/index.rst @@ -6,6 +6,12 @@ directives .. c:function:: int hello(char *name) + :rtype: int + +.. c:function:: MyStruct hello2(char *name) + + :rtype: MyStruct + .. c:member:: float Sphinx.version .. c:var:: int version diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index df3126d38..d00e2cfa8 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -108,6 +108,8 @@ def test_expressions(): exprCheck('int *restrict*') exprCheck('int *(*)(double)') exprCheck('const int*') + exprCheck('__int64') + exprCheck('unsigned __int64') # actual expressions @@ -260,6 +262,8 @@ def test_member_definitions(): check('member', 'double a', {1: 'a'}) check('member', 'unsigned long a', {1: 'a'}) + check('member', '__int64 a', {1: 'a'}) + check('member', 'unsigned __int64 a', {1: 'a'}) check('member', 'int .a', {1: 'a'}) From 4cdff035176b7c97c1379905bb03f46a28592bed Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 15 Mar 2020 00:49:23 +0100 Subject: [PATCH 05/12] C, update CHANGES --- CHANGES | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index d4e8d2ea4..f34d4ae65 100644 --- a/CHANGES +++ b/CHANGES @@ -25,7 +25,7 @@ Incompatible changes * Due to the scoping changes for :rst:dir:`productionlist` some uses of :rst:role:`token` must be modified to include the scope which was previously ignored. -* #6903: Internal data structure of C, Python, reST and standard domains have +* #6903: Internal data structure of Python, reST and standard domains have changed. The node_id is added to the index of objects and modules. Now they contains a pair of docname and node_id for cross reference. * #7276: C++ domain: Non intended behavior is removed such as ``say_hello_`` @@ -38,6 +38,8 @@ Incompatible changes links to ``.. py:function:: say_hello()`` * #7246: py domain: Drop special cross reference helper for exceptions, functions and methods +* The C domain has been rewritten, with additional directives and roles. + The existing ones are now more strict, resulting in new warnings. Deprecated ---------- @@ -91,6 +93,14 @@ Features added using ``:nosearch:`` file-wide metadata * #7142: html theme: Add a theme option: ``pygments_dark_style`` to switch the style of code-blocks in dark mode +* The C domain has been rewritten adding for example: + + - Cross-referencing respecting the current scope. + - Possible to document anonymous entities. + - More specific directives and roles for each type of entitiy, + e.g., handling scoping of enumerators. + - New role :rst:role:`c:expr` for rendering expressions and types + in text. Bugs fixed ---------- @@ -109,6 +119,8 @@ Bugs fixed ``html_link_suffix`` in search results * #7179: std domain: Fix whitespaces are suppressed on referring GenericObject * #7289: console: use bright colors instead of bold +* #1539: C, parse array types. +* #2377: C, parse function pointers even in complex types. Testing -------- From 366d4624c0ecc64c8a64d1bb4094da1d4d4165ae Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 15 Mar 2020 12:39:14 +0100 Subject: [PATCH 06/12] C, bump env version --- sphinx/domains/c.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index a1c96afe5..15e8087df 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -3304,7 +3304,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: return { 'version': 'builtin', - 'env_version': 1, + 'env_version': 2, 'parallel_read_safe': True, 'parallel_write_safe': True, } From bb2f1baa1bfea80a172161b8b4371c8fcaa34e1b Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 15 Mar 2020 15:07:52 +0100 Subject: [PATCH 07/12] C, fix yype issues and remove dead code --- sphinx/domains/c.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 15e8087df..323809966 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -10,7 +10,7 @@ import re from typing import ( - Any, Callable, Dict, Iterator, List, Tuple, Union + Any, Callable, Dict, Iterator, List, Tuple, Type, Union ) from typing import cast @@ -569,7 +569,7 @@ class ASTFunctionParameter(ASTBase): else: return transform(self.arg) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: Any, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) if self.ellipsis: @@ -1479,7 +1479,7 @@ class Symbol: Symbol.debug_indent -= 2 def _symbol_lookup(self, nestedName: ASTNestedName, - onMissingQualifiedSymbol: Callable[["Symbol", ASTIdentifier, Any], "Symbol"], # NOQA + onMissingQualifiedSymbol: Callable[["Symbol", ASTIdentifier], "Symbol"], # NOQA ancestorLookupType: str, matchSelf: bool, recurseInAnon: bool, searchInSiblings: bool) -> SymbolLookupResult: # TODO: further simplification from C++ to C @@ -1507,18 +1507,11 @@ class Symbol: # walk up until we find the first identifier firstName = names[0] while parentSymbol.parent: - if parentSymbol.find_identifier(firstName.identifier, + if parentSymbol.find_identifier(firstName, matchSelf=matchSelf, recurseInAnon=recurseInAnon, searchInSiblings=searchInSiblings): - # if we are in the scope of a constructor but wants to - # reference the class we need to walk one extra up - if (len(names) == 1 and ancestorLookupType == 'class' and matchSelf and - parentSymbol.parent and - parentSymbol.parent.ident == firstName.ident): - pass - else: - break + break parentSymbol = parentSymbol.parent if Symbol.debug_lookup: @@ -2968,7 +2961,7 @@ class CObject(ObjectDescription): env.temp_data['c:last_symbol'] = None return super().run() - def handle_signature(self, sig: str, signode: desc_signature) -> str: + def handle_signature(self, sig: str, signode: desc_signature) -> ASTDeclaration: parentSymbol = self.env.temp_data['c:parent_symbol'] # type: Symbol parser = DefinitionParser(sig, self) @@ -3168,7 +3161,7 @@ class CDomain(Domain): initial_data = { 'root_symbol': Symbol(None, None, None, None), 'objects': {}, # fullname -> docname, node_id, objtype - } # type: Dict[str, Dict[str, Tuple[str, Any]]] + } # type: Dict[str, Union[Symbol, Dict[str, Tuple[str, str, str]]]] @property def objects(self) -> Dict[str, Tuple[str, str, str]]: From 04efdfa917265f0df6bab03aaef95598a56c995a Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 15 Mar 2020 19:54:43 +0100 Subject: [PATCH 08/12] C, style decl specifiers as intented --- sphinx/domains/c.py | 5 +---- tests/roots/test-domain-c/index.rst | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 323809966..85380525f 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -655,10 +655,7 @@ class ASTDeclSpecsSimple(ASTBase): def _add(modifiers: List[Node], text: str) -> None: if len(modifiers) > 0: modifiers.append(nodes.Text(' ')) - # TODO: should probably do - # modifiers.append(addnodes.desc_annotation(text, text)) - # but for now emulate the old output: - modifiers.append(nodes.Text(text)) + modifiers.append(addnodes.desc_annotation(text, text)) for attr in self.attrs: if len(modifiers) > 0: diff --git a/tests/roots/test-domain-c/index.rst b/tests/roots/test-domain-c/index.rst index 22c3bd9e5..7e2c18be9 100644 --- a/tests/roots/test-domain-c/index.rst +++ b/tests/roots/test-domain-c/index.rst @@ -4,7 +4,7 @@ test-domain-c directives ---------- -.. c:function:: int hello(char *name) +.. c:function:: int hello(const char *name) :rtype: int From 7a29b634218a2c0519cd6fb408bc2d663e6619e2 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 15 Mar 2020 20:46:01 +0100 Subject: [PATCH 09/12] C, prepare for multiline rendering --- sphinx/domains/c.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 85380525f..243f57488 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -1215,8 +1215,14 @@ class ASTDeclaration(ASTBase): env: "BuildEnvironment", options: Dict) -> None: verify_description_mode(mode) assert self.symbol - - mainDeclNode = signode + # The caller of the domain added a desc_signature node. + # Always enable multiline: + signode['is_multiline'] = True + # Put each line in a desc_signature_line node. + mainDeclNode = addnodes.desc_signature_line() + mainDeclNode.sphinx_c_tagname = 'declarator' + mainDeclNode['add_permalink'] = not self.symbol.isRedeclaration + signode += mainDeclNode if self.objectType == 'member': pass From 6ec42b071e76bdfd02f3ae00ba87a9ba68b4f230 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Sun, 15 Mar 2020 21:00:42 +0100 Subject: [PATCH 10/12] C, flake and mypy fixes, rename attr in desc_signature_line --- CHANGES | 2 ++ sphinx/addnodes.py | 2 +- sphinx/domains/c.py | 5 ++--- sphinx/domains/cpp.py | 12 +++++------- sphinx/util/cfamily.py | 8 ++++---- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/CHANGES b/CHANGES index f34d4ae65..7abae4b18 100644 --- a/CHANGES +++ b/CHANGES @@ -40,6 +40,8 @@ Incompatible changes functions and methods * The C domain has been rewritten, with additional directives and roles. The existing ones are now more strict, resulting in new warnings. +* The attribute ``sphinx_cpp_tagname`` in the ``desc_signature_line`` node + has been renamed to ``sphinx_line_type``. Deprecated ---------- diff --git a/sphinx/addnodes.py b/sphinx/addnodes.py index fa04e9344..847a6d715 100644 --- a/sphinx/addnodes.py +++ b/sphinx/addnodes.py @@ -119,7 +119,7 @@ class desc_signature_line(nodes.Part, nodes.Inline, nodes.FixedTextElement): It should only be used in a ``desc_signature`` with ``is_multiline`` set. Set ``add_permalink = True`` for the line that should get the permalink. """ - sphinx_cpp_tagname = '' + sphinx_line_type = '' # nodes to use within a desc_signature or desc_signature_line diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 243f57488..49d1cf6a3 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -10,7 +10,7 @@ import re from typing import ( - Any, Callable, Dict, Iterator, List, Tuple, Type, Union + Any, Callable, Dict, Iterator, List, Type, Tuple, Union ) from typing import cast @@ -25,7 +25,6 @@ from sphinx.builders import Builder from sphinx.directives import ObjectDescription from sphinx.domains import Domain, ObjType from sphinx.environment import BuildEnvironment -from sphinx.errors import NoUri from sphinx.locale import _, __ from sphinx.roles import XRefRole from sphinx.util import logging @@ -1220,7 +1219,7 @@ class ASTDeclaration(ASTBase): signode['is_multiline'] = True # Put each line in a desc_signature_line node. mainDeclNode = addnodes.desc_signature_line() - mainDeclNode.sphinx_c_tagname = 'declarator' + mainDeclNode.sphinx_line_type = 'declarator' mainDeclNode['add_permalink'] = not self.symbol.isRedeclaration signode += mainDeclNode diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py index 639ac20b6..5a85d56a6 100644 --- a/sphinx/domains/cpp.py +++ b/sphinx/domains/cpp.py @@ -9,9 +9,8 @@ """ import re -import warnings from typing import ( - Any, Callable, Dict, Iterator, List, Match, Pattern, Tuple, Type, TypeVar, Union + Any, Callable, Dict, Iterator, List, Tuple, Type, TypeVar, Union ) from docutils import nodes, utils @@ -24,7 +23,6 @@ from sphinx.addnodes import desc_signature, pending_xref from sphinx.application import Sphinx from sphinx.builders import Builder from sphinx.config import Config -from sphinx.deprecation import RemovedInSphinx40Warning from sphinx.directives import ObjectDescription from sphinx.domains import Domain, ObjType from sphinx.environment import BuildEnvironment @@ -70,7 +68,7 @@ T = TypeVar('T') Each signature is in a desc_signature node, where all children are desc_signature_line nodes. Each of these lines will have the attribute - 'sphinx_cpp_tagname' set to one of the following (prioritized): + 'sphinx_line_type' set to one of the following (prioritized): - 'declarator', if the line contains the name of the declared object. - 'templateParams', if the line starts a template parameter list, - 'templateParams', if the line has template parameters @@ -1604,7 +1602,7 @@ class ASTTemplateParams(ASTBase): def makeLine(parentNode: desc_signature = parentNode) -> addnodes.desc_signature_line: signode = addnodes.desc_signature_line() parentNode += signode - signode.sphinx_cpp_tagname = 'templateParams' + signode.sphinx_line_type = 'templateParams' return signode if self.isNested: lineNode = parentNode # type: Element @@ -1713,7 +1711,7 @@ class ASTTemplateIntroduction(ASTBase): # Note: 'lineSpec' has no effect on template introductions. signode = addnodes.desc_signature_line() parentNode += signode - signode.sphinx_cpp_tagname = 'templateIntroduction' + signode.sphinx_line_type = 'templateIntroduction' self.concept.describe_signature(signode, 'markType', env, symbol) signode += nodes.Text('{') first = True @@ -3416,7 +3414,7 @@ class ASTDeclaration(ASTBase): signode['is_multiline'] = True # Put each line in a desc_signature_line node. mainDeclNode = addnodes.desc_signature_line() - mainDeclNode.sphinx_cpp_tagname = 'declarator' + mainDeclNode.sphinx_line_type = 'declarator' mainDeclNode['add_permalink'] = not self.symbol.isRedeclaration if self.templatePrefix: diff --git a/sphinx/util/cfamily.py b/sphinx/util/cfamily.py index 87e438851..67fdc114b 100644 --- a/sphinx/util/cfamily.py +++ b/sphinx/util/cfamily.py @@ -1,10 +1,10 @@ """ sphinx.util.cfamily - ~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~ Utility functions common to the C and C++ domains. - :copyright: Copyright 2020-2020 by the Sphinx team, see AUTHORS. + :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -12,7 +12,7 @@ import re import warnings from copy import deepcopy from typing import ( - Any, Callable, Dict, Iterator, List, Match, Pattern, Tuple, Type, TypeVar, Union + Any, Callable, List, Match, Pattern, Tuple ) from sphinx.deprecation import RemovedInSphinx40Warning @@ -244,4 +244,4 @@ class BaseParser: def assert_end(self) -> None: self.skip_ws() if not self.eof: - self.fail('Expected end of definition.') \ No newline at end of file + self.fail('Expected end of definition.') From e4e5ee859880493e23f8deb30ca19d88537e7fa3 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Mon, 16 Mar 2020 19:13:42 +0100 Subject: [PATCH 11/12] C, improve type annotations with related refactoring --- sphinx/domains/c.py | 1057 ++++++++++++++++++++++------------------ sphinx/domains/cpp.py | 6 +- sphinx/util/cfamily.py | 2 +- 3 files changed, 580 insertions(+), 485 deletions(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 49d1cf6a3..2d151df9f 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -19,7 +19,7 @@ from docutils.nodes import Element, Node, TextElement, system_message from docutils.parsers.rst.states import Inliner from sphinx import addnodes -from sphinx.addnodes import pending_xref, desc_signature +from sphinx.addnodes import pending_xref from sphinx.application import Sphinx from sphinx.builders import Builder from sphinx.directives import ObjectDescription @@ -29,7 +29,7 @@ from sphinx.locale import _, __ from sphinx.roles import XRefRole from sphinx.util import logging from sphinx.util.cfamily import ( - NoOldIdError, ASTBase, verify_description_mode, StringifyTransform, + NoOldIdError, ASTBaseBase, verify_description_mode, StringifyTransform, BaseParser, DefinitionError, UnsupportedMultiCharacterCharLiteral, identifier_re, anon_identifier_re, integer_literal_re, octal_literal_re, hex_literal_re, binary_literal_re, float_literal_re, @@ -90,328 +90,16 @@ class _DuplicateSymbolError(Exception): return "Internal C duplicate symbol error:\n%s" % self.symbol.dump(0) -################################################################################ -# Expressions and Literals +class ASTBase(ASTBaseBase): + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + raise NotImplementedError(repr(self)) + + +# Names ################################################################################ -class ASTBooleanLiteral(ASTBase): - def __init__(self, value: bool) -> None: - self.value = value - - def _stringify(self, transform: StringifyTransform) -> str: - if self.value: - return 'true' - else: - return 'false' - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text(str(self))) - - -class ASTNumberLiteral(ASTBase): - def __init__(self, data: str) -> None: - self.data = data - - def _stringify(self, transform: StringifyTransform) -> str: - return self.data - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - txt = str(self) - signode.append(nodes.Text(txt, txt)) - - -class ASTCharLiteral(ASTBase): - def __init__(self, prefix: str, data: str) -> None: - self.prefix = prefix # may be None when no prefix - self.data = data - decoded = data.encode().decode('unicode-escape') - if len(decoded) == 1: - self.value = ord(decoded) - else: - raise UnsupportedMultiCharacterCharLiteral(decoded) - - def _stringify(self, transform: StringifyTransform) -> str: - if self.prefix is None: - return "'" + self.data + "'" - else: - return self.prefix + "'" + self.data + "'" - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - txt = str(self) - signode.append(nodes.Text(txt, txt)) - - -class ASTStringLiteral(ASTBase): - def __init__(self, data: str) -> None: - self.data = data - - def _stringify(self, transform: StringifyTransform) -> str: - return self.data - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - txt = str(self) - signode.append(nodes.Text(txt, txt)) - - -class ASTParenExpr(ASTBase): - def __init__(self, expr): - self.expr = expr - - def _stringify(self, transform: StringifyTransform) -> str: - return '(' + transform(self.expr) + ')' - - def get_id(self, version: int) -> str: - return self.expr.get_id(version) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('(', '(')) - self.expr.describe_signature(signode, mode, env, symbol) - signode.append(nodes.Text(')', ')')) - - -class ASTBinOpExpr(ASTBase): - def __init__(self, exprs, ops): - assert len(exprs) > 0 - assert len(exprs) == len(ops) + 1 - self.exprs = exprs - self.ops = ops - - def _stringify(self, transform: StringifyTransform) -> str: - res = [] - res.append(transform(self.exprs[0])) - for i in range(1, len(self.exprs)): - res.append(' ') - res.append(self.ops[i - 1]) - res.append(' ') - res.append(transform(self.exprs[i])) - return ''.join(res) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - self.exprs[0].describe_signature(signode, mode, env, symbol) - for i in range(1, len(self.exprs)): - signode.append(nodes.Text(' ')) - signode.append(nodes.Text(self.ops[i - 1])) - signode.append(nodes.Text(' ')) - self.exprs[i].describe_signature(signode, mode, env, symbol) - - -class ASTAssignmentExpr(ASTBase): - def __init__(self, exprs, ops): - assert len(exprs) > 0 - assert len(exprs) == len(ops) + 1 - self.exprs = exprs - self.ops = ops - - def _stringify(self, transform: StringifyTransform) -> str: - res = [] - res.append(transform(self.exprs[0])) - for i in range(1, len(self.exprs)): - res.append(' ') - res.append(self.ops[i - 1]) - res.append(' ') - res.append(transform(self.exprs[i])) - return ''.join(res) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - self.exprs[0].describe_signature(signode, mode, env, symbol) - for i in range(1, len(self.exprs)): - signode.append(nodes.Text(' ')) - signode.append(nodes.Text(self.ops[i - 1])) - signode.append(nodes.Text(' ')) - self.exprs[i].describe_signature(signode, mode, env, symbol) - - -class ASTCastExpr(ASTBase): - def __init__(self, typ, expr): - self.typ = typ - self.expr = expr - - def _stringify(self, transform: StringifyTransform) -> str: - res = ['('] - res.append(transform(self.typ)) - res.append(')') - res.append(transform(self.expr)) - return ''.join(res) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('(')) - self.typ.describe_signature(signode, mode, env, symbol) - signode.append(nodes.Text(')')) - self.expr.describe_signature(signode, mode, env, symbol) - - -class ASTUnaryOpExpr(ASTBase): - def __init__(self, op, expr): - self.op = op - self.expr = expr - - def _stringify(self, transform: StringifyTransform) -> str: - return transform(self.op) + transform(self.expr) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text(self.op)) - self.expr.describe_signature(signode, mode, env, symbol) - - -class ASTSizeofType(ASTBase): - def __init__(self, typ): - self.typ = typ - - def _stringify(self, transform: StringifyTransform) -> str: - return "sizeof(" + transform(self.typ) + ")" - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('sizeof(')) - self.typ.describe_signature(signode, mode, env, symbol) - signode.append(nodes.Text(')')) - - -class ASTSizeofExpr(ASTBase): - def __init__(self, expr): - self.expr = expr - - def _stringify(self, transform: StringifyTransform) -> str: - return "sizeof " + transform(self.expr) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('sizeof ')) - self.expr.describe_signature(signode, mode, env, symbol) - - -class ASTAlignofExpr(ASTBase): - def __init__(self, typ): - self.typ = typ - - def _stringify(self, transform: StringifyTransform) -> str: - return "alignof(" + transform(self.typ) + ")" - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('alignof(')) - self.typ.describe_signature(signode, mode, env, symbol) - signode.append(nodes.Text(')')) - - -class ASTPostfixCallExpr(ASTBase): - def __init__(self, lst: Union["ASTParenExprList", "ASTBracedInitList"]) -> None: - self.lst = lst - - def _stringify(self, transform: StringifyTransform) -> str: - return transform(self.lst) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - self.lst.describe_signature(signode, mode, env, symbol) - - -class ASTPostfixArray(ASTBase): - def __init__(self, expr): - self.expr = expr - - def _stringify(self, transform: StringifyTransform) -> str: - return '[' + transform(self.expr) + ']' - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('[')) - self.expr.describe_signature(signode, mode, env, symbol) - signode.append(nodes.Text(']')) - - -class ASTPostfixInc(ASTBase): - def _stringify(self, transform: StringifyTransform) -> str: - return '++' - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('++')) - - -class ASTPostfixDec(ASTBase): - def _stringify(self, transform: StringifyTransform) -> str: - return '--' - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('--')) - - -class ASTPostfixMember(ASTBase): - def __init__(self, name): - self.name = name - - def _stringify(self, transform: StringifyTransform) -> str: - return '.' + transform(self.name) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('.')) - self.name.describe_signature(signode, 'noneIsName', env, symbol) - - -class ASTPostfixMemberOfPointer(ASTBase): - def __init__(self, name): - self.name = name - - def _stringify(self, transform: StringifyTransform) -> str: - return '->' + transform(self.name) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode.append(nodes.Text('->')) - self.name.describe_signature(signode, 'noneIsName', env, symbol) - - -class ASTPostfixExpr(ASTBase): - def __init__(self, prefix, postFixes): - assert len(postFixes) > 0 - self.prefix = prefix - self.postFixes = postFixes - - def _stringify(self, transform: StringifyTransform) -> str: - res = [transform(self.prefix)] - for p in self.postFixes: - res.append(transform(p)) - return ''.join(res) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - self.prefix.describe_signature(signode, mode, env, symbol) - for p in self.postFixes: - p.describe_signature(signode, mode, env, symbol) - - -class ASTFallbackExpr(ASTBase): - def __init__(self, expr): - self.expr = expr - - def _stringify(self, transform: StringifyTransform) -> str: - return self.expr - - def get_id(self, version: int) -> str: - return str(self.expr) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - signode += nodes.Text(self.expr) - - -################################################################################ -# The Rest -################################################################################ - -class ASTIdentifier(ASTBase): +class ASTIdentifier(ASTBaseBase): def __init__(self, identifier: str) -> None: assert identifier is not None assert len(identifier) != 0 @@ -428,8 +116,9 @@ class ASTIdentifier(ASTBase): def get_display_string(self) -> str: return "[anonymous]" if self.is_anon() else self.identifier - def describe_signature(self, signode: Any, mode: str, env: "BuildEnvironment", + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", prefix: str, symbol: "Symbol") -> None: + # note: slightly different signature of describe_signature due to the prefix verify_description_mode(mode) if mode == 'markType': targetText = prefix + self.identifier @@ -478,7 +167,7 @@ class ASTNestedName(ASTBase): else: return res - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) # just print the name part, with template args, not template params @@ -500,7 +189,7 @@ class ASTNestedName(ASTBase): # so it can remove it in inner declarations. dest = signode if mode == 'lastIsName': - dest = addnodes.desc_addname() # type: ignore + dest = addnodes.desc_addname() for i in range(len(names)): ident = names[i] if not first: @@ -520,20 +209,384 @@ class ASTNestedName(ASTBase): raise Exception('Unknown description mode: %s' % mode) -class ASTTrailingTypeSpecFundamental(ASTBase): +################################################################################ +# Expressions +################################################################################ + +class ASTExpression(ASTBase): + pass + + +# Primary expressions +################################################################################ + +class ASTLiteral(ASTExpression): + pass + + +class ASTBooleanLiteral(ASTLiteral): + def __init__(self, value: bool) -> None: + self.value = value + + def _stringify(self, transform: StringifyTransform) -> str: + if self.value: + return 'true' + else: + return 'false' + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text(str(self))) + + +class ASTNumberLiteral(ASTLiteral): + def __init__(self, data: str) -> None: + self.data = data + + def _stringify(self, transform: StringifyTransform) -> str: + return self.data + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + txt = str(self) + signode.append(nodes.Text(txt, txt)) + + +class ASTCharLiteral(ASTLiteral): + def __init__(self, prefix: str, data: str) -> None: + self.prefix = prefix # may be None when no prefix + self.data = data + decoded = data.encode().decode('unicode-escape') + if len(decoded) == 1: + self.value = ord(decoded) + else: + raise UnsupportedMultiCharacterCharLiteral(decoded) + + def _stringify(self, transform: StringifyTransform) -> str: + if self.prefix is None: + return "'" + self.data + "'" + else: + return self.prefix + "'" + self.data + "'" + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + txt = str(self) + signode.append(nodes.Text(txt, txt)) + + +class ASTStringLiteral(ASTLiteral): + def __init__(self, data: str) -> None: + self.data = data + + def _stringify(self, transform: StringifyTransform) -> str: + return self.data + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + txt = str(self) + signode.append(nodes.Text(txt, txt)) + + +class ASTIdExpression(ASTExpression): + def __init__(self, name: ASTNestedName): + # note: this class is basically to cast a nested name as an expression + self.name = name + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.name) + + def get_id(self, version: int) -> str: + return self.name.get_id(version) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.name.describe_signature(signode, mode, env, symbol) + + +class ASTParenExpr(ASTExpression): + def __init__(self, expr): + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return '(' + transform(self.expr) + ')' + + def get_id(self, version: int) -> str: + return self.expr.get_id(version) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('(', '(')) + self.expr.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')', ')')) + + +# Postfix expressions +################################################################################ + +class ASTPostfixOp(ASTBase): + pass + + +class ASTPostfixCallExpr(ASTPostfixOp): + def __init__(self, lst: Union["ASTParenExprList", "ASTBracedInitList"]) -> None: + self.lst = lst + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.lst) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.lst.describe_signature(signode, mode, env, symbol) + + +class ASTPostfixArray(ASTPostfixOp): + def __init__(self, expr): + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return '[' + transform(self.expr) + ']' + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('[')) + self.expr.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(']')) + + +class ASTPostfixInc(ASTPostfixOp): + def _stringify(self, transform: StringifyTransform) -> str: + return '++' + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('++')) + + +class ASTPostfixDec(ASTPostfixOp): + def _stringify(self, transform: StringifyTransform) -> str: + return '--' + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('--')) + + +class ASTPostfixMember(ASTPostfixOp): + def __init__(self, name): + self.name = name + + def _stringify(self, transform: StringifyTransform) -> str: + return '.' + transform(self.name) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('.')) + self.name.describe_signature(signode, 'noneIsName', env, symbol) + + +class ASTPostfixMemberOfPointer(ASTPostfixOp): + def __init__(self, name): + self.name = name + + def _stringify(self, transform: StringifyTransform) -> str: + return '->' + transform(self.name) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('->')) + self.name.describe_signature(signode, 'noneIsName', env, symbol) + + +class ASTPostfixExpr(ASTExpression): + def __init__(self, prefix: ASTExpression, postFixes: List[ASTPostfixOp]): + self.prefix = prefix + self.postFixes = postFixes + + def _stringify(self, transform: StringifyTransform) -> str: + res = [transform(self.prefix)] + for p in self.postFixes: + res.append(transform(p)) + return ''.join(res) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.prefix.describe_signature(signode, mode, env, symbol) + for p in self.postFixes: + p.describe_signature(signode, mode, env, symbol) + + +# Unary expressions +################################################################################ + +class ASTUnaryOpExpr(ASTExpression): + def __init__(self, op: str, expr: ASTExpression): + self.op = op + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return transform(self.op) + transform(self.expr) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text(self.op)) + self.expr.describe_signature(signode, mode, env, symbol) + + +class ASTSizeofType(ASTExpression): + def __init__(self, typ): + self.typ = typ + + def _stringify(self, transform: StringifyTransform) -> str: + return "sizeof(" + transform(self.typ) + ")" + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('sizeof(')) + self.typ.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')')) + + +class ASTSizeofExpr(ASTExpression): + def __init__(self, expr: ASTExpression): + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return "sizeof " + transform(self.expr) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('sizeof ')) + self.expr.describe_signature(signode, mode, env, symbol) + + +class ASTAlignofExpr(ASTExpression): + def __init__(self, typ: "ASTType"): + self.typ = typ + + def _stringify(self, transform: StringifyTransform) -> str: + return "alignof(" + transform(self.typ) + ")" + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('alignof(')) + self.typ.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')')) + + +# Other expressions +################################################################################ + +class ASTCastExpr(ASTExpression): + def __init__(self, typ: "ASTType", expr: ASTExpression): + self.typ = typ + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + res = ['('] + res.append(transform(self.typ)) + res.append(')') + res.append(transform(self.expr)) + return ''.join(res) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode.append(nodes.Text('(')) + self.typ.describe_signature(signode, mode, env, symbol) + signode.append(nodes.Text(')')) + self.expr.describe_signature(signode, mode, env, symbol) + + +class ASTBinOpExpr(ASTBase): + def __init__(self, exprs: List[ASTExpression], ops: List[str]): + assert len(exprs) > 0 + assert len(exprs) == len(ops) + 1 + self.exprs = exprs + self.ops = ops + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + res.append(transform(self.exprs[0])) + for i in range(1, len(self.exprs)): + res.append(' ') + res.append(self.ops[i - 1]) + res.append(' ') + res.append(transform(self.exprs[i])) + return ''.join(res) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.exprs[0].describe_signature(signode, mode, env, symbol) + for i in range(1, len(self.exprs)): + signode.append(nodes.Text(' ')) + signode.append(nodes.Text(self.ops[i - 1])) + signode.append(nodes.Text(' ')) + self.exprs[i].describe_signature(signode, mode, env, symbol) + + +class ASTAssignmentExpr(ASTExpression): + def __init__(self, exprs: List[ASTExpression], ops: List[str]): + assert len(exprs) > 0 + assert len(exprs) == len(ops) + 1 + self.exprs = exprs + self.ops = ops + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + res.append(transform(self.exprs[0])) + for i in range(1, len(self.exprs)): + res.append(' ') + res.append(self.ops[i - 1]) + res.append(' ') + res.append(transform(self.exprs[i])) + return ''.join(res) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + self.exprs[0].describe_signature(signode, mode, env, symbol) + for i in range(1, len(self.exprs)): + signode.append(nodes.Text(' ')) + signode.append(nodes.Text(self.ops[i - 1])) + signode.append(nodes.Text(' ')) + self.exprs[i].describe_signature(signode, mode, env, symbol) + + +class ASTFallbackExpr(ASTExpression): + def __init__(self, expr: str): + self.expr = expr + + def _stringify(self, transform: StringifyTransform) -> str: + return self.expr + + def get_id(self, version: int) -> str: + return str(self.expr) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + signode += nodes.Text(self.expr) + + +################################################################################ +# Types +################################################################################ + +class ASTTrailingTypeSpec(ASTBase): + pass + + +class ASTTrailingTypeSpecFundamental(ASTTrailingTypeSpec): def __init__(self, name: str) -> None: self.name = name def _stringify(self, transform: StringifyTransform) -> str: return self.name - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: signode += nodes.Text(str(self.name)) -class ASTTrailingTypeSpecName(ASTBase): - def __init__(self, prefix: str, nestedName: Any) -> None: +class ASTTrailingTypeSpecName(ASTTrailingTypeSpec): + def __init__(self, prefix: str, nestedName: ASTNestedName) -> None: self.prefix = prefix self.nestedName = nestedName @@ -549,7 +602,7 @@ class ASTTrailingTypeSpecName(ASTBase): res.append(transform(self.nestedName)) return ''.join(res) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: if self.prefix: signode += addnodes.desc_annotation(self.prefix, self.prefix) @@ -558,7 +611,7 @@ class ASTTrailingTypeSpecName(ASTBase): class ASTFunctionParameter(ASTBase): - def __init__(self, arg: Any, ellipsis: bool = False) -> None: + def __init__(self, arg: "ASTTypeWithInit", ellipsis: bool = False) -> None: self.arg = arg self.ellipsis = ellipsis @@ -578,11 +631,11 @@ class ASTFunctionParameter(ASTBase): class ASTParameters(ASTBase): - def __init__(self, args: List[Any]) -> None: + def __init__(self, args: List[ASTFunctionParameter]) -> None: self.args = args @property - def function_params(self) -> Any: + def function_params(self) -> List[ASTFunctionParameter]: return self.args def _stringify(self, transform: StringifyTransform) -> str: @@ -597,7 +650,7 @@ class ASTParameters(ASTBase): res.append(')') return ''.join(res) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) paramlist = addnodes.desc_parameterlist() @@ -611,7 +664,7 @@ class ASTParameters(ASTBase): signode += paramlist -class ASTDeclSpecsSimple(ASTBase): +class ASTDeclSpecsSimple(ASTBaseBase): def __init__(self, storage: str, threadLocal: str, inline: bool, restrict: bool, volatile: bool, const: bool, attrs: List[Any]) -> None: self.storage = storage @@ -675,8 +728,10 @@ class ASTDeclSpecsSimple(ASTBase): class ASTDeclSpecs(ASTBase): - def __init__(self, outer: Any, leftSpecs: ASTDeclSpecsSimple, - rightSpecs: ASTDeclSpecsSimple, trailing: Any) -> None: + def __init__(self, outer: str, + leftSpecs: ASTDeclSpecsSimple, + rightSpecs: ASTDeclSpecsSimple, + trailing: ASTTrailingTypeSpec) -> None: # leftSpecs and rightSpecs are used for output # allSpecs are used for id generation TODO: remove? self.outer = outer @@ -685,10 +740,6 @@ class ASTDeclSpecs(ASTBase): self.allSpecs = self.leftSpecs.mergeWith(self.rightSpecs) self.trailingTypeSpec = trailing - @property - def name(self) -> ASTNestedName: - return self.trailingTypeSpec.name - def _stringify(self, transform: StringifyTransform) -> str: res = [] # type: List[str] l = transform(self.leftSpecs) @@ -705,7 +756,7 @@ class ASTDeclSpecs(ASTBase): res.append(r) return "".join(res) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) modifiers = [] # type: List[Node] @@ -732,8 +783,11 @@ class ASTDeclSpecs(ASTBase): signode += m +# Declarator +################################################################################ + class ASTArray(ASTBase): - def __init__(self, size): + def __init__(self, size: ASTExpression): self.size = size def _stringify(self, transform: StringifyTransform) -> str: @@ -742,7 +796,7 @@ class ASTArray(ASTBase): else: return '[]' - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) signode.append(nodes.Text("[")) @@ -751,8 +805,93 @@ class ASTArray(ASTBase): signode.append(nodes.Text("]")) -class ASTDeclaratorPtr(ASTBase): - def __init__(self, next: Any, restrict: bool, volatile: bool, const: bool, +class ASTDeclarator(ASTBase): + @property + def name(self) -> ASTNestedName: + raise NotImplementedError(repr(self)) + + @property + def function_params(self) -> List[ASTFunctionParameter]: + raise NotImplementedError(repr(self)) + + def require_space_after_declSpecs(self) -> bool: + raise NotImplementedError(repr(self)) + + +class ASTDeclaratorNameParam(ASTDeclarator): + def __init__(self, declId: ASTNestedName, + arrayOps: List[ASTArray], param: ASTParameters) -> None: + self.declId = declId + self.arrayOps = arrayOps + self.param = param + + @property + def name(self) -> ASTNestedName: + return self.declId + + @property + def function_params(self) -> List[ASTFunctionParameter]: + return self.param.function_params + + # ------------------------------------------------------------------------ + + def require_space_after_declSpecs(self) -> bool: + return self.declId is not None + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + if self.declId: + res.append(transform(self.declId)) + for op in self.arrayOps: + res.append(transform(op)) + if self.param: + res.append(transform(self.param)) + return ''.join(res) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + if self.declId: + self.declId.describe_signature(signode, mode, env, symbol) + for op in self.arrayOps: + op.describe_signature(signode, mode, env, symbol) + if self.param: + self.param.describe_signature(signode, mode, env, symbol) + + +class ASTDeclaratorNameBitField(ASTDeclarator): + def __init__(self, declId: ASTNestedName, size: ASTExpression): + self.declId = declId + self.size = size + + @property + def name(self) -> ASTNestedName: + return self.declId + + # ------------------------------------------------------------------------ + + def require_space_after_declSpecs(self) -> bool: + return self.declId is not None + + def _stringify(self, transform: StringifyTransform) -> str: + res = [] + if self.declId: + res.append(transform(self.declId)) + res.append(" : ") + res.append(transform(self.size)) + return ''.join(res) + + def describe_signature(self, signode: TextElement, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + if self.declId: + self.declId.describe_signature(signode, mode, env, symbol) + signode += nodes.Text(' : ', ' : ') + self.size.describe_signature(signode, mode, env, symbol) + + +class ASTDeclaratorPtr(ASTDeclarator): + def __init__(self, next: ASTDeclarator, restrict: bool, volatile: bool, const: bool, attrs: Any) -> None: assert next self.next = next @@ -766,7 +905,7 @@ class ASTDeclaratorPtr(ASTBase): return self.next.name @property - def function_params(self) -> Any: + def function_params(self) -> List[ASTFunctionParameter]: return self.next.function_params def require_space_after_declSpecs(self) -> bool: @@ -796,7 +935,7 @@ class ASTDeclaratorPtr(ASTBase): res.append(transform(self.next)) return ''.join(res) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) signode += nodes.Text("*") @@ -805,7 +944,7 @@ class ASTDeclaratorPtr(ASTBase): if len(self.attrs) > 0 and (self.restrict or self.volatile or self.const): signode += nodes.Text(' ') - def _add_anno(signode: desc_signature, text: str) -> None: + def _add_anno(signode: TextElement, text: str) -> None: signode += addnodes.desc_annotation(text, text) if self.restrict: @@ -824,8 +963,8 @@ class ASTDeclaratorPtr(ASTBase): self.next.describe_signature(signode, mode, env, symbol) -class ASTDeclaratorParen(ASTBase): - def __init__(self, inner: Any, next: Any) -> None: +class ASTDeclaratorParen(ASTDeclarator): + def __init__(self, inner: ASTDeclarator, next: ASTDeclarator) -> None: assert inner assert next self.inner = inner @@ -837,7 +976,7 @@ class ASTDeclaratorParen(ASTBase): return self.inner.name @property - def function_params(self) -> Any: + def function_params(self) -> List[ASTFunctionParameter]: return self.inner.function_params def require_space_after_declSpecs(self) -> bool: @@ -850,7 +989,7 @@ class ASTDeclaratorParen(ASTBase): res.append(transform(self.next)) return ''.join(res) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) signode += nodes.Text('(') @@ -859,87 +998,18 @@ class ASTDeclaratorParen(ASTBase): self.next.describe_signature(signode, "noneIsName", env, symbol) -class ASTDeclaratorNameParam(ASTBase): - def __init__(self, declId: Any, arrayOps: List[Any], param: Any) -> None: - self.declId = declId - self.arrayOps = arrayOps - self.param = param - - @property - def name(self) -> ASTNestedName: - return self.declId - - @property - def function_params(self) -> Any: - return self.param.function_params - - # ------------------------------------------------------------------------ - - def require_space_after_declSpecs(self) -> bool: - return self.declId is not None - - def _stringify(self, transform: StringifyTransform) -> str: - res = [] - if self.declId: - res.append(transform(self.declId)) - for op in self.arrayOps: - res.append(transform(op)) - if self.param: - res.append(transform(self.param)) - return ''.join(res) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - verify_description_mode(mode) - if self.declId: - self.declId.describe_signature(signode, mode, env, symbol) - for op in self.arrayOps: - op.describe_signature(signode, mode, env, symbol) - if self.param: - self.param.describe_signature(signode, mode, env, symbol) - - -class ASTDeclaratorNameBitField(ASTBase): - def __init__(self, declId, size): - self.declId = declId - self.size = size - - @property - def name(self) -> ASTNestedName: - return self.declId - - # ------------------------------------------------------------------------ - - def require_space_after_declSpecs(self) -> bool: - return self.declId is not None - - def _stringify(self, transform: StringifyTransform) -> str: - res = [] - if self.declId: - res.append(transform(self.declId)) - res.append(" : ") - res.append(transform(self.size)) - return ''.join(res) - - def describe_signature(self, signode: desc_signature, mode: str, - env: "BuildEnvironment", symbol: "Symbol") -> None: - verify_description_mode(mode) - if self.declId: - self.declId.describe_signature(signode, mode, env, symbol) - signode += nodes.Text(' : ', ' : ') - self.size.describe_signature(signode, mode, env, symbol) - signode += nodes.Text(self.size, self.size) - +# Initializer +################################################################################ class ASTParenExprList(ASTBase): - def __init__(self, exprs: List[Any]) -> None: + def __init__(self, exprs: List[ASTExpression]) -> None: self.exprs = exprs def _stringify(self, transform: StringifyTransform) -> str: exprs = [transform(e) for e in self.exprs] return '(%s)' % ', '.join(exprs) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) signode.append(nodes.Text('(')) @@ -954,7 +1024,7 @@ class ASTParenExprList(ASTBase): class ASTBracedInitList(ASTBase): - def __init__(self, exprs: List[Any], trailingComma: bool) -> None: + def __init__(self, exprs: List[ASTExpression], trailingComma: bool) -> None: self.exprs = exprs self.trailingComma = trailingComma @@ -963,7 +1033,7 @@ class ASTBracedInitList(ASTBase): trailingComma = ',' if self.trailingComma else '' return '{%s%s}' % (', '.join(exprs), trailingComma) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) signode.append(nodes.Text('{')) @@ -980,7 +1050,8 @@ class ASTBracedInitList(ASTBase): class ASTInitializer(ASTBase): - def __init__(self, value: Any, hasAssign: bool = True) -> None: + def __init__(self, value: Union[ASTBracedInitList, ASTExpression], + hasAssign: bool = True) -> None: self.value = value self.hasAssign = hasAssign @@ -991,7 +1062,7 @@ class ASTInitializer(ASTBase): else: return val - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) if self.hasAssign: @@ -1000,7 +1071,7 @@ class ASTInitializer(ASTBase): class ASTType(ASTBase): - def __init__(self, declSpecs: Any, decl: Any) -> None: + def __init__(self, declSpecs: ASTDeclSpecs, decl: ASTDeclarator) -> None: assert declSpecs assert decl self.declSpecs = declSpecs @@ -1011,7 +1082,7 @@ class ASTType(ASTBase): return self.decl.name @property - def function_params(self) -> Any: + def function_params(self) -> List[ASTFunctionParameter]: return self.decl.function_params def _stringify(self, transform: StringifyTransform) -> str: @@ -1029,7 +1100,7 @@ class ASTType(ASTBase): else: return 'type' - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) self.declSpecs.describe_signature(signode, 'markType', env, symbol) @@ -1044,7 +1115,7 @@ class ASTType(ASTBase): class ASTTypeWithInit(ASTBase): - def __init__(self, type: Any, init: Any) -> None: + def __init__(self, type: ASTType, init: ASTInitializer) -> None: self.type = type self.init = init @@ -1059,7 +1130,7 @@ class ASTTypeWithInit(ASTBase): res.append(transform(self.init)) return ''.join(res) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) self.type.describe_signature(signode, mode, env, symbol) @@ -1067,8 +1138,28 @@ class ASTTypeWithInit(ASTBase): self.init.describe_signature(signode, mode, env, symbol) +class ASTMacroParameter(ASTBase): + def __init__(self, arg: ASTNestedName, ellipsis: bool = False) -> None: + self.arg = arg + self.ellipsis = ellipsis + + def _stringify(self, transform: StringifyTransform) -> str: + if self.ellipsis: + return '...' + else: + return transform(self.arg) + + def describe_signature(self, signode: Any, mode: str, + env: "BuildEnvironment", symbol: "Symbol") -> None: + verify_description_mode(mode) + if self.ellipsis: + signode += nodes.Text('...') + else: + self.arg.describe_signature(signode, mode, env, symbol=symbol) + + class ASTMacro(ASTBase): - def __init__(self, ident: ASTNestedName, args: List[ASTFunctionParameter]) -> None: + def __init__(self, ident: ASTNestedName, args: List[ASTMacroParameter]) -> None: self.ident = ident self.args = args @@ -1090,7 +1181,7 @@ class ASTMacro(ASTBase): res.append(')') return ''.join(res) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) self.ident.describe_signature(signode, mode, env, symbol) @@ -1114,7 +1205,7 @@ class ASTStruct(ASTBase): def _stringify(self, transform: StringifyTransform) -> str: return transform(self.name) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) self.name.describe_signature(signode, mode, env, symbol=symbol) @@ -1130,7 +1221,7 @@ class ASTUnion(ASTBase): def _stringify(self, transform: StringifyTransform) -> str: return transform(self.name) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) self.name.describe_signature(signode, mode, env, symbol=symbol) @@ -1146,14 +1237,14 @@ class ASTEnum(ASTBase): def _stringify(self, transform: StringifyTransform) -> str: return transform(self.name) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) self.name.describe_signature(signode, mode, env, symbol=symbol) class ASTEnumerator(ASTBase): - def __init__(self, name: Any, init: Any) -> None: + def __init__(self, name: ASTNestedName, init: ASTInitializer) -> None: self.name = name self.init = init @@ -1167,7 +1258,7 @@ class ASTEnumerator(ASTBase): res.append(transform(self.init)) return ''.join(res) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", symbol: "Symbol") -> None: verify_description_mode(mode) self.name.describe_signature(signode, mode, env, symbol) @@ -1175,7 +1266,7 @@ class ASTEnumerator(ASTBase): self.init.describe_signature(signode, 'markType', env, symbol) -class ASTDeclaration(ASTBase): +class ASTDeclaration(ASTBaseBase): def __init__(self, objectType: str, directiveType: str, declaration: Any) -> None: self.objectType = objectType self.directiveType = directiveType @@ -1190,7 +1281,7 @@ class ASTDeclaration(ASTBase): return self.declaration.name @property - def function_params(self) -> Any: + def function_params(self) -> List[ASTFunctionParameter]: if self.objectType != 'function': return None return self.declaration.function_params @@ -1210,7 +1301,7 @@ class ASTDeclaration(ASTBase): def _stringify(self, transform: StringifyTransform) -> str: return transform(self.declaration) - def describe_signature(self, signode: desc_signature, mode: str, + def describe_signature(self, signode: TextElement, mode: str, env: "BuildEnvironment", options: Dict) -> None: verify_description_mode(mode) assert self.symbol @@ -1372,14 +1463,14 @@ class Symbol: newChildren.append(sChild) self._children = newChildren - def get_all_symbols(self) -> Iterator[Any]: + def get_all_symbols(self) -> Iterator["Symbol"]: yield self for sChild in self._children: for s in sChild.get_all_symbols(): yield s @property - def children_recurse_anon(self): + def children_recurse_anon(self) -> Iterator["Symbol"]: for c in self._children: yield c if not c.ident.is_anon(): @@ -1891,7 +1982,7 @@ class DefinitionParser(BaseParser): _prefix_keys = ('struct', 'enum', 'union') - def _parse_string(self): + def _parse_string(self) -> str: if self.current_char != '"': return None startPos = self.pos @@ -1970,7 +2061,7 @@ class DefinitionParser(BaseParser): return None - def _parse_literal(self): + def _parse_literal(self) -> ASTLiteral: # -> integer-literal # | character-literal # | floating-literal @@ -2006,7 +2097,7 @@ class DefinitionParser(BaseParser): " resulting in multiple decoded characters.") return None - def _parse_paren_expression(self): + def _parse_paren_expression(self) -> ASTExpression: # "(" expression ")" if self.current_char != '(': return None @@ -2017,21 +2108,24 @@ class DefinitionParser(BaseParser): self.fail("Expected ')' in end of parenthesized expression.") return ASTParenExpr(res) - def _parse_primary_expression(self): + def _parse_primary_expression(self) -> ASTExpression: # literal # "(" expression ")" # id-expression -> we parse this with _parse_nested_name self.skip_ws() - res = self._parse_literal() + res = self._parse_literal() # type: ASTExpression if res is not None: return res res = self._parse_paren_expression() if res is not None: return res - return self._parse_nested_name() + nn = self._parse_nested_name() + if nn is not None: + return ASTIdExpression(nn) + return None def _parse_initializer_list(self, name: str, open: str, close: str - ) -> Tuple[List[Any], bool]: + ) -> Tuple[List[ASTExpression], bool]: # Parse open and close with the actual initializer-list inbetween # -> initializer-clause '...'[opt] # | initializer-list ',' initializer-clause '...'[opt] @@ -2081,7 +2175,7 @@ class DefinitionParser(BaseParser): return None return ASTBracedInitList(exprs, trailingComma) - def _parse_postfix_expression(self): + def _parse_postfix_expression(self) -> ASTPostfixExpr: # -> primary # | postfix "[" expression "]" # | postfix "[" braced-init-list [opt] "]" @@ -2094,7 +2188,7 @@ class DefinitionParser(BaseParser): prefix = self._parse_primary_expression() # and now parse postfixes - postFixes = [] # type: List[Any] + postFixes = [] # type: List[ASTPostfixOp] while True: self.skip_ws() if self.skip_string_and_ws('['): @@ -2134,12 +2228,9 @@ class DefinitionParser(BaseParser): postFixes.append(ASTPostfixCallExpr(lst)) continue break - if len(postFixes) == 0: - return prefix - else: - return ASTPostfixExpr(prefix, postFixes) + return ASTPostfixExpr(prefix, postFixes) - def _parse_unary_expression(self): + def _parse_unary_expression(self) -> ASTExpression: # -> postfix # | "++" cast # | "--" cast @@ -2173,7 +2264,7 @@ class DefinitionParser(BaseParser): return ASTAlignofExpr(typ) return self._parse_postfix_expression() - def _parse_cast_expression(self): + def _parse_cast_expression(self) -> ASTExpression: # -> unary | "(" type-id ")" cast pos = self.pos self.skip_ws() @@ -2196,7 +2287,7 @@ class DefinitionParser(BaseParser): else: return self._parse_unary_expression() - def _parse_logical_or_expression(self): + def _parse_logical_or_expression(self) -> ASTExpression: # logical-or = logical-and || # logical-and = inclusive-or && # inclusive-or = exclusive-or | @@ -2210,10 +2301,10 @@ class DefinitionParser(BaseParser): # pm = cast .*, ->* def _parse_bin_op_expr(self, opId): if opId + 1 == len(_expression_bin_ops): - def parser(): + def parser() -> ASTExpression: return self._parse_cast_expression() else: - def parser(): + def parser() -> ASTExpression: return _parse_bin_op_expr(self, opId + 1) exprs = [] ops = [] @@ -2247,7 +2338,7 @@ class DefinitionParser(BaseParser): # -> "?" expression ":" assignment-expression return None - def _parse_assignment_expression(self): + def _parse_assignment_expression(self) -> ASTExpression: # -> conditional-expression # | logical-or-expression assignment-operator initializer-clause # -> conditional-expression -> @@ -2271,24 +2362,24 @@ class DefinitionParser(BaseParser): oneMore = True if not oneMore: break - if len(ops) == 0: - return orExpr - else: - return ASTAssignmentExpr(exprs, ops) + return ASTAssignmentExpr(exprs, ops) - def _parse_constant_expression(self): + def _parse_constant_expression(self) -> ASTExpression: # -> conditional-expression orExpr = self._parse_logical_or_expression() # TODO: use _parse_conditional_expression_tail return orExpr - def _parse_expression(self): + def _parse_expression(self) -> ASTExpression: # -> assignment-expression # | expression "," assignment-expresion # TODO: actually parse the second production return self._parse_assignment_expression() - def _parse_expression_fallback(self, end, parser, allow=True): + def _parse_expression_fallback( + self, end: List[str], + parser: Callable[[], ASTExpression], + allow: bool = True) -> ASTExpression: # Stupidly "parse" an expression. # 'end' should be a list of characters which ends the expression. @@ -2352,7 +2443,7 @@ class DefinitionParser(BaseParser): break return ASTNestedName(names, rooted) - def _parse_trailing_type_spec(self) -> Any: + def _parse_trailing_type_spec(self) -> ASTTrailingTypeSpec: # fundamental types self.skip_ws() for t in self._simple_fundamental_types: @@ -2428,7 +2519,6 @@ class DefinitionParser(BaseParser): self.fail( 'Expecting "," or ")" in parameters, ' 'got "%s".' % self.current_char) - return ASTParameters(args) def _parse_decl_specs_simple(self, outer: str, typed: bool) -> ASTDeclSpecsSimple: @@ -2505,7 +2595,7 @@ class DefinitionParser(BaseParser): def _parse_declarator_name_suffix( self, named: Union[bool, str], paramMode: str, typed: bool - ) -> Union[ASTDeclaratorNameParam, ASTDeclaratorNameBitField]: + ) -> ASTDeclarator: # now we should parse the name, and then suffixes if named == 'maybe': pos = self.pos @@ -2555,7 +2645,7 @@ class DefinitionParser(BaseParser): param=param) def _parse_declarator(self, named: Union[bool, str], paramMode: str, - typed: bool = True) -> Any: + typed: bool = True) -> ASTDeclarator: # 'typed' here means 'parse return type stuff' if paramMode not in ('type', 'function'): raise Exception( @@ -2717,7 +2807,7 @@ class DefinitionParser(BaseParser): decl = self._parse_declarator(named=named, paramMode=paramMode) return ASTType(declSpecs, decl) - def _parse_type_with_init(self, named: Union[bool, str], outer: str) -> Any: + def _parse_type_with_init(self, named: Union[bool, str], outer: str) -> ASTTypeWithInit: if outer: assert outer in ('type', 'member', 'function') type = self._parse_type(outer=outer, named=named) @@ -2738,7 +2828,7 @@ class DefinitionParser(BaseParser): while 1: self.skip_ws() if self.skip_string('...'): - args.append(ASTFunctionParameter(None, True)) + args.append(ASTMacroParameter(None, True)) self.skip_ws() if not self.skip_string(')'): self.fail('Expected ")" after "..." in macro parameters.') @@ -2746,7 +2836,7 @@ class DefinitionParser(BaseParser): if not self.match(identifier_re): self.fail("Expected identifier in macro parameters.") nn = ASTNestedName([ASTIdentifier(self.matched_text)], rooted=False) - arg = ASTFunctionParameter(nn) + arg = ASTMacroParameter(nn) args.append(arg) self.skip_ws() if self.skip_string_and_ws(','): @@ -2791,6 +2881,7 @@ class DefinitionParser(BaseParser): 'macro', 'struct', 'union', 'enum', 'enumerator', 'type'): raise Exception('Internal error, unknown directiveType "%s".' % directiveType) + declaration = None # type: Any if objectType == 'member': declaration = self._parse_type_with_init(named=True, outer='member') elif objectType == 'function': @@ -2900,7 +2991,7 @@ class CObject(ObjectDescription): docname=self.env.docname) def add_target_and_index(self, ast: ASTDeclaration, sig: str, - signode: desc_signature) -> None: + signode: TextElement) -> None: ids = [] for i in range(1, _max_id + 1): try: @@ -2947,7 +3038,7 @@ class CObject(ObjectDescription): def parse_definition(self, parser: DefinitionParser) -> ASTDeclaration: return parser.parse_declaration(self.object_type, self.objtype) - def describe_signature(self, signode: desc_signature, ast: Any, options: Dict) -> None: + def describe_signature(self, signode: TextElement, ast: Any, options: Dict) -> None: ast.describe_signature(signode, 'lastIsName', self.env, options) def run(self) -> List[Node]: @@ -2963,7 +3054,7 @@ class CObject(ObjectDescription): env.temp_data['c:last_symbol'] = None return super().run() - def handle_signature(self, sig: str, signode: desc_signature) -> ASTDeclaration: + def handle_signature(self, sig: str, signode: TextElement) -> ASTDeclaration: parentSymbol = self.env.temp_data['c:parent_symbol'] # type: Symbol parser = DefinitionParser(sig, self) diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py index 5a85d56a6..c3969fc2e 100644 --- a/sphinx/domains/cpp.py +++ b/sphinx/domains/cpp.py @@ -33,7 +33,7 @@ from sphinx.transforms import SphinxTransform from sphinx.transforms.post_transforms import ReferencesResolver from sphinx.util import logging from sphinx.util.cfamily import ( - NoOldIdError, ASTBase, verify_description_mode, StringifyTransform, + NoOldIdError, ASTBaseBase, verify_description_mode, StringifyTransform, BaseParser, DefinitionError, UnsupportedMultiCharacterCharLiteral, identifier_re, anon_identifier_re, integer_literal_re, octal_literal_re, hex_literal_re, binary_literal_re, float_literal_re, @@ -555,6 +555,10 @@ class _DuplicateSymbolError(Exception): return "Internal C++ duplicate symbol error:\n%s" % self.symbol.dump(0) +class ASTBase(ASTBaseBase): + pass + + ################################################################################ # Attributes ################################################################################ diff --git a/sphinx/util/cfamily.py b/sphinx/util/cfamily.py index 67fdc114b..06d269c7b 100644 --- a/sphinx/util/cfamily.py +++ b/sphinx/util/cfamily.py @@ -77,7 +77,7 @@ class NoOldIdError(Exception): return str(self) -class ASTBase: +class ASTBaseBase: def __eq__(self, other: Any) -> bool: if type(self) is not type(other): return False From f4d0099f2eb550d4782ce26f2fe072dde4156ea5 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Mon, 16 Mar 2020 19:34:49 +0100 Subject: [PATCH 12/12] C, type fix --- sphinx/domains/c.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 2d151df9f..8ab402c58 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -2910,16 +2910,17 @@ class DefinitionParser(BaseParser): self.assert_end() return name - def parse_expression(self): + def parse_expression(self) -> Union[ASTExpression, ASTType]: pos = self.pos + res = None # type: Union[ASTExpression, ASTType] try: - expr = self._parse_expression() + res = self._parse_expression() self.skip_ws() self.assert_end() except DefinitionError as exExpr: self.pos = pos try: - expr = self._parse_type(False) + res = self._parse_type(False) self.skip_ws() self.assert_end() except DefinitionError as exType: @@ -2928,7 +2929,7 @@ class DefinitionParser(BaseParser): errs.append((exExpr, "If expression")) errs.append((exType, "If type")) raise self._make_multi_error(errs, header) - return expr + return res def _make_phony_error_name() -> ASTNestedName: