From 520c0b91a528e7943bc6163a65de7adde387869b Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 29 Sep 2017 00:57:23 +0300 Subject: [PATCH 001/109] test/helpers: Add format_string and format_luav First intended to provide %r functionality like in Python (and also support for %*.*s, but this was not checked), second adds nice table formatting for use in cases similar to screen:snapshot_util(). --- test/helpers.lua | 79 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/test/helpers.lua b/test/helpers.lua index 260f10002e..d5356416af 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -307,6 +307,83 @@ local function dedent(str, leave_indent) return str end +local SUBTBL = { + '\\000', '\\001', '\\002', '\\003', '\\004', + '\\005', '\\006', '\\007', '\\008', '\\t', + '\\n', '\\011', '\\012', '\\r', '\\014', + '\\015', '\\016', '\\017', '\\018', '\\019', + '\\020', '\\021', '\\022', '\\023', '\\024', + '\\025', '\\026', '\\027', '\\028', '\\029', + '\\030', '\\031', +} + +local format_luav + +format_luav = function(v, indent) + local linesep = '\n' + local next_indent = nil + if indent == nil then + indent = '' + linesep = '' + else + next_indent = indent .. ' ' + end + local ret = '' + if type(v) == 'string' then + ret = '\'' .. tostring(v):gsub('[\'\\]', '\\%0'):gsub('[%z\1-\31]', function(match) + return SUBTBL[match:byte()] + end) .. '\'' + elseif type(v) == 'table' then + local processed_keys = {} + ret = '{' .. linesep + for i, subv in ipairs(v) do + ret = ret .. (next_indent or '') .. format_luav(subv, next_indent) .. ',\n' + processed_keys[i] = true + end + for k, subv in pairs(v) do + if not processed_keys[k] then + if type(k) == 'string' and k:match('^[a-zA-Z_][a-zA-Z0-9_]*$') then + ret = ret .. next_indent .. k .. ' = ' + else + ret = ret .. next_indent .. '[' .. format_luav(k) .. '] = ' + end + ret = ret .. format_luav(subv, next_indent) .. ',\n' + end + end + ret = ret .. indent .. '}' + else + -- Not implemented yet + assert(false) + end + return ret +end + +local function format_string(fmt, ...) + local i = 0 + local args = {...} + local function getarg() + i = i + 1 + return args[i] + end + local ret = fmt:gsub('%%[0-9*]*%.?[0-9*]*[cdEefgGiouXxqsr%%]', function(match) + local subfmt = match:gsub('%*', function(match) + return getarg() + end) + local arg = nil + if subfmt:sub(-1) ~= '%' then + arg = getarg() + end + if subfmt:sub(-1) == 'r' then + -- %r is like %q, but it is supposed to single-quote strings and not + -- double-quote them, and also work not only for strings. + subfmt = subfmt:sub(1, -2) .. 's' + arg = format_luav(arg) + end + return subfmt:format(arg) + end) + return ret +end + return { eq = eq, neq = neq, @@ -323,4 +400,6 @@ return { shallowcopy = shallowcopy, concat_tables = concat_tables, dedent = dedent, + format_luav = format_luav, + format_string = format_string, } From 190c8516f534499f89d610fc78a20c025386b2f2 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 26 Sep 2017 23:13:55 +0300 Subject: [PATCH 002/109] unittests: Add a way to print trace on regular error --- test/README.md | 3 +++ test/unit/helpers.lua | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index 01db5960cd..15041b74e8 100644 --- a/test/README.md +++ b/test/README.md @@ -110,3 +110,6 @@ disables tracing (the fastest, but you get no data if tests crash and there was no core dump generated), `1` or empty/undefined leaves only C function cals and returns in the trace (faster then recording everything), `2` records all function calls, returns and lua source lines exuecuted. + +`NVIM_TEST_TRACE_ON_ERROR` (U) (1): makes unit tests yield trace on error in +addition to regular error message. diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index 4b9f185156..a629fca9a2 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -698,7 +698,14 @@ local function check_child_err(rd) local len_s = sc.read(rd, 5) local len = tonumber(len_s) neq(0, len) - local err = sc.read(rd, len + 1) + local err = '' + if os.getenv('NVIM_TEST_TRACE_ON_ERROR') == '1' and #trace ~= 0 then + err = '\nTest failed, trace:\n' .. tracehelp + for _, traceline in ipairs(trace) do + err = err .. traceline + end + end + err = err .. sc.read(rd, len + 1) assert.just_fail(err) end From e479f3b944614f28c42ec597ec473652f3ac9912 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 10 Sep 2017 21:38:43 +0300 Subject: [PATCH 003/109] kvec: Add kv_drop() which is to be used like `(void)kv_pop(kvec)` --- src/nvim/lib/kvec.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/nvim/lib/kvec.h b/src/nvim/lib/kvec.h index 584282d773..4e5fb0d794 100644 --- a/src/nvim/lib/kvec.h +++ b/src/nvim/lib/kvec.h @@ -64,6 +64,14 @@ #define kv_max(v) ((v).capacity) #define kv_last(v) kv_A(v, kv_size(v) - 1) +/// Drop last n items from kvec without resizing +/// +/// Previously spelled as `(void)kv_pop(v)`, repeated n times. +/// +/// @param[out] v Kvec to drop items from. +/// @param[in] n Number of elements to drop. +#define kv_drop(v, n) ((v).size -= (n)) + #define kv_resize(v, s) \ ((v).capacity = (s), \ (v).items = xrealloc((v).items, sizeof((v).items[0]) * (v).capacity)) From ad58e50b45ddb73f0582590b9e96da49f34174d0 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 8 Oct 2017 22:09:12 +0300 Subject: [PATCH 004/109] kvec: Add kv_Z which is like kv_A, but zero is the last value --- src/nvim/lib/kvec.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nvim/lib/kvec.h b/src/nvim/lib/kvec.h index 4e5fb0d794..ee1b890cb9 100644 --- a/src/nvim/lib/kvec.h +++ b/src/nvim/lib/kvec.h @@ -62,7 +62,8 @@ #define kv_pop(v) ((v).items[--(v).size]) #define kv_size(v) ((v).size) #define kv_max(v) ((v).capacity) -#define kv_last(v) kv_A(v, kv_size(v) - 1) +#define kv_Z(v, i) kv_A(v, kv_size(v) - (i) - 1) +#define kv_last(v) kv_Z(v, 0) /// Drop last n items from kvec without resizing /// From 0300c4d10991fb6ce218d45c4fe6d71a73f07d62 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 20 Aug 2017 18:40:22 +0300 Subject: [PATCH 005/109] viml/expressions: Add lexer with some basic tests --- src/nvim/CMakeLists.txt | 2 + src/nvim/viml/parser/expressions.c | 367 ++++++++++++++++++++++ src/nvim/viml/parser/expressions.h | 118 +++++++ src/nvim/viml/parser/parser.h | 129 ++++++++ test/unit/viml/expressions/lexer_spec.lua | 337 ++++++++++++++++++++ 5 files changed, 953 insertions(+) create mode 100644 src/nvim/viml/parser/expressions.c create mode 100644 src/nvim/viml/parser/expressions.h create mode 100644 src/nvim/viml/parser/parser.h create mode 100644 test/unit/viml/expressions/lexer_spec.lua diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index 688912eda6..77e4721853 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -81,6 +81,8 @@ foreach(subdir event eval lua + viml + viml/parser ) if(${subdir} MATCHES "tui" AND NOT FEAT_TUI) continue() diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c new file mode 100644 index 0000000000..0164de3a14 --- /dev/null +++ b/src/nvim/viml/parser/expressions.c @@ -0,0 +1,367 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +/// VimL expression parser + +#include +#include +#include +#include + +#include "nvim/vim.h" +#include "nvim/memory.h" +#include "nvim/types.h" +#include "nvim/charset.h" +#include "nvim/ascii.h" + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/expressions.c.generated.h" +#endif + +/// Character used as a separator in autoload function/variable names. +#define AUTOLOAD_CHAR '#' + +/// Get next token for the VimL expression input +LexExprToken viml_pexpr_next_token(ParserState *const pstate) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + LexExprToken ret = { + .type = kExprLexInvalid, + .start = pstate->pos, + }; + ParserLine pline; + if (!viml_parser_get_remaining_line(pstate, &pline)) { + ret.type = kExprLexEOC; + return ret; + } + if (pline.size <= 0) { + ret.len = 0; + ret.type = kExprLexEOC; + goto viml_pexpr_next_token_adv_return; + } + ret.len = 1; + const uint8_t schar = (uint8_t)pline.data[0]; +#define GET_CCS(ret, pline) \ + do { \ + if (ret.len < pline.size \ + && strchr("?#", pline.data[ret.len]) != NULL) { \ + ret.data.cmp.ccs = \ + (CaseCompareStrategy)pline.data[ret.len]; \ + ret.len++; \ + } else { \ + ret.data.cmp.ccs = kCCStrategyUseOption; \ + } \ + } while (0) + switch (schar) { + // Paired brackets. +#define BRACKET(typ, opning, clsing) \ + case opning: \ + case clsing: { \ + ret.type = typ; \ + ret.data.brc.closing = (schar == clsing); \ + break; \ + } + BRACKET(kExprLexParenthesis, '(', ')') + BRACKET(kExprLexBracket, '[', ']') + BRACKET(kExprLexFigureBrace, '{', '}') +#undef BRACKET + + // Single character tokens without data. +#define CHAR(typ, ch) \ + case ch: { \ + ret.type = typ; \ + break; \ + } + CHAR(kExprLexQuestion, '?') + CHAR(kExprLexColon, ':') + CHAR(kExprLexDot, '.') + CHAR(kExprLexPlus, '+') + CHAR(kExprLexComma, ',') +#undef CHAR + + // Multiplication/division/modulo. +#define MUL(mul_type, ch) \ + case ch: { \ + ret.type = kExprLexMultiplication; \ + ret.data.mul.type = mul_type; \ + break; \ + } + MUL(kExprLexMulMul, '*') + MUL(kExprLexMulDiv, '/') + MUL(kExprLexMulMod, '%') +#undef MUL + +#define CHARREG(typ, cond) \ + do { \ + ret.type = typ; \ + for (; (ret.len < pline.size \ + && cond(pline.data[ret.len])) \ + ; ret.len++) { \ + } \ + } while (0) + + // Whitespace. + case ' ': + case TAB: { + CHARREG(kExprLexSpacing, ascii_iswhite); + break; + } + + // Control character, except for NUL, NL and TAB. + case Ctrl_A: case Ctrl_B: case Ctrl_C: case Ctrl_D: case Ctrl_E: + case Ctrl_F: case Ctrl_G: case Ctrl_H: + + case Ctrl_K: case Ctrl_L: case Ctrl_M: case Ctrl_N: case Ctrl_O: + case Ctrl_P: case Ctrl_Q: case Ctrl_R: case Ctrl_S: case Ctrl_T: + case Ctrl_U: case Ctrl_V: case Ctrl_W: case Ctrl_X: case Ctrl_Y: + case Ctrl_Z: { +#define ISCTRL(schar) (schar < ' ') + CHARREG(kExprLexInvalid, ISCTRL); + ret.data.err.type = kExprLexSpacing; + ret.data.err.msg = + _("E15: Invalid control character present in input: %.*s"); + break; +#undef ISCTRL + } + + // Number. + // Note: determining whether dot is (not) a part of a float needs more + // context, so lexer does not do this. + // FIXME: Resolve ambiguity by additional argument. + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': case '8': case '9': { + CHARREG(kExprLexNumber, ascii_isdigit); + break; + } + + // Environment variable. + case '$': { + CHARREG(kExprLexEnv, vim_isIDc); + break; + } + + // Normal variable/function name. + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': + case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': + case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': + case 'v': case 'w': case 'x': case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': + case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': + case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': + case 'V': case 'W': case 'X': case 'Y': case 'Z': + case '_': { +#define ISWORD_OR_AUTOLOAD(x) \ + (ASCII_ISALNUM(x) || (x) == AUTOLOAD_CHAR || (x) == '_') +#define ISWORD(x) \ + (ASCII_ISALNUM(x) || (x) == '_') + ret.data.var.scope = 0; + ret.data.var.autoload = false; + CHARREG(kExprLexPlainIdentifier, ISWORD); + // "is" and "isnot" operators. + if ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0) + || (ret.len == 5 && memcmp(pline.data, "isnot", 5) == 0)) { + ret.type = kExprLexComparison; + ret.data.cmp.type = kExprLexCmpIdentical; + ret.data.cmp.inv = (ret.len == 5); + GET_CCS(ret, pline); + // Scope: `s:`, etc. + } else if (ret.len == 1 + && pline.size > 1 + && strchr("sgvbwtla", schar) != NULL + && pline.data[ret.len] == ':') { + ret.len++; + ret.data.var.scope = schar; + CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); + ret.data.var.autoload = ( + memchr(pline.data + 2, AUTOLOAD_CHAR, ret.len - 2) + != NULL); + // Previous CHARREG stopped at autoload character in order to make it + // possible to detect `is#`. Continue now with autoload characters + // included. + // + // Warning: there is ambiguity for the lexer: `is#Foo(1)` is a call of + // function `is#Foo()`, `1is#Foo(1)` is a comparison `1 is# Foo(1)`. This + // needs to be resolved on the higher level where context is available. + } else if (pline.size > ret.len + && pline.data[ret.len] == AUTOLOAD_CHAR) { + ret.data.var.autoload = true; + CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); + } + break; +#undef ISWORD_OR_AUTOLOAD +#undef ISWORD + } +#undef CHARREG + + // Option. + case '&': { +#define OPTNAMEMISS(ret) \ + do { \ + ret.type = kExprLexInvalid; \ + ret.data.err.type = kExprLexOption; \ + ret.data.err.msg = _("E112: Option name missing: %.*s"); \ + } while (0) + if (pline.size > 1 && pline.data[1] == '&') { + ret.type = kExprLexAnd; + ret.len++; + break; + } + if (pline.size == 1 || !ASCII_ISALPHA(pline.data[1])) { + OPTNAMEMISS(ret); + break; + } + ret.type = kExprLexOption; + if (pline.size > 2 + && pline.data[2] == ':' + && strchr("gl", pline.data[1]) != NULL) { + ret.len += 2; + ret.data.opt.scope = (pline.data[1] == 'g' + ? kExprLexOptGlobal + : kExprLexOptLocal); + ret.data.opt.name = pline.data + 3; + } else { + ret.data.opt.scope = kExprLexOptUnspecified; + ret.data.opt.name = pline.data + 1; + } + const char *p = ret.data.opt.name; + const char *const e = pline.data + pline.size; + if (e - p >= 4 && p[0] == 't' && p[1] == '_') { + ret.data.opt.len = 4; + ret.len += 4; + } else { + for (; p < e && ASCII_ISALPHA(*p); p++) { + } + ret.data.opt.len = (size_t)(p - ret.data.opt.name); + if (ret.data.opt.len == 0) { + OPTNAMEMISS(ret); + } else { + ret.len += ret.data.opt.len; + } + } + break; +#undef OPTNAMEMISS + } + + // Register. + case '@': { + ret.type = kExprLexRegister; + if (pline.size > 1) { + ret.len++; + ret.data.reg.name = (uint8_t)pline.data[1]; + } else { + ret.data.reg.name = -1; + } + break; + } + + // Single quoted string. + case '\'': { + ret.type = kExprLexSingleQuotedString; + ret.data.str.closed = false; + for (; ret.len < pline.size && !ret.data.str.closed; ret.len++) { + if (pline.data[ret.len] == '\'') { + if (ret.len + 1 < pline.size && pline.data[ret.len + 1] == '\'') { + ret.len++; + } else { + ret.data.str.closed = true; + } + } + } + break; + } + + // Double quoted string. + case '"': { + ret.type = kExprLexDoubleQuotedString; + ret.data.str.closed = false; + for (; ret.len < pline.size && !ret.data.str.closed; ret.len++) { + if (pline.data[ret.len] == '\\') { + if (ret.len + 1 < pline.size) { + ret.len++; + } + } else if (pline.data[ret.len] == '"') { + ret.data.str.closed = true; + } + } + break; + } + + // Unary not, (un)equality and regex (not) match comparison operators. + case '!': + case '=': { + if (pline.size == 1) { +viml_pexpr_next_token_invalid_comparison: + ret.type = (schar == '!' ? kExprLexNot : kExprLexInvalid); + if (ret.type == kExprLexInvalid) { + ret.data.err.msg = _("E15: Expected == or =~: %.*s"); + ret.data.err.type = kExprLexComparison; + } + break; + } + ret.type = kExprLexComparison; + ret.data.cmp.inv = (schar == '!'); + if (pline.data[1] == '=') { + ret.data.cmp.type = kExprLexCmpEqual; + ret.len++; + } else if (pline.data[1] == '~') { + ret.data.cmp.type = kExprLexCmpMatches; + ret.len++; + } else { + goto viml_pexpr_next_token_invalid_comparison; + } + GET_CCS(ret, pline); + break; + } + + // Less/greater [or equal to] comparison operators. + case '>': + case '<': { + ret.type = kExprLexComparison; + const bool haseqsign = (pline.size > 1 && pline.data[1] == '='); + if (haseqsign) { + ret.len++; + } + GET_CCS(ret, pline); + ret.data.cmp.inv = (schar == '<'); + ret.data.cmp.type = ((ret.data.cmp.inv ^ haseqsign) + ? kExprLexCmpGreaterOrEqual + : kExprLexCmpGreater); + break; + } + + // Minus sign or arrow from lambdas. + case '-': { + if (pline.size > 1 && pline.data[1] == '>') { + ret.len++; + ret.type = kExprLexArrow; + } else { + ret.type = kExprLexMinus; + } + break; + } + + // Expression end because Ex command ended. + case NUL: + case NL: { + ret.type = kExprLexEOC; + break; + } + + // Everything else is not valid. + default: { + ret.len = (size_t)utfc_ptr2len_len((const char_u *)pline.data, + (int)pline.size); + ret.type = kExprLexInvalid; + ret.data.err.type = kExprLexPlainIdentifier; + ret.data.err.msg = _("E15: Unidentified character: %.*s"); + break; + } + } +#undef GET_CCS +viml_pexpr_next_token_adv_return: + viml_parser_advance(pstate, ret.len); + return ret; +} diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h new file mode 100644 index 0000000000..52354760a5 --- /dev/null +++ b/src/nvim/viml/parser/expressions.h @@ -0,0 +1,118 @@ +#ifndef NVIM_VIML_PARSER_EXPRESSIONS_H +#define NVIM_VIML_PARSER_EXPRESSIONS_H + +#include +#include + +#include "nvim/types.h" +#include "nvim/viml/parser/parser.h" + +// Defines whether to ignore case: +// == kCCStrategyUseOption +// ==# kCCStrategyMatchCase +// ==? kCCStrategyIgnoreCase +typedef enum { + kCCStrategyUseOption = 0, // 0 for xcalloc + kCCStrategyMatchCase = '#', + kCCStrategyIgnoreCase = '?', +} CaseCompareStrategy; + +/// Lexer token type +typedef enum { + kExprLexInvalid = 0, ///< Invalid token, indicaten an error. + kExprLexMissing, ///< Missing token, for use in parser. + kExprLexSpacing, ///< Spaces, tabs, newlines, etc. + kExprLexEOC, ///< End of command character: NL, |, just end of stream. + + kExprLexQuestion, ///< Question mark, for use in ternary. + kExprLexColon, ///< Colon, for use in ternary. + kExprLexOr, ///< Logical or operator. + kExprLexAnd, ///< Logical and operator. + kExprLexComparison, ///< One of the comparison operators. + kExprLexPlus, ///< Plus sign. + kExprLexMinus, ///< Minus sign. + kExprLexDot, ///< Dot: either concat or subscript, also part of the float. + kExprLexMultiplication, ///< Multiplication, division or modulo operator. + + kExprLexNot, ///< Not: !. + + kExprLexNumber, ///< Integer number literal, or part of a float. + kExprLexSingleQuotedString, ///< Single quoted string literal. + kExprLexDoubleQuotedString, ///< Double quoted string literal. + kExprLexOption, ///< &optionname option value. + kExprLexRegister, ///< @r register value. + kExprLexEnv, ///< Environment $variable value. + kExprLexPlainIdentifier, ///< Identifier without scope: `abc`, `foo#bar`. + + kExprLexBracket, ///< Bracket, either opening or closing. + kExprLexFigureBrace, ///< Figure brace, either opening or closing. + kExprLexParenthesis, ///< Parenthesis, either opening or closing. + kExprLexComma, ///< Comma. + kExprLexArrow, ///< Arrow, like from lambda expressions. +} LexExprTokenType; + +/// Lexer token +typedef struct { + ParserPosition start; + size_t len; + LexExprTokenType type; + union { + struct { + enum { + kExprLexCmpEqual, ///< Equality, unequality. + kExprLexCmpMatches, ///< Matches regex, not matches regex. + kExprLexCmpGreater, ///< `>` or `<=` + kExprLexCmpGreaterOrEqual, ///< `>=` or `<`. + kExprLexCmpIdentical, ///< `is` or `isnot` + } type; ///< Comparison type. + CaseCompareStrategy ccs; ///< Case comparison strategy. + bool inv; ///< True if comparison is to be inverted. + } cmp; ///< For kExprLexComparison. + + struct { + enum { + kExprLexMulMul, ///< Real multiplication. + kExprLexMulDiv, ///< Division. + kExprLexMulMod, ///< Modulo. + } type; ///< Multiplication type. + } mul; ///< For kExprLexMultiplication. + + struct { + bool closing; ///< True if bracket/etc is a closing one. + } brc; ///< For brackets/braces/parenthesis. + + struct { + int name; ///< Register name, may be -1 if name not present. + } reg; ///< For kExprLexRegister. + + struct { + bool closed; ///< True if quote was closed. + } str; ///< For kExprLexSingleQuotedString and kExprLexDoubleQuotedString. + + struct { + const char *name; ///< Option name start. + size_t len; ///< Option name length. + enum { + kExprLexOptUnspecified = 0, + kExprLexOptGlobal = 1, + kExprLexOptLocal = 2, + } scope; ///< Option scope: &l:, &g: or not specified. + } opt; ///< Option properties. + + struct { + int scope; ///< Scope character or 0 if not present. + bool autoload; ///< Has autoload characters. + } var; ///< For kExprLexPlainIdentifier + + struct { + LexExprTokenType type; ///< Suggested type for parsing incorrect code. + const char *msg; ///< Error message. + } err; ///< For kExprLexInvalid + } data; ///< Additional data, if needed. +} LexExprToken; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/expressions.h.generated.h" +#endif + +#endif // NVIM_VIML_PARSER_EXPRESSIONS_H diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h new file mode 100644 index 0000000000..ec582294e1 --- /dev/null +++ b/src/nvim/viml/parser/parser.h @@ -0,0 +1,129 @@ +#ifndef NVIM_VIML_PARSER_PARSER_H +#define NVIM_VIML_PARSER_PARSER_H + +#include +#include +#include + +#include "nvim/lib/kvec.h" +#include "nvim/func_attr.h" + +/// One parsed line +typedef struct { + const char *data; ///< Parsed line pointer + size_t size; ///< Parsed line size +} ParserLine; + +/// Line getter type for parser +/// +/// Line getter must return {NULL, 0} for EOF. +typedef void (*ParserLineGetter)(void *cookie, ParserLine *ret_pline); + +/// Parser position in the input +typedef struct { + size_t line; ///< Line index in ParserInputReader.lines. + size_t col; ///< Byte index in the line. +} ParserPosition; + +/// Parser state item. +typedef struct { + enum { + kPTopStateParsingCommand = 0, + kPTopStateParsingExpression, + } type; + union { + struct { + enum { + kExprUnknown = 0, + } type; + } expr; + } data; +} ParserStateItem; + +/// Structure defining input reader +typedef struct { + /// Function used to get next line. + ParserLineGetter get_line; + /// Data for get_line function. + void *cookie; + /// All lines obtained by get_line. + kvec_withinit_t(ParserLine, 4) lines; +} ParserInputReader; + +/// Highlighted region definition +/// +/// Note: one chunk may highlight only one line. +typedef struct { + ParserPosition start; ///< Start of the highlight: line and column. + size_t end_col; ///< End column, points to the start of the next character. + const char *group; ///< Highlight group. +} ParserHighlightChunk; + +/// Highlighting defined by a parser +typedef kvec_withinit_t(ParserHighlightChunk, 16) ParserHighlight; + +/// Structure defining parser state +typedef struct { + /// Line reader. + ParserInputReader reader; + /// Position up to which input was parsed. + ParserPosition pos; + /// Parser state stack. + kvec_withinit_t(ParserStateItem, 16) stack; + /// Highlighting support. + ParserHighlight *colors; + /// True if line continuation can be used. + bool can_continuate; +} ParserState; + +static inline bool viml_parser_get_remaining_line(ParserState *const pstate, + ParserLine *const ret_pline) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_WARN_UNUSED_RESULT REAL_FATTR_NONNULL_ALL; + +/// Get currently parsed line, shifted to pstate->pos.col +/// +/// @param pstate Parser state to operate on. +/// +/// @return True if there is a line, false in case of EOF. +static inline bool viml_parser_get_remaining_line(ParserState *const pstate, + ParserLine *const ret_pline) +{ + const size_t num_lines = kv_size(pstate->reader.lines); + if (pstate->pos.line == num_lines) { + pstate->reader.get_line(pstate->reader.cookie, ret_pline); + kvi_push(pstate->reader.lines, *ret_pline); + } else { + *ret_pline = kv_last(pstate->reader.lines); + } + assert(pstate->pos.line == kv_size(pstate->reader.lines) - 1); + if (ret_pline->data != NULL) { + ret_pline->data += pstate->pos.col; + ret_pline->size -= pstate->pos.col; + } + return ret_pline->data != NULL; +} + +static inline void viml_parser_advance(ParserState *const pstate, + const size_t len) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ALL; + +/// Advance position by a given number of bytes +/// +/// At maximum advances to the next line. +/// +/// @param pstate Parser state to advance. +/// @param[in] len Number of bytes to advance. +static inline void viml_parser_advance(ParserState *const pstate, + const size_t len) +{ + assert(pstate->pos.line == kv_size(pstate->reader.lines) - 1); + const ParserLine pline = kv_last(pstate->reader.lines); + if (pstate->pos.col + len >= pline.size) { + pstate->pos.line++; + pstate->pos.col = 0; + } else { + pstate->pos.col += len; + } +} + +#endif // NVIM_VIML_PARSER_PARSER_H diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua new file mode 100644 index 0000000000..bf5afe4eeb --- /dev/null +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -0,0 +1,337 @@ +local helpers = require('test.unit.helpers')(after_each) +local itp = helpers.gen_itp(it) + +local child_call_once = helpers.child_call_once +local cimport = helpers.cimport +local ffi = helpers.ffi +local eq = helpers.eq + +local lib = cimport('./src/nvim/viml/parser/expressions.h') + +local eltkn_type_tab, eltkn_cmp_type_tab, ccs_tab, eltkn_mul_type_tab +local eltkn_opt_scope_tab +child_call_once(function() + eltkn_type_tab = { + [tonumber(lib.kExprLexInvalid)] = 'Invalid', + [tonumber(lib.kExprLexMissing)] = 'Missing', + [tonumber(lib.kExprLexSpacing)] = 'Spacing', + [tonumber(lib.kExprLexEOC)] = 'EOC', + + [tonumber(lib.kExprLexQuestion)] = 'Question', + [tonumber(lib.kExprLexColon)] = 'Colon', + [tonumber(lib.kExprLexOr)] = 'Or', + [tonumber(lib.kExprLexAnd)] = 'And', + [tonumber(lib.kExprLexComparison)] = 'Comparison', + [tonumber(lib.kExprLexPlus)] = 'Plus', + [tonumber(lib.kExprLexMinus)] = 'Minus', + [tonumber(lib.kExprLexDot)] = 'Dot', + [tonumber(lib.kExprLexMultiplication)] = 'Multiplication', + + [tonumber(lib.kExprLexNot)] = 'Not', + + [tonumber(lib.kExprLexNumber)] = 'Number', + [tonumber(lib.kExprLexSingleQuotedString)] = 'SingleQuotedString', + [tonumber(lib.kExprLexDoubleQuotedString)] = 'DoubleQuotedString', + [tonumber(lib.kExprLexOption)] = 'Option', + [tonumber(lib.kExprLexRegister)] = 'Register', + [tonumber(lib.kExprLexEnv)] = 'Env', + [tonumber(lib.kExprLexPlainIdentifier)] = 'PlainIdentifier', + + [tonumber(lib.kExprLexBracket)] = 'Bracket', + [tonumber(lib.kExprLexFigureBrace)] = 'FigureBrace', + [tonumber(lib.kExprLexParenthesis)] = 'Parenthesis', + [tonumber(lib.kExprLexComma)] = 'Comma', + [tonumber(lib.kExprLexArrow)] = 'Arrow', + } + + eltkn_cmp_type_tab = { + [tonumber(lib.kExprLexCmpEqual)] = 'Equal', + [tonumber(lib.kExprLexCmpMatches)] = 'Matches', + [tonumber(lib.kExprLexCmpGreater)] = 'Greater', + [tonumber(lib.kExprLexCmpGreaterOrEqual)] = 'GreaterOrEqual', + [tonumber(lib.kExprLexCmpIdentical)] = 'Identical', + } + + ccs_tab = { + [tonumber(lib.kCCStrategyUseOption)] = 'UseOption', + [tonumber(lib.kCCStrategyMatchCase)] = 'MatchCase', + [tonumber(lib.kCCStrategyIgnoreCase)] = 'IgnoreCase', + } + + eltkn_mul_type_tab = { + [tonumber(lib.kExprLexMulMul)] = 'Mul', + [tonumber(lib.kExprLexMulDiv)] = 'Div', + [tonumber(lib.kExprLexMulMod)] = 'Mod', + } + + eltkn_opt_scope_tab = { + [tonumber(lib.kExprLexOptUnspecified)] = 'Unspecified', + [tonumber(lib.kExprLexOptGlobal)] = 'Global', + [tonumber(lib.kExprLexOptLocal)] = 'Local', + } +end) + +local function array_size(arr) + return ffi.sizeof(arr) / ffi.sizeof(arr[0]) +end + +local function kvi_size(kvi) + return array_size(kvi.init_array) +end + +local function kvi_init(kvi) + kvi.capacity = kvi_size(kvi) + kvi.items = kvi.init_array + return kvi +end + +local function kvi_new(ct) + return kvi_init(ffi.new(ct)) +end + +local function new_pstate(strings) + local strings_idx = 0 + local function get_line(_, ret_pline) + strings_idx = strings_idx + 1 + local str = strings[strings_idx] + local data, size + if type(str) == 'string' then + data = str + size = #str + elseif type(str) == 'nil' then + data = nil + size = 0 + elseif type(str) == 'table' then + data = str.data + size = str.size + elseif type(str) == 'function' then + data, size = str() + size = size or 0 + end + ret_pline.data = data + ret_pline.size = size + end + local pline_init = { + data = nil, + size = 0, + } + local state = { + reader = { + get_line = get_line, + cookie = nil, + }, + pos = { line = 0, col = 0 }, + colors = kvi_new('ParserHighlight'), + can_continuate = false, + } + local ret = ffi.new('ParserState', state) + kvi_init(ret.reader.lines) + kvi_init(ret.stack) + return ret +end + +local function conv_enum(etab, eval) + local n = tonumber(eval) + return etab[n] or n +end + +local function conv_eltkn_type(typ) + return conv_enum(eltkn_type_tab, typ) +end + +local function pline2lua(pline) + return ffi.string(pline.data, pline.size) +end + +local bracket_types = { + Bracket = true, + FigureBrace = true, + Parenthesis = true, +} + +local function intchar2lua(ch) + ch = tonumber(ch) + return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch +end + +local function eltkn2lua(pstate, tkn) + local ret = { + type = conv_eltkn_type(tkn.type), + len = tonumber(tkn.len), + start = { line = tonumber(tkn.start.line), col = tonumber(tkn.start.col) }, + } + if ret.start.line < pstate.reader.lines.size then + local pstr = pline2lua(pstate.reader.lines.items[ret.start.line]) + if ret.start.col >= #pstr then + ret.error = 'start.col >= #pstr' + else + ret.str = pstr:sub(ret.start.col + 1, ret.start.col + ret.len) + if #(ret.str) ~= ret.len then + ret.error = '#str /= len' + end + end + else + ret.error = 'start.line >= pstate.reader.lines.size' + end + if ret.type == 'Comparison' then + ret.data = { + type = conv_enum(eltkn_cmp_type_tab, tkn.data.cmp.type), + ccs = conv_enum(ccs_tab, tkn.data.cmp.ccs), + inv = (not not tkn.data.cmp.inv), + } + elseif ret.type == 'Multiplication' then + ret.data = { type = conv_enum(eltkn_mul_type_tab, tkn.data.mul.type) } + elseif bracket_types[ret.type] then + ret.data = { closing = (not not tkn.data.brc.closing) } + elseif ret.type == 'Register' then + ret.data = { name = intchar2lua(tkn.data.reg.name) } + elseif (ret.type == 'SingleQuotedString' + or ret.type == 'DoubleQuotedString') then + ret.data = { closed = (not not tkn.data.str.closed) } + elseif ret.type == 'Option' then + ret.data = { + scope = conv_enum(eltkn_opt_scope_tab, tkn.data.opt.scope), + name = ffi.string(tkn.data.opt.name, tkn.data.opt.len), + } + elseif ret.type == 'PlainIdentifier' then + ret.data = { + scope = intchar2lua(tkn.data.var.scope), + autoload = (not not tkn.data.var.autoload), + } + elseif ret.type == 'Invalid' then + ret.data = { error = ffi.string(tkn.data.err.msg) } + end + return ret, tkn +end + +local function next_eltkn(pstate) + return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, false)) +end + +describe('Expressions lexer', function() + itp('works (single tokens)', function() + local function singl_eltkn_test(typ, str, data) + local pstate = new_pstate({str}) + eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, + next_eltkn(pstate)) + if not ( + typ == 'Spacing' + or (typ == 'Register' and str == '@') + or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString') + and not data.closed) + ) then + pstate = new_pstate({str .. ' '}) + eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, + next_eltkn(pstate)) + end + pstate = new_pstate({'x' .. str}) + pstate.pos.col = 1 + eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ}, + next_eltkn(pstate)) + end + singl_eltkn_test('Parenthesis', '(', {closing=false}) + singl_eltkn_test('Parenthesis', ')', {closing=true}) + singl_eltkn_test('Bracket', '[', {closing=false}) + singl_eltkn_test('Bracket', ']', {closing=true}) + singl_eltkn_test('FigureBrace', '{', {closing=false}) + singl_eltkn_test('FigureBrace', '}', {closing=true}) + singl_eltkn_test('Question', '?') + singl_eltkn_test('Colon', ':') + singl_eltkn_test('Dot', '.') + singl_eltkn_test('Plus', '+') + singl_eltkn_test('Comma', ',') + singl_eltkn_test('Multiplication', '*', {type='Mul'}) + singl_eltkn_test('Multiplication', '/', {type='Div'}) + singl_eltkn_test('Multiplication', '%', {type='Mod'}) + singl_eltkn_test('Spacing', ' \t\t \t\t') + singl_eltkn_test('Spacing', ' ') + singl_eltkn_test('Spacing', '\t') + singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'}) + singl_eltkn_test('Number', '0123') + singl_eltkn_test('Number', '0') + singl_eltkn_test('Number', '9') + singl_eltkn_test('Env', '$abc') + singl_eltkn_test('Env', '$') + singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', '_test', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', '_test_foo', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 't', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test5', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 't0', {autoload=false, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0}) + singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0}) + singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0}) + local function scope_test(scope) + singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope}) + singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope}) + end + scope_test('s') + scope_test('g') + scope_test('v') + scope_test('b') + scope_test('w') + scope_test('t') + scope_test('l') + scope_test('a') + local function comparison_test(op, inv_op, cmp_type) + singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'}) + singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'}) + singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'}) + singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'}) + singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'}) + singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'}) + end + comparison_test('is', 'isnot', 'Identical') + singl_eltkn_test('And', '&&') + singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'}) + singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'}) + singl_eltkn_test('Option', '&t_\r\r', {scope='Unspecified', name='t_\r\r'}) + singl_eltkn_test('Option', '&t_\t\t', {scope='Unspecified', name='t_\t\t'}) + singl_eltkn_test('Option', '&t_ ', {scope='Unspecified', name='t_ '}) + singl_eltkn_test('Option', '&g:opt', {scope='Global', name='opt'}) + singl_eltkn_test('Option', '&l:opt', {scope='Local', name='opt'}) + singl_eltkn_test('Invalid', '&l:', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Invalid', '&g:', {error='E112: Option name missing: %.*s'}) + singl_eltkn_test('Register', '@', {name=-1}) + singl_eltkn_test('Register', '@a', {name='a'}) + singl_eltkn_test('Register', '@\r', {name=13}) + singl_eltkn_test('Register', '@ ', {name=' '}) + singl_eltkn_test('Register', '@\t', {name=9}) + singl_eltkn_test('SingleQuotedString', '\'test', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'test\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'x\'\'\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'x\'', {closed=true}) + singl_eltkn_test('SingleQuotedString', '\'\'\'', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'x\'\'', {closed=false}) + singl_eltkn_test('SingleQuotedString', '\'\'\'x', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"test', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"test"', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\""', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"x\\""', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\"x"', {closed=true}) + singl_eltkn_test('DoubleQuotedString', '"\\"', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"x\\"', {closed=false}) + singl_eltkn_test('DoubleQuotedString', '"\\"x', {closed=false}) + singl_eltkn_test('Not', '!') + singl_eltkn_test('Invalid', '=', {error='E15: Expected == or =~: %.*s'}) + comparison_test('==', '!=', 'Equal') + comparison_test('=~', '!~', 'Matches') + comparison_test('>', '<=', 'Greater') + comparison_test('>=', '<', 'GreaterOrEqual') + singl_eltkn_test('Minus', '-') + singl_eltkn_test('Arrow', '->') + singl_eltkn_test('EOC', '\0') + singl_eltkn_test('EOC', '\n') + singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'}) + + local pstate = new_pstate({{data=nil, size=0}}) + eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'}, + next_eltkn(pstate)) + + local pstate = new_pstate({''}) + eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'}, + next_eltkn(pstate)) + end) +end) From 2d8b9937deae3731143f4ea44e5c41715fe1363a Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 20 Aug 2017 20:40:59 +0300 Subject: [PATCH 006/109] viml/parser: Handle encoding conversions --- src/nvim/viml/parser/expressions.c | 13 +++++++-- src/nvim/viml/parser/parser.h | 34 +++++++++++++++++++++-- test/unit/viml/expressions/lexer_spec.lua | 7 +++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 0164de3a14..c29fac9cb4 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -25,8 +25,13 @@ #define AUTOLOAD_CHAR '#' /// Get next token for the VimL expression input -LexExprToken viml_pexpr_next_token(ParserState *const pstate) - FUNC_ATTR_WARN_UNUSED_RESULT +/// +/// @param pstate Parser state. +/// @param[in] peek If true, do not advance pstate cursor. +/// +/// @return Next token. +LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { LexExprToken ret = { .type = kExprLexInvalid, @@ -362,6 +367,8 @@ viml_pexpr_next_token_invalid_comparison: } #undef GET_CCS viml_pexpr_next_token_adv_return: - viml_parser_advance(pstate, ret.len); + if (!peek) { + viml_parser_advance(pstate, ret.len); + } return ret; } diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h index ec582294e1..231e43b4c7 100644 --- a/src/nvim/viml/parser/parser.h +++ b/src/nvim/viml/parser/parser.h @@ -7,11 +7,13 @@ #include "nvim/lib/kvec.h" #include "nvim/func_attr.h" +#include "nvim/mbyte.h" /// One parsed line typedef struct { const char *data; ///< Parsed line pointer size_t size; ///< Parsed line size + bool allocated; ///< True if line may be freed. } ParserLine; /// Line getter type for parser @@ -48,6 +50,8 @@ typedef struct { void *cookie; /// All lines obtained by get_line. kvec_withinit_t(ParserLine, 4) lines; + /// Conversion, for :scriptencoding. + vimconv_T conv; } ParserInputReader; /// Highlighted region definition @@ -76,6 +80,33 @@ typedef struct { bool can_continuate; } ParserState; +static inline void viml_preader_get_line(ParserInputReader *const preader, + ParserLine *const ret_pline) + REAL_FATTR_NONNULL_ALL; + +/// Get one line from ParserInputReader +static inline void viml_preader_get_line(ParserInputReader *const preader, + ParserLine *const ret_pline) +{ + ParserLine pline; + preader->get_line(preader->cookie, &pline); + if (preader->conv.vc_type != CONV_NONE && pline.size) { + ParserLine cpline = { + .allocated = true, + .size = pline.size, + }; + cpline.data = (char *)string_convert(&preader->conv, + (char_u *)pline.data, + &cpline.size); + if (pline.allocated) { + xfree((void *)pline.data); + } + pline = cpline; + } + kvi_push(preader->lines, pline); + *ret_pline = pline; +} + static inline bool viml_parser_get_remaining_line(ParserState *const pstate, ParserLine *const ret_pline) REAL_FATTR_ALWAYS_INLINE REAL_FATTR_WARN_UNUSED_RESULT REAL_FATTR_NONNULL_ALL; @@ -90,8 +121,7 @@ static inline bool viml_parser_get_remaining_line(ParserState *const pstate, { const size_t num_lines = kv_size(pstate->reader.lines); if (pstate->pos.line == num_lines) { - pstate->reader.get_line(pstate->reader.cookie, ret_pline); - kvi_push(pstate->reader.lines, *ret_pline); + viml_preader_get_line(&pstate->reader, ret_pline); } else { *ret_pline = kv_last(pstate->reader.lines); } diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index bf5afe4eeb..c877ce4bbf 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -110,15 +110,22 @@ local function new_pstate(strings) end ret_pline.data = data ret_pline.size = size + ret_pline.allocated = false end local pline_init = { data = nil, size = 0, + allocated = false, } local state = { reader = { get_line = get_line, cookie = nil, + conv = { + vc_type = 0, + vc_factor = 1, + vc_fail = false, + }, }, pos = { line = 0, col = 0 }, colors = kvi_new('ParserHighlight'), From 1265da028882d9877a5ebbd3f3f52cb4b52a4b94 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 3 Sep 2017 19:53:41 +0300 Subject: [PATCH 007/109] viml/parser: Add helper functions for highlighting --- src/nvim/viml/parser/parser.h | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h index 231e43b4c7..a17edac403 100644 --- a/src/nvim/viml/parser/parser.h +++ b/src/nvim/viml/parser/parser.h @@ -156,4 +156,33 @@ static inline void viml_parser_advance(ParserState *const pstate, } } +static inline void viml_parser_highlight(ParserState *const pstate, + const ParserPosition start, + const size_t end_col, + const char *const group) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ALL; + +/// Record highlighting of some region of text +/// +/// @param pstate Parser state to work with. +/// @param[in] start Start position of the highlight. +/// @param[in] len Highlighting chunk length. +/// @param[in] group Highlight group. +static inline void viml_parser_highlight(ParserState *const pstate, + const ParserPosition start, + const size_t len, + const char *const group) +{ + if (pstate->colors == NULL) { + return; + } + // TODO(ZyX-I): May do some assert() sanitizing here. + // TODO(ZyX-I): May join chunks. + kvi_push(*pstate->colors, ((ParserHighlightChunk) { + .start = start, + .end_col = start.col + len, + .group = group, + })); +} + #endif // NVIM_VIML_PARSER_PARSER_H From 919223c23ae3c8c904f35e7d605b1cf14d44a5f0 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 3 Sep 2017 19:57:24 +0300 Subject: [PATCH 008/109] unittests: Move some functions into helpers modules --- test/unit/helpers.lua | 28 ++++++ test/unit/viml/expressions/lexer_spec.lua | 104 +++------------------- test/unit/viml/helpers.lua | 97 ++++++++++++++++++++ 3 files changed, 135 insertions(+), 94 deletions(-) create mode 100644 test/unit/viml/helpers.lua diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index a629fca9a2..a5ca7b069b 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -760,6 +760,29 @@ end cimport('./src/nvim/types.h', './src/nvim/main.h', './src/nvim/os/time.h') +local function conv_enum(etab, eval) + local n = tonumber(eval) + return etab[n] or n +end + +local function array_size(arr) + return ffi.sizeof(arr) / ffi.sizeof(arr[0]) +end + +local function kvi_size(kvi) + return array_size(kvi.init_array) +end + +local function kvi_init(kvi) + kvi.capacity = kvi_size(kvi) + kvi.items = kvi.init_array + return kvi +end + +local function kvi_new(ct) + return kvi_init(ffi.new(ct)) +end + local module = { cimport = cimport, cppimport = cppimport, @@ -780,6 +803,11 @@ local module = { child_call_once = child_call_once, child_cleanup_once = child_cleanup_once, sc = sc, + conv_enum = conv_enum, + array_size = array_sive, + kvi_size = kvi_size, + kvi_init = kvi_init, + kvi_new = kvi_new, } return function() return module diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index c877ce4bbf..32182f650d 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -1,11 +1,18 @@ local helpers = require('test.unit.helpers')(after_each) +local viml_helpers = require('test.unit.viml.helpers') local itp = helpers.gen_itp(it) local child_call_once = helpers.child_call_once +local conv_enum = helpers.conv_enum local cimport = helpers.cimport local ffi = helpers.ffi local eq = helpers.eq +local pline2lua = viml_helpers.pline2lua +local new_pstate = viml_helpers.new_pstate +local intchar2lua = viml_helpers.intchar2lua +local pstate_set_str = viml_helpers.pstate_set_str + local lib = cimport('./src/nvim/viml/parser/expressions.h') local eltkn_type_tab, eltkn_cmp_type_tab, ccs_tab, eltkn_mul_type_tab @@ -71,114 +78,23 @@ child_call_once(function() } end) -local function array_size(arr) - return ffi.sizeof(arr) / ffi.sizeof(arr[0]) -end - -local function kvi_size(kvi) - return array_size(kvi.init_array) -end - -local function kvi_init(kvi) - kvi.capacity = kvi_size(kvi) - kvi.items = kvi.init_array - return kvi -end - -local function kvi_new(ct) - return kvi_init(ffi.new(ct)) -end - -local function new_pstate(strings) - local strings_idx = 0 - local function get_line(_, ret_pline) - strings_idx = strings_idx + 1 - local str = strings[strings_idx] - local data, size - if type(str) == 'string' then - data = str - size = #str - elseif type(str) == 'nil' then - data = nil - size = 0 - elseif type(str) == 'table' then - data = str.data - size = str.size - elseif type(str) == 'function' then - data, size = str() - size = size or 0 - end - ret_pline.data = data - ret_pline.size = size - ret_pline.allocated = false - end - local pline_init = { - data = nil, - size = 0, - allocated = false, - } - local state = { - reader = { - get_line = get_line, - cookie = nil, - conv = { - vc_type = 0, - vc_factor = 1, - vc_fail = false, - }, - }, - pos = { line = 0, col = 0 }, - colors = kvi_new('ParserHighlight'), - can_continuate = false, - } - local ret = ffi.new('ParserState', state) - kvi_init(ret.reader.lines) - kvi_init(ret.stack) - return ret -end - -local function conv_enum(etab, eval) - local n = tonumber(eval) - return etab[n] or n -end - local function conv_eltkn_type(typ) return conv_enum(eltkn_type_tab, typ) end -local function pline2lua(pline) - return ffi.string(pline.data, pline.size) -end - local bracket_types = { Bracket = true, FigureBrace = true, Parenthesis = true, } -local function intchar2lua(ch) - ch = tonumber(ch) - return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch -end - local function eltkn2lua(pstate, tkn) local ret = { type = conv_eltkn_type(tkn.type), - len = tonumber(tkn.len), - start = { line = tonumber(tkn.start.line), col = tonumber(tkn.start.col) }, } - if ret.start.line < pstate.reader.lines.size then - local pstr = pline2lua(pstate.reader.lines.items[ret.start.line]) - if ret.start.col >= #pstr then - ret.error = 'start.col >= #pstr' - else - ret.str = pstr:sub(ret.start.col + 1, ret.start.col + ret.len) - if #(ret.str) ~= ret.len then - ret.error = '#str /= len' - end - end - else - ret.error = 'start.line >= pstate.reader.lines.size' + pstate_set_str(pstate, tkn.start, tkn.len, ret) + if not ret.error and (#(ret.str) ~= ret.len) then + ret.error = '#str /= len' end if ret.type == 'Comparison' then ret.data = { diff --git a/test/unit/viml/helpers.lua b/test/unit/viml/helpers.lua new file mode 100644 index 0000000000..2cb60499eb --- /dev/null +++ b/test/unit/viml/helpers.lua @@ -0,0 +1,97 @@ +local helpers = require('test.unit.helpers')(nil) + +local ffi = helpers.ffi +local kvi_new = helpers.kvi_new +local kvi_init = helpers.kvi_init + +local function new_pstate(strings) + local strings_idx = 0 + local function get_line(_, ret_pline) + strings_idx = strings_idx + 1 + local str = strings[strings_idx] + local data, size + if type(str) == 'string' then + data = str + size = #str + elseif type(str) == 'nil' then + data = nil + size = 0 + elseif type(str) == 'table' then + data = str.data + size = str.size + elseif type(str) == 'function' then + data, size = str() + size = size or 0 + end + ret_pline.data = data + ret_pline.size = size + ret_pline.allocated = false + end + local pline_init = { + data = nil, + size = 0, + allocated = false, + } + local state = { + reader = { + get_line = get_line, + cookie = nil, + conv = { + vc_type = 0, + vc_factor = 1, + vc_fail = false, + }, + }, + pos = { line = 0, col = 0 }, + colors = kvi_new('ParserHighlight'), + can_continuate = false, + } + local ret = ffi.new('ParserState', state) + kvi_init(ret.reader.lines) + kvi_init(ret.stack) + return ret +end + +local function intchar2lua(ch) + ch = tonumber(ch) + return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch +end + +local function pline2lua(pline) + return ffi.string(pline.data, pline.size) +end + +local function pstate_str(pstate, start, len) + local str = nil + local err = nil + if start.line < pstate.reader.lines.size then + local pstr = pline2lua(pstate.reader.lines.items[start.line]) + if start.col >= #pstr then + err = 'start.col >= #pstr' + else + str = pstr:sub(tonumber(start.col) + 1, tonumber(start.col + len)) + end + else + err = 'start.line >= pstate.reader.lines.size' + end + return str, err +end + +local function pstate_set_str(pstate, start, len, ret) + ret = ret or {} + ret.start = { + line = tonumber(start.line), + col = tonumber(start.col) + } + ret.len = tonumber(len) + ret.str, ret.error = pstate_str(pstate, start, len) + return ret +end + +return { + pline2lua = pline2lua, + pstate_str = pstate_str, + new_pstate = new_pstate, + intchar2lua = intchar2lua, + pstate_set_str = pstate_set_str, +} From 430e516d3ac1235c1ee3009a8a36089bf278440e Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 3 Sep 2017 21:58:16 +0300 Subject: [PATCH 009/109] viml/parser/expressions: Start creating expressions parser Currently supported nodes: - Register as it is one of the simplest value nodes (even numbers are not that simple with that dot handling). - Plus, both unary and binary. - Parenthesis, both nesting and calling. Note regarding unit tests: it stores data for AST in highlighting in strings in place of tables because luassert fails to do a good job at representing big tables. Squashing a bunch of data into a single string simply yields more readable result. --- src/nvim/viml/parser/expressions.c | 625 +++++++++++++++ src/nvim/viml/parser/expressions.h | 74 ++ test/unit/eval/helpers.lua | 5 +- test/unit/helpers.lua | 28 + test/unit/viml/expressions/parser_spec.lua | 887 +++++++++++++++++++++ 5 files changed, 1615 insertions(+), 4 deletions(-) create mode 100644 test/unit/viml/expressions/parser_spec.lua diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index c29fac9cb4..b54f2eb237 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -13,10 +13,18 @@ #include "nvim/types.h" #include "nvim/charset.h" #include "nvim/ascii.h" +#include "nvim/lib/kvec.h" #include "nvim/viml/parser/expressions.h" #include "nvim/viml/parser/parser.h" +typedef kvec_withinit_t(ExprASTNode **, 16) ExprASTStack; + +typedef enum { + kELvlOperator, ///< Operators: function call, subscripts, binary operators, … + kELvlValue, ///< Actual value: literals, variables, nested expressions. +} ExprASTLevel; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.c.generated.h" #endif @@ -144,6 +152,7 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek) // Environment variable. case '$': { + // FIXME: Parser function can’t be thread-safe with vim_isIDc. CHARREG(kExprLexEnv, vim_isIDc); break; } @@ -183,6 +192,7 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek) ret.data.var.autoload = ( memchr(pline.data + 2, AUTOLOAD_CHAR, ret.len - 2) != NULL); + // FIXME: Resolve ambiguity with an argument to the lexer function. // Previous CHARREG stopped at autoload character in order to make it // possible to detect `is#`. Continue now with autoload characters // included. @@ -372,3 +382,618 @@ viml_pexpr_next_token_adv_return: } return ret; } + +// start = s ternary_expr s EOC +// ternary_expr = binop_expr +// ( s Question s ternary_expr s Colon s ternary_expr s )? +// binop_expr = unaryop_expr ( binop unaryop_expr )? +// unaryop_expr = ( unaryop )? subscript_expr +// subscript_expr = subscript_expr subscript +// | value_expr +// subscript = Bracket('[') s ternary_expr s Bracket(']') +// | s Parenthesis('(') call_args Parenthesis(')') +// | Dot ( PlainIdentifier | Number )+ +// # Note: `s` before Parenthesis('(') is only valid if preceding subscript_expr +// # is PlainIdentifier +// value_expr = ( float | Number +// | DoubleQuotedString | SingleQuotedString +// | paren_expr +// | list_literal +// | lambda_literal +// | dict_literal +// | Environment +// | Option +// | Register +// | var ) +// float = Number Dot Number ( PlainIdentifier('e') ( Plus | Minus )? Number )? +// # Note: `1.2.3` is concat and not float. `"abc".2.3` is also concat without +// # floats. +// paren_expr = Parenthesis('(') s ternary_expr s Parenthesis(')') +// list_literal = Bracket('[') s +// ( ternary_expr s Comma s )* +// ternary_expr? s +// Bracket(']') +// dict_literal = FigureBrace('{') s +// ( ternary_expr s Colon s ternary_expr s Comma s )* +// ( ternary_expr s Colon s ternary_expr s )? +// FigureBrace('}') +// lambda_literal = FigureBrace('{') s +// ( PlainIdentifier s Comma s )* +// PlainIdentifier s +// Arrow s +// ternary_expr s +// FigureBrace('}') +// var = varchunk+ +// varchunk = PlainIdentifier +// | Comparison("is" | "is#" | "isnot" | "isnot#") +// | FigureBrace('{') s ternary_expr s FigureBrace('}') +// call_args = ( s ternary_expr s Comma s )* s ternary_expr? s +// binop = s ( Plus | Minus | Dot +// | Comparison +// | Multiplication +// | Or +// | And ) s +// unaryop = s ( Not | Plus | Minus ) s +// s = Spacing? +// +// Binary operator precedence and associativity: +// +// Operator | Precedence | Associativity +// ---------+------------+----------------- +// || | 2 | left +// && | 3 | left +// cmp* | 4 | not associative +// + - . | 5 | left +// * / % | 6 | left +// +// * comparison operators: +// +// == ==# ==? != !=# !=? +// =~ =~# =~? !~ !~# !~? +// > ># >? <= <=# <=? +// < <# = >=# >=? +// is is# is? isnot isnot# isnot? +// +// Used highlighting groups and assumed linkage: +// +// NVimInvalid -> Error +// NVimInvalidValue -> NVimInvalid +// NVimInvalidOperator -> NVimInvalid +// NVimInvalidDelimiter -> NVimInvalid +// +// NVimOperator -> Operator +// NVimUnaryOperator -> NVimOperator +// NVimBinaryOperator -> NVimOperator +// NVimComparisonOperator -> NVimOperator +// NVimTernaryOperator -> NVimOperator +// +// NVimParenthesis -> Delimiter +// +// NVimInvalidSpacing -> NVimInvalid +// NVimInvalidTernaryOperator -> NVimInvalidOperator +// NVimInvalidRegister -> NVimInvalidValue +// NVimInvalidClosingBracket -> NVimInvalidDelimiter +// NVimInvalidSpacing -> NVimInvalid +// +// NVimUnaryPlus -> NVimUnaryOperator +// NVimBinaryPlus -> NVimBinaryOperator +// NVimRegister -> SpecialChar +// NVimNestingParenthesis -> NVimParenthesis +// NVimCallingParenthesis -> NVimParenthesis + +/// Allocate a new node and set some of the values +/// +/// @param[in] type Node type to allocate. +/// @param[in] level Node level to allocate +static inline ExprASTNode *viml_pexpr_new_node(const ExprASTNodeType type) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_MALLOC +{ + ExprASTNode *ret = xmalloc(sizeof(*ret)); + ret->type = type; + ret->children = NULL; + ret->next = NULL; + return ret; +} + +typedef enum { + kEOpLvlInvalid = 0, + kEOpLvlParens, + kEOpLvlTernary, + kEOpLvlOr, + kEOpLvlAnd, + kEOpLvlComparison, + kEOpLvlAddition, ///< Addition, subtraction and concatenation. + kEOpLvlMultiplication, ///< Multiplication, division and modulo. + kEOpLvlUnary, ///< Unary operations: not, minus, plus. + kEOpLvlSubscript, ///< Subscripts. + kEOpLvlValue, ///< Values: literals, variables, nested expressions, … +} ExprOpLvl; + +typedef enum { + kEOpAssNo= 'n', ///< Not associative / not applicable. + kEOpAssLeft = 'l', ///< Left associativity. + kEOpAssRight = 'r', ///< Right associativity. +} ExprOpAssociativity; + +static const ExprOpLvl node_type_to_op_lvl[] = { + [kExprNodeMissing] = kEOpLvlInvalid, + [kExprNodeOpMissing] = kEOpLvlMultiplication, + + [kExprNodeNested] = kEOpLvlParens, + [kExprNodeComplexIdentifier] = kEOpLvlParens, + + [kExprNodeTernary] = kEOpLvlTernary, + + [kExprNodeBinaryPlus] = kEOpLvlAddition, + + [kExprNodeUnaryPlus] = kEOpLvlUnary, + + [kExprNodeSubscript] = kEOpLvlSubscript, + [kExprNodeCall] = kEOpLvlSubscript, + + [kExprNodeRegister] = kEOpLvlValue, + [kExprNodeListLiteral] = kEOpLvlValue, + [kExprNodePlainIdentifier] = kEOpLvlValue, +}; + +static const ExprOpAssociativity node_type_to_op_ass[] = { + [kExprNodeMissing] = kEOpAssNo, + [kExprNodeOpMissing] = kEOpAssNo, + + [kExprNodeNested] = kEOpAssNo, + [kExprNodeComplexIdentifier] = kEOpAssLeft, + + [kExprNodeTernary] = kEOpAssNo, + + [kExprNodeBinaryPlus] = kEOpAssLeft, + + [kExprNodeUnaryPlus] = kEOpAssNo, + + [kExprNodeSubscript] = kEOpAssLeft, + [kExprNodeCall] = kEOpAssLeft, + + [kExprNodeRegister] = kEOpAssNo, + [kExprNodeListLiteral] = kEOpAssNo, + [kExprNodePlainIdentifier] = kEOpAssNo, +}; + +#ifdef UNIT_TESTING +#include +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_ast_stack( + const ExprASTStack *const ast_stack, + const char *const msg) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE +{ + fprintf(stderr, "\n%sstack: %zu:\n", msg, kv_size(*ast_stack)); + for (size_t i = 0; i < kv_size(*ast_stack); i++) { + const ExprASTNode *const *const eastnode_p = ( + (const ExprASTNode *const *)kv_A(*ast_stack, i)); + if (*eastnode_p == NULL) { + fprintf(stderr, "- %p : NULL\n", (void *)eastnode_p); + } else { + fprintf(stderr, "- %p : %p : %c : %zu:%zu:%zu\n", + (void *)eastnode_p, (void *)(*eastnode_p), (*eastnode_p)->type, + (*eastnode_p)->start.line, (*eastnode_p)->start.col, + (*eastnode_p)->len); + } + } +} +#define PSTACK(msg) \ + viml_pexpr_debug_print_ast_stack(&ast_stack, #msg) +#define PSTACK_P(msg) \ + viml_pexpr_debug_print_ast_stack(ast_stack, #msg) +#endif + +/// Handle binary operator +/// +/// This function is responsible for handling priority levels as well. +static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, + ExprASTNode *const bop_node, + ExprASTLevel *const want_level_p) + FUNC_ATTR_NONNULL_ALL +{ + ExprASTNode **top_node_p = NULL; + ExprASTNode *top_node; + ExprOpLvl top_node_lvl; + ExprOpAssociativity top_node_ass; + assert(kv_size(*ast_stack)); + const ExprOpLvl bop_node_lvl = node_type_to_op_lvl[bop_node->type]; + do { + ExprASTNode **new_top_node_p = kv_last(*ast_stack); + ExprASTNode *new_top_node = *new_top_node_p; + assert(new_top_node != NULL); + const ExprOpLvl new_top_node_lvl = node_type_to_op_lvl[new_top_node->type]; + const ExprOpAssociativity new_top_node_ass = ( + node_type_to_op_ass[new_top_node->type]); + if (top_node_p != NULL + && ((bop_node_lvl > new_top_node_lvl + || (bop_node_lvl == new_top_node_lvl + && new_top_node_ass == kEOpAssNo)))) { + break; + } + kv_drop(*ast_stack, 1); + top_node_p = new_top_node_p; + top_node = new_top_node; + top_node_lvl = new_top_node_lvl; + top_node_ass = new_top_node_ass; + } while (kv_size(*ast_stack)); + // FIXME Handle right and no associativity correctly + *top_node_p = bop_node; + bop_node->children = top_node; + assert(bop_node->children->next == NULL); + kvi_push(*ast_stack, top_node_p); + kvi_push(*ast_stack, &bop_node->children->next); + *want_level_p = kELvlValue; +} + +/// Get highlight group name +#define HL(g) (is_invalid ? "NVimInvalid" #g : "NVim" #g) + +/// Highlight current token with the given group +#define HL_CUR_TOKEN(g) \ + viml_parser_highlight(pstate, cur_token.start, cur_token.len, \ + HL(g)) + +/// Allocate new node, saving some values +#define NEW_NODE(type) \ + viml_pexpr_new_node(type) + +/// Set position of the given node to position from the given token +/// +/// @param cur_node Node to modify. +/// @param cur_token Token to set position from. +#define POS_FROM_TOKEN(cur_node, cur_token) \ + do { \ + cur_node->start = cur_token.start; \ + cur_node->len = cur_token.len; \ + } while (0) + +/// Allocate new node and set its position from the current token +/// +/// If previous token happened to contain spacing then it will be included. +/// +/// @param cur_node Variable to save allocated node to. +/// @param typ Node type. +#define NEW_NODE_WITH_CUR_POS(cur_node, typ) \ + do { \ + cur_node = NEW_NODE(typ); \ + POS_FROM_TOKEN(cur_node, cur_token); \ + if (prev_token.type == kExprLexSpacing) { \ + cur_node->start = prev_token.start; \ + cur_node->len += prev_token.len; \ + } \ + } while (0) + +// TODO(ZyX-I): actual condition +/// Check whether it is possible to have next expression after current +/// +/// For :echo: `:echo @a @a` is a valid expression. `:echo (@a @a)` is not. +#define MAY_HAVE_NEXT_EXPR \ + (kv_size(ast_stack) == 1) + +/// Record missing operator: for things like +/// +/// :echo @a @a +/// +/// (allowed) or +/// +/// :echo (@a @a) +/// +/// (parsed as OpMissing(@a, @a)). +#define OP_MISSING \ + do { \ + if (flags & kExprFlagsMulti && MAY_HAVE_NEXT_EXPR) { \ + /* Multiple expressions allowed, return without calling */ \ + /* viml_parser_advance(). */ \ + goto viml_pexpr_parse_end; \ + } else { \ + assert(*top_node_p != NULL); \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, _("E15: Missing operator: %.*s")); \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOpMissing); \ + cur_node->len = 0; \ + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_level); \ + is_invalid = true; \ + goto viml_pexpr_parse_process_token; \ + } \ + } while (0) + +/// Set AST error, unless AST already is not correct +/// +/// @param[out] ret_ast AST to set error in. +/// @param[in] pstate Parser state, used to get error message argument. +/// @param[in] msg Error message, assumed to be already translated and +/// containing a single %token "%.*s". +/// @param[in] start Position at which error occurred. +static inline void east_set_error(ExprAST *const ret_ast, + const ParserState *const pstate, + const char *const msg, + const ParserPosition start) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE +{ + if (!ret_ast->correct) { + return; + } + const ParserLine pline = pstate->reader.lines.items[start.line]; + ret_ast->correct = false; + ret_ast->err.msg = msg; + ret_ast->err.arg_len = (int)(pline.size - start.col); + ret_ast->err.arg = pline.data + start.col; +} + +/// Set error from the given kExprLexInvalid token and given message +#define ERROR_FROM_TOKEN_AND_MSG(cur_token, msg) \ + east_set_error(&ast, pstate, msg, cur_token.start) + +/// Set error from the given kExprLexInvalid token +#define ERROR_FROM_TOKEN(cur_token) \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, cur_token.data.err.msg) + +/// Parse one VimL expression +/// +/// @param pstate Parser state. +/// @param[in] flags Additional flags, see ExprParserFlags +/// +/// @return Parsed AST. +ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL +{ + ExprAST ast = { + .correct = true, + .err = { + .msg = NULL, + .arg_len = 0, + .arg = NULL, + }, + .root = NULL, + }; + ExprASTStack ast_stack; + kvi_init(ast_stack); + kvi_push(ast_stack, &ast.root); + // Expressions stack: + // 1. *last is NULL if want_level is kExprLexValue. Indicates where expression + // is to be put. + // 2. *last is not NULL otherwise, indicates current expression to be used as + // an operator argument. + ExprASTLevel want_level = kELvlValue; + LexExprToken prev_token = { .type = kExprLexMissing }; + bool highlighted_prev_spacing = false; + do { + LexExprToken cur_token = viml_pexpr_next_token(pstate, true); + if (cur_token.type == kExprLexEOC) { + if (flags & kExprFlagsDisallowEOC) { + if (cur_token.len == 0) { + // It is end of string, break. + break; + } else { + // It is NL, NUL or bar. + // + // Note: `=1 | 2` actually yields 1 in Vim without any + // errors. This will be changed here. + cur_token.type = kExprLexInvalid; + cur_token.data.err.msg = _("E15: Unexpected EOC character: %.*s"); + const ParserLine pline = ( + pstate->reader.lines.items[cur_token.start.line]); + const char eoc_char = pline.data[cur_token.start.col]; + cur_token.data.err.type = ((eoc_char == NUL || eoc_char == NL) + ? kExprLexSpacing + : kExprLexOr); + } + } else { + break; + } + } + LexExprTokenType tok_type = cur_token.type; + const bool token_invalid = (tok_type == kExprLexInvalid); + bool is_invalid = token_invalid; +viml_pexpr_parse_process_token: + if (tok_type == kExprLexSpacing) { + if (is_invalid) { + viml_parser_highlight(pstate, cur_token.start, cur_token.len, + HL(Spacing)); + } else { + // Do not do anything: let regular spacing be highlighted as normal. + // This also allows later to highlight spacing as invalid. + } + goto viml_pexpr_parse_cycle_end; + } else if (is_invalid && prev_token.type == kExprLexSpacing + && !highlighted_prev_spacing) { + viml_parser_highlight(pstate, prev_token.start, prev_token.len, + HL(Spacing)); + is_invalid = false; + highlighted_prev_spacing = true; + } + ExprASTNode **const top_node_p = kv_last(ast_stack); + ExprASTNode *cur_node = NULL; + // Keep these two asserts separate for debugging purposes. + assert(want_level == kELvlValue || *top_node_p != NULL); + assert(want_level != kELvlValue || *top_node_p == NULL); + switch (tok_type) { + case kExprLexEOC: { + assert(false); + } + case kExprLexInvalid: { + ERROR_FROM_TOKEN(cur_token); + tok_type = cur_token.data.err.type; + goto viml_pexpr_parse_process_token; + } + case kExprLexRegister: { + if (want_level == kELvlValue) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeRegister); + cur_node->data.reg.name = cur_token.data.reg.name; + *top_node_p = cur_node; + want_level = kELvlOperator; + viml_parser_highlight(pstate, cur_token.start, cur_token.len, + HL(Register)); + } else { + // Register in operator position: e.g. @a @a + OP_MISSING; + } + break; + } + case kExprLexPlus: { + if (want_level == kELvlValue) { + // Value level: assume unary plus + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnaryPlus); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + HL_CUR_TOKEN(UnaryPlus); + } else if (want_level < kELvlValue) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeBinaryPlus); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_level); + HL_CUR_TOKEN(BinaryPlus); + } + want_level = kELvlValue; + break; + } + case kExprLexParenthesis: { + if (cur_token.data.brc.closing) { + if (want_level == kELvlValue) { + if (kv_size(ast_stack) > 1) { + const ExprASTNode *const prev_top_node = *kv_Z(ast_stack, 1); + if (prev_top_node->type == kExprNodeCall) { + // Function call without arguments, this is not an error. + // But further code does not expect NULL nodes. + kv_drop(ast_stack, 1); + goto viml_pexpr_parse_no_paren_closing_error; + } + } + is_invalid = true; + ERROR_FROM_TOKEN_AND_MSG(cur_token, _("E15: Expected value: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); + cur_node->len = 0; + *top_node_p = cur_node; + } else { + // Always drop the topmost value: when want_level != kELvlValue + // topmost item on stack is a *finished* left operand, which may as + // well be "(@a)" which needs not be finished. + kv_drop(ast_stack, 1); + } +viml_pexpr_parse_no_paren_closing_error: {} + ExprASTNode **new_top_node_p = NULL; + while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeNested + && (*new_top_node_p)->type != kExprNodeCall))) { + new_top_node_p = kv_pop(ast_stack); + } + if (new_top_node_p != NULL + && ((*new_top_node_p)->type == kExprNodeNested + || (*new_top_node_p)->type == kExprNodeCall)) { + if ((*new_top_node_p)->type == kExprNodeNested) { + HL_CUR_TOKEN(NestingParenthesis); + } else { + HL_CUR_TOKEN(CallingParenthesis); + } + } else { + // “Always drop the topmost value” branch has got rid of the single + // value stack had, so there is nothing known to enclose. Correct + // this. + if (new_top_node_p == NULL) { + new_top_node_p = top_node_p; + } + is_invalid = true; + HL_CUR_TOKEN(NestingParenthesis); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing parenthesis: %.*s")); + cur_node = NEW_NODE(kExprNodeNested); + cur_node->start = cur_token.start; + cur_node->len = 0; + // Unexpected closing parenthesis, assume that it was wanted to + // enclose everything in (). + cur_node->children = *new_top_node_p; + *new_top_node_p = cur_node; + assert(cur_node->next == NULL); + } + kvi_push(ast_stack, new_top_node_p); + want_level = kELvlOperator; + } else { + if (want_level == kELvlValue) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeNested); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + HL_CUR_TOKEN(NestingParenthesis); + } else if (want_level == kELvlOperator) { + if (prev_token.type == kExprLexSpacing) { + // For some reason "function (args)" is a function call, but + // "(funcref) (args)" is not. AFAIR this somehow involves + // compatibility and Bram was commenting that this is + // intentionally inconsistent and he is not very happy with the + // situation himself. + if ((*top_node_p)->type != kExprNodePlainIdentifier + && (*top_node_p)->type != kExprNodeComplexIdentifier) { + OP_MISSING; + } + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCall); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_level); + HL_CUR_TOKEN(CallingParenthesis); + } else { + // Currently it is impossible to reach this. + assert(false); + } + want_level = kELvlValue; + } + break; + } + } +viml_pexpr_parse_cycle_end: + prev_token = cur_token; + highlighted_prev_spacing = false; + viml_parser_advance(pstate, cur_token.len); + } while (true); +viml_pexpr_parse_end: + if (want_level == kELvlValue) { + east_set_error(&ast, pstate, _("E15: Expected value: %.*s"), pstate->pos); + } else if (kv_size(ast_stack) != 1) { + // Something may be wrong, check whether it really is. + + // Pointer to ast.root must never be dropped, so “!= 1” is expected to be + // the same as “> 1”. + assert(kv_size(ast_stack)); + // Topmost stack item must be a *finished* value, so it must not be + // analyzed. E.g. it may contain an already finished nested expression. + kv_drop(ast_stack, 1); + while (ast.correct && kv_size(ast_stack)) { + const ExprASTNode *const cur_node = (*kv_pop(ast_stack)); + // This should only happen when want_level == kELvlValue. + assert(cur_node != NULL); + switch (cur_node->type) { + case kExprNodeOpMissing: + case kExprNodeMissing: { + // Error should’ve been already reported. + break; + } + case kExprNodeCall: { + // TODO(ZyX-I): Rehighlight as invalid? + east_set_error( + &ast, pstate, + _("E116: Missing closing parenthesis for function call: %.*s"), + cur_node->start); + break; + } + case kExprNodeNested: { + // TODO(ZyX-I): Rehighlight as invalid? + east_set_error( + &ast, pstate, + _("E110: Missing closing parenthesis for nested expression" + ": %.*s"), + cur_node->start); + break; + } + case kExprNodeBinaryPlus: + case kExprNodeUnaryPlus: + case kExprNodeRegister: { + // It is OK to see these in the stack. + break; + } + // TODO(ZyX-I): handle other values + } + } + } + kvi_destroy(ast_stack); + return ast; +} + +#undef NEW_NODE +#undef HL diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 52354760a5..13888562df 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -111,6 +111,80 @@ typedef struct { } data; ///< Additional data, if needed. } LexExprToken; +/// Expression AST node type +typedef enum { + kExprNodeMissing = 'X', + kExprNodeOpMissing = '_', + kExprNodeTernary = '?', ///< Ternary operator, valid one has three children. + kExprNodeRegister = '@', ///< Register, no children. + kExprNodeSubscript = 's', ///< Subscript, should have two or three children. + kExprNodeListLiteral = 'l', ///< List literal, any number of children. + kExprNodeUnaryPlus = 'p', + kExprNodeBinaryPlus = '+', + kExprNodeNested = 'e', ///< Nested parenthesised expression. + kExprNodeCall = 'c', ///< Function call. + /// Plain identifier: simple variable/function name + /// + /// Looks like "string", "g:Foo", etc: consists from a single + /// kExprLexPlainIdentifier token. + kExprNodePlainIdentifier = 'i', + /// Complex identifier: variable/function name with curly braces + kExprNodeComplexIdentifier = 'I', +} ExprASTNodeType; + +typedef struct expr_ast_node ExprASTNode; + +/// Structure representing one AST node +struct expr_ast_node { + ExprASTNodeType type; ///< Node type. + /// Node children: e.g. for 1 + 2 nodes 1 and 2 will be children of +. + ExprASTNode *children; + /// Next node: e.g. for 1 + 2 child nodes 1 and 2 are put into a single-linked + /// list: `(+)->children` references only node 1, node 2 is in + /// `(+)->children->next`. + ExprASTNode *next; + ParserPosition start; + size_t len; + union { + struct { + int name; ///< Register name, may be -1 if name not present. + } reg; ///< For kExprNodeRegister. + } data; +}; + +enum { + /// Allow multiple expressions in a row: e.g. for :echo + /// + /// Parser will still parse only one of them though. + kExprFlagsMulti = (1 << 0), + /// Allow NL, NUL and bar to be EOC + /// + /// When parsing expressions input by user bar is assumed to be a binary + /// operator and other two are spacings. + kExprFlagsDisallowEOC = (1 << 1), + /// Print errors when encountered + /// + /// Without the flag they are only taken into account when parsing. + kExprFlagsPrintError = (1 << 2), +} ExprParserFlags; + +/// Structure representing complety AST for one expression +typedef struct { + /// True if represented AST is correct and can be executed. Incorrect ones may + /// still be used for completion, or in linters. + bool correct; + /// When AST is not correct this message will be printed. + /// + /// Uses `emsgf(msg, arg_len, arg);`, `msg` is assumed to contain only `%.*s`. + struct { + const char *msg; + int arg_len; + const char *arg; + } err; + /// Root node of the AST. + ExprASTNode *root; +} ExprAST; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.h.generated.h" #endif diff --git a/test/unit/eval/helpers.lua b/test/unit/eval/helpers.lua index 5bc482216e..d7399182f7 100644 --- a/test/unit/eval/helpers.lua +++ b/test/unit/eval/helpers.lua @@ -1,5 +1,6 @@ local helpers = require('test.unit.helpers')(nil) +local ptr2key = helpers.ptr2key local cimport = helpers.cimport local to_cstr = helpers.to_cstr local ffi = helpers.ffi @@ -91,10 +92,6 @@ local function populate_partial(pt, lua_pt, processed) return pt end -local ptr2key = function(ptr) - return tostring(ptr) -end - local lst2tbl local dct2tbl diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index a5ca7b069b..d3d14a5ca2 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -783,6 +783,31 @@ local function kvi_new(ct) return kvi_init(ffi.new(ct)) end +local function make_enum_conv_tab(lib, values, skip_pref, set_cb) + child_call_once(function() + local ret = {} + for _, v in ipairs(values) do + local str_v = v + if v:sub(1, #skip_pref) == skip_pref then + str_v = v:sub(#skip_pref + 1) + end + ret[tonumber(lib[v])] = str_v + end + set_cb(ret) + end) +end + +local function ptr2addr(ptr) + return tonumber(ffi.cast('intptr_t', ffi.cast('void *', ptr))) +end + +local s = ffi.new('char[64]', {0}) + +local function ptr2key(ptr) + ffi.C.snprintf(s, ffi.sizeof(s), '%p', ffi.cast('void *', ptr)) + return ffi.string(s) +end + local module = { cimport = cimport, cppimport = cppimport, @@ -808,6 +833,9 @@ local module = { kvi_size = kvi_size, kvi_init = kvi_init, kvi_new = kvi_new, + make_enum_conv_tab = make_enum_conv_tab, + ptr2addr = ptr2addr, + ptr2key = ptr2key, } return function() return module diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua new file mode 100644 index 0000000000..c4bb067391 --- /dev/null +++ b/test/unit/viml/expressions/parser_spec.lua @@ -0,0 +1,887 @@ +local helpers = require('test.unit.helpers')(after_each) +local viml_helpers = require('test.unit.viml.helpers') +local itp = helpers.gen_itp(it) + +local make_enum_conv_tab = helpers.make_enum_conv_tab +local child_call_once = helpers.child_call_once +local conv_enum = helpers.conv_enum +local ptr2key = helpers.ptr2key +local cimport = helpers.cimport +local ffi = helpers.ffi +local eq = helpers.eq + +local pline2lua = viml_helpers.pline2lua +local new_pstate = viml_helpers.new_pstate +local intchar2lua = viml_helpers.intchar2lua +local pstate_set_str = viml_helpers.pstate_set_str + +local lib = cimport('./src/nvim/viml/parser/expressions.h') + +local east_node_type_tab +make_enum_conv_tab(lib, { + 'kExprNodeMissing', + 'kExprNodeOpMissing', + 'kExprNodeTernary', + 'kExprNodeRegister', + 'kExprNodeSubscript', + 'kExprNodeListLiteral', + 'kExprNodeUnaryPlus', + 'kExprNodeBinaryPlus', + 'kExprNodeNested', + 'kExprNodeCall', + 'kExprNodePlainIdentifier', + 'kExprNodeComplexIdentifier', +}, 'kExprNode', function(ret) east_node_type_tab = ret end) + +local function conv_east_node_type(typ) + return conv_enum(east_node_type_tab, typ) +end + +local eastnodelist2lua + +local function eastnode2lua(pstate, eastnode, checked_nodes) + local key = ptr2key(eastnode) + if checked_nodes[key] then + checked_nodes[key].duplicate_key = key + return { duplicate = key } + end + local typ = conv_east_node_type(eastnode.type) + local ret = {} + checked_nodes[key] = ret + ret.children = eastnodelist2lua(pstate, eastnode.children, checked_nodes) + local str = pstate_set_str(pstate, eastnode.start, eastnode.len) + local ret_str + if str.error then + ret_str = 'error:' .. str.error + else + ret_str = ('%u:%u:%s'):format(str.start.line, str.start.col, str.str) + end + if typ == 'Register' then + typ = typ .. ('(name=%s)'):format( + tostring(intchar2lua(eastnode.data.reg.name))) + end + ret_str = typ .. ':' .. ret_str + local can_simplify = true + for k, v in pairs(ret) do + can_simplify = false + end + if can_simplify then + ret = ret_str + else + ret[1] = ret_str + end + return ret +end + +eastnodelist2lua = function(pstate, eastnode, checked_nodes) + local ret = {} + while eastnode ~= nil do + ret[#ret + 1] = eastnode2lua(pstate, eastnode, checked_nodes) + eastnode = eastnode.next + end + if #ret == 0 then + ret = nil + end + return ret +end + +local function east2lua(pstate, east) + local checked_nodes = {} + return { + err = (not east.correct) and { + msg = ffi.string(east.err.msg), + arg = ('%u:%s'):format( + tonumber(east.err.arg_len), + ffi.string(east.err.arg, east.err.arg_len)), + } or nil, + ast = eastnodelist2lua(pstate, east.root, checked_nodes), + } +end + +local function phl2lua(pstate) + local ret = {} + for i = 0, (tonumber(pstate.colors.size) - 1) do + local chunk = pstate.colors.items[i] + local chunk_tbl = pstate_set_str( + pstate, chunk.start, chunk.end_col - chunk.start.col, { + group = ffi.string(chunk.group), + }) + chunk_str = ('%s:%u:%u:%s'):format( + chunk_tbl.group, + chunk_tbl.start.line, + chunk_tbl.start.col, + chunk_tbl.str) + ret[i + 1] = chunk_str + end + return ret +end + +child_call_once(function() + assert:set_parameter('TableFormatLevel', 1000000) +end) + +describe('Expressions parser', function() + itp('works', function() + local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) + local pstate = new_pstate({str}) + local east = lib.viml_pexpr_parse(pstate, flags) + local ast = east2lua(pstate, east) + eq(exp_ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exp_highlighting_fs) do + exp_highlighting[i], next_col = h(next_col) + end + eq(exp_highlighting, phl2lua(pstate)) + end + end + local function hl(group, str, shift) + return function(next_col) + local col = next_col + (shift or 0) + return (('%s:%u:%u:%s'):format( + 'NVim' .. group, + 0, + col, + str)), (col + #str) + end + end + check_parsing('@a', 0, { + ast = { + 'Register(name=a):0:0:@a', + }, + }, { + hl('Register', '@a'), + }) + check_parsing('+@a', 0, { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + }) + check_parsing('@a+@b', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+@b+@c', 0, { + ast = { + { + 'BinaryPlus:0:5:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + 'Register(name=c):0:6:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing('+@a+@b', 0, { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:4:@b', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('+@a++@b', 0, { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'UnaryPlus:0:4:+', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a@b', 0, { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:2:@b', + }, + }, + }, + err = { + arg = '2:@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidRegister', '@b'), + }) + check_parsing(' @a \t @b', 0, { + ast = { + { + 'OpMissing:0:3:', + children = { + 'Register(name=a):0:0: @a', + 'Register(name=b):0:3: \t @b', + }, + }, + }, + err = { + arg = '2:@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a', 1), + hl('InvalidSpacing', ' \t '), + hl('Register', '@b'), + }) + check_parsing('+', 0, { + ast = { + 'UnaryPlus:0:0:+', + }, + err = { + arg = '0:', + msg = 'E15: Expected value: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + }) + check_parsing(' +', 0, { + ast = { + 'UnaryPlus:0:0: +', + }, + err = { + arg = '0:', + msg = 'E15: Expected value: %.*s', + }, + }, { + hl('UnaryPlus', '+', 1), + }) + check_parsing('@a+ ', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '0:', + msg = 'E15: Expected value: %.*s', + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + }) + check_parsing('(@a)', 0, { + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + }) + check_parsing('()', 0, { + ast = { + { + 'Nested:0:0:(', + children = { + 'Missing:0:1:', + }, + }, + }, + err = { + arg = '1:)', + msg = 'E15: Expected value: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing(')', 0, { + ast = { + { + 'Nested:0:0:', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = '1:)', + msg = 'E15: Expected value: %.*s', + }, + }, { + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+)', 0, { + ast = { + { + 'Nested:0:1:', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = '1:)', + msg = 'E15: Expected value: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+@a(@b)', 0, { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + { + 'Call:0:3:(', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+@b(@c)', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a()', 0, { + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a ()', 0, { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:2: (', + children = { + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = '2:()', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidSpacing', ' '), + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing( + '@a + (@b)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing( + '@a + (+@b)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'UnaryPlus:0:6:+', + children = { + 'Register(name=b):0:7:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing( + '@a + (@b + @c)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + }) + check_parsing('(@a)+@b', 0, { + ast = { + { + 'BinaryPlus:0:4:+', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:5:@b', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+(@b)(@c)', 0, { + -- 01234567890 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:7:(', + children = { + { + 'Nested:0:3:(', + children = { 'Register(name=b):0:4:@b' }, + }, + 'Register(name=c):0:8:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))(@c)', 0, { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:9:(', + children = { + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))+@c', 0, { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:9:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing( + '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', 0, {--[[ + | | | | | | | | || | | || | | ||| || || || || + 000000000011111111112222222222333333333344444444445555555 + 012345678901234567890123456789012345678901234567890123456 + ]] + ast = {{ + 'BinaryPlus:0:31: +', + children = { + { + 'BinaryPlus:0:23: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=d):0:16: @d', + 'Register(name=e):0:20:@e', + }, + }, + }, + }, + { + 'Nested:0:25: (', + children = { + { + 'UnaryPlus:0:27:+', + children = { + 'Register(name=f):0:28:@f', + }, + }, + }, + }, + }, + }, + { + 'Call:0:53:(', + children = { + { + 'Nested:0:33: (', + children = { + { + 'Call:0:48:(', + children = { + { + 'Call:0:44:(', + children = { + { + 'Nested:0:35:(', + children = { + { + 'UnaryPlus:0:36:+', + children = { + { + 'Call:0:39:(', + children = { + 'Register(name=g):0:37:@g', + 'Register(name=h):0:40:@h', + }, + }, + }, + }, + }, + }, + 'Register(name=j):0:45:@j', + }, + }, + 'Register(name=k):0:49:@k', + }, + }, + }, + }, + 'Register(name=l):0:54:@l', + }, + }, + }, + }}, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('NestingParenthesis', '('), + hl('UnaryPlus', '+'), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@k'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@l'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a)', 0, { + -- 012 + ast = { + { + 'Nested:0:2:', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '1:)', + msg = 'E15: Unexpected closing parenthesis: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('(@a', 0, { + -- 012 + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '3:(@a', + msg = 'E110: Missing closing parenthesis for nested expression: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + }) + check_parsing('@a(@b', 0, { + -- 01234 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + err = { + arg = '3:(@b', + msg = 'E116: Missing closing parenthesis for function call: %.*s', + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + }) + end) +end) From 7c97f783935ec122fbf0d7d070c00804738abd6a Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 11 Sep 2017 01:27:46 +0300 Subject: [PATCH 010/109] klee: Start preparing for klee tests First stage: something compiling without klee, but with a buch of dirty hacks - done. Second stage: something running under klee, able to emit useful results, but still using dirty hacks - done. Third stage: make CMake care about clang argumnets - not done, may be omitted if proves to be too hard. Not that klee can be run on CI in any case. --- src/nvim/lib/ringbuf.h | 27 +++ test/symbolic/klee/nvim/charset.c | 10 + test/symbolic/klee/nvim/garray.c | 195 +++++++++++++++++++ test/symbolic/klee/nvim/gettext.c | 4 + test/symbolic/klee/nvim/mbyte.c | 18 ++ test/symbolic/klee/nvim/memory.c | 90 +++++++++ test/symbolic/klee/run.sh | 69 +++++++ test/symbolic/klee/viml_expressions_lexer.c | 86 ++++++++ test/symbolic/klee/viml_expressions_parser.c | 102 ++++++++++ 9 files changed, 601 insertions(+) create mode 100644 test/symbolic/klee/nvim/charset.c create mode 100644 test/symbolic/klee/nvim/garray.c create mode 100644 test/symbolic/klee/nvim/gettext.c create mode 100644 test/symbolic/klee/nvim/mbyte.c create mode 100644 test/symbolic/klee/nvim/memory.c create mode 100755 test/symbolic/klee/run.sh create mode 100644 test/symbolic/klee/viml_expressions_lexer.c create mode 100644 test/symbolic/klee/viml_expressions_parser.c diff --git a/src/nvim/lib/ringbuf.h b/src/nvim/lib/ringbuf.h index 12b75ec65a..e63eae70b0 100644 --- a/src/nvim/lib/ringbuf.h +++ b/src/nvim/lib/ringbuf.h @@ -15,6 +15,7 @@ #ifndef NVIM_LIB_RINGBUF_H #define NVIM_LIB_RINGBUF_H +#include #include #include #include @@ -73,6 +74,32 @@ typedef struct { \ RBType *buf_end; \ } TypeName##RingBuffer; +/// Dummy item free macros, for use in RINGBUF_INIT +/// +/// This macros actually does nothing. +/// +/// @param[in] item Item to be freed. +#define RINGBUF_DUMMY_FREE(item) + +/// Static ring buffer +/// +/// @warning Ring buffers created with this macros must neither be freed nor +/// deallocated. +/// +/// @param scope Ring buffer scope. +/// @param TypeName Ring buffer type name. +/// @param RBType Type of the single ring buffer element. +/// @param varname Variable name. +/// @param rbsize Ring buffer size. +#define RINGBUF_STATIC(scope, TypeName, RBType, varname, rbsize) \ +static RBType _##varname##_buf[rbsize]; \ +scope TypeName##RingBuffer varname = { \ + .buf = _##varname##_buf, \ + .next = _##varname##_buf, \ + .first = NULL, \ + .buf_end = _##varname##_buf + rbsize - 1, \ +}; + /// Initialize a new ring buffer /// /// @param TypeName Ring buffer type name. Actual type name will be diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c new file mode 100644 index 0000000000..a40488920e --- /dev/null +++ b/test/symbolic/klee/nvim/charset.c @@ -0,0 +1,10 @@ +#include + +#include "nvim/ascii.h" +#include "nvim/macros.h" +#include "nvim/charset.h" + +bool vim_isIDc(int c) +{ + return ASCII_ISALNUM(c); +} diff --git a/test/symbolic/klee/nvim/garray.c b/test/symbolic/klee/nvim/garray.c new file mode 100644 index 0000000000..260870c3c2 --- /dev/null +++ b/test/symbolic/klee/nvim/garray.c @@ -0,0 +1,195 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +/// @file garray.c +/// +/// Functions for handling growing arrays. + +#include +#include + +#include "nvim/vim.h" +#include "nvim/ascii.h" +#include "nvim/log.h" +#include "nvim/memory.h" +#include "nvim/path.h" +#include "nvim/garray.h" +#include "nvim/strings.h" + +// #include "nvim/globals.h" +#include "nvim/memline.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "garray.c.generated.h" +#endif + +/// Clear an allocated growing array. +void ga_clear(garray_T *gap) +{ + xfree(gap->ga_data); + + // Initialize growing array without resetting itemsize or growsize + gap->ga_data = NULL; + gap->ga_maxlen = 0; + gap->ga_len = 0; +} + +/// Clear a growing array that contains a list of strings. +/// +/// @param gap +void ga_clear_strings(garray_T *gap) +{ + GA_DEEP_CLEAR_PTR(gap); +} + +/// Initialize a growing array. +/// +/// @param gap +/// @param itemsize +/// @param growsize +void ga_init(garray_T *gap, int itemsize, int growsize) +{ + gap->ga_data = NULL; + gap->ga_maxlen = 0; + gap->ga_len = 0; + gap->ga_itemsize = itemsize; + ga_set_growsize(gap, growsize); +} + +/// A setter for the growsize that guarantees it will be at least 1. +/// +/// @param gap +/// @param growsize +void ga_set_growsize(garray_T *gap, int growsize) +{ + if (growsize < 1) { + WLOG("trying to set an invalid ga_growsize: %d", growsize); + gap->ga_growsize = 1; + } else { + gap->ga_growsize = growsize; + } +} + +/// Make room in growing array "gap" for at least "n" items. +/// +/// @param gap +/// @param n +void ga_grow(garray_T *gap, int n) +{ + if (gap->ga_maxlen - gap->ga_len >= n) { + // the garray still has enough space, do nothing + return; + } + + if (gap->ga_growsize < 1) { + WLOG("ga_growsize(%d) is less than 1", gap->ga_growsize); + } + + // the garray grows by at least growsize + if (n < gap->ga_growsize) { + n = gap->ga_growsize; + } + int new_maxlen = gap->ga_len + n; + + size_t new_size = (size_t)(gap->ga_itemsize * new_maxlen); + size_t old_size = (size_t)(gap->ga_itemsize * gap->ga_maxlen); + + // reallocate and clear the new memory + char *pp = xrealloc(gap->ga_data, new_size); + memset(pp + old_size, 0, new_size - old_size); + + gap->ga_maxlen = new_maxlen; + gap->ga_data = pp; +} + +/// For a growing array that contains a list of strings: concatenate all the +/// strings with sep as separator. +/// +/// @param gap +/// @param sep +/// +/// @returns the concatenated strings +char_u *ga_concat_strings_sep(const garray_T *gap, const char *sep) + FUNC_ATTR_NONNULL_RET +{ + const size_t nelem = (size_t) gap->ga_len; + const char **strings = gap->ga_data; + + if (nelem == 0) { + return (char_u *) xstrdup(""); + } + + size_t len = 0; + for (size_t i = 0; i < nelem; i++) { + len += strlen(strings[i]); + } + + // add some space for the (num - 1) separators + len += (nelem - 1) * strlen(sep); + char *const ret = xmallocz(len); + + char *s = ret; + for (size_t i = 0; i < nelem - 1; i++) { + s = xstpcpy(s, strings[i]); + s = xstpcpy(s, sep); + } + strcpy(s, strings[nelem - 1]); + + return (char_u *) ret; +} + +/// For a growing array that contains a list of strings: concatenate all the +/// strings with a separating comma. +/// +/// @param gap +/// +/// @returns the concatenated strings +char_u* ga_concat_strings(const garray_T *gap) FUNC_ATTR_NONNULL_RET +{ + return ga_concat_strings_sep(gap, ","); +} + +/// Concatenate a string to a growarray which contains characters. +/// When "s" is NULL does not do anything. +/// +/// WARNING: +/// - Does NOT copy the NUL at the end! +/// - The parameter may not overlap with the growing array +/// +/// @param gap +/// @param s +void ga_concat(garray_T *gap, const char_u *restrict s) +{ + if (s == NULL) { + return; + } + + ga_concat_len(gap, (const char *restrict) s, strlen((char *) s)); +} + +/// Concatenate a string to a growarray which contains characters +/// +/// @param[out] gap Growarray to modify. +/// @param[in] s String to concatenate. +/// @param[in] len String length. +void ga_concat_len(garray_T *const gap, const char *restrict s, + const size_t len) + FUNC_ATTR_NONNULL_ALL +{ + if (len) { + ga_grow(gap, (int) len); + char *data = gap->ga_data; + memcpy(data + gap->ga_len, s, len); + gap->ga_len += (int) len; + } +} + +/// Append one byte to a growarray which contains bytes. +/// +/// @param gap +/// @param c +void ga_append(garray_T *gap, char c) +{ + GA_APPEND(char, gap, c); +} + diff --git a/test/symbolic/klee/nvim/gettext.c b/test/symbolic/klee/nvim/gettext.c new file mode 100644 index 0000000000..b9cc98d770 --- /dev/null +++ b/test/symbolic/klee/nvim/gettext.c @@ -0,0 +1,4 @@ +char *gettext(const char *s) +{ + return (char *)s; +} diff --git a/test/symbolic/klee/nvim/mbyte.c b/test/symbolic/klee/nvim/mbyte.c new file mode 100644 index 0000000000..394d17b700 --- /dev/null +++ b/test/symbolic/klee/nvim/mbyte.c @@ -0,0 +1,18 @@ +#include + +#include "nvim/types.h" +#include "nvim/mbyte.h" +#include "nvim/ascii.h" + +char_u *string_convert(const vimconv_T *conv, char_u *data, size_t *size) +{ + return NULL; +} + +int utfc_ptr2len_len(const char_u *p, int size) +{ + if (size < 1 || *p == NUL) { + return 0; + } + return 1; +} diff --git a/test/symbolic/klee/nvim/memory.c b/test/symbolic/klee/nvim/memory.c new file mode 100644 index 0000000000..7e924a410a --- /dev/null +++ b/test/symbolic/klee/nvim/memory.c @@ -0,0 +1,90 @@ +#include +#include + +#include "nvim/lib/ringbuf.h" + +enum { RB_SIZE = 1024 }; + +typedef struct { + void *ptr; + size_t size; +} AllocRecord; + +RINGBUF_TYPEDEF(AllocRecords, AllocRecord) +RINGBUF_INIT(AllocRecords, arecs, AllocRecord, RINGBUF_DUMMY_FREE) +RINGBUF_STATIC(static, AllocRecords, AllocRecord, arecs, RB_SIZE) + +size_t allocated_memory = 0; + +void *xmalloc(const size_t size) +{ + void *ret = malloc(size); + allocated_memory += size; + assert(arecs_rb_length(&arecs) < RB_SIZE); + arecs_rb_push(&arecs, (AllocRecord) { + .ptr = ret, + .size = size, + }); + return ret; +} + +void xfree(void *const p) +{ + RINGBUF_FORALL(&arecs, AllocRecord, arec) { + if (arec->ptr == p) { + allocated_memory -= arec->size; + arecs_rb_remove(&arecs, arecs_rb_find_idx(&arecs, arec)); + return; + } + } + assert(false); +} + +void *xrealloc(void *const p, size_t new_size) +{ + void *ret = realloc(p, new_size); + RINGBUF_FORALL(&arecs, AllocRecord, arec) { + if (arec->ptr == p) { + allocated_memory -= arec->size; + allocated_memory += new_size; + arec->ptr = ret; + arec->size = new_size; + return ret; + } + } + assert(false); + return (void *)(intptr_t)1; +} + +char *xstrdup(const char *str) + FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_RET + FUNC_ATTR_NONNULL_ALL +{ + return xmemdupz(str, strlen(str)); +} + +void *xmallocz(size_t size) + FUNC_ATTR_MALLOC FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT +{ + size_t total_size = size + 1; + assert(total_size > size); + + void *ret = xmalloc(total_size); + ((char *)ret)[size] = 0; + + return ret; +} + +char *xstpcpy(char *restrict dst, const char *restrict src) + FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL +{ + const size_t len = strlen(src); + return (char *)memcpy(dst, src, len + 1) + len; +} + +void *xmemdupz(const void *data, size_t len) + FUNC_ATTR_MALLOC FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT + FUNC_ATTR_NONNULL_ALL +{ + return memcpy(xmallocz(len), data, len); +} diff --git a/test/symbolic/klee/run.sh b/test/symbolic/klee/run.sh new file mode 100755 index 0000000000..388903c234 --- /dev/null +++ b/test/symbolic/klee/run.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +set -e +set -x +test -z "$POSH_VERSION" && set -u + +PROJECT_SOURCE_DIR=. +PROJECT_BINARY_DIR="$PROJECT_SOURCE_DIR/build" +KLEE_TEST_DIR="$PROJECT_SOURCE_DIR/test/symbolic/klee" +KLEE_BIN_DIR="$PROJECT_BINARY_DIR/klee" +KLEE_OUT_DIR="$KLEE_BIN_DIR/out" + +main() { + local compile= + if test "$1" = "-c" ; then + compile=1 + shift + fi + local test="$1" ; shift + + local includes= + includes="$includes -I$KLEE_TEST_DIR" + includes="$includes -I/home/klee/klee_src/include" + includes="$includes -I$PROJECT_SOURCE_DIR/src" + includes="$includes -I$PROJECT_BINARY_DIR/src/nvim/auto" + includes="$includes -I$PROJECT_BINARY_DIR/include" + includes="$includes -I$PROJECT_BINARY_DIR/config" + includes="$includes -I/host-includes" + + local defines= + defines="$defines -DMIN_LOG_LEVEL=9999" + defines="$defines -DINCLUDE_GENERATED_DECLARATIONS" + + test -z "$compile" && defines="$defines -DUSE_KLEE" + + test -d "$KLEE_BIN_DIR" || mkdir -p "$KLEE_BIN_DIR" + + if test -z "$compile" ; then + test -d "$KLEE_OUT_DIR" && rm -r "$KLEE_OUT_DIR" + local line1='cd /image' + line1="$line1 && $(echo clang \ + $includes $defines \ + -o "$KLEE_BIN_DIR/a.bc" -emit-llvm -g -c \ + "$KLEE_TEST_DIR/$test.c")" + line1="$line1 && klee --libc=uclibc --posix-runtime " + line1="$line1 '--output-dir=$KLEE_OUT_DIR' '$KLEE_BIN_DIR/a.bc'" + local line2="for t in '$KLEE_OUT_DIR'/*.err" + line2="$line2 ; do ktest-tool --write-ints" + line2="$line2 \"\$(printf '%s' \"\$t\" | sed -e 's@.[^/]*\$@.out@')\"" + line2="$line2 ; done" + printf '%s\n%s\n' "$line1" "$line2" | \ + docker run \ + --volume "$(cd "$PROJECT_SOURCE_DIR" && pwd)":/image \ + --volume "/usr/include":/host-includes \ + --interactive \ + --rm \ + --ulimit='stack=-1:-1' \ + klee/klee \ + /bin/sh -x + else + clang \ + $includes $defines \ + -o "$KLEE_BIN_DIR/$test" \ + -O0 -g \ + "$KLEE_TEST_DIR/$test.c" + fi +} + +main "$@" diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c new file mode 100644 index 0000000000..e3b5aa80cc --- /dev/null +++ b/test/symbolic/klee/viml_expressions_lexer.c @@ -0,0 +1,86 @@ +#ifdef USE_KLEE +# include +#else +# include +#endif +#include +#include +#include + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/mbyte.h" + +#include "nvim/memory.c" +#include "nvim/mbyte.c" +#include "nvim/charset.c" +#include "nvim/garray.c" +#include "nvim/gettext.c" +#include "nvim/viml/parser/expressions.c" + +#define INPUT_SIZE 7 + +uint8_t avoid_optimizing_out; + +void simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} + +int main(const int argc, const char *const *const argv, + const char *const *const environ) +{ + char input[INPUT_SIZE]; + uint8_t shift; + const bool peek = false; + avoid_optimizing_out = argc; + +#ifdef USE_KLEE + klee_make_symbolic(input, sizeof(input), "input"); + klee_make_symbolic(&shift, sizeof(shift), "shift"); + klee_assume(shift < INPUT_SIZE); +#endif + + ParserLine plines[] = { + { +#ifdef USE_KLEE + .data = &input[shift], + .size = sizeof(input) - shift, +#else + .data = (const char *)&argv[1], + .size = strlen(argv[1]), +#endif + .allocated = false, + }, + { + .data = NULL, + .size = 0, + .allocated = false, + }, + }; +#ifdef USE_KLEE + assert(plines[0].size <= INPUT_SIZE); + assert((plines[0].data[0] != 5) | (plines[0].data[0] != argc)); +#endif + ParserLine *cur_pline = &plines[0]; + + ParserState pstate = { + .reader = { + .get_line = simple_get_line, + .cookie = &cur_pline, + .lines = KV_INITIAL_VALUE, + .conv.vc_type = CONV_NONE, + }, + .pos = { 0, 0 }, + .colors = NULL, + .can_continuate = false, + }; + kvi_init(pstate.reader.lines); + + LexExprToken token = viml_pexpr_next_token(&pstate, peek); + assert((pstate.pos.line == 0) + ? (pstate.pos.col > 0) + : (pstate.pos.line == 1 && pstate.pos.col == 0)); +} diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c new file mode 100644 index 0000000000..2bad1adc53 --- /dev/null +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -0,0 +1,102 @@ +#ifdef USE_KLEE +# error UNFINISHED +# include +#else +# include +#endif +#include +#include +#include + +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/mbyte.h" + +#include "nvim/memory.c" +#include "nvim/mbyte.c" +#include "nvim/charset.c" +#include "nvim/garray.c" +#include "nvim/gettext.c" +#include "nvim/viml/parser/expressions.c" + +#define INPUT_SIZE 50 + +uint8_t avoid_optimizing_out; + +void simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} + +int main(const int argc, const char *const *const argv, + const char *const *const environ) +{ + char input[INPUT_SIZE]; + uint8_t shift; + int flags; + const bool peek = false; + avoid_optimizing_out = argc; + +#ifndef USE_KLEE + sscanf(argv[2], "%d", &flags); +#endif + +#ifdef USE_KLEE + klee_make_symbolic(input, sizeof(input), "input"); + klee_make_symbolic(&shift, sizeof(shift), "shift"); + klee_make_symbolic(&flags, sizeof{flags}, "flags"); + klee_assume(shift < INPUT_SIZE); + klee_assume( + flags <= kExprFlagsMulti|kExprFlagsDisallowEOC|kExprFlagsPrintError); +#endif + + ParserLine plines[] = { + { +#ifdef USE_KLEE + .data = &input[shift], + .size = sizeof(input) - shift, +#else + .data = argv[1], + .size = strlen(argv[1]), +#endif + .allocated = false, + }, + { + .data = NULL, + .size = 0, + .allocated = false, + }, + }; +#ifdef USE_KLEE + assert(plines[0].size <= INPUT_SIZE); + assert((plines[0].data[0] != 5) | (plines[0].data[0] != argc)); +#endif + ParserLine *cur_pline = &plines[0]; + + ParserState pstate = { + .reader = { + .get_line = simple_get_line, + .cookie = &cur_pline, + .lines = KV_INITIAL_VALUE, + .conv.vc_type = CONV_NONE, + }, + .pos = { 0, 0 }, + .colors = NULL, + .can_continuate = false, + }; + kvi_init(pstate.reader.lines); + + const ExprAST ast = viml_pexpr_parse(&pstate, flags); + assert(ast.root != NULL + || plines[0].size == 0); + assert(ast.root != NULL || !ast.correct); + assert(ast.correct + || (ast.err.msg != NULL + && ast.err.arg != NULL + && ast.err.arg >= plines[0].data + && ((size_t)(ast.err.arg - plines[0].data) + ast.err.arg_len + <= plines[0].size))); + // FIXME: free memory and assert no memory leaks +} From 7980614650f0aedb39bf88466e5bd3ce90429cc1 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 17 Sep 2017 17:33:03 +0300 Subject: [PATCH 011/109] viml/parser/expressions: Add support for figure braces (three kinds) --- src/nvim/viml/parser/expressions.c | 644 +++++++++++-- src/nvim/viml/parser/expressions.h | 35 + test/unit/viml/expressions/parser_spec.lua | 1018 +++++++++++++++++++- 3 files changed, 1596 insertions(+), 101 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index b54f2eb237..f4cfed3113 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -20,10 +20,22 @@ typedef kvec_withinit_t(ExprASTNode **, 16) ExprASTStack; +/// Which nodes may be wanted typedef enum { - kELvlOperator, ///< Operators: function call, subscripts, binary operators, … - kELvlValue, ///< Actual value: literals, variables, nested expressions. -} ExprASTLevel; + /// Operators: function call, subscripts, binary operators, … + /// + /// For unrestricted expressions. + kENodeOperator, + /// Values: literals, variables, nested expressions, unary operators. + /// + /// For unrestricted expressions as well, implies that top item in AST stack + /// points to NULL. + kENodeValue, + /// Argument: only allows simple argument names. + kENodeArgument, + /// Argument separator: only allows commas. + kENodeArgumentSeparator, +} ExprASTWantedNode; #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.c.generated.h" @@ -456,6 +468,8 @@ viml_pexpr_next_token_adv_return: // // Used highlighting groups and assumed linkage: // +// NVimInternalError -> highlight as fg:red/bg:red +// // NVimInvalid -> Error // NVimInvalidValue -> NVimInvalid // NVimInvalidOperator -> NVimInvalid @@ -469,11 +483,33 @@ viml_pexpr_next_token_adv_return: // // NVimParenthesis -> Delimiter // +// NVimComma -> Delimiter +// NVimArrow -> Delimiter +// +// NVimLambda -> Delimiter +// NVimDict -> Delimiter +// NVimCurly -> Delimiter +// +// NVimIdentifier -> Identifier +// NVimIdentifierScope -> NVimIdentifier +// NVimIdentifierScopeDelimiter -> NVimIdentifier +// +// NVimFigureBrace -> NVimInternalError +// +// NVimInvalidComma -> NVimInvalidDelimiter // NVimInvalidSpacing -> NVimInvalid // NVimInvalidTernaryOperator -> NVimInvalidOperator // NVimInvalidRegister -> NVimInvalidValue // NVimInvalidClosingBracket -> NVimInvalidDelimiter // NVimInvalidSpacing -> NVimInvalid +// NVimInvalidArrow -> NVimInvalidDelimiter +// NVimInvalidLambda -> NVimInvalidDelimiter +// NVimInvalidDict -> NVimInvalidDelimiter +// NVimInvalidCurly -> NVimInvalidDelimiter +// NVimInvalidFigureBrace -> NVimInvalidDelimiter +// NVimInvalidIdentifier -> NVimInvalidValue +// NVimInvalidIdentifierScope -> NVimInvalidValue +// NVimInvalidIdentifierScopeDelimiter -> NVimInvalidValue // // NVimUnaryPlus -> NVimUnaryOperator // NVimBinaryPlus -> NVimBinaryOperator @@ -498,6 +534,9 @@ static inline ExprASTNode *viml_pexpr_new_node(const ExprASTNodeType type) typedef enum { kEOpLvlInvalid = 0, kEOpLvlParens, + kEOpLvlArrow, + kEOpLvlComma, + kEOpLvlColon, kEOpLvlTernary, kEOpLvlOr, kEOpLvlAnd, @@ -506,6 +545,7 @@ typedef enum { kEOpLvlMultiplication, ///< Multiplication, division and modulo. kEOpLvlUnary, ///< Unary operations: not, minus, plus. kEOpLvlSubscript, ///< Subscripts. + kEOpLvlComplexIdentifier, ///< Plain identifier, curly braces name. kEOpLvlValue, ///< Values: literals, variables, nested expressions, … } ExprOpLvl; @@ -520,7 +560,16 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeOpMissing] = kEOpLvlMultiplication, [kExprNodeNested] = kEOpLvlParens, - [kExprNodeComplexIdentifier] = kEOpLvlParens, + + [kExprNodeUnknownFigure] = kEOpLvlParens, + [kExprNodeLambda] = kEOpLvlParens, + [kExprNodeDictLiteral] = kEOpLvlParens, + + [kExprNodeArrow] = kEOpLvlArrow, + + [kExprNodeComma] = kEOpLvlComma, + + [kExprNodeColon] = kEOpLvlColon, [kExprNodeTernary] = kEOpLvlTernary, @@ -531,9 +580,12 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeSubscript] = kEOpLvlSubscript, [kExprNodeCall] = kEOpLvlSubscript, + [kExprNodeComplexIdentifier] = kEOpLvlComplexIdentifier, + [kExprNodePlainIdentifier] = kEOpLvlComplexIdentifier, + [kExprNodeCurlyBracesIdentifier] = kEOpLvlComplexIdentifier, + [kExprNodeRegister] = kEOpLvlValue, [kExprNodeListLiteral] = kEOpLvlValue, - [kExprNodePlainIdentifier] = kEOpLvlValue, }; static const ExprOpAssociativity node_type_to_op_ass[] = { @@ -541,7 +593,24 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeOpMissing] = kEOpAssNo, [kExprNodeNested] = kEOpAssNo, - [kExprNodeComplexIdentifier] = kEOpAssLeft, + + [kExprNodeUnknownFigure] = kEOpAssLeft, + [kExprNodeLambda] = kEOpAssNo, + [kExprNodeDictLiteral] = kEOpAssNo, + + // Does not really matter. + [kExprNodeArrow] = kEOpAssNo, + + [kExprNodeColon] = kEOpAssNo, + + // Right associativity for comma because this means easier access to arguments + // list, etc: for "[a, b, c, d]" you can access "a" in one step if it is + // represented as "list(comma(a, comma(b, comma(c, d))))" then if it is + // "list(comma(comma(comma(a, b), c), d))" in which case you will need to + // traverse all three comma() structures. And with comma operator (including + // actual comma operator from C which is not present in VimL) nobody cares + // about associativity, only about order of execution. + [kExprNodeComma] = kEOpAssRight, [kExprNodeTernary] = kEOpAssNo, @@ -552,14 +621,31 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeSubscript] = kEOpAssLeft, [kExprNodeCall] = kEOpAssLeft, + [kExprNodePlainIdentifier] = kEOpAssLeft, + [kExprNodeComplexIdentifier] = kEOpAssLeft, + [kExprNodeCurlyBracesIdentifier] = kEOpAssLeft, + [kExprNodeRegister] = kEOpAssNo, [kExprNodeListLiteral] = kEOpAssNo, - [kExprNodePlainIdentifier] = kEOpAssNo, }; #ifdef UNIT_TESTING #include REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_ast_node( + const ExprASTNode *const *const eastnode_p, + const char *const prefix) +{ + if (*eastnode_p == NULL) { + fprintf(stderr, "%s %p : NULL\n", prefix, (void *)eastnode_p); + } else { + fprintf(stderr, "%s %p : %p : %c : %zu:%zu:%zu\n", + prefix, (void *)eastnode_p, (void *)(*eastnode_p), + (*eastnode_p)->type, (*eastnode_p)->start.line, + (*eastnode_p)->start.col, (*eastnode_p)->len); + } +} +REAL_FATTR_UNUSED static inline void viml_pexpr_debug_print_ast_stack( const ExprASTStack *const ast_stack, const char *const msg) @@ -567,22 +653,17 @@ static inline void viml_pexpr_debug_print_ast_stack( { fprintf(stderr, "\n%sstack: %zu:\n", msg, kv_size(*ast_stack)); for (size_t i = 0; i < kv_size(*ast_stack); i++) { - const ExprASTNode *const *const eastnode_p = ( - (const ExprASTNode *const *)kv_A(*ast_stack, i)); - if (*eastnode_p == NULL) { - fprintf(stderr, "- %p : NULL\n", (void *)eastnode_p); - } else { - fprintf(stderr, "- %p : %p : %c : %zu:%zu:%zu\n", - (void *)eastnode_p, (void *)(*eastnode_p), (*eastnode_p)->type, - (*eastnode_p)->start.line, (*eastnode_p)->start.col, - (*eastnode_p)->len); - } + viml_pexpr_debug_print_ast_node( + (const ExprASTNode *const *)kv_A(*ast_stack, i), + "-"); } } #define PSTACK(msg) \ viml_pexpr_debug_print_ast_stack(&ast_stack, #msg) #define PSTACK_P(msg) \ viml_pexpr_debug_print_ast_stack(ast_stack, #msg) +#define PNODE_P(eastnode_p, msg) \ + viml_pexpr_debug_print_ast_node((const ExprASTNode *const *)ast_stack, #msg) #endif /// Handle binary operator @@ -590,7 +671,7 @@ static inline void viml_pexpr_debug_print_ast_stack( /// This function is responsible for handling priority levels as well. static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, ExprASTNode *const bop_node, - ExprASTLevel *const want_level_p) + ExprASTWantedNode *const want_node_p) FUNC_ATTR_NONNULL_ALL { ExprASTNode **top_node_p = NULL; @@ -599,6 +680,9 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, ExprOpAssociativity top_node_ass; assert(kv_size(*ast_stack)); const ExprOpLvl bop_node_lvl = node_type_to_op_lvl[bop_node->type]; +#ifndef NDEBUG + const ExprOpAssociativity bop_node_ass = node_type_to_op_ass[bop_node->type]; +#endif do { ExprASTNode **new_top_node_p = kv_last(*ast_stack); ExprASTNode *new_top_node = *new_top_node_p; @@ -606,6 +690,8 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, const ExprOpLvl new_top_node_lvl = node_type_to_op_lvl[new_top_node->type]; const ExprOpAssociativity new_top_node_ass = ( node_type_to_op_ass[new_top_node->type]); + assert(bop_node_lvl != new_top_node_lvl + || bop_node_ass == new_top_node_ass); if (top_node_p != NULL && ((bop_node_lvl > new_top_node_lvl || (bop_node_lvl == new_top_node_lvl @@ -617,14 +703,60 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, top_node = new_top_node; top_node_lvl = new_top_node_lvl; top_node_ass = new_top_node_ass; + if (bop_node_lvl == top_node_lvl && top_node_ass == kEOpAssRight) { + break; + } } while (kv_size(*ast_stack)); - // FIXME Handle right and no associativity correctly - *top_node_p = bop_node; - bop_node->children = top_node; - assert(bop_node->children->next == NULL); - kvi_push(*ast_stack, top_node_p); - kvi_push(*ast_stack, &bop_node->children->next); - *want_level_p = kELvlValue; + // FIXME: Handle no associativity + if (top_node_ass == kEOpAssLeft || top_node_lvl != bop_node_lvl) { + // outer(op(x,y)) -> outer(new_op(op(x,y),*)) + // + // Before: top_node_p = outer(*), points to op(x,y) + // Other stack elements unknown + // + // After: top_node_p = outer(*), points to new_op(op(x,y)) + // &bop_node->children->next = new_op(op(x,y),*), points to NULL + *top_node_p = bop_node; + bop_node->children = top_node; + assert(bop_node->children->next == NULL); + kvi_push(*ast_stack, top_node_p); + kvi_push(*ast_stack, &bop_node->children->next); + } else { + assert(top_node_lvl == bop_node_lvl && top_node_ass == kEOpAssRight); + assert(top_node->children != NULL && top_node->children->next != NULL); + // outer(op(x,y)) -> outer(op(x,new_op(y,*))) + // + // Before: top_node_p = outer(*), points to op(x,y) + // Other stack elements unknown + // + // After: top_node_p = outer(*), points to op(x,new_op(y)) + // &top_node->children->next = op(x,*), points to new_op(y) + // &bop_node->children->next = new_op(y,*), points to NULL + bop_node->children = top_node->children->next; + top_node->children->next = bop_node; + assert(bop_node->children->next == NULL); + kvi_push(*ast_stack, top_node_p); + kvi_push(*ast_stack, &top_node->children->next); + kvi_push(*ast_stack, &bop_node->children->next); + } + *want_node_p = (*want_node_p == kENodeArgumentSeparator + ? kENodeArgument + : kENodeValue); +} + +/// ParserPosition literal based on ParserPosition pos with columns shifted +/// +/// Function does not check whether remaining position is valid. +/// +/// @param[in] pos Position to shift. +/// @param[in] shift Number of bytes to shift. +/// +/// @return Shifted position. +static inline ParserPosition shifted_pos(const ParserPosition pos, + const size_t shift) + FUNC_ATTR_CONST FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT +{ + return (ParserPosition) { .line = pos.line, .col = pos.col + shift }; } /// Get highlight group name @@ -692,12 +824,25 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, ERROR_FROM_TOKEN_AND_MSG(cur_token, _("E15: Missing operator: %.*s")); \ NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOpMissing); \ cur_node->len = 0; \ - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_level); \ - is_invalid = true; \ + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); \ goto viml_pexpr_parse_process_token; \ } \ } while (0) +/// Record missing value: for things like "* 5" +/// +/// @param[in] msg Error message. +#define ADD_VALUE_IF_MISSING(msg) \ + do { \ + if (want_node == kENodeValue) { \ + ERROR_FROM_TOKEN_AND_MSG(cur_token, (msg)); \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); \ + cur_node->len = 0; \ + *top_node_p = cur_node; \ + want_node = kENodeOperator; \ + } \ + } while (0) + /// Set AST error, unless AST already is not correct /// /// @param[out] ret_ast AST to set error in. @@ -721,14 +866,42 @@ static inline void east_set_error(ExprAST *const ret_ast, ret_ast->err.arg = pline.data + start.col; } -/// Set error from the given kExprLexInvalid token and given message +/// Set error from the given token and given message #define ERROR_FROM_TOKEN_AND_MSG(cur_token, msg) \ - east_set_error(&ast, pstate, msg, cur_token.start) + do { \ + is_invalid = true; \ + east_set_error(&ast, pstate, msg, cur_token.start); \ + } while (0) + +/// Like #ERROR_FROM_TOKEN_AND_MSG, but gets position from a node +#define ERROR_FROM_NODE_AND_MSG(node, msg) \ + do { \ + is_invalid = true; \ + east_set_error(&ast, pstate, msg, node->start); \ + } while (0) /// Set error from the given kExprLexInvalid token #define ERROR_FROM_TOKEN(cur_token) \ ERROR_FROM_TOKEN_AND_MSG(cur_token, cur_token.data.err.msg) +/// Select figure brace type, altering highlighting as well if needed +/// +/// @param[out] node Node to modify type. +/// @param[in] new_type New type, one of ExprASTNodeType values without +/// kExprNode prefix. +/// @param[in] hl Corresponding highlighting, passed as an argument to #HL. +#define SELECT_FIGURE_BRACE_TYPE(node, new_type, hl) \ + do { \ + ExprASTNode *const node_ = (node); \ + assert(node_->type == kExprNodeUnknownFigure \ + || node_->type == kExprNode##new_type); \ + node_->type = kExprNode##new_type; \ + if (pstate->colors) { \ + kv_A(*pstate->colors, node_->data.fig.opening_hl_idx).group = \ + HL(hl); \ + } \ + } while (0) + /// Parse one VimL expression /// /// @param pstate Parser state. @@ -751,13 +924,15 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) kvi_init(ast_stack); kvi_push(ast_stack, &ast.root); // Expressions stack: - // 1. *last is NULL if want_level is kExprLexValue. Indicates where expression + // 1. *last is NULL if want_node is kExprLexValue. Indicates where expression // is to be put. // 2. *last is not NULL otherwise, indicates current expression to be used as // an operator argument. - ExprASTLevel want_level = kELvlValue; + ExprASTWantedNode want_node = kENodeValue; LexExprToken prev_token = { .type = kExprLexMissing }; bool highlighted_prev_spacing = false; + // Lambda node, valid when parsing lambda arguments only. + ExprASTNode *lambda_node = NULL; do { LexExprToken cur_token = viml_pexpr_next_token(pstate, true); if (cur_token.type == kExprLexEOC) { @@ -789,8 +964,7 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) viml_pexpr_parse_process_token: if (tok_type == kExprLexSpacing) { if (is_invalid) { - viml_parser_highlight(pstate, cur_token.start, cur_token.len, - HL(Spacing)); + HL_CUR_TOKEN(Spacing); } else { // Do not do anything: let regular spacing be highlighted as normal. // This also allows later to highlight spacing as invalid. @@ -803,11 +977,44 @@ viml_pexpr_parse_process_token: is_invalid = false; highlighted_prev_spacing = true; } + const ParserLine pline = pstate->reader.lines.items[cur_token.start.line]; ExprASTNode **const top_node_p = kv_last(ast_stack); ExprASTNode *cur_node = NULL; - // Keep these two asserts separate for debugging purposes. - assert(want_level == kELvlValue || *top_node_p != NULL); - assert(want_level != kELvlValue || *top_node_p == NULL); + assert((want_node == kENodeValue || want_node == kENodeArgument) + == (*top_node_p == NULL)); + if ((want_node == kENodeArgumentSeparator + && tok_type != kExprLexComma + && tok_type != kExprLexArrow) + || (want_node == kENodeArgument + && !(tok_type == kExprLexPlainIdentifier + && cur_token.data.var.scope == 0 + && !cur_token.data.var.autoload) + && tok_type != kExprLexArrow)) { + lambda_node->data.fig.type_guesses.allow_lambda = false; + if (lambda_node->children != NULL + && lambda_node->children->type == kExprNodeComma) { + // If lambda has comma child this means that parser has already seen at + // least "{arg1,", so node cannot possibly be anything, but lambda. + + // Vim may give E121 or E720 in this case, but it does not look right to + // have either because both are results of reevaluation possibly-lambda + // node as a dictionary and here this is not going to happen. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected lambda arguments list or arrow: %.*s")); + } else { + // Else it may appear that possibly-lambda node is actually a dictionary + // or curly-braces-name identifier. + lambda_node = NULL; + if (want_node == kENodeArgumentSeparator) { + want_node = kENodeOperator; + } else { + want_node = kENodeValue; + } + } + } + assert(lambda_node == NULL + || want_node == kENodeArgumentSeparator + || want_node == kENodeArgument); switch (tok_type) { case kExprLexEOC: { assert(false); @@ -818,13 +1025,12 @@ viml_pexpr_parse_process_token: goto viml_pexpr_parse_process_token; } case kExprLexRegister: { - if (want_level == kELvlValue) { + if (want_node == kENodeValue) { NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeRegister); cur_node->data.reg.name = cur_token.data.reg.name; *top_node_p = cur_node; - want_level = kELvlOperator; - viml_parser_highlight(pstate, cur_token.start, cur_token.len, - HL(Register)); + want_node = kENodeOperator; + HL_CUR_TOKEN(Register); } else { // Register in operator position: e.g. @a @a OP_MISSING; @@ -832,23 +1038,343 @@ viml_pexpr_parse_process_token: break; } case kExprLexPlus: { - if (want_level == kELvlValue) { + if (want_node == kENodeValue) { // Value level: assume unary plus NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnaryPlus); *top_node_p = cur_node; kvi_push(ast_stack, &cur_node->children); HL_CUR_TOKEN(UnaryPlus); - } else if (want_level < kELvlValue) { + } else { NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeBinaryPlus); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_level); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); HL_CUR_TOKEN(BinaryPlus); } - want_level = kELvlValue; + want_node = kENodeValue; + break; + } + case kExprLexComma: { + assert(want_node != kENodeArgument); + if (want_node == kENodeValue) { + // Value level: comma appearing here is not valid. + // Note: in Vim string(,x) will give E116, this is not the case here. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected value, got comma: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); + cur_node->len = 0; + *top_node_p = cur_node; + want_node = (want_node == kENodeArgument + ? kENodeArgumentSeparator + : kENodeOperator); + } + if (want_node == kENodeArgumentSeparator) { + assert(lambda_node->data.fig.type_guesses.allow_lambda); + assert(lambda_node != NULL); + SELECT_FIGURE_BRACE_TYPE(lambda_node, Lambda, Lambda); + } + if (kv_size(ast_stack) < 2) { + goto viml_pexpr_parse_invalid_comma; + } + for (size_t i = 1; i < kv_size(ast_stack); i++) { + const ExprASTNode *const *const eastnode_p = + (const ExprASTNode *const *)kv_Z(ast_stack, i); + if (!((*eastnode_p)->type == kExprNodeComma + || ((*eastnode_p)->type == kExprNodeColon + && i == 1)) + || i == kv_size(ast_stack) - 1) { + switch ((*eastnode_p)->type) { + case kExprNodeLambda: { + assert(want_node == kENodeArgumentSeparator); + break; + } + case kExprNodeDictLiteral: + case kExprNodeListLiteral: + case kExprNodeCall: { + break; + } + default: { +viml_pexpr_parse_invalid_comma: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Comma outside of call, lambda or literal: %.*s")); + break; + } + } + break; + } + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComma); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + HL_CUR_TOKEN(Comma); + break; + } + case kExprLexColon: { + ADD_VALUE_IF_MISSING(_("E15: Expected value, got colon: %.*s")); + if (kv_size(ast_stack) < 2) { + goto viml_pexpr_parse_invalid_colon; + } + for (size_t i = 1; i < kv_size(ast_stack); i++) { + ExprASTNode *const *const eastnode_p = + (ExprASTNode *const *)kv_Z(ast_stack, i); + if ((*eastnode_p)->type != kExprNodeColon + || i == kv_size(ast_stack) - 1) { + switch ((*eastnode_p)->type) { + case kExprNodeUnknownFigure: { + SELECT_FIGURE_BRACE_TYPE((*eastnode_p), DictLiteral, Dict); + break; + } + case kExprNodeComma: + case kExprNodeDictLiteral: + case kExprNodeTernary: { + break; + } + default: { +viml_pexpr_parse_invalid_colon: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Colon outside of dictionary or ternary operator: " + "%.*s")); + break; + } + } + break; + } + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + // FIXME: Handle ternary operator. + HL_CUR_TOKEN(Colon); + want_node = kENodeValue; + break; + } + case kExprLexFigureBrace: { + if (cur_token.data.brc.closing) { + ExprASTNode **new_top_node_p = NULL; + // Always drop the topmost value: + // + // 1. When want_node != kENodeValue topmost item on stack is + // a *finished* left operand, which may as well be "{@a}" which + // needs not be finished again. + // 2. Otherwise it is pointing to NULL what nobody wants. + kv_drop(ast_stack, 1); + if (!kv_size(ast_stack)) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnknownFigure); + cur_node->data.fig.type_guesses.allow_lambda = false; + cur_node->data.fig.type_guesses.allow_dict = false; + cur_node->data.fig.type_guesses.allow_ident = false; + cur_node->len = 0; + if (want_node != kENodeValue) { + cur_node->children = *top_node_p; + } + *top_node_p = cur_node; + goto viml_pexpr_parse_figure_brace_closing_error; + } + if (want_node == kENodeValue) { + if ((*kv_last(ast_stack))->type != kExprNodeUnknownFigure + && (*kv_last(ast_stack))->type != kExprNodeComma) { + // kv_last being UnknownFigure may occur for empty dictionary + // literal, while Comma is expected in case of non-empty one. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected value, got closing figure brace: %.*s")); + } + } else { + if (!kv_size(ast_stack)) { + new_top_node_p = top_node_p; + goto viml_pexpr_parse_figure_brace_closing_error; + } + } + do { + new_top_node_p = kv_pop(ast_stack); + } while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeUnknownFigure + && (*new_top_node_p)->type != kExprNodeDictLiteral + && ((*new_top_node_p)->type + != kExprNodeCurlyBracesIdentifier) + && (*new_top_node_p)->type != kExprNodeLambda))); + ExprASTNode *new_top_node = *new_top_node_p; + switch (new_top_node->type) { + case kExprNodeUnknownFigure: { + if (new_top_node->children == NULL) { + // No children of curly braces node indicates empty dictionary. + + // Should actually be kENodeArgument, but that was changed + // earlier. + assert(want_node == kENodeValue); + assert(new_top_node->data.fig.type_guesses.allow_dict); + SELECT_FIGURE_BRACE_TYPE(new_top_node, DictLiteral, Dict); + HL_CUR_TOKEN(Dict); + } else if (new_top_node->data.fig.type_guesses.allow_ident) { + SELECT_FIGURE_BRACE_TYPE(new_top_node, CurlyBracesIdentifier, + Curly); + HL_CUR_TOKEN(Curly); + } else { + // If by this time type of the node has not already been + // guessed, but it definitely is not a curly braces name then + // it is invalid for sure. + ERROR_FROM_NODE_AND_MSG( + new_top_node, + _("E15: Don't know what figure brace means: %.*s")); + if (pstate->colors) { + // Will reset to NVimInvalidFigureBrace. + kv_A(*pstate->colors, + new_top_node->data.fig.opening_hl_idx).group = ( + HL(FigureBrace)); + } + HL_CUR_TOKEN(FigureBrace); + } + break; + } + case kExprNodeDictLiteral: { + HL_CUR_TOKEN(Dict); + break; + } + case kExprNodeCurlyBracesIdentifier: { + HL_CUR_TOKEN(Curly); + break; + } + case kExprNodeLambda: { + HL_CUR_TOKEN(Lambda); + break; + } + default: { +viml_pexpr_parse_figure_brace_closing_error: + assert(!kv_size(ast_stack)); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing figure brace: %.*s")); + HL_CUR_TOKEN(FigureBrace); + break; + } + } + kvi_push(ast_stack, new_top_node_p); + want_node = kENodeOperator; + } else { + if (want_node == kENodeValue) { + HL_CUR_TOKEN(FigureBrace); + // Value: may be any of lambda, dictionary literal and curly braces + // name. + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnknownFigure); + cur_node->data.fig.type_guesses.allow_lambda = true; + cur_node->data.fig.type_guesses.allow_dict = true; + cur_node->data.fig.type_guesses.allow_ident = true; + if (pstate->colors) { + cur_node->data.fig.opening_hl_idx = kv_size(*pstate->colors) - 1; + } + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + want_node = kENodeArgument; + lambda_node = cur_node; + } else { + // Operator: may only be curly braces name, but only under certain + // conditions. + + // First condition is that there is no space before {. + if (prev_token.type == kExprLexSpacing) { + OP_MISSING; + } + switch ((*top_node_p)->type) { + // Second is that previous node is one of the identifiers: + // complex, plain, curly braces. + + // TODO(ZyX-I): Extend syntax to allow ${expr}. This is needed to + // handle environment variables like those bash uses for + // `export -f`: their names consist not only of alphanumeric + // characetrs. + case kExprNodeComplexIdentifier: + case kExprNodePlainIdentifier: + case kExprNodeCurlyBracesIdentifier: { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComplexIdentifier); + cur_node->len = 0; + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + ExprASTNode *const new_top_node = *kv_last(ast_stack); + assert(new_top_node->next == NULL); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCurlyBracesIdentifier); + new_top_node->next = cur_node; + kvi_push(ast_stack, &cur_node->children); + HL_CUR_TOKEN(Curly); + break; + } + default: { + OP_MISSING; + break; + } + } + } + } + break; + } + case kExprLexArrow: { + if (want_node == kENodeArgumentSeparator + || want_node == kENodeArgument) { + if (want_node == kENodeArgument) { + kv_drop(ast_stack, 1); + } + assert(kv_size(ast_stack) >= 1); + while ((*kv_last(ast_stack))->type != kExprNodeLambda + && (*kv_last(ast_stack))->type != kExprNodeUnknownFigure) { + kv_drop(ast_stack, 1); + } + assert((*kv_last(ast_stack)) == lambda_node); + SELECT_FIGURE_BRACE_TYPE(lambda_node, Lambda, Lambda); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); + if (lambda_node->children == NULL) { + assert(want_node == kENodeArgument); + lambda_node->children = cur_node; + kvi_push(ast_stack, &lambda_node->children); + } else { + assert(lambda_node->children->next == NULL); + lambda_node->children->next = cur_node; + kvi_push(ast_stack, &lambda_node->children->next); + } + kvi_push(ast_stack, &cur_node->children); + lambda_node = NULL; + } else { + // Only first branch is valid. + is_invalid = true; + ADD_VALUE_IF_MISSING(_("E15: Unexpected arrow: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + } + want_node = kENodeValue; + HL_CUR_TOKEN(Arrow); + break; + } + case kExprLexPlainIdentifier: { + if (want_node == kENodeValue || want_node == kENodeArgument) { + want_node = (want_node == kENodeArgument + ? kENodeArgumentSeparator + : kENodeOperator); + // FIXME: It is not valid to have scope inside complex identifier, + // check that. + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainIdentifier); + cur_node->data.var.scope = cur_token.data.var.scope; + const size_t scope_shift = (cur_token.data.var.scope == 0 + ? 0 + : 2); + cur_node->data.var.ident = (pline.data + cur_token.start.col + + scope_shift); + cur_node->data.var.ident_len = cur_token.len - scope_shift; + *top_node_p = cur_node; + if (scope_shift) { + viml_parser_highlight(pstate, cur_token.start, 1, + HL(IdentifierScope)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), 1, + HL(IdentifierScopeDelimiter)); + } + if (scope_shift < cur_token.len) { + viml_parser_highlight(pstate, shifted_pos(cur_token.start, + scope_shift), + cur_token.len - scope_shift, + HL(Identifier)); + } + } else { + OP_MISSING; + } break; } case kExprLexParenthesis: { if (cur_token.data.brc.closing) { - if (want_level == kELvlValue) { + if (want_node == kENodeValue) { if (kv_size(ast_stack) > 1) { const ExprASTNode *const prev_top_node = *kv_Z(ast_stack, 1); if (prev_top_node->type == kExprNodeCall) { @@ -858,15 +1384,15 @@ viml_pexpr_parse_process_token: goto viml_pexpr_parse_no_paren_closing_error; } } - is_invalid = true; - ERROR_FROM_TOKEN_AND_MSG(cur_token, _("E15: Expected value: %.*s")); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected value, got parenthesis: %.*s")); NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); cur_node->len = 0; *top_node_p = cur_node; } else { - // Always drop the topmost value: when want_level != kELvlValue + // Always drop the topmost value: when want_node != kENodeValue // topmost item on stack is a *finished* left operand, which may as - // well be "(@a)" which needs not be finished. + // well be "(@a)" which needs not be finished again. kv_drop(ast_stack, 1); } viml_pexpr_parse_no_paren_closing_error: {} @@ -892,10 +1418,9 @@ viml_pexpr_parse_no_paren_closing_error: {} if (new_top_node_p == NULL) { new_top_node_p = top_node_p; } - is_invalid = true; - HL_CUR_TOKEN(NestingParenthesis); ERROR_FROM_TOKEN_AND_MSG( cur_token, _("E15: Unexpected closing parenthesis: %.*s")); + HL_CUR_TOKEN(NestingParenthesis); cur_node = NEW_NODE(kExprNodeNested); cur_node->start = cur_token.start; cur_node->len = 0; @@ -906,14 +1431,14 @@ viml_pexpr_parse_no_paren_closing_error: {} assert(cur_node->next == NULL); } kvi_push(ast_stack, new_top_node_p); - want_level = kELvlOperator; + want_node = kENodeOperator; } else { - if (want_level == kELvlValue) { + if (want_node == kENodeValue) { NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeNested); *top_node_p = cur_node; kvi_push(ast_stack, &cur_node->children); HL_CUR_TOKEN(NestingParenthesis); - } else if (want_level == kELvlOperator) { + } else if (want_node == kENodeOperator) { if (prev_token.type == kExprLexSpacing) { // For some reason "function (args)" is a function call, but // "(funcref) (args)" is not. AFAIR this somehow involves @@ -926,13 +1451,13 @@ viml_pexpr_parse_no_paren_closing_error: {} } } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCall); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_level); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); HL_CUR_TOKEN(CallingParenthesis); } else { // Currently it is impossible to reach this. assert(false); } - want_level = kELvlValue; + want_node = kENodeValue; } break; } @@ -943,8 +1468,9 @@ viml_pexpr_parse_cycle_end: viml_parser_advance(pstate, cur_token.len); } while (true); viml_pexpr_parse_end: - if (want_level == kELvlValue) { - east_set_error(&ast, pstate, _("E15: Expected value: %.*s"), pstate->pos); + if (want_node == kENodeValue) { + east_set_error(&ast, pstate, _("E15: Expected value, got EOC: %.*s"), + pstate->pos); } else if (kv_size(ast_stack) != 1) { // Something may be wrong, check whether it really is. @@ -956,7 +1482,7 @@ viml_pexpr_parse_end: kv_drop(ast_stack, 1); while (ast.correct && kv_size(ast_stack)) { const ExprASTNode *const cur_node = (*kv_pop(ast_stack)); - // This should only happen when want_level == kELvlValue. + // This should only happen when want_node == kENodeValue. assert(cur_node != NULL); switch (cur_node->type) { case kExprNodeOpMissing: diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 13888562df..13640ec137 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -2,6 +2,7 @@ #define NVIM_VIML_PARSER_EXPRESSIONS_H #include +#include #include #include "nvim/types.h" @@ -130,6 +131,17 @@ typedef enum { kExprNodePlainIdentifier = 'i', /// Complex identifier: variable/function name with curly braces kExprNodeComplexIdentifier = 'I', + /// Figure brace expression which is not yet known + /// + /// May resolve to any of kExprNodeDictLiteral, kExprNodeLambda or + /// kExprNodeCurlyBracesIdentifier. + kExprNodeUnknownFigure = '{', + kExprNodeLambda = '\\', ///< Lambda. + kExprNodeDictLiteral = 'd', ///< Dictionary literal. + kExprNodeCurlyBracesIdentifier= '}', ///< Part of the curly braces name. + kExprNodeComma = ',', ///< Comma “operator”. + kExprNodeColon = ':', ///< Colon “operator”. + kExprNodeArrow = '>', ///< Arrow “operator”. } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -149,6 +161,27 @@ struct expr_ast_node { struct { int name; ///< Register name, may be -1 if name not present. } reg; ///< For kExprNodeRegister. + struct { + /// Which nodes UnknownFigure can’t possibly represent. + struct { + /// True if UnknownFigure may actually represent dictionary literal. + bool allow_dict; + /// True if UnknownFigure may actually represent lambda. + bool allow_lambda; + /// True if UnknownFigure may actually be part of curly braces name. + bool allow_ident; + } type_guesses; + /// Highlight chunk index, used for rehighlighting if needed + size_t opening_hl_idx; + } fig; ///< For kExprNodeUnknownFigure. + struct { + int scope; ///< Scope character or 0 if not present. + /// Actual identifier without scope. + /// + /// Points to inside parser reader state. + const char *ident; + size_t ident_len; ///< Actual identifier length. + } var; } data; }; @@ -166,6 +199,8 @@ enum { /// /// Without the flag they are only taken into account when parsing. kExprFlagsPrintError = (1 << 2), + // WARNING: whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_parser.c. } ExprParserFlags; /// Structure representing complety AST for one expression diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index c4bb067391..63b8baa8a8 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -31,6 +31,13 @@ make_enum_conv_tab(lib, { 'kExprNodeCall', 'kExprNodePlainIdentifier', 'kExprNodeComplexIdentifier', + 'kExprNodeUnknownFigure', + 'kExprNodeLambda', + 'kExprNodeDictLiteral', + 'kExprNodeCurlyBracesIdentifier', + 'kExprNodeComma', + 'kExprNodeColon', + 'kExprNodeArrow', }, 'kExprNode', function(ret) east_node_type_tab = ret end) local function conv_east_node_type(typ) @@ -59,6 +66,16 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) if typ == 'Register' then typ = typ .. ('(name=%s)'):format( tostring(intchar2lua(eastnode.data.reg.name))) + elseif typ == 'PlainIdentifier' then + typ = typ .. ('(scope=%s,ident=%s)'):format( + tostring(intchar2lua(eastnode.data.var.scope)), + ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len)) + elseif (typ == 'UnknownFigure' or typ == 'DictLiteral' + or typ == 'CurlyBracesIdentifier' or typ == 'Lambda') then + typ = typ .. ('(%s)'):format( + (eastnode.data.fig.type_guesses.allow_lambda and '\\' or '-') + .. (eastnode.data.fig.type_guesses.allow_dict and 'd' or '-') + .. (eastnode.data.fig.type_guesses.allow_ident and 'i' or '-')) end ret_str = typ .. ':' .. ret_str local can_simplify = true @@ -90,8 +107,7 @@ local function east2lua(pstate, east) return { err = (not east.correct) and { msg = ffi.string(east.err.msg), - arg = ('%u:%s'):format( - tonumber(east.err.arg_len), + arg = ('%s'):format( ffi.string(east.err.arg, east.err.arg_len)), } or nil, ast = eastnodelist2lua(pstate, east.root, checked_nodes), @@ -121,31 +137,31 @@ child_call_once(function() end) describe('Expressions parser', function() - itp('works', function() - local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) - local pstate = new_pstate({str}) - local east = lib.viml_pexpr_parse(pstate, flags) - local ast = east2lua(pstate, east) - eq(exp_ast, ast) - if exp_highlighting_fs then - local exp_highlighting = {} - local next_col = 0 - for i, h in ipairs(exp_highlighting_fs) do - exp_highlighting[i], next_col = h(next_col) - end - eq(exp_highlighting, phl2lua(pstate)) + local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) + local pstate = new_pstate({str}) + local east = lib.viml_pexpr_parse(pstate, flags) + local ast = east2lua(pstate, east) + eq(exp_ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exp_highlighting_fs) do + exp_highlighting[i], next_col = h(next_col) end + eq(exp_highlighting, phl2lua(pstate)) end - local function hl(group, str, shift) - return function(next_col) - local col = next_col + (shift or 0) - return (('%s:%u:%u:%s'):format( - 'NVim' .. group, - 0, - col, - str)), (col + #str) - end + end + local function hl(group, str, shift) + return function(next_col) + local col = next_col + (shift or 0) + return (('%s:%u:%u:%s'):format( + 'NVim' .. group, + 0, + col, + str)), (col + #str) end + end + itp('works with + and @a', function() check_parsing('@a', 0, { ast = { 'Register(name=a):0:0:@a', @@ -263,7 +279,7 @@ describe('Expressions parser', function() }, }, err = { - arg = '2:@b', + arg = '@b', msg = 'E15: Missing operator: %.*s', }, }, { @@ -281,7 +297,7 @@ describe('Expressions parser', function() }, }, err = { - arg = '2:@b', + arg = '@b', msg = 'E15: Missing operator: %.*s', }, }, { @@ -294,8 +310,8 @@ describe('Expressions parser', function() 'UnaryPlus:0:0:+', }, err = { - arg = '0:', - msg = 'E15: Expected value: %.*s', + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', }, }, { hl('UnaryPlus', '+'), @@ -305,8 +321,8 @@ describe('Expressions parser', function() 'UnaryPlus:0:0: +', }, err = { - arg = '0:', - msg = 'E15: Expected value: %.*s', + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', }, }, { hl('UnaryPlus', '+', 1), @@ -321,13 +337,15 @@ describe('Expressions parser', function() }, }, err = { - arg = '0:', - msg = 'E15: Expected value: %.*s', + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', }, }, { hl('Register', '@a'), hl('BinaryPlus', '+'), }) + end) + itp('works with @a, + and parenthesis', function() check_parsing('(@a)', 0, { ast = { { @@ -352,8 +370,8 @@ describe('Expressions parser', function() }, }, err = { - arg = '1:)', - msg = 'E15: Expected value: %.*s', + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', }, }, { hl('NestingParenthesis', '('), @@ -369,8 +387,8 @@ describe('Expressions parser', function() }, }, err = { - arg = '1:)', - msg = 'E15: Expected value: %.*s', + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', }, }, { hl('InvalidNestingParenthesis', ')'), @@ -390,8 +408,8 @@ describe('Expressions parser', function() }, }, err = { - arg = '1:)', - msg = 'E15: Expected value: %.*s', + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', }, }, { hl('UnaryPlus', '+'), @@ -473,7 +491,7 @@ describe('Expressions parser', function() }, }, err = { - arg = '2:()', + arg = '()', msg = 'E15: Missing operator: %.*s', }, }, { @@ -838,7 +856,7 @@ describe('Expressions parser', function() }, }, err = { - arg = '1:)', + arg = ')', msg = 'E15: Unexpected closing parenthesis: %.*s', }, }, { @@ -856,7 +874,7 @@ describe('Expressions parser', function() }, }, err = { - arg = '3:(@a', + arg = '(@a', msg = 'E110: Missing closing parenthesis for nested expression: %.*s', }, }, { @@ -875,7 +893,7 @@ describe('Expressions parser', function() }, }, err = { - arg = '3:(@b', + arg = '(@b', msg = 'E116: Missing closing parenthesis for function call: %.*s', }, }, { @@ -884,4 +902,920 @@ describe('Expressions parser', function() hl('Register', '@b'), }) end) + itp('works with identifiers', function() + check_parsing('var', 0, { + ast = { + 'PlainIdentifier(scope=0,ident=var):0:0:var', + }, + }, { + hl('Identifier', 'var'), + }) + check_parsing('g:var', 0, { + ast = { + 'PlainIdentifier(scope=g,ident=var):0:0:g:var', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Identifier', 'var'), + }) + check_parsing('g:', 0, { + ast = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + }) + end) + itp('works with curly braces', function() + check_parsing('{}', 0, { + ast = { + 'DictLiteral(-di):0:0:{', + }, + }, { + hl('Dict', '{'), + hl('Dict', '}'), + }) + check_parsing('{a}', 0, { + -- 012 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Identifier', 'a'), + hl('Curly', '}'), + }) + check_parsing('{a:b}', 0, { + -- 012 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Identifier', 'b'), + hl('Curly', '}'), + }) + check_parsing('{a:@b}', 0, { + -- 012345 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'OpMissing:0:3:', + children={ + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, + }, + err = { + arg = '@b}', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidRegister', '@b'), + hl('Curly', '}'), + }) + check_parsing('{@a}', 0, { + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{->@a}', 0, { + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Arrow:0:1:->', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{->@a+@b}', 0, { + -- 012345678 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Arrow:0:1:->', + children = { + { + 'BinaryPlus:0:5:+', + children = { + 'Register(name=a):0:3:@a', + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('Lambda', '}'), + }) + check_parsing('{a->@a}', 0, { + -- 012345678 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2:->', + children = { + 'Register(name=a):0:4:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->@a}', 0, { + -- 012345678 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'Register(name=a):0:6:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c->@a}', 0, { + -- 01234567890 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + { + 'Arrow:0:6:->', + children = { + 'Register(name=a):0:8:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('Comma', ','), + hl('Identifier', 'c'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d->@a}', 0, { + -- 0123456789012 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:8:->', + children = { + 'Register(name=a):0:10:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('Comma', ','), + hl('Identifier', 'c'), + hl('Comma', ','), + hl('Identifier', 'd'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d,->@a}', 0, { + -- 01234567890123 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:9:->', + children = { + 'Register(name=a):0:11:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('Comma', ','), + hl('Identifier', 'c'), + hl('Comma', ','), + hl('Identifier', 'd'), + hl('Comma', ','), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->{c,d->{e,f->@a}}}', 0, { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + 'Lambda(\\di):0:6:{', + children = { + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + 'PlainIdentifier(scope=0,ident=d):0:9:d', + }, + }, + { + 'Arrow:0:10:->', + children = { + { + 'Lambda(\\di):0:12:{', + children = { + { + 'Comma:0:14:,', + children = { + 'PlainIdentifier(scope=0,ident=e):0:13:e', + 'PlainIdentifier(scope=0,ident=f):0:15:f', + }, + }, + { + 'Arrow:0:16:->', + children = { + 'Register(name=a):0:18:@a', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('Identifier', 'c'), + hl('Comma', ','), + hl('Identifier', 'd'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('Identifier', 'e'), + hl('Comma', ','), + hl('Identifier', 'f'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + hl('Lambda', '}'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->c,d}', 0, { + -- 0123456789 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',d}', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('Arrow', '->'), + hl('Identifier', 'c'), + hl('InvalidComma', ','), + hl('Identifier', 'd'), + hl('Lambda', '}'), + }) + check_parsing('a,b,c,d', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('InvalidComma', ','), + hl('Identifier', 'b'), + hl('InvalidComma', ','), + hl('Identifier', 'c'), + hl('InvalidComma', ','), + hl('Identifier', 'd'), + }) + check_parsing('a,b,c,d,', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d,', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('InvalidComma', ','), + hl('Identifier', 'b'), + hl('InvalidComma', ','), + hl('Identifier', 'c'), + hl('InvalidComma', ','), + hl('Identifier', 'd'), + hl('InvalidComma', ','), + }) + check_parsing(',', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:0:,', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ',', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('InvalidComma', ','), + }) + check_parsing('{,a->@a}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Arrow:0:3:->', + children = { + { + 'Comma:0:1:,', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:2:a', + }, + }, + 'Register(name=a):0:5:@a', + }, + }, + }, + }, + }, + err = { + arg = ',a->@a}', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('Curly', '{'), + hl('InvalidComma', ','), + hl('Identifier', 'a'), + hl('InvalidArrow', '->'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('}', 0, { + -- 0123456789 + ast = { + 'UnknownFigure(---):0:0:', + }, + err = { + arg = '}', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidFigureBrace', '}'), + }) + check_parsing('{->}', 0, { + -- 0123456789 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + 'Arrow:0:1:->', + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,b}', 0, { + -- 0123456789 + ast = { + { + 'Lambda(-di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,}', 0, { + -- 0123456789 + ast = { + { + 'Lambda(-di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('InvalidLambda', '}'), + }) + check_parsing('{@a:@b}', 0, { + -- 0123456789 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d}', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,}', 0, { + -- 01234567890123456789 + -- 0 1 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', 0, { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + { + 'Colon:0:21::', + children = { + 'Register(name=g):0:19:@g', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Register', '@g'), + hl('Colon', ':'), + hl('InvalidDict', '}'), + }) + check_parsing('{@a:@b,}', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Dict', '}'), + }) + end) + -- FIXME: Test sequence of arrows inside and outside lambdas. + -- FIXME: Test multiple arguments calling. + -- FIXME: Test autoload character and scope in lambda arguments. end) From d4782fb1ca05e76095086bdcbc8dcea47f532d00 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 26 Sep 2017 00:28:34 +0300 Subject: [PATCH 012/109] viml/parser/expressions: Make commas actually work when calling --- src/nvim/viml/parser/expressions.c | 99 ++++++++++++---------- test/unit/viml/expressions/parser_spec.lua | 42 +++++++++ 2 files changed, 97 insertions(+), 44 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index f4cfed3113..b9abf4a067 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -395,6 +395,43 @@ viml_pexpr_next_token_adv_return: return ret; } +#ifdef UNIT_TESTING +#include +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_ast_node( + const ExprASTNode *const *const eastnode_p, + const char *const prefix) +{ + if (*eastnode_p == NULL) { + fprintf(stderr, "%s %p : NULL\n", prefix, (void *)eastnode_p); + } else { + fprintf(stderr, "%s %p : %p : %c : %zu:%zu:%zu\n", + prefix, (void *)eastnode_p, (void *)(*eastnode_p), + (*eastnode_p)->type, (*eastnode_p)->start.line, + (*eastnode_p)->start.col, (*eastnode_p)->len); + } +} +REAL_FATTR_UNUSED +static inline void viml_pexpr_debug_print_ast_stack( + const ExprASTStack *const ast_stack, + const char *const msg) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE +{ + fprintf(stderr, "\n%sstack: %zu:\n", msg, kv_size(*ast_stack)); + for (size_t i = 0; i < kv_size(*ast_stack); i++) { + viml_pexpr_debug_print_ast_node( + (const ExprASTNode *const *)kv_A(*ast_stack, i), + "-"); + } +} +#define PSTACK(msg) \ + viml_pexpr_debug_print_ast_stack(&ast_stack, #msg) +#define PSTACK_P(msg) \ + viml_pexpr_debug_print_ast_stack(ast_stack, #msg) +#define PNODE_P(eastnode_p, msg) \ + viml_pexpr_debug_print_ast_node((const ExprASTNode *const *)ast_stack, #msg) +#endif + // start = s ternary_expr s EOC // ternary_expr = binop_expr // ( s Question s ternary_expr s Colon s ternary_expr s )? @@ -560,6 +597,9 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeOpMissing] = kEOpLvlMultiplication, [kExprNodeNested] = kEOpLvlParens, + // Note: it is kEOpLvlSubscript for “binary operator” itself, but + // kEOpLvlParens when it comes to inside the parenthesis. + [kExprNodeCall] = kEOpLvlParens, [kExprNodeUnknownFigure] = kEOpLvlParens, [kExprNodeLambda] = kEOpLvlParens, @@ -578,7 +618,6 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeUnaryPlus] = kEOpLvlUnary, [kExprNodeSubscript] = kEOpLvlSubscript, - [kExprNodeCall] = kEOpLvlSubscript, [kExprNodeComplexIdentifier] = kEOpLvlComplexIdentifier, [kExprNodePlainIdentifier] = kEOpLvlComplexIdentifier, @@ -593,6 +632,7 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeOpMissing] = kEOpAssNo, [kExprNodeNested] = kEOpAssNo, + [kExprNodeCall] = kEOpAssNo, [kExprNodeUnknownFigure] = kEOpAssLeft, [kExprNodeLambda] = kEOpAssNo, @@ -619,7 +659,6 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeUnaryPlus] = kEOpAssNo, [kExprNodeSubscript] = kEOpAssLeft, - [kExprNodeCall] = kEOpAssLeft, [kExprNodePlainIdentifier] = kEOpAssLeft, [kExprNodeComplexIdentifier] = kEOpAssLeft, @@ -629,43 +668,6 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeListLiteral] = kEOpAssNo, }; -#ifdef UNIT_TESTING -#include -REAL_FATTR_UNUSED -static inline void viml_pexpr_debug_print_ast_node( - const ExprASTNode *const *const eastnode_p, - const char *const prefix) -{ - if (*eastnode_p == NULL) { - fprintf(stderr, "%s %p : NULL\n", prefix, (void *)eastnode_p); - } else { - fprintf(stderr, "%s %p : %p : %c : %zu:%zu:%zu\n", - prefix, (void *)eastnode_p, (void *)(*eastnode_p), - (*eastnode_p)->type, (*eastnode_p)->start.line, - (*eastnode_p)->start.col, (*eastnode_p)->len); - } -} -REAL_FATTR_UNUSED -static inline void viml_pexpr_debug_print_ast_stack( - const ExprASTStack *const ast_stack, - const char *const msg) - FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE -{ - fprintf(stderr, "\n%sstack: %zu:\n", msg, kv_size(*ast_stack)); - for (size_t i = 0; i < kv_size(*ast_stack); i++) { - viml_pexpr_debug_print_ast_node( - (const ExprASTNode *const *)kv_A(*ast_stack, i), - "-"); - } -} -#define PSTACK(msg) \ - viml_pexpr_debug_print_ast_stack(&ast_stack, #msg) -#define PSTACK_P(msg) \ - viml_pexpr_debug_print_ast_stack(ast_stack, #msg) -#define PNODE_P(eastnode_p, msg) \ - viml_pexpr_debug_print_ast_node((const ExprASTNode *const *)ast_stack, #msg) -#endif - /// Handle binary operator /// /// This function is responsible for handling priority levels as well. @@ -679,17 +681,24 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, ExprOpLvl top_node_lvl; ExprOpAssociativity top_node_ass; assert(kv_size(*ast_stack)); - const ExprOpLvl bop_node_lvl = node_type_to_op_lvl[bop_node->type]; +#define NODE_LVL(typ) \ + (bop_node->type == kExprNodeCall && typ == kExprNodeCall \ + ? kEOpLvlSubscript \ + : node_type_to_op_lvl[typ]) +#define NODE_ASS(typ) \ + (bop_node->type == kExprNodeCall && typ == kExprNodeCall \ + ? kEOpAssLeft \ + : node_type_to_op_ass[typ]) + const ExprOpLvl bop_node_lvl = NODE_LVL(bop_node->type); #ifndef NDEBUG - const ExprOpAssociativity bop_node_ass = node_type_to_op_ass[bop_node->type]; + const ExprOpAssociativity bop_node_ass = NODE_ASS(bop_node->type); #endif do { ExprASTNode **new_top_node_p = kv_last(*ast_stack); ExprASTNode *new_top_node = *new_top_node_p; assert(new_top_node != NULL); - const ExprOpLvl new_top_node_lvl = node_type_to_op_lvl[new_top_node->type]; - const ExprOpAssociativity new_top_node_ass = ( - node_type_to_op_ass[new_top_node->type]); + const ExprOpLvl new_top_node_lvl = NODE_LVL(new_top_node->type); + const ExprOpAssociativity new_top_node_ass = NODE_ASS(new_top_node->type); assert(bop_node_lvl != new_top_node_lvl || bop_node_ass == new_top_node_ass); if (top_node_p != NULL @@ -742,6 +751,8 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, *want_node_p = (*want_node_p == kENodeArgumentSeparator ? kENodeArgument : kENodeValue); +#undef NODE_ASS +#undef NODE_LVL } /// ParserPosition literal based on ParserPosition pos with columns shifted diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 63b8baa8a8..4d4a5f1007 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -901,6 +901,48 @@ describe('Expressions parser', function() hl('CallingParenthesis', '('), hl('Register', '@b'), }) + check_parsing('@a(@b, @c, @d, @e)', 0, { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Comma:0:5:,', + children = { + 'Register(name=b):0:3:@b', + { + 'Comma:0:9:,', + children = { + 'Register(name=c):0:6: @c', + { + 'Comma:0:13:,', + children = { + 'Register(name=d):0:10: @d', + 'Register(name=e):0:14: @e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c', 1), + hl('Comma', ','), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('CallingParenthesis', ')'), + }) end) itp('works with identifiers', function() check_parsing('var', 0, { From 3cc65ac054976ef7520f0247b430ebef2f9537b7 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 26 Sep 2017 00:52:40 +0300 Subject: [PATCH 013/109] viml/parser/expressions: Make commas actually work when calling --- src/nvim/viml/parser/expressions.c | 24 ++--- test/unit/viml/expressions/parser_spec.lua | 115 +++++++++++++++++++++ 2 files changed, 125 insertions(+), 14 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index b9abf4a067..7bee779c49 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -681,24 +681,22 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, ExprOpLvl top_node_lvl; ExprOpAssociativity top_node_ass; assert(kv_size(*ast_stack)); -#define NODE_LVL(typ) \ - (bop_node->type == kExprNodeCall && typ == kExprNodeCall \ - ? kEOpLvlSubscript \ - : node_type_to_op_lvl[typ]) -#define NODE_ASS(typ) \ - (bop_node->type == kExprNodeCall && typ == kExprNodeCall \ - ? kEOpAssLeft \ - : node_type_to_op_ass[typ]) - const ExprOpLvl bop_node_lvl = NODE_LVL(bop_node->type); + const ExprOpLvl bop_node_lvl = (bop_node->type == kExprNodeCall + ? kEOpLvlSubscript + : node_type_to_op_lvl[bop_node->type]); #ifndef NDEBUG - const ExprOpAssociativity bop_node_ass = NODE_ASS(bop_node->type); + const ExprOpAssociativity bop_node_ass = ( + bop_node->type == kExprNodeCall + ? kEOpAssLeft + : node_type_to_op_ass[bop_node->type]); #endif do { ExprASTNode **new_top_node_p = kv_last(*ast_stack); ExprASTNode *new_top_node = *new_top_node_p; assert(new_top_node != NULL); - const ExprOpLvl new_top_node_lvl = NODE_LVL(new_top_node->type); - const ExprOpAssociativity new_top_node_ass = NODE_ASS(new_top_node->type); + const ExprOpLvl new_top_node_lvl = node_type_to_op_lvl[new_top_node->type]; + const ExprOpAssociativity new_top_node_ass = ( + node_type_to_op_ass[new_top_node->type]); assert(bop_node_lvl != new_top_node_lvl || bop_node_ass == new_top_node_ass); if (top_node_p != NULL @@ -751,8 +749,6 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, *want_node_p = (*want_node_p == kENodeArgumentSeparator ? kENodeArgument : kENodeValue); -#undef NODE_ASS -#undef NODE_LVL } /// ParserPosition literal based on ParserPosition pos with columns shifted diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 4d4a5f1007..b747a40e27 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -943,6 +943,121 @@ describe('Expressions parser', function() hl('Register', '@e', 1), hl('CallingParenthesis', ')'), }) + check_parsing('@a(@b(@c))', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + { + 'Call:0:8:(', + children = { + 'Register(name=c):0:6:@c', + { + 'Comma:0:15:,', + children = { + { + 'Call:0:11:(', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=f):0:16: @f', + { + 'Comma:0:26:,', + children = { + { + 'Call:0:22:(', + children = { + 'Register(name=g):0:20:@g', + 'Register(name=h):0:23:@h', + }, + }, + { + 'Call:0:30:(', + children = { + 'Register(name=i):0:27: @i', + 'Register(name=j):0:31:@j', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', '('), + hl('Register', '@d'), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@f', 1), + hl('CallingParenthesis', '('), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@i', 1), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) end) itp('works with identifiers', function() check_parsing('var', 0, { From 0987d3b10f36202e9f0289b50298e69aaf2fa4d2 Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 26 Sep 2017 01:22:13 +0300 Subject: [PATCH 014/109] viml/parser/expressions: Make curly braces name actually work --- src/nvim/viml/parser/expressions.c | 116 +++++-- test/unit/viml/expressions/parser_spec.lua | 384 ++++++++++++++++++++- 2 files changed, 450 insertions(+), 50 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 7bee779c49..cabf2dac58 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -909,6 +909,55 @@ static inline void east_set_error(ExprAST *const ret_ast, } \ } while (0) +/// Add identifier which should constitute complex identifier node +/// +/// This one is to be called only in case want_node is kENodeOperator. +/// +/// @param new_ident_node_code Code used to create a new identifier node and +/// update want_node and ast_stack, without +/// a trailing semicolon. +/// @param hl Highlighting name to use, passed as an argument to #HL. +#define ADD_IDENT(new_ident_node_code, hl) \ + do { \ + assert(want_node == kENodeOperator); \ + /* Operator: may only be curly braces name, but only under certain */ \ + /* conditions. */ \ +\ + /* First condition is that there is no space before a part of complex */ \ + /* identifier. */ \ + if (prev_token.type == kExprLexSpacing) { \ + OP_MISSING; \ + } \ + switch ((*top_node_p)->type) { \ + /* Second is that previous node is one of the identifiers: */ \ + /* complex, plain, curly braces. */ \ +\ + /* TODO(ZyX-I): Extend syntax to allow ${expr}. This is needed to */ \ + /* handle environment variables like those bash uses for */ \ + /* `export -f`: their names consist not only of alphanumeric */ \ + /* characetrs. */ \ + case kExprNodeComplexIdentifier: \ + case kExprNodePlainIdentifier: \ + case kExprNodeCurlyBracesIdentifier: { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComplexIdentifier); \ + cur_node->len = 0; \ + cur_node->children = *top_node_p; \ + *top_node_p = cur_node; \ + kvi_push(ast_stack, &cur_node->children->next); \ + ExprASTNode **const new_top_node_p = kv_last(ast_stack); \ + assert(*new_top_node_p == NULL); \ + new_ident_node_code; \ + *new_top_node_p = cur_node; \ + HL_CUR_TOKEN(hl); \ + break; \ + } \ + default: { \ + OP_MISSING; \ + break; \ + } \ + } \ + } while (0) + /// Parse one VimL expression /// /// @param pstate Parser state. @@ -1272,40 +1321,18 @@ viml_pexpr_parse_figure_brace_closing_error: want_node = kENodeArgument; lambda_node = cur_node; } else { - // Operator: may only be curly braces name, but only under certain - // conditions. - - // First condition is that there is no space before {. - if (prev_token.type == kExprLexSpacing) { - OP_MISSING; - } - switch ((*top_node_p)->type) { - // Second is that previous node is one of the identifiers: - // complex, plain, curly braces. - - // TODO(ZyX-I): Extend syntax to allow ${expr}. This is needed to - // handle environment variables like those bash uses for - // `export -f`: their names consist not only of alphanumeric - // characetrs. - case kExprNodeComplexIdentifier: - case kExprNodePlainIdentifier: - case kExprNodeCurlyBracesIdentifier: { - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComplexIdentifier); - cur_node->len = 0; - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); - ExprASTNode *const new_top_node = *kv_last(ast_stack); - assert(new_top_node->next == NULL); - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCurlyBracesIdentifier); - new_top_node->next = cur_node; - kvi_push(ast_stack, &cur_node->children); - HL_CUR_TOKEN(Curly); - break; - } - default: { - OP_MISSING; - break; - } - } + ADD_IDENT( + do { + NEW_NODE_WITH_CUR_POS(cur_node, + kExprNodeCurlyBracesIdentifier); + cur_node->data.fig.opening_hl_idx = kv_size(*pstate->colors); + cur_node->data.fig.type_guesses.allow_lambda = false; + cur_node->data.fig.type_guesses.allow_dict = false; + cur_node->data.fig.type_guesses.allow_ident = true; + kvi_push(ast_stack, &cur_node->children); + want_node = kENodeValue; + } while (0), + Curly); } } break; @@ -1351,8 +1378,6 @@ viml_pexpr_parse_figure_brace_closing_error: want_node = (want_node == kENodeArgument ? kENodeArgumentSeparator : kENodeOperator); - // FIXME: It is not valid to have scope inside complex identifier, - // check that. NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainIdentifier); cur_node->data.var.scope = cur_token.data.var.scope; const size_t scope_shift = (cur_token.data.var.scope == 0 @@ -1374,8 +1399,22 @@ viml_pexpr_parse_figure_brace_closing_error: cur_token.len - scope_shift, HL(Identifier)); } + // FIXME: Actually, g{foo}g:foo is valid: "1?g{foo}g:foo" is like + // "g{foo}g" and not an error. } else { - OP_MISSING; + if (cur_token.data.var.scope == 0) { + ADD_IDENT( + do { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainIdentifier); + cur_node->data.var.scope = cur_token.data.var.scope; + cur_node->data.var.ident = pline.data + cur_token.start.col; + cur_node->data.var.ident_len = cur_token.len; + want_node = kENodeOperator; + } while (0), + Identifier); + } else { + OP_MISSING; + } } break; } @@ -1453,7 +1492,8 @@ viml_pexpr_parse_no_paren_closing_error: {} // intentionally inconsistent and he is not very happy with the // situation himself. if ((*top_node_p)->type != kExprNodePlainIdentifier - && (*top_node_p)->type != kExprNodeComplexIdentifier) { + && (*top_node_p)->type != kExprNodeComplexIdentifier + && (*top_node_p)->type != kExprNodeCurlyBracesIdentifier) { OP_MISSING; } } diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index b747a40e27..eec4cb5bd9 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -1059,7 +1059,7 @@ describe('Expressions parser', function() hl('CallingParenthesis', ')'), }) end) - itp('works with identifiers', function() + itp('works with variable names, including curly braces ones', function() check_parsing('var', 0, { ast = { 'PlainIdentifier(scope=0,ident=var):0:0:var', @@ -1084,16 +1084,6 @@ describe('Expressions parser', function() hl('IdentifierScope', 'g'), hl('IdentifierScopeDelimiter', ':'), }) - end) - itp('works with curly braces', function() - check_parsing('{}', 0, { - ast = { - 'DictLiteral(-di):0:0:{', - }, - }, { - hl('Dict', '{'), - hl('Dict', '}'), - }) check_parsing('{a}', 0, { -- 012 ast = { @@ -1167,6 +1157,209 @@ describe('Expressions parser', function() hl('Register', '@a'), hl('Curly', '}'), }) + check_parsing('{@a}{@b}', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'CurlyBracesIdentifier(--i):0:4:{', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('Register', '@b'), + hl('Curly', '}'), + }) + check_parsing('g:{@a}', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}_test', 0, { + -- 012345678 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:4:_test', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('Identifier', '_test'), + }) + check_parsing('g:{@a}_test', 0, { + -- 01234567890 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('Identifier', '_test'), + }) + check_parsing('g:{@a}_test()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:11:(', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('Identifier', '_test'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a} ()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:4: (', + children = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('g:{@a} ()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:6: (', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + end) + itp('works with lambdas and dictionaries', function() + check_parsing('{}', 0, { + ast = { + 'DictLiteral(-di):0:0:{', + }, + }, { + hl('Dict', '{'), + hl('Dict', '}'), + }) check_parsing('{->@a}', 0, { ast = { { @@ -1971,8 +2164,175 @@ describe('Expressions parser', function() hl('Comma', ','), hl('Dict', '}'), }) + check_parsing('{({f -> g})(@h)(@i)}', 0, { + -- 01234567890123456789 + -- 0 1 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Call:0:15:(', + children = { + { + 'Call:0:11:(', + children = { + { + 'Nested:0:1:(', + children = { + { + 'Lambda(\\di):0:2:{', + children = { + 'PlainIdentifier(scope=0,ident=f):0:3:f', + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:7: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:12:@h', + }, + }, + 'Register(name=i):0:16:@i', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('NestingParenthesis', '('), + hl('Lambda', '{'), + hl('Identifier', 'f'), + hl('Arrow', '->', 1), + hl('Identifier', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + }) + -- FIXME the below should not crash + check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', 0, { + -- 01234567890123456789012345678901234567890123456 + -- 0 1 2 3 4 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + { + 'Call:0:37:(', + children = { + { + 'Lambda(\\di):0:3:{', + children = { + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + { + 'Arrow:0:8: ->', + children = { + { + 'BinaryPlus:0:19: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + 'Register(name=d):0:11: @d', + 'Register(name=e):0:16: @e', + }, + }, + { + 'Call:0:32:(', + children = { + { + 'NestingParenthesis:0:21: (', + children = { + { + 'Lambda(\\di):0:23:{', + children = { + 'PlainIdentifier(scope=0,ident=f):0:24:f', + { + 'Arrow:0:25: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:28: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:33:@h', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=i):0:38:@i', + }, + }, + 'PlainIdentifier(scope=0,ident=j):0:42:j', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:42:_test', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Lambda', '{'), + hl('Identifier', 'b'), + hl('Comma', ','), + hl('Identifier', 'c', 1), + hl('Arrow', '->', 1), + hl('Register', '@d', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '('), + hl('Lambda', '{'), + hl('Identifier', 'f'), + hl('Arrow', '->', 1), + hl('Identifier', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('Identifier', 'j'), + }) end) -- FIXME: Test sequence of arrows inside and outside lambdas. - -- FIXME: Test multiple arguments calling. -- FIXME: Test autoload character and scope in lambda arguments. end) From 9fa8f7fc0a24371f7956450d840bdae8a2fc9a51 Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 28 Sep 2017 00:40:25 +0300 Subject: [PATCH 015/109] viml/parser/expressions: Add a way to adjust lexer It also adds support for kExprLexOr which for some reason was forgotten. It was only made sure that KLEE test compiles in non-KLEE mode, not that something works or that KLEE is able to run tests. --- src/nvim/viml/parser/expressions.c | 105 ++++++--- src/nvim/viml/parser/expressions.h | 28 +++ test/symbolic/klee/nvim/memory.c | 8 + test/symbolic/klee/viml_expressions_lexer.c | 24 +- test/unit/viml/expressions/lexer_spec.lua | 247 ++++++++++++++++---- 5 files changed, 323 insertions(+), 89 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index cabf2dac58..3027c0046b 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -47,10 +47,10 @@ typedef enum { /// Get next token for the VimL expression input /// /// @param pstate Parser state. -/// @param[in] peek If true, do not advance pstate cursor. +/// @param[in] flags Flags, @see LexExprFlags. /// /// @return Next token. -LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek) +LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { LexExprToken ret = { @@ -153,12 +153,33 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek) } // Number. - // Note: determining whether dot is (not) a part of a float needs more - // context, so lexer does not do this. - // FIXME: Resolve ambiguity by additional argument. case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { + ret.data.num.is_float = false; CHARREG(kExprLexNumber, ascii_isdigit); + if (flags & kELFlagAllowFloat) { + if (pline.size > ret.len + 1 + && pline.data[ret.len] == '.' + && ascii_isdigit(pline.data[ret.len + 1])) { + ret.len++; + ret.data.num.is_float = true; + CHARREG(kExprLexNumber, ascii_isdigit); + if (pline.size > ret.len + 1 + && (pline.data[ret.len] == 'e' + || pline.data[ret.len] == 'E') + && ((pline.size > ret.len + 2 + && (pline.data[ret.len + 1] == '+' + || pline.data[ret.len + 1] == '-') + && ascii_isdigit(pline.data[ret.len + 2])) + || ascii_isdigit(pline.data[ret.len + 1]))) { + ret.len++; + if (pline.data[ret.len] == '+' || pline.data[ret.len] == '-') { + ret.len++; + } + CHARREG(kExprLexNumber, ascii_isdigit); + } + } + } break; } @@ -187,8 +208,9 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek) ret.data.var.autoload = false; CHARREG(kExprLexPlainIdentifier, ISWORD); // "is" and "isnot" operators. - if ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0) - || (ret.len == 5 && memcmp(pline.data, "isnot", 5) == 0)) { + if (!(flags & kELFlagIsNotCmp) + && ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0) + || (ret.len == 5 && memcmp(pline.data, "isnot", 5) == 0))) { ret.type = kExprLexComparison; ret.data.cmp.type = kExprLexCmpIdentical; ret.data.cmp.inv = (ret.len == 5); @@ -197,14 +219,14 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const bool peek) } else if (ret.len == 1 && pline.size > 1 && strchr("sgvbwtla", schar) != NULL - && pline.data[ret.len] == ':') { + && pline.data[ret.len] == ':' + && !(flags & kELFlagForbidScope)) { ret.len++; ret.data.var.scope = schar; CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); ret.data.var.autoload = ( memchr(pline.data + 2, AUTOLOAD_CHAR, ret.len - 2) != NULL); - // FIXME: Resolve ambiguity with an argument to the lexer function. // Previous CHARREG stopped at autoload character in order to make it // possible to detect `is#`. Continue now with autoload characters // included. @@ -373,7 +395,30 @@ viml_pexpr_next_token_invalid_comparison: // Expression end because Ex command ended. case NUL: case NL: { - ret.type = kExprLexEOC; + if (flags & kELFlagForbidEOC) { + ret.type = kExprLexInvalid; + ret.data.err.msg = _("E15: Unexpected EOC character: %.*s"); + ret.data.err.type = kExprLexSpacing; + } else { + ret.type = kExprLexEOC; + } + break; + } + + case '|': { + if (pline.size >= 2 && pline.data[ret.len] == '|') { + // "||" is or. + ret.len++; + ret.type = kExprLexOr; + } else if (flags & kELFlagForbidEOC) { + // Note: `=1 | 2` actually yields 1 in Vim without any + // errors. This will be changed here. + ret.type = kExprLexInvalid; + ret.data.err.msg = _("E15: Unexpected EOC character: %.*s"); + ret.data.err.type = kExprLexOr; + } else { + ret.type = kExprLexEOC; + } break; } @@ -389,7 +434,7 @@ viml_pexpr_next_token_invalid_comparison: } #undef GET_CCS viml_pexpr_next_token_adv_return: - if (!peek) { + if (!(flags & kELFlagPeek)) { viml_parser_advance(pstate, ret.len); } return ret; @@ -990,34 +1035,28 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) // Lambda node, valid when parsing lambda arguments only. ExprASTNode *lambda_node = NULL; do { - LexExprToken cur_token = viml_pexpr_next_token(pstate, true); + const int want_node_to_lexer_flags[] = { + [kENodeValue] = kELFlagIsNotCmp, + [kENodeOperator] = kELFlagForbidScope, + [kENodeArgument] = kELFlagIsNotCmp, + [kENodeArgumentSeparator] = kELFlagForbidScope, + }; + // FIXME Determine when (not) to allow floating-point numbers. + const int lexer_additional_flags = ( + kELFlagPeek + | ((flags & kExprFlagsDisallowEOC) ? kELFlagForbidEOC : 0)); + LexExprToken cur_token = viml_pexpr_next_token( + pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags); if (cur_token.type == kExprLexEOC) { - if (flags & kExprFlagsDisallowEOC) { - if (cur_token.len == 0) { - // It is end of string, break. - break; - } else { - // It is NL, NUL or bar. - // - // Note: `=1 | 2` actually yields 1 in Vim without any - // errors. This will be changed here. - cur_token.type = kExprLexInvalid; - cur_token.data.err.msg = _("E15: Unexpected EOC character: %.*s"); - const ParserLine pline = ( - pstate->reader.lines.items[cur_token.start.line]); - const char eoc_char = pline.data[cur_token.start.col]; - cur_token.data.err.type = ((eoc_char == NUL || eoc_char == NL) - ? kExprLexSpacing - : kExprLexOr); - } - } else { - break; - } + break; } LexExprTokenType tok_type = cur_token.type; const bool token_invalid = (tok_type == kExprLexInvalid); bool is_invalid = token_invalid; viml_pexpr_parse_process_token: + // May use different flags this time. + cur_token = viml_pexpr_next_token( + pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags); if (tok_type == kExprLexSpacing) { if (is_invalid) { HL_CUR_TOKEN(Spacing); diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 13640ec137..64abab9e41 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -109,9 +109,37 @@ typedef struct { LexExprTokenType type; ///< Suggested type for parsing incorrect code. const char *msg; ///< Error message. } err; ///< For kExprLexInvalid + + struct { + bool is_float; ///< True if number is a floating-point. + } num; ///< For kExprLexNumber } data; ///< Additional data, if needed. } LexExprToken; +typedef enum { + /// If set, “pointer” to the current byte in pstate will not be shifted + kELFlagPeek = (1 << 0), + /// Determines whether scope is allowed to come before the identifier + kELFlagForbidScope = (1 << 1), + /// Determines whether floating-point numbers are allowed + /// + /// I.e. whether dot is a decimal point separator or is not a part of + /// a number at all. + kELFlagAllowFloat = (1 << 2), + /// Determines whether `is` and `isnot` are seen as comparison operators + /// + /// If set they are supposed to be just regular identifiers. + kELFlagIsNotCmp = (1 << 3), + /// Determines whether EOC tokens are allowed + /// + /// If set then it will yield Invalid token with E15 in place of EOC one if + /// “EOC” is something like "|". It is fine with emitting EOC at the end of + /// string still, with or without this flag set. + kELFlagForbidEOC = (1 << 4), + // WARNING: whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_lexer.c. +} LexExprFlags; + /// Expression AST node type typedef enum { kExprNodeMissing = 'X', diff --git a/test/symbolic/klee/nvim/memory.c b/test/symbolic/klee/nvim/memory.c index 7e924a410a..1f9cdce6c0 100644 --- a/test/symbolic/klee/nvim/memory.c +++ b/test/symbolic/klee/nvim/memory.c @@ -15,11 +15,16 @@ RINGBUF_INIT(AllocRecords, arecs, AllocRecord, RINGBUF_DUMMY_FREE) RINGBUF_STATIC(static, AllocRecords, AllocRecord, arecs, RB_SIZE) size_t allocated_memory = 0; +size_t ever_allocated_memory = 0; + +size_t allocated_memory_limit = SIZE_MAX; void *xmalloc(const size_t size) { void *ret = malloc(size); allocated_memory += size; + ever_allocated_memory += size; + assert(allocated_memory <= allocated_memory_limit); assert(arecs_rb_length(&arecs) < RB_SIZE); arecs_rb_push(&arecs, (AllocRecord) { .ptr = ret, @@ -47,6 +52,9 @@ void *xrealloc(void *const p, size_t new_size) if (arec->ptr == p) { allocated_memory -= arec->size; allocated_memory += new_size; + if (new_size > arec->size) { + ever_allocated_memory += (new_size - arec->size); + } arec->ptr = ret; arec->size = new_size; return ret; diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c index e3b5aa80cc..67f3eb7faa 100644 --- a/test/symbolic/klee/viml_expressions_lexer.c +++ b/test/symbolic/klee/viml_expressions_lexer.c @@ -34,13 +34,20 @@ int main(const int argc, const char *const *const argv, { char input[INPUT_SIZE]; uint8_t shift; - const bool peek = false; + int flags; avoid_optimizing_out = argc; +#ifndef USE_KLEE + sscanf(argv[2], "%d", &flags); +#endif + #ifdef USE_KLEE klee_make_symbolic(input, sizeof(input), "input"); klee_make_symbolic(&shift, sizeof(shift), "shift"); + klee_make_symbolic(&flags, sizeof(flags), "flags"); klee_assume(shift < INPUT_SIZE); + klee_assume(flags <= (kELFlagPeek|kELFlagAllowFloat|kELFlagForbidEOC + |kELFlagForbidScope|kELFlagIsNotCmp)); #endif ParserLine plines[] = { @@ -79,8 +86,15 @@ int main(const int argc, const char *const *const argv, }; kvi_init(pstate.reader.lines); - LexExprToken token = viml_pexpr_next_token(&pstate, peek); - assert((pstate.pos.line == 0) - ? (pstate.pos.col > 0) - : (pstate.pos.line == 1 && pstate.pos.col == 0)); + allocated_memory_limit = 0; + LexExprToken token = viml_pexpr_next_token(&pstate, flags); + if (flags & kELFlagPeek) { + assert(pstate.pos.line == 0 && pstate.pos.col == 0); + } else { + assert((pstate.pos.line == 0) + ? (pstate.pos.col > 0) + : (pstate.pos.line == 1 && pstate.pos.col == 0)); + } + assert(allocated_memory == 0); + assert(ever_allocated_memory == 0); } diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index 32182f650d..972478c2e5 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -1,5 +1,6 @@ local helpers = require('test.unit.helpers')(after_each) local viml_helpers = require('test.unit.viml.helpers') +local global_helpers = require('test.helpers') local itp = helpers.gen_itp(it) local child_call_once = helpers.child_call_once @@ -13,6 +14,8 @@ local new_pstate = viml_helpers.new_pstate local intchar2lua = viml_helpers.intchar2lua local pstate_set_str = viml_helpers.pstate_set_str +local shallowcopy = global_helpers.shallowcopy + local lib = cimport('./src/nvim/viml/parser/expressions.h') local eltkn_type_tab, eltkn_cmp_type_tab, ccs_tab, eltkn_mul_type_tab @@ -121,37 +124,81 @@ local function eltkn2lua(pstate, tkn) scope = intchar2lua(tkn.data.var.scope), autoload = (not not tkn.data.var.autoload), } + elseif ret.type == 'Number' then + ret.data = { + is_float = (not not tkn.data.num.is_float), + } elseif ret.type == 'Invalid' then ret.data = { error = ffi.string(tkn.data.err.msg) } end return ret, tkn end -local function next_eltkn(pstate) - return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, false)) +local function next_eltkn(pstate, flags) + return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, flags)) end describe('Expressions lexer', function() - itp('works (single tokens)', function() - local function singl_eltkn_test(typ, str, data) - local pstate = new_pstate({str}) - eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, - next_eltkn(pstate)) - if not ( - typ == 'Spacing' - or (typ == 'Register' and str == '@') - or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString') - and not data.closed) - ) then - pstate = new_pstate({str .. ' '}) - eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, - next_eltkn(pstate)) + local flags = 0 + local should_advance = true + local function check_advance(pstate, bytes_to_advance, initial_col) + local tgt = initial_col + bytes_to_advance + if should_advance then + if pstate.reader.lines.items[0].size == tgt then + eq(1, pstate.pos.line) + eq(0, pstate.pos.col) + else + eq(0, pstate.pos.line) + eq(tgt, pstate.pos.col) end - pstate = new_pstate({'x' .. str}) - pstate.pos.col = 1 - eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ}, - next_eltkn(pstate)) + else + eq(0, pstate.pos.line) + eq(initial_col, pstate.pos.col) end + end + local function singl_eltkn_test(typ, str, data) + local pstate = new_pstate({str}) + eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 0) + if not ( + typ == 'Spacing' + or (typ == 'Register' and str == '@') + or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString') + and not data.closed) + ) then + pstate = new_pstate({str .. ' '}) + eq({data=data, len=#str, start={col=0, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 0) + end + pstate = new_pstate({'x' .. str}) + pstate.pos.col = 1 + eq({data=data, len=#str, start={col=1, line=0}, str=str, type=typ}, + next_eltkn(pstate, flags)) + check_advance(pstate, #str, 1) + end + local function scope_test(scope) + singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope}) + singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope}) + end + local function comparison_test(op, inv_op, cmp_type) + singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'}) + singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'}) + singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'}) + singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'}) + singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'}) + singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'}) + end + local function simple_test(pstate_arg, exp_type, exp_len, exp) + local pstate = new_pstate(pstate_arg) + local exp = shallowcopy(exp) + exp.type = exp_type + exp.len = exp_len or #(pstate_arg[0]) + exp.start = { col = 0, line = 0 } + eq(exp, next_eltkn(pstate, flags)) + end + local function stable_tests() singl_eltkn_test('Parenthesis', '(', {closing=false}) singl_eltkn_test('Parenthesis', ')', {closing=true}) singl_eltkn_test('Bracket', '[', {closing=false}) @@ -170,9 +217,9 @@ describe('Expressions lexer', function() singl_eltkn_test('Spacing', ' ') singl_eltkn_test('Spacing', '\t') singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'}) - singl_eltkn_test('Number', '0123') - singl_eltkn_test('Number', '0') - singl_eltkn_test('Number', '9') + singl_eltkn_test('Number', '0123', {is_float=false}) + singl_eltkn_test('Number', '0', {is_float=false}) + singl_eltkn_test('Number', '9', {is_float=false}) singl_eltkn_test('Env', '$abc') singl_eltkn_test('Env', '$') singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0}) @@ -184,28 +231,8 @@ describe('Expressions lexer', function() singl_eltkn_test('PlainIdentifier', 'test#var', {autoload=true, scope=0}) singl_eltkn_test('PlainIdentifier', 'test#var#val###', {autoload=true, scope=0}) singl_eltkn_test('PlainIdentifier', 't#####', {autoload=true, scope=0}) - local function scope_test(scope) - singl_eltkn_test('PlainIdentifier', scope .. ':test#var', {autoload=true, scope=scope}) - singl_eltkn_test('PlainIdentifier', scope .. ':', {autoload=false, scope=scope}) - end - scope_test('s') - scope_test('g') - scope_test('v') - scope_test('b') - scope_test('w') - scope_test('t') - scope_test('l') - scope_test('a') - local function comparison_test(op, inv_op, cmp_type) - singl_eltkn_test('Comparison', op, {type=cmp_type, inv=false, ccs='UseOption'}) - singl_eltkn_test('Comparison', inv_op, {type=cmp_type, inv=true, ccs='UseOption'}) - singl_eltkn_test('Comparison', op .. '#', {type=cmp_type, inv=false, ccs='MatchCase'}) - singl_eltkn_test('Comparison', inv_op .. '#', {type=cmp_type, inv=true, ccs='MatchCase'}) - singl_eltkn_test('Comparison', op .. '?', {type=cmp_type, inv=false, ccs='IgnoreCase'}) - singl_eltkn_test('Comparison', inv_op .. '?', {type=cmp_type, inv=true, ccs='IgnoreCase'}) - end - comparison_test('is', 'isnot', 'Identical') singl_eltkn_test('And', '&&') + singl_eltkn_test('Or', '||') singl_eltkn_test('Invalid', '&', {error='E112: Option name missing: %.*s'}) singl_eltkn_test('Option', '&opt', {scope='Unspecified', name='opt'}) singl_eltkn_test('Option', '&t_xx', {scope='Unspecified', name='t_xx'}) @@ -245,16 +272,134 @@ describe('Expressions lexer', function() comparison_test('>=', '<', 'GreaterOrEqual') singl_eltkn_test('Minus', '-') singl_eltkn_test('Arrow', '->') + singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'}) + simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'}) + simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'}) + simple_test({'2.'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.x'}, 'Number', 1, {data={is_float=false}, str='2'}) + end + + local function regular_scope_tests() + scope_test('s') + scope_test('g') + scope_test('v') + scope_test('b') + scope_test('w') + scope_test('t') + scope_test('l') + scope_test('a') + + simple_test({'g:'}, 'PlainIdentifier', 2, {data={scope='g', autoload=false}, str='g:'}) + simple_test({'g:is#foo'}, 'PlainIdentifier', 8, {data={scope='g', autoload=true}, str='g:is#foo'}) + simple_test({'g:isnot#foo'}, 'PlainIdentifier', 11, {data={scope='g', autoload=true}, str='g:isnot#foo'}) + end + + local function regular_is_tests() + comparison_test('is', 'isnot', 'Identical') + + simple_test({'is'}, 'Comparison', 2, {data={type='Identical', inv=false, ccs='UseOption'}, str='is'}) + simple_test({'isnot'}, 'Comparison', 5, {data={type='Identical', inv=true, ccs='UseOption'}, str='isnot'}) + simple_test({'is?'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='IgnoreCase'}, str='is?'}) + simple_test({'isnot?'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='IgnoreCase'}, str='isnot?'}) + simple_test({'is#'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'}) + simple_test({'isnot#'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'}) + simple_test({'is#foo'}, 'Comparison', 3, {data={type='Identical', inv=false, ccs='MatchCase'}, str='is#'}) + simple_test({'isnot#foo'}, 'Comparison', 6, {data={type='Identical', inv=true, ccs='MatchCase'}, str='isnot#'}) + end + + local function regular_number_tests() + simple_test({'2.0'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0x'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false}, str='2'}) + end + + local function regular_eoc_tests() + singl_eltkn_test('EOC', '|') singl_eltkn_test('EOC', '\0') singl_eltkn_test('EOC', '\n') - singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'}) + end - local pstate = new_pstate({{data=nil, size=0}}) - eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'}, - next_eltkn(pstate)) + itp('works (single tokens, zero flags)', function() + stable_tests() - local pstate = new_pstate({''}) - eq({len=0, error='start.col >= #pstr', start={col=0, line=0}, type='EOC'}, - next_eltkn(pstate)) + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + regular_number_tests() + end) + itp('peeks', function() + flags = tonumber(lib.kELFlagPeek) + should_advance = false + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + regular_number_tests() + end) + itp('forbids scope', function() + flags = tonumber(lib.kELFlagForbidScope) + stable_tests() + + regular_eoc_tests() + regular_is_tests() + regular_number_tests() + + simple_test({'g:'}, 'PlainIdentifier', 1, {data={scope=0, autoload=false}, str='g'}) + end) + itp('allows floats', function() + flags = tonumber(lib.kELFlagAllowFloat) + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_is_tests() + + simple_test({'2.0'}, 'Number', 3, {data={is_float=true}, str='2.0'}) + simple_test({'2.0x'}, 'Number', 3, {data={is_float=true}, str='2.0'}) + simple_test({'2.0e'}, 'Number', 3, {data={is_float=true}, str='2.0'}) + simple_test({'2.0e+'}, 'Number', 3, {data={is_float=true}, str='2.0'}) + simple_test({'2.0e-'}, 'Number', 3, {data={is_float=true}, str='2.0'}) + simple_test({'2.0e+x'}, 'Number', 3, {data={is_float=true}, str='2.0'}) + simple_test({'2.0e-x'}, 'Number', 3, {data={is_float=true}, str='2.0'}) + simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true}, str='2.0e5'}) + simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true}, str='2.0e+5'}) + simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true}, str='2.0e-5'}) + end) + itp('treats `is` as an identifier', function() + flags = tonumber(lib.kELFlagIsNotCmp) + stable_tests() + + regular_eoc_tests() + regular_scope_tests() + regular_number_tests() + + simple_test({'is'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'}) + simple_test({'isnot'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'}) + simple_test({'is?'}, 'PlainIdentifier', 2, {data={scope=0, autoload=false}, str='is'}) + simple_test({'isnot?'}, 'PlainIdentifier', 5, {data={scope=0, autoload=false}, str='isnot'}) + simple_test({'is#'}, 'PlainIdentifier', 3, {data={scope=0, autoload=true}, str='is#'}) + simple_test({'isnot#'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='isnot#'}) + simple_test({'is#foo'}, 'PlainIdentifier', 6, {data={scope=0, autoload=true}, str='is#foo'}) + simple_test({'isnot#foo'}, 'PlainIdentifier', 9, {data={scope=0, autoload=true}, str='isnot#foo'}) + end) + itp('forbids EOC', function() + flags = tonumber(lib.kELFlagForbidEOC) + stable_tests() + + regular_scope_tests() + regular_is_tests() + regular_number_tests() + + singl_eltkn_test('Invalid', '|', {error='E15: Unexpected EOC character: %.*s'}) + singl_eltkn_test('Invalid', '\0', {error='E15: Unexpected EOC character: %.*s'}) + singl_eltkn_test('Invalid', '\n', {error='E15: Unexpected EOC character: %.*s'}) end) end) From f2650660819ddeae97c26114a97aff9608bd226e Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 29 Sep 2017 01:13:35 +0300 Subject: [PATCH 016/109] =?UTF-8?q?unittests:=20Add=20support=20for=20dump?= =?UTF-8?q?ing=20=E2=80=9Cexpected=E2=80=9D=20state?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Purpose is similar to that of `screen:snapshot_util()`, but in different domain. --- test/unit/viml/expressions/parser_spec.lua | 53 +++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index eec4cb5bd9..51db2dd2d9 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -1,5 +1,6 @@ local helpers = require('test.unit.helpers')(after_each) local viml_helpers = require('test.unit.viml.helpers') +local global_helpers = require('test.helpers') local itp = helpers.gen_itp(it) local make_enum_conv_tab = helpers.make_enum_conv_tab @@ -15,8 +16,51 @@ local new_pstate = viml_helpers.new_pstate local intchar2lua = viml_helpers.intchar2lua local pstate_set_str = viml_helpers.pstate_set_str +local format_string = global_helpers.format_string +local format_luav = global_helpers.format_luav + local lib = cimport('./src/nvim/viml/parser/expressions.h') +local function format_check(expr, flags, ast, hls) + -- That forces specific order. + print( format_string('\ncheck_parsing(%r, %u, {', expr, flags)) + local digits = ' -- ' + local digits2 = ' -- ' + for i = 0, #expr - 1 do + if i % 10 == 0 then + digits2 = ('%s%10u'):format(digits2, i / 10) + end + digits = ('%s%u'):format(digits, i % 10) + end + print(digits) + if #expr > 10 then + print(digits2) + end + print(' ast = ' .. format_luav(ast.ast, ' ') .. ',') + if ast.err then + print(' err = {') + print(' arg = ' .. format_luav(ast.err.arg) .. ',') + print(' msg = ' .. format_luav(ast.err.msg) .. ',') + print(' },') + end + print('}, {') + local next_col = 0 + for _, v in ipairs(hls) do + local group, line, col, str = v:match('NVim([a-zA-Z]+):(%d+):(%d+):(.*)') + col = tonumber(col) + line = tonumber(line) + assert(line == 0) + local col_shift = col - next_col + assert(col_shift >= 0) + next_col = col + #str + print(format_string(' hl(%r, %r%s),', + group, + str, + (col_shift == 0 and '' or (', %u'):format(col_shift)))) + end + print('})') +end + local east_node_type_tab make_enum_conv_tab(lib, { 'kExprNodeMissing', @@ -137,10 +181,15 @@ child_call_once(function() end) describe('Expressions parser', function() - local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) + local function check_parsing(str, flags, exp_ast, exp_highlighting_fs, + print_exp) local pstate = new_pstate({str}) local east = lib.viml_pexpr_parse(pstate, flags) local ast = east2lua(pstate, east) + local hls = phl2lua(pstate) + if print_exp then + format_check(str, flags, ast, hls) + end eq(exp_ast, ast) if exp_highlighting_fs then local exp_highlighting = {} @@ -148,7 +197,7 @@ describe('Expressions parser', function() for i, h in ipairs(exp_highlighting_fs) do exp_highlighting[i], next_col = h(next_col) end - eq(exp_highlighting, phl2lua(pstate)) + eq(exp_highlighting, hls) end end local function hl(group, str, shift) From f33543377e39fc62ab063ca57c716984fb07aea1 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 30 Sep 2017 21:34:34 +0300 Subject: [PATCH 017/109] viml/parser/expressions: Add a way to represent tokens from C code --- src/nvim/viml/parser/expressions.c | 179 +++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 3027c0046b..fc64fee140 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -440,6 +440,176 @@ viml_pexpr_next_token_adv_return: return ret; } +#ifdef UNIT_TESTING +static const char *const eltkn_type_tab[] = { + [kExprLexInvalid] = "Invalid", + [kExprLexMissing] = "Missing", + [kExprLexSpacing] = "Spacing", + [kExprLexEOC] = "EOC", + + [kExprLexQuestion] = "Question", + [kExprLexColon] = "Colon", + [kExprLexOr] = "Or", + [kExprLexAnd] = "And", + [kExprLexComparison] = "Comparison", + [kExprLexPlus] = "Plus", + [kExprLexMinus] = "Minus", + [kExprLexDot] = "Dot", + [kExprLexMultiplication] = "Multiplication", + + [kExprLexNot] = "Not", + + [kExprLexNumber] = "Number", + [kExprLexSingleQuotedString] = "SingleQuotedString", + [kExprLexDoubleQuotedString] = "DoubleQuotedString", + [kExprLexOption] = "Option", + [kExprLexRegister] = "Register", + [kExprLexEnv] = "Env", + [kExprLexPlainIdentifier] = "PlainIdentifier", + + [kExprLexBracket] = "Bracket", + [kExprLexFigureBrace] = "FigureBrace", + [kExprLexParenthesis] = "Parenthesis", + [kExprLexComma] = "Comma", + [kExprLexArrow] = "Arrow", +}; + +static const char *const eltkn_cmp_type_tab[] = { + [kExprLexCmpEqual] = "Equal", + [kExprLexCmpMatches] = "Matches", + [kExprLexCmpGreater] = "Greater", + [kExprLexCmpGreaterOrEqual] = "GreaterOrEqual", + [kExprLexCmpIdentical] = "Identical", +}; + +static const char *const ccs_tab[] = { + [kCCStrategyUseOption] = "UseOption", + [kCCStrategyMatchCase] = "MatchCase", + [kCCStrategyIgnoreCase] = "IgnoreCase", +}; + +static const char *const eltkn_mul_type_tab[] = { + [kExprLexMulMul] = "Mul", + [kExprLexMulDiv] = "Div", + [kExprLexMulMod] = "Mod", +}; + +static const char *const eltkn_opt_scope_tab[] = { + [kExprLexOptUnspecified] = "Unspecified", + [kExprLexOptGlobal] = "Global", + [kExprLexOptLocal] = "Local", +}; + +/// Represent `int` character as a string +/// +/// Converts +/// - ASCII digits into '{digit}' +/// - ASCII printable characters into a single-character strings +/// - everything else to numbers. +/// +/// @param[in] ch Character to convert. +/// +/// @return Converted string, stored in a static buffer (overriden after each +/// call). +static const char *intchar2str(const int ch) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + static char buf[sizeof(int) * 3 + 1]; + if (' ' <= ch && ch < 0x7f) { + if (ascii_isdigit(ch)) { + buf[0] = '\''; + buf[1] = (char)ch; + buf[2] = '\''; + buf[3] = NUL; + } else { + buf[0] = (char)ch; + buf[1] = NUL; + } + } else { + snprintf(buf, sizeof(buf), "%i", ch); + } + return buf; +} + +/// Represent token as a string +/// +/// Intended for testing and debugging purposes. +/// +/// @param[in] pstate Parser state, needed to get token string from it. May be +/// NULL, in which case in place of obtaining part of the +/// string represented by token only token length is +/// returned. +/// @param[in] token Token to represent. +/// @param[out] ret_size Return string size, for cases like NULs inside +/// a string. May be NULL. +/// +/// @return Token represented in a string form, in a static buffer (overwritten +/// on each call). +const char *viml_pexpr_repr_token(const ParserState *const pstate, + const LexExprToken token, + size_t *const ret_size) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + static char ret[1024]; + char *p = ret; + const char *const e = &ret[1024] - 1; +#define ADDSTR(...) \ + do { \ + p += snprintf(p, (size_t)(sizeof(ret) - (size_t)(p - ret)), __VA_ARGS__); \ + if (p >= e) { \ + goto viml_pexpr_repr_token_end; \ + } \ + } while (0) + ADDSTR("%zu:%zu:%s", token.start.line, token.start.col, + eltkn_type_tab[token.type]); + switch (token.type) { +#define TKNARGS(tkn_type, ...) \ + case tkn_type: { \ + ADDSTR(__VA_ARGS__); \ + break; \ + } + TKNARGS(kExprLexComparison, "(type=%s,ccs=%s,inv=%i)", + eltkn_cmp_type_tab[token.data.cmp.type], + ccs_tab[token.data.cmp.ccs], + (int)token.data.cmp.inv) + TKNARGS(kExprLexMultiplication, "(type=%s)", + eltkn_mul_type_tab[token.data.mul.type]) + TKNARGS(kExprLexRegister, "(name=%s)", intchar2str(token.data.reg.name)) + case kExprLexDoubleQuotedString: + TKNARGS(kExprLexSingleQuotedString, "(closed=%i)", + (int)token.data.str.closed) + TKNARGS(kExprLexOption, "(scope=%s,name=%.*s)", + eltkn_opt_scope_tab[token.data.opt.scope], + (int)token.data.opt.len, token.data.opt.name) + TKNARGS(kExprLexPlainIdentifier, "(scope=%s,autoload=%i)", + intchar2str(token.data.var.scope), (int)token.data.var.autoload) + TKNARGS(kExprLexNumber, "(is_float=%i)", (int)token.data.num.is_float) + TKNARGS(kExprLexInvalid, "(msg=%s)", token.data.err.msg) + default: { + // No additional arguments. + break; + } +#undef TKNARGS + } + if (pstate == NULL) { + ADDSTR("::%zu", token.len); + } else { + *p++ = ':'; + memmove( + p, &pstate->reader.lines.items[token.start.line].data[token.start.col], + token.len); + p += token.len; + *p = NUL; + } +#undef ADDSTR +viml_pexpr_repr_token_end: + if (ret_size != NULL) { + *ret_size = (size_t)(p - ret); + } + return ret; +} +#endif + #ifdef UNIT_TESTING #include REAL_FATTR_UNUSED @@ -469,12 +639,21 @@ static inline void viml_pexpr_debug_print_ast_stack( "-"); } } + +static inline void viml_pexpr_debug_print_token( + const ParserState *const pstate, const LexExprToken token) + FUNC_ATTR_ALWAYS_INLINE +{ + fprintf(stderr, "\ntkn: %s\n", viml_pexpr_repr_token(pstate, token, NULL)); +} #define PSTACK(msg) \ viml_pexpr_debug_print_ast_stack(&ast_stack, #msg) #define PSTACK_P(msg) \ viml_pexpr_debug_print_ast_stack(ast_stack, #msg) #define PNODE_P(eastnode_p, msg) \ viml_pexpr_debug_print_ast_node((const ExprASTNode *const *)ast_stack, #msg) +#define PTOKEN(tkn) \ + viml_pexpr_debug_print_token(pstate, tkn) #endif // start = s ternary_expr s EOC From 3735537a508c5690c4622ebe450e6f3f15706670 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 1 Oct 2017 15:54:46 +0300 Subject: [PATCH 018/109] viml/parser/expressions: Fix call inside nested parenthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It may have incorrectly tried to call everything because of essentially “value” nodes being treated as not such. --- src/nvim/viml/parser/expressions.c | 13 +++--- test/unit/viml/expressions/parser_spec.lua | 49 ++++++++++++++++++---- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index fc64fee140..1713d0c89f 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -612,6 +612,7 @@ viml_pexpr_repr_token_end: #ifdef UNIT_TESTING #include + REAL_FATTR_UNUSED static inline void viml_pexpr_debug_print_ast_node( const ExprASTNode *const *const eastnode_p, @@ -626,6 +627,7 @@ static inline void viml_pexpr_debug_print_ast_node( (*eastnode_p)->start.col, (*eastnode_p)->len); } } + REAL_FATTR_UNUSED static inline void viml_pexpr_debug_print_ast_stack( const ExprASTStack *const ast_stack, @@ -640,6 +642,7 @@ static inline void viml_pexpr_debug_print_ast_stack( } } +REAL_FATTR_UNUSED static inline void viml_pexpr_debug_print_token( const ParserState *const pstate, const LexExprToken token) FUNC_ATTR_ALWAYS_INLINE @@ -794,6 +797,7 @@ static inline ExprASTNode *viml_pexpr_new_node(const ExprASTNodeType type) typedef enum { kEOpLvlInvalid = 0, + kEOpLvlComplexIdentifier, kEOpLvlParens, kEOpLvlArrow, kEOpLvlComma, @@ -806,7 +810,6 @@ typedef enum { kEOpLvlMultiplication, ///< Multiplication, division and modulo. kEOpLvlUnary, ///< Unary operations: not, minus, plus. kEOpLvlSubscript, ///< Subscripts. - kEOpLvlComplexIdentifier, ///< Plain identifier, curly braces name. kEOpLvlValue, ///< Values: literals, variables, nested expressions, … } ExprOpLvl; @@ -843,10 +846,10 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeSubscript] = kEOpLvlSubscript, - [kExprNodeComplexIdentifier] = kEOpLvlComplexIdentifier, - [kExprNodePlainIdentifier] = kEOpLvlComplexIdentifier, [kExprNodeCurlyBracesIdentifier] = kEOpLvlComplexIdentifier, + [kExprNodeComplexIdentifier] = kEOpLvlValue, + [kExprNodePlainIdentifier] = kEOpLvlValue, [kExprNodeRegister] = kEOpLvlValue, [kExprNodeListLiteral] = kEOpLvlValue, }; @@ -884,10 +887,10 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeSubscript] = kEOpAssLeft, - [kExprNodePlainIdentifier] = kEOpAssLeft, - [kExprNodeComplexIdentifier] = kEOpAssLeft, [kExprNodeCurlyBracesIdentifier] = kEOpAssLeft, + [kExprNodeComplexIdentifier] = kEOpAssLeft, + [kExprNodePlainIdentifier] = kEOpAssNo, [kExprNodeRegister] = kEOpAssNo, [kExprNodeListLiteral] = kEOpAssNo, }; diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 51db2dd2d9..a2b76ccf8d 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -2269,7 +2269,43 @@ describe('Expressions parser', function() hl('CallingParenthesis', ')'), hl('Curly', '}'), }) - -- FIXME the below should not crash + check_parsing('a:{b()}c', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:7:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + { + 'Call:0:4:(', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Identifier', 'b'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('Identifier', 'c'), + }) check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', 0, { -- 01234567890123456789012345678901234567890123456 -- 0 1 2 3 4 @@ -2277,9 +2313,9 @@ describe('Expressions parser', function() { 'ComplexIdentifier:0:2:', children = { - 'PlainIdentifier(scope=a,ident=):0:0:g:', + 'PlainIdentifier(scope=a,ident=):0:0:a:', { - 'ComplexIdentifier:0:6:', + 'ComplexIdentifier:0:42:', children = { { 'CurlyBracesIdentifier(--i):0:2:{', @@ -2314,7 +2350,7 @@ describe('Expressions parser', function() 'Call:0:32:(', children = { { - 'NestingParenthesis:0:21: (', + 'Nested:0:21: (', children = { { 'Lambda(\\di):0:23:{', @@ -2342,10 +2378,9 @@ describe('Expressions parser', function() 'Register(name=i):0:38:@i', }, }, - 'PlainIdentifier(scope=0,ident=j):0:42:j', }, }, - 'PlainIdentifier(scope=0,ident=_test):0:42:_test', + 'PlainIdentifier(scope=0,ident=j):0:42:j', }, }, }, @@ -2364,7 +2399,7 @@ describe('Expressions parser', function() hl('BinaryPlus', '+', 1), hl('Register', '@e', 1), hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '('), + hl('NestingParenthesis', '(', 1), hl('Lambda', '{'), hl('Identifier', 'f'), hl('Arrow', '->', 1), From 9e721031d597bfa435da03597939191970f7a918 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 1 Oct 2017 16:50:46 +0300 Subject: [PATCH 019/109] viml/parser/expressions: Fix determining invalid commas/colons --- src/nvim/viml/parser/expressions.c | 167 ++++++++++++++------- src/nvim/viml/parser/expressions.h | 8 +- test/unit/viml/expressions/parser_spec.lua | 78 +++++++++- 3 files changed, 194 insertions(+), 59 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 1713d0c89f..c283241cb4 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -13,6 +13,7 @@ #include "nvim/types.h" #include "nvim/charset.h" #include "nvim/ascii.h" +#include "nvim/assert.h" #include "nvim/lib/kvec.h" #include "nvim/viml/parser/expressions.h" @@ -37,6 +38,32 @@ typedef enum { kENodeArgumentSeparator, } ExprASTWantedNode; +/// Operator priority level +typedef enum { + kEOpLvlInvalid = 0, + kEOpLvlComplexIdentifier, + kEOpLvlParens, + kEOpLvlArrow, + kEOpLvlComma, + kEOpLvlColon, + kEOpLvlTernary, + kEOpLvlOr, + kEOpLvlAnd, + kEOpLvlComparison, + kEOpLvlAddition, ///< Addition, subtraction and concatenation. + kEOpLvlMultiplication, ///< Multiplication, division and modulo. + kEOpLvlUnary, ///< Unary operations: not, minus, plus. + kEOpLvlSubscript, ///< Subscripts. + kEOpLvlValue, ///< Values: literals, variables, nested expressions, … +} ExprOpLvl; + +/// Operator associativity +typedef enum { + kEOpAssNo= 'n', ///< Not associative / not applicable. + kEOpAssLeft = 'l', ///< Left associativity. + kEOpAssRight = 'r', ///< Right associativity. +} ExprOpAssociativity; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.c.generated.h" #endif @@ -747,6 +774,7 @@ static inline void viml_pexpr_debug_print_token( // // NVimParenthesis -> Delimiter // +// NVimColon -> Delimiter // NVimComma -> Delimiter // NVimArrow -> Delimiter // @@ -895,6 +923,32 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeListLiteral] = kEOpAssNo, }; +/// Get AST node priority level +/// +/// Used primary to reduce line length, so keep the name short. +/// +/// @param[in] node Node to get priority for. +/// +/// @return Node priority level. +static inline ExprOpLvl node_lvl(const ExprASTNode node) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + return node_type_to_op_lvl[node.type]; +} + +/// Get AST node associativity, to be used for operator nodes primary +/// +/// Used primary to reduce line length, so keep the name short. +/// +/// @param[in] node Node to get priority for. +/// +/// @return Node associativity. +static inline ExprOpAssociativity node_ass(const ExprASTNode node) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + return node_type_to_op_ass[node.type]; +} + /// Handle binary operator /// /// This function is responsible for handling priority levels as well. @@ -910,20 +964,19 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, assert(kv_size(*ast_stack)); const ExprOpLvl bop_node_lvl = (bop_node->type == kExprNodeCall ? kEOpLvlSubscript - : node_type_to_op_lvl[bop_node->type]); + : node_lvl(*bop_node)); #ifndef NDEBUG const ExprOpAssociativity bop_node_ass = ( bop_node->type == kExprNodeCall ? kEOpAssLeft - : node_type_to_op_ass[bop_node->type]); + : node_ass(*bop_node)); #endif do { ExprASTNode **new_top_node_p = kv_last(*ast_stack); ExprASTNode *new_top_node = *new_top_node_p; assert(new_top_node != NULL); - const ExprOpLvl new_top_node_lvl = node_type_to_op_lvl[new_top_node->type]; - const ExprOpAssociativity new_top_node_ass = ( - node_type_to_op_ass[new_top_node->type]); + const ExprOpLvl new_top_node_lvl = node_lvl(*new_top_node); + const ExprOpAssociativity new_top_node_ass = node_ass(*new_top_node); assert(bop_node_lvl != new_top_node_lvl || bop_node_ass == new_top_node_ass); if (top_node_p != NULL @@ -1352,31 +1405,30 @@ viml_pexpr_parse_process_token: goto viml_pexpr_parse_invalid_comma; } for (size_t i = 1; i < kv_size(ast_stack); i++) { - const ExprASTNode *const *const eastnode_p = - (const ExprASTNode *const *)kv_Z(ast_stack, i); - if (!((*eastnode_p)->type == kExprNodeComma - || ((*eastnode_p)->type == kExprNodeColon - && i == 1)) - || i == kv_size(ast_stack) - 1) { - switch ((*eastnode_p)->type) { - case kExprNodeLambda: { - assert(want_node == kENodeArgumentSeparator); - break; - } - case kExprNodeDictLiteral: - case kExprNodeListLiteral: - case kExprNodeCall: { - break; - } - default: { -viml_pexpr_parse_invalid_comma: - ERROR_FROM_TOKEN_AND_MSG( - cur_token, - _("E15: Comma outside of call, lambda or literal: %.*s")); - break; - } - } + ExprASTNode *const *const eastnode_p = + (ExprASTNode *const *)kv_Z(ast_stack, i); + const ExprASTNodeType eastnode_type = (*eastnode_p)->type; + const ExprOpLvl eastnode_lvl = node_lvl(**eastnode_p); + if (eastnode_type == kExprNodeLambda) { + assert(want_node == kENodeArgumentSeparator); break; + } else if (eastnode_type == kExprNodeDictLiteral + || eastnode_type == kExprNodeListLiteral + || eastnode_type == kExprNodeCall) { + break; + } else if (eastnode_type == kExprNodeComma + || eastnode_type == kExprNodeColon + || eastnode_lvl > kEOpLvlComma) { + // Do nothing + } else { +viml_pexpr_parse_invalid_comma: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Comma outside of call, lambda or literal: %.*s")); + break; + } + if (i == kv_size(ast_stack) - 1) { + goto viml_pexpr_parse_invalid_comma; } } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComma); @@ -1389,37 +1441,48 @@ viml_pexpr_parse_invalid_comma: if (kv_size(ast_stack) < 2) { goto viml_pexpr_parse_invalid_colon; } + bool is_ternary = false; + bool can_be_ternary = true; for (size_t i = 1; i < kv_size(ast_stack); i++) { ExprASTNode *const *const eastnode_p = (ExprASTNode *const *)kv_Z(ast_stack, i); - if ((*eastnode_p)->type != kExprNodeColon - || i == kv_size(ast_stack) - 1) { - switch ((*eastnode_p)->type) { - case kExprNodeUnknownFigure: { - SELECT_FIGURE_BRACE_TYPE((*eastnode_p), DictLiteral, Dict); - break; - } - case kExprNodeComma: - case kExprNodeDictLiteral: - case kExprNodeTernary: { - break; - } - default: { -viml_pexpr_parse_invalid_colon: - ERROR_FROM_TOKEN_AND_MSG( - cur_token, - _("E15: Colon outside of dictionary or ternary operator: " - "%.*s")); - break; - } - } + const ExprASTNodeType eastnode_type = (*eastnode_p)->type; + const ExprOpLvl eastnode_lvl = node_lvl(**eastnode_p); + STATIC_ASSERT(kEOpLvlTernary > kEOpLvlComma, + "Unexpected operator priorities"); + if (can_be_ternary && eastnode_lvl == kEOpLvlTernary) { + assert(eastnode_type == kExprNodeTernary); + is_ternary = true; break; + } else if (eastnode_type == kExprNodeUnknownFigure) { + SELECT_FIGURE_BRACE_TYPE(*eastnode_p, DictLiteral, Dict); + break; + } else if (eastnode_type == kExprNodeDictLiteral + || eastnode_type == kExprNodeComma) { + break; + } else if (eastnode_lvl > kEOpLvlTernary) { + // Do nothing + } else if (eastnode_lvl > kEOpLvlComma) { + can_be_ternary = false; + } else { +viml_pexpr_parse_invalid_colon: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Colon outside of dictionary or ternary operator: " + "%.*s")); + break; + } + if (i == kv_size(ast_stack) - 1) { + goto viml_pexpr_parse_invalid_colon; } } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); - // FIXME: Handle ternary operator. - HL_CUR_TOKEN(Colon); + if (is_ternary) { + HL_CUR_TOKEN(TernaryColon); + } else { + HL_CUR_TOKEN(Colon); + } want_node = kENodeValue; break; } diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 64abab9e41..01a51e4eda 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -144,10 +144,10 @@ typedef enum { typedef enum { kExprNodeMissing = 'X', kExprNodeOpMissing = '_', - kExprNodeTernary = '?', ///< Ternary operator, valid one has three children. - kExprNodeRegister = '@', ///< Register, no children. - kExprNodeSubscript = 's', ///< Subscript, should have two or three children. - kExprNodeListLiteral = 'l', ///< List literal, any number of children. + kExprNodeTernary = '?', ///< Ternary operator. + kExprNodeRegister = '@', ///< Register. + kExprNodeSubscript = 's', ///< Subscript. + kExprNodeListLiteral = 'l', ///< List literal. kExprNodeUnaryPlus = 'p', kExprNodeBinaryPlus = '+', kExprNodeNested = 'e', ///< Nested parenthesised expression. diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index a2b76ccf8d..1f734c3c2a 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -181,14 +181,14 @@ child_call_once(function() end) describe('Expressions parser', function() - local function check_parsing(str, flags, exp_ast, exp_highlighting_fs, - print_exp) + local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) local pstate = new_pstate({str}) local east = lib.viml_pexpr_parse(pstate, flags) local ast = east2lua(pstate, east) local hls = phl2lua(pstate) - if print_exp then + if exp_ast == nil then format_check(str, flags, ast, hls) + return end eq(exp_ast, ast) if exp_highlighting_fs then @@ -2416,6 +2416,78 @@ describe('Expressions parser', function() hl('Curly', '}'), hl('Identifier', 'j'), }) + check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:8: :', + children = { + { + 'BinaryPlus:0:3: +', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:5: @b', + }, + }, + { + 'BinaryPlus:0:13: +', + children = { + 'Register(name=c):0:10: @c', + 'Register(name=d):0:15: @d', + }, + }, + }, + }, + { + 'Colon:0:27: :', + children = { + { + 'BinaryPlus:0:22: +', + children = { + 'Register(name=e):0:19: @e', + 'Register(name=f):0:24: @f', + }, + }, + { + 'BinaryPlus:0:32: +', + children = { + 'Register(name=g):0:29: @g', + 'Register(name=i):0:34: @i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('Register', '@b', 1), + hl('Colon', ':', 1), + hl('Register', '@c', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@f', 1), + hl('Colon', ':', 1), + hl('Register', '@g', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@i', 1), + hl('Dict', '}'), + }) end) -- FIXME: Test sequence of arrows inside and outside lambdas. -- FIXME: Test autoload character and scope in lambda arguments. From 6144e26eb920a90b0db22bd7afcac0b9e0734ed6 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 1 Oct 2017 22:35:41 +0300 Subject: [PATCH 020/109] viml/parser/expressions: Add support for ternary operator --- src/nvim/viml/parser/expressions.c | 87 +-- src/nvim/viml/parser/expressions.h | 6 +- test/unit/viml/expressions/parser_spec.lua | 651 ++++++++++++++++++++- 3 files changed, 705 insertions(+), 39 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index c283241cb4..41c77c5c88 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -46,6 +46,7 @@ typedef enum { kEOpLvlArrow, kEOpLvlComma, kEOpLvlColon, + kEOpLvlTernaryValue, kEOpLvlTernary, kEOpLvlOr, kEOpLvlAnd, @@ -770,7 +771,8 @@ static inline void viml_pexpr_debug_print_token( // NVimUnaryOperator -> NVimOperator // NVimBinaryOperator -> NVimOperator // NVimComparisonOperator -> NVimOperator -// NVimTernaryOperator -> NVimOperator +// NVimTernary -> NVimOperator +// NVimTernaryColon -> NVimTernary // // NVimParenthesis -> Delimiter // @@ -790,7 +792,8 @@ static inline void viml_pexpr_debug_print_token( // // NVimInvalidComma -> NVimInvalidDelimiter // NVimInvalidSpacing -> NVimInvalid -// NVimInvalidTernaryOperator -> NVimInvalidOperator +// NVimInvalidTernary -> NVimInvalidOperator +// NVimInvalidTernaryColon -> NVimInvalidTernary // NVimInvalidRegister -> NVimInvalidValue // NVimInvalidClosingBracket -> NVimInvalidDelimiter // NVimInvalidSpacing -> NVimInvalid @@ -823,30 +826,6 @@ static inline ExprASTNode *viml_pexpr_new_node(const ExprASTNodeType type) return ret; } -typedef enum { - kEOpLvlInvalid = 0, - kEOpLvlComplexIdentifier, - kEOpLvlParens, - kEOpLvlArrow, - kEOpLvlComma, - kEOpLvlColon, - kEOpLvlTernary, - kEOpLvlOr, - kEOpLvlAnd, - kEOpLvlComparison, - kEOpLvlAddition, ///< Addition, subtraction and concatenation. - kEOpLvlMultiplication, ///< Multiplication, division and modulo. - kEOpLvlUnary, ///< Unary operations: not, minus, plus. - kEOpLvlSubscript, ///< Subscripts. - kEOpLvlValue, ///< Values: literals, variables, nested expressions, … -} ExprOpLvl; - -typedef enum { - kEOpAssNo= 'n', ///< Not associative / not applicable. - kEOpAssLeft = 'l', ///< Left associativity. - kEOpAssRight = 'r', ///< Right associativity. -} ExprOpAssociativity; - static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeMissing] = kEOpLvlInvalid, [kExprNodeOpMissing] = kEOpLvlMultiplication, @@ -868,6 +847,8 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeTernary] = kEOpLvlTernary, + [kExprNodeTernaryValue] = kEOpLvlTernaryValue, + [kExprNodeBinaryPlus] = kEOpLvlAddition, [kExprNodeUnaryPlus] = kEOpLvlUnary, @@ -907,7 +888,9 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { // about associativity, only about order of execution. [kExprNodeComma] = kEOpAssRight, - [kExprNodeTernary] = kEOpAssNo, + [kExprNodeTernary] = kEOpAssRight, + + [kExprNodeTernaryValue] = kEOpAssRight, [kExprNodeBinaryPlus] = kEOpAssLeft, @@ -1450,9 +1433,20 @@ viml_pexpr_parse_invalid_comma: const ExprOpLvl eastnode_lvl = node_lvl(**eastnode_p); STATIC_ASSERT(kEOpLvlTernary > kEOpLvlComma, "Unexpected operator priorities"); - if (can_be_ternary && eastnode_lvl == kEOpLvlTernary) { - assert(eastnode_type == kExprNodeTernary); + if (can_be_ternary && eastnode_type == kExprNodeTernaryValue + && !(*eastnode_p)->data.ter.got_colon) { + kv_drop(ast_stack, i); + (*eastnode_p)->start = cur_token.start; + (*eastnode_p)->len = cur_token.len; + if (prev_token.type == kExprLexSpacing) { + (*eastnode_p)->start = prev_token.start; + (*eastnode_p)->len += prev_token.len; + } is_ternary = true; + (*eastnode_p)->data.ter.got_colon = true; + assert((*eastnode_p)->children != NULL); + assert((*eastnode_p)->children->next == NULL); + kvi_push(ast_stack, &(*eastnode_p)->children->next); break; } else if (eastnode_type == kExprNodeUnknownFigure) { SELECT_FIGURE_BRACE_TYPE(*eastnode_p, DictLiteral, Dict); @@ -1460,7 +1454,7 @@ viml_pexpr_parse_invalid_comma: } else if (eastnode_type == kExprNodeDictLiteral || eastnode_type == kExprNodeComma) { break; - } else if (eastnode_lvl > kEOpLvlTernary) { + } else if (eastnode_lvl >= kEOpLvlTernaryValue) { // Do nothing } else if (eastnode_lvl > kEOpLvlComma) { can_be_ternary = false; @@ -1476,11 +1470,11 @@ viml_pexpr_parse_invalid_colon: goto viml_pexpr_parse_invalid_colon; } } - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); if (is_ternary) { HL_CUR_TOKEN(TernaryColon); } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); HL_CUR_TOKEN(Colon); } want_node = kENodeValue; @@ -1683,8 +1677,6 @@ viml_pexpr_parse_figure_brace_closing_error: cur_token.len - scope_shift, HL(Identifier)); } - // FIXME: Actually, g{foo}g:foo is valid: "1?g{foo}g:foo" is like - // "g{foo}g" and not an error. } else { if (cur_token.data.var.scope == 0) { ADD_IDENT( @@ -1792,6 +1784,21 @@ viml_pexpr_parse_no_paren_closing_error: {} } break; } + case kExprLexQuestion: { + ADD_VALUE_IF_MISSING(_("E15: Expected value, got question mark: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeTernary); + viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + HL_CUR_TOKEN(Ternary); + ExprASTNode *ter_val_node; + NEW_NODE_WITH_CUR_POS(ter_val_node, kExprNodeTernaryValue); + ter_val_node->data.ter.got_colon = false; + assert(cur_node->children != NULL); + assert(cur_node->children->next == NULL); + assert(kv_last(ast_stack) == &cur_node->children->next); + *kv_last(ast_stack) = ter_val_node; + kvi_push(ast_stack, &ter_val_node->children); + break; + } } viml_pexpr_parse_cycle_end: prev_token = cur_token; @@ -1815,6 +1822,7 @@ viml_pexpr_parse_end: const ExprASTNode *const cur_node = (*kv_pop(ast_stack)); // This should only happen when want_node == kENodeValue. assert(cur_node != NULL); + // TODO(ZyX-I): Rehighlight as invalid? switch (cur_node->type) { case kExprNodeOpMissing: case kExprNodeMissing: { @@ -1822,7 +1830,6 @@ viml_pexpr_parse_end: break; } case kExprNodeCall: { - // TODO(ZyX-I): Rehighlight as invalid? east_set_error( &ast, pstate, _("E116: Missing closing parenthesis for function call: %.*s"), @@ -1830,7 +1837,6 @@ viml_pexpr_parse_end: break; } case kExprNodeNested: { - // TODO(ZyX-I): Rehighlight as invalid? east_set_error( &ast, pstate, _("E110: Missing closing parenthesis for nested expression" @@ -1844,6 +1850,15 @@ viml_pexpr_parse_end: // It is OK to see these in the stack. break; } + case kExprNodeTernaryValue: { + if (!cur_node->data.ter.got_colon) { + // Actually Vim throws E109 in more cases. + east_set_error( + &ast, pstate, _("E109: Missing ':' after '?': %.*s"), + cur_node->start); + } + break; + } // TODO(ZyX-I): handle other values } } diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 01a51e4eda..cf0907850a 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -145,6 +145,7 @@ typedef enum { kExprNodeMissing = 'X', kExprNodeOpMissing = '_', kExprNodeTernary = '?', ///< Ternary operator. + kExprNodeTernaryValue = 'C', ///< Ternary operator, colon. kExprNodeRegister = '@', ///< Register. kExprNodeSubscript = 's', ///< Subscript. kExprNodeListLiteral = 'l', ///< List literal. @@ -209,7 +210,10 @@ struct expr_ast_node { /// Points to inside parser reader state. const char *ident; size_t ident_len; ///< Actual identifier length. - } var; + } var; ///< For kExprNodePlainIdentifier. + struct { + bool got_colon; ///< True if colon was seen. + } ter; ///< For kExprNodeTernaryValue. } data; }; diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 1f734c3c2a..ea37d64662 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -66,6 +66,7 @@ make_enum_conv_tab(lib, { 'kExprNodeMissing', 'kExprNodeOpMissing', 'kExprNodeTernary', + 'kExprNodeTernaryValue', 'kExprNodeRegister', 'kExprNodeSubscript', 'kExprNodeListLiteral', @@ -2489,6 +2490,652 @@ describe('Expressions parser', function() hl('Dict', '}'), }) end) - -- FIXME: Test sequence of arrows inside and outside lambdas. - -- FIXME: Test autoload character and scope in lambda arguments. + itp('works with ternary operator', function() + check_parsing('a ? b : c', 0, { + -- 012345678 + ast = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Ternary', '?', 1), + hl('Identifier', 'b', 1), + hl('TernaryColon', ':', 1), + hl('Identifier', 'c', 1), + }) + check_parsing('@a?@b?@c:@d:@e', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:11::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:8::', + children = { + 'Register(name=c):0:6:@c', + 'Register(name=d):0:9:@d', + }, + }, + }, + }, + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('TernaryColon', ':'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b:@c?@d:@e', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:5::', + children = { + 'Register(name=b):0:3:@b', + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', 0, { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:29::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:20::', + children = { + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + { + 'Ternary:0:14:?', + children = { + 'Register(name=e):0:12:@e', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=f):0:15:@f', + 'Register(name=g):0:18:@g', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Ternary:0:23:?', + children = { + 'Register(name=h):0:21:@h', + { + 'TernaryValue:0:26::', + children = { + 'Register(name=i):0:24:@i', + 'Register(name=j):0:27:@j', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=k):0:30:@k', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + hl('Ternary', '?'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('TernaryColon', ':'), + hl('Register', '@h'), + hl('Ternary', '?'), + hl('Register', '@i'), + hl('TernaryColon', ':'), + hl('Register', '@j'), + hl('TernaryColon', ':'), + hl('Register', '@k'), + }) + check_parsing('?', 0, { + -- 0 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + 'TernaryValue:0:0:?', + }, + }, + }, + err = { + arg = '?', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + }) + + check_parsing('?:', 0, { + -- 01 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = '?:', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + }) + + check_parsing('?::', 0, { + -- 012 + ast = { + { + 'Colon:0:2::', + children = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + 'Missing:0:2:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '?::', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + hl('InvalidColon', ':'), + }) + + check_parsing('a?b', 0, { + -- 012 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '?b', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('Ternary', '?'), + hl('Identifier', 'b'), + }) + check_parsing('a?b:', 0, { + -- 0123 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, + }, + err = { + arg = '?b:', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + }) + + check_parsing('a?b::c', 0, { + -- 012345 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:4::', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('TernaryColon', ':'), + hl('Identifier', 'c'), + }) + + check_parsing('a?b :', 0, { + -- 01234 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('Ternary', '?'), + hl('Identifier', 'b'), + hl('TernaryColon', ':', 1), + }) + + check_parsing('(@a?@b:@c)?@d:@e', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:13::', + children = { + 'Register(name=d):0:11:@d', + 'Register(name=e):0:14:@e', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + + check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', 0, { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:21::', + children = { + { + 'Nested:0:11:(', + children = { + { + 'Ternary:0:14:?', + children = { + 'Register(name=d):0:12:@d', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=e):0:15:@e', + 'Register(name=f):0:18:@f', + }, + }, + }, + }, + }, + }, + { + 'Nested:0:22:(', + children = { + { + 'Ternary:0:25:?', + children = { + 'Register(name=g):0:23:@g', + { + 'TernaryValue:0:28::', + children = { + 'Register(name=h):0:26:@h', + 'Register(name=i):0:29:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('NestingParenthesis', '('), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('TernaryColon', ':'), + hl('NestingParenthesis', '('), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', 0, { + -- 0123456789012345678901234567 + -- 0 1 2 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:19::', + children = { + { + 'Ternary:0:13:?', + children = { + 'Register(name=d):0:11:@d', + { + 'TernaryValue:0:16::', + children = { + 'Register(name=e):0:14:@e', + 'Register(name=f):0:17:@f', + }, + }, + }, + }, + { + 'Ternary:0:22:?', + children = { + 'Register(name=g):0:20:@g', + { + 'TernaryValue:0:25::', + children = { + 'Register(name=h):0:23:@h', + 'Register(name=i):0:26:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + }) + check_parsing('a?b{cdef}g:h', 0, { + -- 012345678901 + -- 0 1 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:10::', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'ComplexIdentifier:0:9:', + children = { + { + 'CurlyBracesIdentifier(--i):0:3:{', + children = { + 'PlainIdentifier(scope=0,ident=cdef):0:4:cdef', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:9:g', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=h):0:11:h', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Ternary', '?'), + hl('Identifier', 'b'), + hl('Curly', '{'), + hl('Identifier', 'cdef'), + hl('Curly', '}'), + hl('Identifier', 'g'), + hl('TernaryColon', ':'), + hl('Identifier', 'h'), + }) + end) end) From 6791c574209c83570746c139d93f8e6a6b9cd135 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 2 Oct 2017 01:22:35 +0300 Subject: [PATCH 021/109] viml/parser/expressions: Make sure that arrows outside lambda throw --- src/nvim/viml/parser/expressions.c | 3 +- test/unit/viml/expressions/parser_spec.lua | 190 +++++++++++++++++++++ 2 files changed, 192 insertions(+), 1 deletion(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 41c77c5c88..982465055e 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1642,8 +1642,9 @@ viml_pexpr_parse_figure_brace_closing_error: lambda_node = NULL; } else { // Only first branch is valid. - is_invalid = true; ADD_VALUE_IF_MISSING(_("E15: Unexpected arrow: %.*s")); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Arrow outside of lambda: %.*s")); NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); } diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index ea37d64662..2c80b437dc 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -2489,6 +2489,196 @@ describe('Expressions parser', function() hl('Register', '@i', 1), hl('Dict', '}'), }) + check_parsing('-> -> ->', 0, { + -- 01234567 + ast = { + { + 'Arrow:0:0:->', + children = { + 'Missing:0:0:', + { + 'Arrow:0:2: ->', + children = { + 'Missing:0:2:', + { + 'Arrow:0:5: ->', + children = { + 'Missing:0:5:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> -> ->', + msg = 'E15: Unexpected arrow: %.*s', + }, + }, { + hl('InvalidArrow', '->'), + hl('InvalidArrow', '->', 1), + hl('InvalidArrow', '->', 1), + }) + check_parsing('a -> b -> c -> d', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Arrow:0:1: ->', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Arrow:0:6: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + { + 'Arrow:0:11: ->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:9: c', + 'PlainIdentifier(scope=0,ident=d):0:14: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> b -> c -> d', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('InvalidArrow', '->', 1), + hl('Identifier', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('Identifier', 'c', 1), + hl('InvalidArrow', '->', 1), + hl('Identifier', 'd', 1), + }) + check_parsing('{a -> b -> c}', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2: ->', + children = { + { + 'Arrow:0:7: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:5: b', + 'PlainIdentifier(scope=0,ident=c):0:10: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> c}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Arrow', '->', 1), + hl('Identifier', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('Identifier', 'c', 1), + hl('Lambda', '}'), + }) + check_parsing('{a: -> b}', 0, { + -- 012345678 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Arrow:0:3: ->', + children = { + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'PlainIdentifier(scope=0,ident=b):0:6: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidArrow', '->', 1), + hl('Identifier', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a:b -> b}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Identifier', 'b'), + hl('InvalidArrow', '->', 1), + hl('Identifier', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a#b -> b}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=a#b):0:1:a#b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('Identifier', 'a#b'), + hl('InvalidArrow', '->', 1), + hl('Identifier', 'b', 1), + hl('Curly', '}'), + }) end) itp('works with ternary operator', function() check_parsing('a ? b : c', 0, { From 6168e1127c1c80a3810854649b0776146545043b Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 2 Oct 2017 02:41:55 +0300 Subject: [PATCH 022/109] viml/parser/expressions: Add support for comparison operators --- src/nvim/viml/parser/expressions.c | 131 +++++--- src/nvim/viml/parser/expressions.h | 45 ++- test/symbolic/klee/viml_expressions_parser.c | 8 +- test/unit/viml/expressions/lexer_spec.lua | 25 +- test/unit/viml/expressions/parser_spec.lua | 325 ++++++++++++++++++- test/unit/viml/helpers.lua | 31 ++ 6 files changed, 483 insertions(+), 82 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 982465055e..8e6f991e03 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -102,7 +102,7 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) if (ret.len < pline.size \ && strchr("?#", pline.data[ret.len]) != NULL) { \ ret.data.cmp.ccs = \ - (CaseCompareStrategy)pline.data[ret.len]; \ + (ExprCaseCompareStrategy)pline.data[ret.len]; \ ret.len++; \ } else { \ ret.data.cmp.ccs = kCCStrategyUseOption; \ @@ -240,7 +240,7 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) && ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0) || (ret.len == 5 && memcmp(pline.data, "isnot", 5) == 0))) { ret.type = kExprLexComparison; - ret.data.cmp.type = kExprLexCmpIdentical; + ret.data.cmp.type = kExprCmpIdentical; ret.data.cmp.inv = (ret.len == 5); GET_CCS(ret, pline); // Scope: `s:`, etc. @@ -381,10 +381,10 @@ viml_pexpr_next_token_invalid_comparison: ret.type = kExprLexComparison; ret.data.cmp.inv = (schar == '!'); if (pline.data[1] == '=') { - ret.data.cmp.type = kExprLexCmpEqual; + ret.data.cmp.type = kExprCmpEqual; ret.len++; } else if (pline.data[1] == '~') { - ret.data.cmp.type = kExprLexCmpMatches; + ret.data.cmp.type = kExprCmpMatches; ret.len++; } else { goto viml_pexpr_next_token_invalid_comparison; @@ -404,8 +404,8 @@ viml_pexpr_next_token_invalid_comparison: GET_CCS(ret, pline); ret.data.cmp.inv = (schar == '<'); ret.data.cmp.type = ((ret.data.cmp.inv ^ haseqsign) - ? kExprLexCmpGreaterOrEqual - : kExprLexCmpGreater); + ? kExprCmpGreaterOrEqual + : kExprCmpGreater); break; } @@ -503,11 +503,11 @@ static const char *const eltkn_type_tab[] = { }; static const char *const eltkn_cmp_type_tab[] = { - [kExprLexCmpEqual] = "Equal", - [kExprLexCmpMatches] = "Matches", - [kExprLexCmpGreater] = "Greater", - [kExprLexCmpGreaterOrEqual] = "GreaterOrEqual", - [kExprLexCmpIdentical] = "Identical", + [kExprCmpEqual] = "Equal", + [kExprCmpMatches] = "Matches", + [kExprCmpGreater] = "Greater", + [kExprCmpGreaterOrEqual] = "GreaterOrEqual", + [kExprCmpIdentical] = "Identical", }; static const char *const ccs_tab[] = { @@ -770,7 +770,8 @@ static inline void viml_pexpr_debug_print_token( // NVimOperator -> Operator // NVimUnaryOperator -> NVimOperator // NVimBinaryOperator -> NVimOperator -// NVimComparisonOperator -> NVimOperator +// NVimComparisonOperator -> NVimBinaryOperator +// NVimComparisonOperatorModifier -> NVimComparisonOperator // NVimTernary -> NVimOperator // NVimTernaryColon -> NVimTernary // @@ -805,6 +806,8 @@ static inline void viml_pexpr_debug_print_token( // NVimInvalidIdentifier -> NVimInvalidValue // NVimInvalidIdentifierScope -> NVimInvalidValue // NVimInvalidIdentifierScopeDelimiter -> NVimInvalidValue +// NVimInvalidComparisonOperator -> NVimInvalidOperator +// NVimInvalidComparisonOperatorModifier -> NVimInvalidComparisonOperator // // NVimUnaryPlus -> NVimUnaryOperator // NVimBinaryPlus -> NVimBinaryOperator @@ -849,6 +852,8 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeTernaryValue] = kEOpLvlTernaryValue, + [kExprNodeComparison] = kEOpLvlComparison, + [kExprNodeBinaryPlus] = kEOpLvlAddition, [kExprNodeUnaryPlus] = kEOpLvlUnary, @@ -892,6 +897,8 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeTernaryValue] = kEOpAssRight, + [kExprNodeComparison] = kEOpAssRight, + [kExprNodeBinaryPlus] = kEOpAssLeft, [kExprNodeUnaryPlus] = kEOpAssNo, @@ -935,11 +942,23 @@ static inline ExprOpAssociativity node_ass(const ExprASTNode node) /// Handle binary operator /// /// This function is responsible for handling priority levels as well. -static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, +/// +/// @param[in] pstate Parser state, used for error reporting. +/// @param ast_stack AST stack. May be popped of some values and will +/// definitely receive new ones. +/// @param bop_node New node to handle. +/// @param[out] want_node_p New value of want_node. +/// @param[out] ast_err Location where error is saved, if any. +/// +/// @return True if no errors occurred, false otherwise. +static bool viml_pexpr_handle_bop(const ParserState *const pstate, + ExprASTStack *const ast_stack, ExprASTNode *const bop_node, - ExprASTWantedNode *const want_node_p) + ExprASTWantedNode *const want_node_p, + ExprASTError *const ast_err) FUNC_ATTR_NONNULL_ALL { + bool ret = true; ExprASTNode **top_node_p = NULL; ExprASTNode *top_node; ExprOpLvl top_node_lvl; @@ -977,7 +996,6 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, break; } } while (kv_size(*ast_stack)); - // FIXME: Handle no associativity if (top_node_ass == kEOpAssLeft || top_node_lvl != bop_node_lvl) { // outer(op(x,y)) -> outer(new_op(op(x,y),*)) // @@ -1008,10 +1026,18 @@ static void viml_pexpr_handle_bop(ExprASTStack *const ast_stack, kvi_push(*ast_stack, top_node_p); kvi_push(*ast_stack, &top_node->children->next); kvi_push(*ast_stack, &bop_node->children->next); + // TODO(ZyX-I): Make this not error, but treat like Python does + if (bop_node->type == kExprNodeComparison) { + east_set_error(pstate, ast_err, + _("E15: Operator is not associative: %.*s"), + bop_node->start); + ret = false; + } } *want_node_p = (*want_node_p == kENodeArgumentSeparator ? kENodeArgument : kENodeValue); + return ret; } /// ParserPosition literal based on ParserPosition pos with columns shifted @@ -1074,6 +1100,13 @@ static inline ParserPosition shifted_pos(const ParserPosition pos, #define MAY_HAVE_NEXT_EXPR \ (kv_size(ast_stack) == 1) +/// Add operator node +/// +/// @param[in] cur_node Node to add. +#define ADD_OP_NODE(cur_node) \ + is_invalid |= !viml_pexpr_handle_bop(pstate, &ast_stack, cur_node, \ + &want_node, &ast.err) + /// Record missing operator: for things like /// /// :echo @a @a @@ -1094,7 +1127,7 @@ static inline ParserPosition shifted_pos(const ParserPosition pos, ERROR_FROM_TOKEN_AND_MSG(cur_token, _("E15: Missing operator: %.*s")); \ NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOpMissing); \ cur_node->len = 0; \ - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); \ + ADD_OP_NODE(cur_node); \ goto viml_pexpr_parse_process_token; \ } \ } while (0) @@ -1120,34 +1153,33 @@ static inline ParserPosition shifted_pos(const ParserPosition pos, /// @param[in] msg Error message, assumed to be already translated and /// containing a single %token "%.*s". /// @param[in] start Position at which error occurred. -static inline void east_set_error(ExprAST *const ret_ast, - const ParserState *const pstate, +static inline void east_set_error(const ParserState *const pstate, + ExprASTError *const ret_ast_err, const char *const msg, const ParserPosition start) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_ALWAYS_INLINE { - if (!ret_ast->correct) { + if (ret_ast_err->msg != NULL) { return; } const ParserLine pline = pstate->reader.lines.items[start.line]; - ret_ast->correct = false; - ret_ast->err.msg = msg; - ret_ast->err.arg_len = (int)(pline.size - start.col); - ret_ast->err.arg = pline.data + start.col; + ret_ast_err->msg = msg; + ret_ast_err->arg_len = (int)(pline.size - start.col); + ret_ast_err->arg = pline.data + start.col; } /// Set error from the given token and given message #define ERROR_FROM_TOKEN_AND_MSG(cur_token, msg) \ do { \ is_invalid = true; \ - east_set_error(&ast, pstate, msg, cur_token.start); \ + east_set_error(pstate, &ast.err, msg, cur_token.start); \ } while (0) /// Like #ERROR_FROM_TOKEN_AND_MSG, but gets position from a node #define ERROR_FROM_NODE_AND_MSG(node, msg) \ do { \ is_invalid = true; \ - east_set_error(&ast, pstate, msg, node->start); \ + east_set_error(pstate, &ast.err, msg, node->start); \ } while (0) /// Set error from the given kExprLexInvalid token @@ -1231,7 +1263,6 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { ExprAST ast = { - .correct = true, .err = { .msg = NULL, .arg_len = 0, @@ -1359,12 +1390,38 @@ viml_pexpr_parse_process_token: HL_CUR_TOKEN(UnaryPlus); } else { NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeBinaryPlus); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + ADD_OP_NODE(cur_node); HL_CUR_TOKEN(BinaryPlus); } want_node = kENodeValue; break; } + case kExprLexComparison: { + ADD_VALUE_IF_MISSING( + _("E15: Expected value, got comparison operator: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComparison); + if (cur_token.type == kExprLexInvalid) { + cur_node->data.cmp.ccs = kCCStrategyUseOption; + cur_node->data.cmp.type = kExprCmpEqual; + cur_node->data.cmp.inv = false; + } else { + cur_node->data.cmp.ccs = cur_token.data.cmp.ccs; + cur_node->data.cmp.type = cur_token.data.cmp.type; + cur_node->data.cmp.inv = cur_token.data.cmp.inv; + } + ADD_OP_NODE(cur_node); + if (cur_token.data.cmp.ccs != kCCStrategyUseOption) { + viml_parser_highlight(pstate, cur_token.start, cur_token.len - 1, + HL(ComparisonOperator)); + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, cur_token.len - 1), 1, + HL(ComparisonOperatorModifier)); + } else { + HL_CUR_TOKEN(ComparisonOperator); + } + want_node = kENodeValue; + break; + } case kExprLexComma: { assert(want_node != kENodeArgument); if (want_node == kENodeValue) { @@ -1415,7 +1472,7 @@ viml_pexpr_parse_invalid_comma: } } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeComma); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + ADD_OP_NODE(cur_node); HL_CUR_TOKEN(Comma); break; } @@ -1474,7 +1531,7 @@ viml_pexpr_parse_invalid_colon: HL_CUR_TOKEN(TernaryColon); } else { NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + ADD_OP_NODE(cur_node); HL_CUR_TOKEN(Colon); } want_node = kENodeValue; @@ -1646,7 +1703,7 @@ viml_pexpr_parse_figure_brace_closing_error: ERROR_FROM_TOKEN_AND_MSG( cur_token, _("E15: Arrow outside of lambda: %.*s")); NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + ADD_OP_NODE(cur_node); } want_node = kENodeValue; HL_CUR_TOKEN(Arrow); @@ -1775,7 +1832,7 @@ viml_pexpr_parse_no_paren_closing_error: {} } } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCall); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + ADD_OP_NODE(cur_node); HL_CUR_TOKEN(CallingParenthesis); } else { // Currently it is impossible to reach this. @@ -1788,7 +1845,7 @@ viml_pexpr_parse_no_paren_closing_error: {} case kExprLexQuestion: { ADD_VALUE_IF_MISSING(_("E15: Expected value, got question mark: %.*s")); NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeTernary); - viml_pexpr_handle_bop(&ast_stack, cur_node, &want_node); + ADD_OP_NODE(cur_node); HL_CUR_TOKEN(Ternary); ExprASTNode *ter_val_node; NEW_NODE_WITH_CUR_POS(ter_val_node, kExprNodeTernaryValue); @@ -1808,7 +1865,7 @@ viml_pexpr_parse_cycle_end: } while (true); viml_pexpr_parse_end: if (want_node == kENodeValue) { - east_set_error(&ast, pstate, _("E15: Expected value, got EOC: %.*s"), + east_set_error(pstate, &ast.err, _("E15: Expected value, got EOC: %.*s"), pstate->pos); } else if (kv_size(ast_stack) != 1) { // Something may be wrong, check whether it really is. @@ -1819,7 +1876,7 @@ viml_pexpr_parse_end: // Topmost stack item must be a *finished* value, so it must not be // analyzed. E.g. it may contain an already finished nested expression. kv_drop(ast_stack, 1); - while (ast.correct && kv_size(ast_stack)) { + while (ast.err.msg == NULL && kv_size(ast_stack)) { const ExprASTNode *const cur_node = (*kv_pop(ast_stack)); // This should only happen when want_node == kENodeValue. assert(cur_node != NULL); @@ -1832,14 +1889,14 @@ viml_pexpr_parse_end: } case kExprNodeCall: { east_set_error( - &ast, pstate, + pstate, &ast.err, _("E116: Missing closing parenthesis for function call: %.*s"), cur_node->start); break; } case kExprNodeNested: { east_set_error( - &ast, pstate, + pstate, &ast.err, _("E110: Missing closing parenthesis for nested expression" ": %.*s"), cur_node->start); @@ -1855,7 +1912,7 @@ viml_pexpr_parse_end: if (!cur_node->data.ter.got_colon) { // Actually Vim throws E109 in more cases. east_set_error( - &ast, pstate, _("E109: Missing ':' after '?': %.*s"), + pstate, &ast.err, _("E109: Missing ':' after '?': %.*s"), cur_node->start); } break; diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index cf0907850a..8ca3ceacb9 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -16,7 +16,7 @@ typedef enum { kCCStrategyUseOption = 0, // 0 for xcalloc kCCStrategyMatchCase = '#', kCCStrategyIgnoreCase = '?', -} CaseCompareStrategy; +} ExprCaseCompareStrategy; /// Lexer token type typedef enum { @@ -52,6 +52,14 @@ typedef enum { kExprLexArrow, ///< Arrow, like from lambda expressions. } LexExprTokenType; +typedef enum { + kExprCmpEqual, ///< Equality, unequality. + kExprCmpMatches, ///< Matches regex, not matches regex. + kExprCmpGreater, ///< `>` or `<=` + kExprCmpGreaterOrEqual, ///< `>=` or `<`. + kExprCmpIdentical, ///< `is` or `isnot` +} ExprComparisonType; + /// Lexer token typedef struct { ParserPosition start; @@ -59,14 +67,8 @@ typedef struct { LexExprTokenType type; union { struct { - enum { - kExprLexCmpEqual, ///< Equality, unequality. - kExprLexCmpMatches, ///< Matches regex, not matches regex. - kExprLexCmpGreater, ///< `>` or `<=` - kExprLexCmpGreaterOrEqual, ///< `>=` or `<`. - kExprLexCmpIdentical, ///< `is` or `isnot` - } type; ///< Comparison type. - CaseCompareStrategy ccs; ///< Case comparison strategy. + ExprComparisonType type; ///< Comparison type. + ExprCaseCompareStrategy ccs; ///< Case comparison strategy. bool inv; ///< True if comparison is to be inverted. } cmp; ///< For kExprLexComparison. @@ -171,6 +173,7 @@ typedef enum { kExprNodeComma = ',', ///< Comma “operator”. kExprNodeColon = ':', ///< Colon “operator”. kExprNodeArrow = '>', ///< Arrow “operator”. + kExprNodeComparison = '=', ///< Various comparison operators. } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -214,6 +217,11 @@ struct expr_ast_node { struct { bool got_colon; ///< True if colon was seen. } ter; ///< For kExprNodeTernaryValue. + struct { + ExprComparisonType type; ///< Comparison type. + ExprCaseCompareStrategy ccs; ///< Case comparison strategy. + bool inv; ///< True if comparison is to be inverted. + } cmp; ///< For kExprNodeComparison. } data; }; @@ -235,19 +243,22 @@ enum { // viml_expressions_parser.c. } ExprParserFlags; +/// AST error definition +typedef struct { + /// Error message. Must contain a single printf format atom: %.*s. + const char *msg; + /// Error message argument: points to the location of the error. + const char *arg; + /// Message argument length: length till the end of string. + int arg_len; +} ExprASTError; + /// Structure representing complety AST for one expression typedef struct { - /// True if represented AST is correct and can be executed. Incorrect ones may - /// still be used for completion, or in linters. - bool correct; /// When AST is not correct this message will be printed. /// /// Uses `emsgf(msg, arg_len, arg);`, `msg` is assumed to contain only `%.*s`. - struct { - const char *msg; - int arg_len; - const char *arg; - } err; + ExprASTError err; /// Root node of the AST. ExprASTNode *root; } ExprAST; diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index 2bad1adc53..5ad592b99f 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -91,12 +91,6 @@ int main(const int argc, const char *const *const argv, const ExprAST ast = viml_pexpr_parse(&pstate, flags); assert(ast.root != NULL || plines[0].size == 0); - assert(ast.root != NULL || !ast.correct); - assert(ast.correct - || (ast.err.msg != NULL - && ast.err.arg != NULL - && ast.err.arg >= plines[0].data - && ((size_t)(ast.err.arg - plines[0].data) + ast.err.arg_len - <= plines[0].size))); + assert(ast.root != NULL || ast.err.msg); // FIXME: free memory and assert no memory leaks } diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index 972478c2e5..d201d54526 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -1,7 +1,7 @@ local helpers = require('test.unit.helpers')(after_each) -local viml_helpers = require('test.unit.viml.helpers') local global_helpers = require('test.helpers') local itp = helpers.gen_itp(it) +local viml_helpers = require('test.unit.viml.helpers') local child_call_once = helpers.child_call_once local conv_enum = helpers.conv_enum @@ -9,17 +9,18 @@ local cimport = helpers.cimport local ffi = helpers.ffi local eq = helpers.eq +local conv_ccs = viml_helpers.conv_ccs local pline2lua = viml_helpers.pline2lua local new_pstate = viml_helpers.new_pstate local intchar2lua = viml_helpers.intchar2lua +local conv_cmp_type = viml_helpers.conv_cmp_type local pstate_set_str = viml_helpers.pstate_set_str local shallowcopy = global_helpers.shallowcopy local lib = cimport('./src/nvim/viml/parser/expressions.h') -local eltkn_type_tab, eltkn_cmp_type_tab, ccs_tab, eltkn_mul_type_tab -local eltkn_opt_scope_tab +local eltkn_type_tab, eltkn_mul_type_tab, eltkn_opt_scope_tab child_call_once(function() eltkn_type_tab = { [tonumber(lib.kExprLexInvalid)] = 'Invalid', @@ -54,20 +55,6 @@ child_call_once(function() [tonumber(lib.kExprLexArrow)] = 'Arrow', } - eltkn_cmp_type_tab = { - [tonumber(lib.kExprLexCmpEqual)] = 'Equal', - [tonumber(lib.kExprLexCmpMatches)] = 'Matches', - [tonumber(lib.kExprLexCmpGreater)] = 'Greater', - [tonumber(lib.kExprLexCmpGreaterOrEqual)] = 'GreaterOrEqual', - [tonumber(lib.kExprLexCmpIdentical)] = 'Identical', - } - - ccs_tab = { - [tonumber(lib.kCCStrategyUseOption)] = 'UseOption', - [tonumber(lib.kCCStrategyMatchCase)] = 'MatchCase', - [tonumber(lib.kCCStrategyIgnoreCase)] = 'IgnoreCase', - } - eltkn_mul_type_tab = { [tonumber(lib.kExprLexMulMul)] = 'Mul', [tonumber(lib.kExprLexMulDiv)] = 'Div', @@ -101,8 +88,8 @@ local function eltkn2lua(pstate, tkn) end if ret.type == 'Comparison' then ret.data = { - type = conv_enum(eltkn_cmp_type_tab, tkn.data.cmp.type), - ccs = conv_enum(ccs_tab, tkn.data.cmp.ccs), + type = conv_cmp_type(tkn.data.cmp.type), + ccs = conv_ccs(tkn.data.cmp.ccs), inv = (not not tkn.data.cmp.inv), } elseif ret.type == 'Multiplication' then diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 2c80b437dc..efa88455e4 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -1,7 +1,7 @@ local helpers = require('test.unit.helpers')(after_each) -local viml_helpers = require('test.unit.viml.helpers') local global_helpers = require('test.helpers') local itp = helpers.gen_itp(it) +local viml_helpers = require('test.unit.viml.helpers') local make_enum_conv_tab = helpers.make_enum_conv_tab local child_call_once = helpers.child_call_once @@ -11,9 +11,11 @@ local cimport = helpers.cimport local ffi = helpers.ffi local eq = helpers.eq +local conv_ccs = viml_helpers.conv_ccs local pline2lua = viml_helpers.pline2lua local new_pstate = viml_helpers.new_pstate local intchar2lua = viml_helpers.intchar2lua +local conv_cmp_type = viml_helpers.conv_cmp_type local pstate_set_str = viml_helpers.pstate_set_str local format_string = global_helpers.format_string @@ -83,6 +85,7 @@ make_enum_conv_tab(lib, { 'kExprNodeComma', 'kExprNodeColon', 'kExprNodeArrow', + 'kExprNodeComparison', }, 'kExprNode', function(ret) east_node_type_tab = ret end) local function conv_east_node_type(typ) @@ -121,6 +124,10 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) (eastnode.data.fig.type_guesses.allow_lambda and '\\' or '-') .. (eastnode.data.fig.type_guesses.allow_dict and 'd' or '-') .. (eastnode.data.fig.type_guesses.allow_ident and 'i' or '-')) + elseif typ == 'Comparison' then + typ = typ .. ('(type=%s,inv=%u,ccs=%s)'):format( + conv_cmp_type(eastnode.data.cmp.type), eastnode.data.cmp.inv and 1 or 0, + conv_ccs(eastnode.data.cmp.ccs)) end ret_str = typ .. ':' .. ret_str local can_simplify = true @@ -150,7 +157,7 @@ end local function east2lua(pstate, east) local checked_nodes = {} return { - err = (not east.correct) and { + err = east.err.msg ~= nil and { msg = ffi.string(east.err.msg), arg = ('%s'):format( ffi.string(east.err.arg, east.err.arg_len)), @@ -3328,4 +3335,318 @@ describe('Expressions parser', function() hl('Identifier', 'h'), }) end) + itp('works with comparison operators', function() + check_parsing('a == b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:1: ==', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '==', 1), + hl('Identifier', 'b', 1), + }) + + check_parsing('a ==? b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=IgnoreCase):0:1: ==?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '==', 1), + hl('ComparisonOperatorModifier', '?'), + hl('Identifier', 'b', 1), + }) + + check_parsing('a ==# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=MatchCase):0:1: ==#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '==', 1), + hl('ComparisonOperatorModifier', '#'), + hl('Identifier', 'b', 1), + }) + + check_parsing('a !=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=1,ccs=MatchCase):0:1: !=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '!=', 1), + hl('ComparisonOperatorModifier', '#'), + hl('Identifier', 'b', 1), + }) + + check_parsing('a <=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Greater,inv=1,ccs=MatchCase):0:1: <=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '<=', 1), + hl('ComparisonOperatorModifier', '#'), + hl('Identifier', 'b', 1), + }) + + check_parsing('a >=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=0,ccs=MatchCase):0:1: >=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '>=', 1), + hl('ComparisonOperatorModifier', '#'), + hl('Identifier', 'b', 1), + }) + + check_parsing('a ># b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Greater,inv=0,ccs=MatchCase):0:1: >#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '>', 1), + hl('ComparisonOperatorModifier', '#'), + hl('Identifier', 'b', 1), + }) + + check_parsing('a <# b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:1: <#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '<', 1), + hl('ComparisonOperatorModifier', '#'), + hl('Identifier', 'b', 1), + }) + + check_parsing('a is#b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=MatchCase):0:1: is#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', 'is', 1), + hl('ComparisonOperatorModifier', '#'), + hl('Identifier', 'b'), + }) + + check_parsing('a is?b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=IgnoreCase):0:1: is?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', 'is', 1), + hl('ComparisonOperatorModifier', '?'), + hl('Identifier', 'b'), + }) + + check_parsing('a isnot b', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=Identical,inv=1,ccs=UseOption):0:1: isnot', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', 'isnot', 1), + hl('Identifier', 'b', 1), + }) + + check_parsing('a < b < c', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:5: <', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + err = { + arg = ' < c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('ComparisonOperator', '<', 1), + hl('Identifier', 'b', 1), + hl('InvalidComparisonOperator', '<', 1), + hl('Identifier', 'c', 1), + }) + check_parsing('a += b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3:=', + children = { + { + 'BinaryPlus:0:1: +', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Missing:0:3:', + }, + }, + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + err = { + arg = '= b', + msg = 'E15: Expected == or =~: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('BinaryPlus', '+', 1), + hl('InvalidComparisonOperator', '='), + hl('Identifier', 'b', 1), + }) + check_parsing('a + b == c + d', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:5: ==', + children = { + { + 'BinaryPlus:0:1: +', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + { + 'BinaryPlus:0:10: +', + children = { + 'PlainIdentifier(scope=0,ident=c):0:8: c', + 'PlainIdentifier(scope=0,ident=d):0:12: d', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('BinaryPlus', '+', 1), + hl('Identifier', 'b', 1), + hl('ComparisonOperator', '==', 1), + hl('Identifier', 'c', 1), + hl('BinaryPlus', '+', 1), + hl('Identifier', 'd', 1), + }) + check_parsing('+ a == + b', 0, { + -- 0123456789 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3: ==', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1: a', + }, + }, + { + 'UnaryPlus:0:6: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:8: b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Identifier', 'a', 1), + hl('ComparisonOperator', '==', 1), + hl('UnaryPlus', '+', 1), + hl('Identifier', 'b', 1), + }) + end) end) diff --git a/test/unit/viml/helpers.lua b/test/unit/viml/helpers.lua index 2cb60499eb..0b92be2654 100644 --- a/test/unit/viml/helpers.lua +++ b/test/unit/viml/helpers.lua @@ -1,8 +1,13 @@ local helpers = require('test.unit.helpers')(nil) local ffi = helpers.ffi +local cimport = helpers.cimport local kvi_new = helpers.kvi_new local kvi_init = helpers.kvi_init +local conv_enum = helpers.conv_enum +local make_enum_conv_tab = helpers.make_enum_conv_tab + +local lib = cimport('./src/nvim/viml/parser/expressions.h') local function new_pstate(strings) local strings_idx = 0 @@ -88,10 +93,36 @@ local function pstate_set_str(pstate, start, len, ret) return ret end +local eltkn_cmp_type_tab +make_enum_conv_tab(lib, { + 'kExprCmpEqual', + 'kExprCmpMatches', + 'kExprCmpGreater', + 'kExprCmpGreaterOrEqual', + 'kExprCmpIdentical', +}, 'kExprCmp', function(ret) eltkn_cmp_type_tab = ret end) + +local function conv_cmp_type(typ) + return conv_enum(eltkn_cmp_type_tab, typ) +end + +local ccs_tab +make_enum_conv_tab(lib, { + 'kCCStrategyUseOption', + 'kCCStrategyMatchCase', + 'kCCStrategyIgnoreCase', +}, 'kCCStrategy', function(ret) ccs_tab = ret end) + +local function conv_ccs(ccs) + return conv_enum(ccs_tab, ccs) +end + return { + conv_ccs = conv_ccs, pline2lua = pline2lua, pstate_str = pstate_str, new_pstate = new_pstate, intchar2lua = intchar2lua, + conv_cmp_type = conv_cmp_type, pstate_set_str = pstate_set_str, } From 0bc4e2237960712426da3774c1430f5874c49aea Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 3 Oct 2017 00:39:40 +0300 Subject: [PATCH 023/109] viml/parser/expressions: Forbid dot or alpha characters after a float This is basically what Vim already does, in addition to forbidding floats should there be a concat immediately before it. --- src/nvim/viml/parser/expressions.c | 6 ++++++ test/unit/viml/expressions/lexer_spec.lua | 21 +++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 8e6f991e03..8c95d1db14 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -186,6 +186,7 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) ret.data.num.is_float = false; CHARREG(kExprLexNumber, ascii_isdigit); if (flags & kELFlagAllowFloat) { + const LexExprToken non_float_ret = ret; if (pline.size > ret.len + 1 && pline.data[ret.len] == '.' && ascii_isdigit(pline.data[ret.len + 1])) { @@ -207,6 +208,11 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) CHARREG(kExprLexNumber, ascii_isdigit); } } + if (pline.size > ret.len + && (pline.data[ret.len] == '.' + || ASCII_ISALPHA(pline.data[ret.len]))) { + ret = non_float_ret; + } } break; } diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index d201d54526..bd8045632e 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -264,6 +264,15 @@ describe('Expressions lexer', function() simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'}) simple_test({'2.'}, 'Number', 1, {data={is_float=false}, str='2'}) simple_test({'2.x'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.2.'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0x'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false}, str='2'}) end local function regular_scope_tests() @@ -296,12 +305,6 @@ describe('Expressions lexer', function() local function regular_number_tests() simple_test({'2.0'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0x'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false}, str='2'}) simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false}, str='2'}) simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false}, str='2'}) simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false}, str='2'}) @@ -350,12 +353,6 @@ describe('Expressions lexer', function() regular_is_tests() simple_test({'2.0'}, 'Number', 3, {data={is_float=true}, str='2.0'}) - simple_test({'2.0x'}, 'Number', 3, {data={is_float=true}, str='2.0'}) - simple_test({'2.0e'}, 'Number', 3, {data={is_float=true}, str='2.0'}) - simple_test({'2.0e+'}, 'Number', 3, {data={is_float=true}, str='2.0'}) - simple_test({'2.0e-'}, 'Number', 3, {data={is_float=true}, str='2.0'}) - simple_test({'2.0e+x'}, 'Number', 3, {data={is_float=true}, str='2.0'}) - simple_test({'2.0e-x'}, 'Number', 3, {data={is_float=true}, str='2.0'}) simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true}, str='2.0e5'}) simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true}, str='2.0e+5'}) simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true}, str='2.0e-5'}) From 163792e9b9854fe046ada3233dec0fd0f6c55737 Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 6 Oct 2017 01:19:43 +0300 Subject: [PATCH 024/109] viml/parser/expressions: Make lexer parse numbers, support non-decimal --- src/nvim/viml/parser/expressions.c | 146 +++++++++++++++-- src/nvim/viml/parser/expressions.h | 6 + test/symbolic/klee/nvim/charset.c | 165 ++++++++++++++++++++ test/symbolic/klee/viml_expressions_lexer.c | 6 +- test/unit/viml/expressions/lexer_spec.lua | 73 ++++++--- 5 files changed, 362 insertions(+), 34 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 8c95d1db14..5d892fb8f8 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -15,10 +15,13 @@ #include "nvim/ascii.h" #include "nvim/assert.h" #include "nvim/lib/kvec.h" +#include "nvim/eval/typval.h" #include "nvim/viml/parser/expressions.h" #include "nvim/viml/parser/parser.h" +#define vim_str2nr(s, ...) vim_str2nr((const char_u *)(s), __VA_ARGS__) + typedef kvec_withinit_t(ExprASTNode **, 16) ExprASTStack; /// Which nodes may be wanted @@ -72,6 +75,43 @@ typedef enum { /// Character used as a separator in autoload function/variable names. #define AUTOLOAD_CHAR '#' +/// Scale number by a given factor +/// +/// Used to apply exponent to a number. Idea taken from uClibc. +/// +/// @param[in] num Number to scale. Does not bother doing anything if it is +/// zero. +/// @param[in] base Base, should be 10 since non-decimal floating-point +/// numbers are not supported. +/// @param[in] exponent Exponent to scale by. +/// @param[in] exponent_negative True if exponent is negative. +static inline float_T scale_number(const float_T num, + const uint8_t base, + const uvarnumber_T exponent, + const bool exponent_negative) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_CONST +{ + if (num == 0 || exponent == 0) { + return num; + } + assert(base); + uvarnumber_T exp = exponent; + float_T p_base = (float_T)base; + float_T ret = num; + while (exp) { + if (exp & 1) { + if (exponent_negative) { + ret /= p_base; + } else { + ret *= p_base; + } + } + exp >>= 1; + p_base *= p_base; + } + return ret; +} + /// Get next token for the VimL expression input /// /// @param pstate Parser state. @@ -184,6 +224,11 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { ret.data.num.is_float = false; + ret.data.num.base = 10; + size_t frac_start = 0; + size_t exp_start = 0; + size_t frac_end = 0; + bool exp_negative = false; CHARREG(kExprLexNumber, ascii_isdigit); if (flags & kELFlagAllowFloat) { const LexExprToken non_float_ret = ret; @@ -191,8 +236,18 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) && pline.data[ret.len] == '.' && ascii_isdigit(pline.data[ret.len + 1])) { ret.len++; + frac_start = ret.len; + frac_end = ret.len; ret.data.num.is_float = true; - CHARREG(kExprLexNumber, ascii_isdigit); + for (; ret.len < pline.size && ascii_isdigit(pline.data[ret.len]) + ; ret.len++) { + // A small optimization: trailing zeroes in fractional part do not + // add anything to significand, so it is useless to include them in + // frac_end. + if (pline.data[ret.len] != '0') { + frac_end = ret.len + 1; + } + } if (pline.size > ret.len + 1 && (pline.data[ret.len] == 'e' || pline.data[ret.len] == 'E') @@ -202,9 +257,11 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) && ascii_isdigit(pline.data[ret.len + 2])) || ascii_isdigit(pline.data[ret.len + 1]))) { ret.len++; - if (pline.data[ret.len] == '+' || pline.data[ret.len] == '-') { + if (pline.data[ret.len] == '+' + || (exp_negative = (pline.data[ret.len] == '-'))) { ret.len++; } + exp_start = ret.len; CHARREG(kExprLexNumber, ascii_isdigit); } } @@ -214,6 +271,58 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) ret = non_float_ret; } } + // TODO(ZyX-I): detect overflows + if (ret.data.num.is_float) { + // Vim used to use string2float here which in turn uses strtod(). There + // are two problems with this approach: + // 1. strtod() is locale-dependent. Not sure how it is worked around so + // that I do not see relevant bugs, but it still does not look like + // a good idea. + // 2. strtod() does not accept length argument. + // + // The below variant of parsing floats was recognized as acceptable + // because it is basically how uClibc does the thing: it generates + // a number ignoring decimal point (but recording its position), then + // uses recorded position to scale number down when processing exponent. + float_T significand_part = 0; + uvarnumber_T exp_part = 0; + const size_t frac_size = (size_t)(frac_end - frac_start); + for (size_t i = 0; i < frac_end; i++) { + if (i == frac_start - 1) { + continue; + } + significand_part = significand_part * 10 + (pline.data[i] - '0'); + } + if (exp_start) { + vim_str2nr(pline.data + exp_start, NULL, NULL, 0, NULL, &exp_part, + (int)(ret.len - exp_start)); + } + if (exp_negative) { + exp_part += frac_size; + } else { + if (exp_part < frac_size) { + exp_negative = true; + exp_part = frac_size - exp_part; + } else { + exp_part -= frac_size; + } + } + ret.data.num.val.floating = scale_number(significand_part, 10, exp_part, + exp_negative); + } else { + int len; + int prep; + vim_str2nr(pline.data, &prep, &len, STR2NR_ALL, NULL, + &ret.data.num.val.integer, (int)pline.size); + ret.len = (size_t)len; + const uint8_t bases[] = { + [0] = 10, + ['0'] = 8, + ['x'] = 16, ['X'] = 16, + ['b'] = 2, ['B'] = 2, + }; + ret.data.num.base = bases[prep]; + } break; } @@ -474,7 +583,6 @@ viml_pexpr_next_token_adv_return: return ret; } -#ifdef UNIT_TESTING static const char *const eltkn_type_tab[] = { [kExprLexInvalid] = "Invalid", [kExprLexMissing] = "Missing", @@ -617,7 +725,12 @@ const char *viml_pexpr_repr_token(const ParserState *const pstate, (int)token.data.opt.len, token.data.opt.name) TKNARGS(kExprLexPlainIdentifier, "(scope=%s,autoload=%i)", intchar2str(token.data.var.scope), (int)token.data.var.autoload) - TKNARGS(kExprLexNumber, "(is_float=%i)", (int)token.data.num.is_float) + TKNARGS(kExprLexNumber, "(is_float=%i,base=%i,val=%lg)", + (int)token.data.num.is_float, + (int)token.data.num.base, + (double)(token.data.num.is_float + ? token.data.num.val.floating + : token.data.num.val.integer)) TKNARGS(kExprLexInvalid, "(msg=%s)", token.data.err.msg) default: { // No additional arguments. @@ -642,7 +755,6 @@ viml_pexpr_repr_token_end: } return ret; } -#endif #ifdef UNIT_TESTING #include @@ -776,8 +888,10 @@ static inline void viml_pexpr_debug_print_token( // NVimOperator -> Operator // NVimUnaryOperator -> NVimOperator // NVimBinaryOperator -> NVimOperator +// // NVimComparisonOperator -> NVimBinaryOperator // NVimComparisonOperatorModifier -> NVimComparisonOperator +// // NVimTernary -> NVimOperator // NVimTernaryColon -> NVimTernary // @@ -795,8 +909,21 @@ static inline void viml_pexpr_debug_print_token( // NVimIdentifierScope -> NVimIdentifier // NVimIdentifierScopeDelimiter -> NVimIdentifier // +// NVimIdentifierKey -> Identifier +// // NVimFigureBrace -> NVimInternalError // +// NVimUnaryPlus -> NVimUnaryOperator +// NVimBinaryPlus -> NVimBinaryOperator +// NVimConcatOrSubscript -> NVimBinaryOperator +// +// NVimRegister -> SpecialChar +// NVimNumber -> Number +// NVimFloat -> NVimNumber +// +// NVimNestingParenthesis -> NVimParenthesis +// NVimCallingParenthesis -> NVimParenthesis +// // NVimInvalidComma -> NVimInvalidDelimiter // NVimInvalidSpacing -> NVimInvalid // NVimInvalidTernary -> NVimInvalidOperator @@ -814,12 +941,9 @@ static inline void viml_pexpr_debug_print_token( // NVimInvalidIdentifierScopeDelimiter -> NVimInvalidValue // NVimInvalidComparisonOperator -> NVimInvalidOperator // NVimInvalidComparisonOperatorModifier -> NVimInvalidComparisonOperator -// -// NVimUnaryPlus -> NVimUnaryOperator -// NVimBinaryPlus -> NVimBinaryOperator -// NVimRegister -> SpecialChar -// NVimNestingParenthesis -> NVimParenthesis -// NVimCallingParenthesis -> NVimParenthesis +// NVimInvalidNumber -> NVimInvalidValue +// NVimInvalidFloat -> NVimInvalidValue +// NVimInvalidIdentifierKey -> NVimInvalidIdentifier /// Allocate a new node and set some of the values /// diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 8ca3ceacb9..29903490bb 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -7,6 +7,7 @@ #include "nvim/types.h" #include "nvim/viml/parser/parser.h" +#include "nvim/eval/typval.h" // Defines whether to ignore case: // == kCCStrategyUseOption @@ -113,6 +114,11 @@ typedef struct { } err; ///< For kExprLexInvalid struct { + union { + float_T floating; + uvarnumber_T integer; + } val; ///< Number value. + uint8_t base; ///< Base: 2, 8, 10 or 16. bool is_float; ///< True if number is a floating-point. } num; ///< For kExprLexNumber } data; ///< Additional data, if needed. diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c index a40488920e..409d7d443c 100644 --- a/test/symbolic/klee/nvim/charset.c +++ b/test/symbolic/klee/nvim/charset.c @@ -3,8 +3,173 @@ #include "nvim/ascii.h" #include "nvim/macros.h" #include "nvim/charset.h" +#include "nvim/eval/typval.h" +#include "nvim/vim.h" bool vim_isIDc(int c) { return ASCII_ISALNUM(c); } + +int hex2nr(int c) +{ + if ((c >= 'a') && (c <= 'f')) { + return c - 'a' + 10; + } + + if ((c >= 'A') && (c <= 'F')) { + return c - 'A' + 10; + } + return c - '0'; +} + +void vim_str2nr(const char_u *const start, int *const prep, int *const len, + const int what, varnumber_T *const nptr, + uvarnumber_T *const unptr, const int maxlen) +{ + const char_u *ptr = start; + int pre = 0; // default is decimal + bool negative = false; + uvarnumber_T un = 0; + + if (ptr[0] == '-') { + negative = true; + ptr++; + } + + // Recognize hex, octal and bin. + if ((ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9') + && (maxlen == 0 || maxlen > 1)) { + pre = ptr[1]; + + if ((what & STR2NR_HEX) + && ((pre == 'X') || (pre == 'x')) + && ascii_isxdigit(ptr[2]) + && (maxlen == 0 || maxlen > 2)) { + // hexadecimal + ptr += 2; + } else if ((what & STR2NR_BIN) + && ((pre == 'B') || (pre == 'b')) + && ascii_isbdigit(ptr[2]) + && (maxlen == 0 || maxlen > 2)) { + // binary + ptr += 2; + } else { + // decimal or octal, default is decimal + pre = 0; + + if (what & STR2NR_OCT) { + // Don't interpret "0", "08" or "0129" as octal. + for (int n = 1; ascii_isdigit(ptr[n]); ++n) { + if (ptr[n] > '7') { + // can't be octal + pre = 0; + break; + } + if (ptr[n] >= '0') { + // assume octal + pre = '0'; + } + if (n == maxlen) { + break; + } + } + } + } + } + + // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. + int n = 1; + if ((pre == 'B') || (pre == 'b') || what == STR2NR_BIN + STR2NR_FORCE) { + // bin + if (pre != 0) { + n += 2; // skip over "0b" + } + while ('0' <= *ptr && *ptr <= '1') { + // avoid ubsan error for overflow + if (un < UVARNUMBER_MAX / 2) { + un = 2 * un + (uvarnumber_T)(*ptr - '0'); + } else { + un = UVARNUMBER_MAX; + } + ptr++; + if (n++ == maxlen) { + break; + } + } + } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) { + // octal + while ('0' <= *ptr && *ptr <= '7') { + // avoid ubsan error for overflow + if (un < UVARNUMBER_MAX / 8) { + un = 8 * un + (uvarnumber_T)(*ptr - '0'); + } else { + un = UVARNUMBER_MAX; + } + ptr++; + if (n++ == maxlen) { + break; + } + } + } else if ((pre == 'X') || (pre == 'x') + || what == STR2NR_HEX + STR2NR_FORCE) { + // hex + if (pre != 0) { + n += 2; // skip over "0x" + } + while (ascii_isxdigit(*ptr)) { + // avoid ubsan error for overflow + if (un < UVARNUMBER_MAX / 16) { + un = 16 * un + (uvarnumber_T)hex2nr(*ptr); + } else { + un = UVARNUMBER_MAX; + } + ptr++; + if (n++ == maxlen) { + break; + } + } + } else { + // decimal + while (ascii_isdigit(*ptr)) { + // avoid ubsan error for overflow + if (un < UVARNUMBER_MAX / 10) { + un = 10 * un + (uvarnumber_T)(*ptr - '0'); + } else { + un = UVARNUMBER_MAX; + } + ptr++; + if (n++ == maxlen) { + break; + } + } + } + + if (prep != NULL) { + *prep = pre; + } + + if (len != NULL) { + *len = (int)(ptr - start); + } + + if (nptr != NULL) { + if (negative) { // account for leading '-' for decimal numbers + // avoid ubsan error for overflow + if (un > VARNUMBER_MAX) { + *nptr = VARNUMBER_MIN; + } else { + *nptr = -(varnumber_T)un; + } + } else { + if (un > VARNUMBER_MAX) { + un = VARNUMBER_MAX; + } + *nptr = (varnumber_T)un; + } + } + + if (unptr != NULL) { + *unptr = un; + } +} diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c index 67f3eb7faa..cddc1cb2f1 100644 --- a/test/symbolic/klee/viml_expressions_lexer.c +++ b/test/symbolic/klee/viml_expressions_lexer.c @@ -2,6 +2,7 @@ # include #else # include +# include #endif #include #include @@ -56,7 +57,7 @@ int main(const int argc, const char *const *const argv, .data = &input[shift], .size = sizeof(input) - shift, #else - .data = (const char *)&argv[1], + .data = (const char *)argv[1], .size = strlen(argv[1]), #endif .allocated = false, @@ -97,4 +98,7 @@ int main(const int argc, const char *const *const argv, } assert(allocated_memory == 0); assert(ever_allocated_memory == 0); +#ifndef USE_KLEE + fprintf(stderr, "tkn: %s\n", viml_pexpr_repr_token(&pstate, token, NULL)); +#endif } diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index bd8045632e..f180d8ceff 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -114,7 +114,11 @@ local function eltkn2lua(pstate, tkn) elseif ret.type == 'Number' then ret.data = { is_float = (not not tkn.data.num.is_float), + base = tonumber(tkn.data.num.base), } + ret.data.val = tonumber(tkn.data.num.is_float + and tkn.data.num.val.floating + or tkn.data.num.val.integer) elseif ret.type == 'Invalid' then ret.data = { error = ffi.string(tkn.data.err.msg) } end @@ -204,9 +208,20 @@ describe('Expressions lexer', function() singl_eltkn_test('Spacing', ' ') singl_eltkn_test('Spacing', '\t') singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'}) - singl_eltkn_test('Number', '0123', {is_float=false}) - singl_eltkn_test('Number', '0', {is_float=false}) - singl_eltkn_test('Number', '9', {is_float=false}) + singl_eltkn_test('Number', '0123', {is_float=false, base=8, val=83}) + singl_eltkn_test('Number', '01234567', {is_float=false, base=8, val=342391}) + singl_eltkn_test('Number', '012345678', {is_float=false, base=10, val=12345678}) + singl_eltkn_test('Number', '0x123', {is_float=false, base=16, val=291}) + singl_eltkn_test('Number', '0x56FF', {is_float=false, base=16, val=22271}) + singl_eltkn_test('Number', '0xabcdef', {is_float=false, base=16, val=11259375}) + singl_eltkn_test('Number', '0xABCDEF', {is_float=false, base=16, val=11259375}) + singl_eltkn_test('Number', '0x0', {is_float=false, base=16, val=0}) + singl_eltkn_test('Number', '00', {is_float=false, base=8, val=0}) + singl_eltkn_test('Number', '0b0', {is_float=false, base=2, val=0}) + singl_eltkn_test('Number', '0b010111', {is_float=false, base=2, val=23}) + singl_eltkn_test('Number', '0b100111', {is_float=false, base=2, val=39}) + singl_eltkn_test('Number', '0', {is_float=false, base=10, val=0}) + singl_eltkn_test('Number', '9', {is_float=false, base=10, val=9}) singl_eltkn_test('Env', '$abc') singl_eltkn_test('Env', '$') singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0}) @@ -262,17 +277,21 @@ describe('Expressions lexer', function() singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'}) simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'}) simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'}) - simple_test({'2.'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.x'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.2.'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0x'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'}) + simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'}) + simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'}) end local function regular_scope_tests() @@ -304,10 +323,10 @@ describe('Expressions lexer', function() end local function regular_number_tests() - simple_test({'2.0'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false}, str='2'}) - simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false}, str='2'}) + simple_test({'2.0'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) end local function regular_eoc_tests() @@ -352,10 +371,20 @@ describe('Expressions lexer', function() regular_scope_tests() regular_is_tests() - simple_test({'2.0'}, 'Number', 3, {data={is_float=true}, str='2.0'}) - simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true}, str='2.0e5'}) - simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true}, str='2.0e+5'}) - simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true}, str='2.0e-5'}) + simple_test({'2.2'}, 'Number', 3, {data={is_float=true, base=10, val=2.2}, str='2.2'}) + simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true, base=10, val=2e5}, str='2.0e5'}) + simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true, base=10, val=2e5}, str='2.0e+5'}) + simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true, base=10, val=2e-5}, str='2.0e-5'}) + simple_test({'2.500000e-5'}, 'Number', 11, {data={is_float=true, base=10, val=2.5e-5}, str='2.500000e-5'}) + simple_test({'2.5555e2'}, 'Number', 8, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e2'}) + simple_test({'2.5555e+2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e+2'}) + simple_test({'2.5555e-2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e-2}, str='2.5555e-2'}) + simple_test({{data='2.5e-5', size=3}}, + 'Number', 3, {data={is_float=true, base=10, val=2.5}, str='2.5'}) + simple_test({{data='2.5e5', size=4}}, + 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'}) + simple_test({{data='2.5e-50', size=6}}, + 'Number', 6, {data={is_float=true, base=10, val=2.5e-5}, str='2.5e-5'}) end) itp('treats `is` as an identifier', function() flags = tonumber(lib.kELFlagIsNotCmp) From 21a5ce033c5a853bed3204ea9f0f7a3cfc1d164f Mon Sep 17 00:00:00 2001 From: ZyX Date: Tue, 3 Oct 2017 01:30:02 +0300 Subject: [PATCH 025/109] viml/parser/expressions: Add support for the dot operator and numbers --- src/nvim/viml/parser/expressions.c | 108 +++++++- src/nvim/viml/parser/expressions.h | 23 +- test/unit/viml/expressions/parser_spec.lua | 294 +++++++++++++++++++++ 3 files changed, 416 insertions(+), 9 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 5d892fb8f8..4babf4312c 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -915,7 +915,8 @@ static inline void viml_pexpr_debug_print_token( // // NVimUnaryPlus -> NVimUnaryOperator // NVimBinaryPlus -> NVimBinaryOperator -// NVimConcatOrSubscript -> NVimBinaryOperator +// NVimConcat -> NVimBinaryOperator +// NVimConcatOrSubscript -> NVimConcat // // NVimRegister -> SpecialChar // NVimNumber -> Number @@ -971,6 +972,7 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeUnknownFigure] = kEOpLvlParens, [kExprNodeLambda] = kEOpLvlParens, [kExprNodeDictLiteral] = kEOpLvlParens, + [kExprNodeListLiteral] = kEOpLvlParens, [kExprNodeArrow] = kEOpLvlArrow, @@ -985,17 +987,21 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeComparison] = kEOpLvlComparison, [kExprNodeBinaryPlus] = kEOpLvlAddition, + [kExprNodeConcat] = kEOpLvlAddition, [kExprNodeUnaryPlus] = kEOpLvlUnary, + [kExprNodeConcatOrSubscript] = kEOpLvlSubscript, [kExprNodeSubscript] = kEOpLvlSubscript, [kExprNodeCurlyBracesIdentifier] = kEOpLvlComplexIdentifier, [kExprNodeComplexIdentifier] = kEOpLvlValue, [kExprNodePlainIdentifier] = kEOpLvlValue, + [kExprNodePlainKey] = kEOpLvlValue, [kExprNodeRegister] = kEOpLvlValue, - [kExprNodeListLiteral] = kEOpLvlValue, + [kExprNodeInteger] = kEOpLvlValue, + [kExprNodeFloat] = kEOpLvlValue, }; static const ExprOpAssociativity node_type_to_op_ass[] = { @@ -1008,6 +1014,7 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeUnknownFigure] = kEOpAssLeft, [kExprNodeLambda] = kEOpAssNo, [kExprNodeDictLiteral] = kEOpAssNo, + [kExprNodeListLiteral] = kEOpAssNo, // Does not really matter. [kExprNodeArrow] = kEOpAssNo, @@ -1030,17 +1037,21 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeComparison] = kEOpAssRight, [kExprNodeBinaryPlus] = kEOpAssLeft, + [kExprNodeConcat] = kEOpAssLeft, [kExprNodeUnaryPlus] = kEOpAssNo, + [kExprNodeConcatOrSubscript] = kEOpAssLeft, [kExprNodeSubscript] = kEOpAssLeft, [kExprNodeCurlyBracesIdentifier] = kEOpAssLeft, [kExprNodeComplexIdentifier] = kEOpAssLeft, [kExprNodePlainIdentifier] = kEOpAssNo, + [kExprNodePlainKey] = kEOpAssNo, [kExprNodeRegister] = kEOpAssNo, - [kExprNodeListLiteral] = kEOpAssNo, + [kExprNodeInteger] = kEOpAssNo, + [kExprNodeFloat] = kEOpAssNo, }; /// Get AST node priority level @@ -1420,10 +1431,20 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) [kENodeArgument] = kELFlagIsNotCmp, [kENodeArgumentSeparator] = kELFlagForbidScope, }; - // FIXME Determine when (not) to allow floating-point numbers. + const bool is_concat_or_subscript = ( + want_node == kENodeValue + && kv_size(ast_stack) > 1 + && (*kv_Z(ast_stack, 1))->type == kExprNodeConcatOrSubscript); const int lexer_additional_flags = ( kELFlagPeek - | ((flags & kExprFlagsDisallowEOC) ? kELFlagForbidEOC : 0)); + | ((flags & kExprFlagsDisallowEOC) ? kELFlagForbidEOC : 0) + | ((want_node == kENodeValue + && (kv_size(ast_stack) == 1 + || ((*kv_Z(ast_stack, 1))->type != kExprNodeConcat + && ((*kv_Z(ast_stack, 1))->type + != kExprNodeConcatOrSubscript)))) + ? kELFlagAllowFloat + : 0)); LexExprToken cur_token = viml_pexpr_next_token( pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags); if (cur_token.type == kExprLexEOC) { @@ -1456,11 +1477,42 @@ viml_pexpr_parse_process_token: ExprASTNode *cur_node = NULL; assert((want_node == kENodeValue || want_node == kENodeArgument) == (*top_node_p == NULL)); + // Note: in Vim whether expression "cond?d.a:2" is valid depends both on + // "cond" and whether "d" is a dictionary: expression is valid if condition + // is true and "d" is a dictionary (with "a" key or it will complain about + // missing one, but this is not relevant); if any of the requirements is + // broken then this thing is parsed as "d . a:2" yielding missing colon + // error. This parser does not allow such ambiguity, especially because it + // simply can’t: whether "d" is a dictionary is not known at the parsing + // time. + // + // Here example will always contain a concat with "a:2" sucking colon, + // making expression invalid both because there is no longer a spare colon + // for ternary and because concatenating dictionary with anything is not + // valid. There are more cases when this will make a difference though. + const bool node_is_key = ( + is_concat_or_subscript + && (cur_token.type == kExprLexPlainIdentifier + ? (!cur_token.data.var.autoload + && cur_token.data.var.scope == 0) + : (cur_token.type == kExprLexNumber)) + && prev_token.type != kExprLexSpacing); + if (is_concat_or_subscript && !node_is_key) { + // Note: in Vim "d. a" (this is the reason behind `prev_token.type != + // kExprLexSpacing` part of the condition) as well as any other "d.{expr}" + // where "{expr}" does not look like a key is invalid whenever "d" happens + // to be a dictionary. Since parser has no idea whether preceding + // expression is actually a dictionary it can’t outright reject anything, + // so it turns kExprNodeConcatOrSubscript into kExprNodeConcat instead, + // which will yield different errors then Vim does in a number of + // circumstances, and in any case runtime and not parse time errors. + (*kv_Z(ast_stack, 1))->type = kExprNodeConcat; + } if ((want_node == kENodeArgumentSeparator && tok_type != kExprLexComma && tok_type != kExprLexArrow) || (want_node == kENodeArgument - && !(tok_type == kExprLexPlainIdentifier + && !(cur_token.type == kExprLexPlainIdentifier && cur_token.data.var.scope == 0 && !cur_token.data.var.autoload) && tok_type != kExprLexArrow)) { @@ -1844,7 +1896,10 @@ viml_pexpr_parse_figure_brace_closing_error: want_node = (want_node == kENodeArgument ? kENodeArgumentSeparator : kENodeOperator); - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainIdentifier); + NEW_NODE_WITH_CUR_POS(cur_node, + (node_is_key + ? kExprNodePlainKey + : kExprNodePlainIdentifier)); cur_node->data.var.scope = cur_token.data.var.scope; const size_t scope_shift = (cur_token.data.var.scope == 0 ? 0 @@ -1854,6 +1909,7 @@ viml_pexpr_parse_figure_brace_closing_error: cur_node->data.var.ident_len = cur_token.len - scope_shift; *top_node_p = cur_node; if (scope_shift) { + assert(!node_is_key); viml_parser_highlight(pstate, cur_token.start, 1, HL(IdentifierScope)); viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), 1, @@ -1863,7 +1919,9 @@ viml_pexpr_parse_figure_brace_closing_error: viml_parser_highlight(pstate, shifted_pos(cur_token.start, scope_shift), cur_token.len - scope_shift, - HL(Identifier)); + (node_is_key + ? HL(IdentifierKey) + : HL(Identifier))); } } else { if (cur_token.data.var.scope == 0) { @@ -1882,6 +1940,40 @@ viml_pexpr_parse_figure_brace_closing_error: } break; } + case kExprLexNumber: { + if (want_node != kENodeValue) { + OP_MISSING; + } + if (node_is_key) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainKey); + cur_node->data.var.ident = pline.data + cur_token.start.col; + cur_node->data.var.ident_len = cur_token.len; + HL_CUR_TOKEN(IdentifierKey); + } else if (cur_token.data.num.is_float) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeFloat); + cur_node->data.flt.value = cur_token.data.num.val.floating; + HL_CUR_TOKEN(Float); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeInteger); + cur_node->data.num.value = cur_token.data.num.val.integer; + HL_CUR_TOKEN(Number); + } + want_node = kENodeOperator; + *top_node_p = cur_node; + break; + } + case kExprLexDot: { + ADD_VALUE_IF_MISSING(_("E15: Unexpected dot: %.*s")); + if (prev_token.type == kExprLexSpacing) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeConcat); + HL_CUR_TOKEN(Concat); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeConcatOrSubscript); + HL_CUR_TOKEN(ConcatOrSubscript); + } + ADD_OP_NODE(cur_node); + break; + } case kExprLexParenthesis: { if (cur_token.data.brc.closing) { if (want_node == kENodeValue) { diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 29903490bb..0d496c87ba 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -166,6 +166,8 @@ typedef enum { /// Looks like "string", "g:Foo", etc: consists from a single /// kExprLexPlainIdentifier token. kExprNodePlainIdentifier = 'i', + /// Plain dictionary key, for use with kExprNodeConcatOrSubscript + kExprNodePlainKey = 'k', /// Complex identifier: variable/function name with curly braces kExprNodeComplexIdentifier = 'I', /// Figure brace expression which is not yet known @@ -180,6 +182,19 @@ typedef enum { kExprNodeColon = ':', ///< Colon “operator”. kExprNodeArrow = '>', ///< Arrow “operator”. kExprNodeComparison = '=', ///< Various comparison operators. + /// Concat operator + /// + /// To be only used in cases when it is known for sure it is not a subscript. + kExprNodeConcat = '.', + /// Concat or subscript operator + /// + /// For cases when it is not obvious whether expression is a concat or + /// a subscript. May only have either number or plain identifier as the second + /// child. To make it easier to avoid curly braces in place of + /// kExprNodePlainIdentifier node kExprNodePlainKey is used. + kExprNodeConcatOrSubscript = 'S', + kExprNodeInteger = '0', ///< Integral number. + kExprNodeFloat = '1', ///< Floating-point number. } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -219,7 +234,7 @@ struct expr_ast_node { /// Points to inside parser reader state. const char *ident; size_t ident_len; ///< Actual identifier length. - } var; ///< For kExprNodePlainIdentifier. + } var; ///< For kExprNodePlainIdentifier and kExprNodePlainKey. struct { bool got_colon; ///< True if colon was seen. } ter; ///< For kExprNodeTernaryValue. @@ -228,6 +243,12 @@ struct expr_ast_node { ExprCaseCompareStrategy ccs; ///< Case comparison strategy. bool inv; ///< True if comparison is to be inverted. } cmp; ///< For kExprNodeComparison. + struct { + uvarnumber_T value; + } num; ///< For kExprNodeInteger. + struct { + float_T value; + } flt; ///< For kExprNodeFloat. } data; }; diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index efa88455e4..8d96c29db7 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -77,6 +77,7 @@ make_enum_conv_tab(lib, { 'kExprNodeNested', 'kExprNodeCall', 'kExprNodePlainIdentifier', + 'kExprNodePlainKey', 'kExprNodeComplexIdentifier', 'kExprNodeUnknownFigure', 'kExprNodeLambda', @@ -86,6 +87,10 @@ make_enum_conv_tab(lib, { 'kExprNodeColon', 'kExprNodeArrow', 'kExprNodeComparison', + 'kExprNodeConcat', + 'kExprNodeConcatOrSubscript', + 'kExprNodeInteger', + 'kExprNodeFloat', }, 'kExprNode', function(ret) east_node_type_tab = ret end) local function conv_east_node_type(typ) @@ -118,6 +123,9 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) typ = typ .. ('(scope=%s,ident=%s)'):format( tostring(intchar2lua(eastnode.data.var.scope)), ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len)) + elseif typ == 'PlainKey' then + typ = typ .. ('(key=%s)'):format( + ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len)) elseif (typ == 'UnknownFigure' or typ == 'DictLiteral' or typ == 'CurlyBracesIdentifier' or typ == 'Lambda') then typ = typ .. ('(%s)'):format( @@ -128,6 +136,10 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) typ = typ .. ('(type=%s,inv=%u,ccs=%s)'):format( conv_cmp_type(eastnode.data.cmp.type), eastnode.data.cmp.inv and 1 or 0, conv_ccs(eastnode.data.cmp.ccs)) + elseif typ == 'Integer' then + typ = typ .. ('(val=%u)'):format(tonumber(eastnode.data.num.value)) + elseif typ == 'Float' then + typ = typ .. ('(val=%e)'):format(tonumber(eastnode.data.flt.value)) end ret_str = typ .. ':' .. ret_str local can_simplify = true @@ -190,6 +202,8 @@ end) describe('Expressions parser', function() local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) + flags = flags or 0 + local pstate = new_pstate({str}) local east = lib.viml_pexpr_parse(pstate, flags) local ast = east2lua(pstate, east) @@ -3649,4 +3663,284 @@ describe('Expressions parser', function() hl('Identifier', 'b', 1), }) end) + itp('works with concat/subscript', function() + check_parsing('.', 0, { + -- 0 + ast = { + { + 'ConcatOrSubscript:0:0:.', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = '.', + msg = 'E15: Unexpected dot: %.*s', + }, + }, { + hl('InvalidConcatOrSubscript', '.'), + }) + + check_parsing('a.', 0, { + -- 01 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('ConcatOrSubscript', '.'), + }) + + check_parsing('a.b', 0, { + -- 012 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=b):0:2:b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', 'b'), + }) + + check_parsing('1.2', 0, { + -- 012 + ast = { + 'Float(val=1.200000e+00):0:0:1.2', + }, + }, { + hl('Float', '1.2'), + }) + + check_parsing('1.2 + 1.3e-5', 0, { + -- 012345678901 + -- 0 1 + ast = { + { + 'BinaryPlus:0:3: +', + children = { + 'Float(val=1.200000e+00):0:0:1.2', + 'Float(val=1.300000e-05):0:5: 1.3e-5', + }, + }, + }, + }, { + hl('Float', '1.2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('a . 1.2 + 1.3e-5', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'BinaryPlus:0:7: +', + children = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + 'Float(val=1.300000e-05):0:9: 1.3e-5', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('1.3e-5 + 1.2 . a', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:12: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'Float(val=1.200000e+00):0:8: 1.2', + }, + }, + 'PlainIdentifier(scope=0,ident=a):0:14: a', + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.2', 1), + hl('Concat', '.', 1), + hl('Identifier', 'a', 1), + }) + + check_parsing('1.3e-5 + a . 1.2', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:10: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'PlainIdentifier(scope=0,ident=a):0:8: a', + }, + }, + { + 'ConcatOrSubscript:0:14:.', + children = { + 'Integer(val=1):0:12: 1', + 'PlainKey(key=2):0:15:2', + }, + }, + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('Identifier', 'a', 1), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('1.2.3', 0, { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'Integer(val=1):0:0:1', + 'PlainKey(key=2):0:2:2', + }, + }, + 'PlainKey(key=3):0:4:3', + }, + }, + }, + }, { + hl('Number', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '3'), + }) + + check_parsing('a.1.2', 0, { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=1):0:2:1', + }, + }, + 'PlainKey(key=2):0:4:2', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('a . 1.2', 0, { + -- 0123456 + ast = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('+a . +b', 0, { + -- 0123456 + ast = { + { + 'Concat:0:2: .', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'UnaryPlus:0:4: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:6:b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Identifier', 'a'), + hl('Concat', '.', 1), + hl('UnaryPlus', '+', 1), + hl('Identifier', 'b'), + }) + end) end) From e45e519495832e3d1d0fde1e32723d4140c5fc65 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 8 Oct 2017 01:19:58 +0300 Subject: [PATCH 026/109] viml/parser/expressions: Error out on multiple colons in a row --- src/nvim/viml/parser/expressions.c | 4 +- test/unit/viml/expressions/parser_spec.lua | 73 ++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 4babf4312c..fffae8833d 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1691,8 +1691,10 @@ viml_pexpr_parse_invalid_comma: SELECT_FIGURE_BRACE_TYPE(*eastnode_p, DictLiteral, Dict); break; } else if (eastnode_type == kExprNodeDictLiteral - || eastnode_type == kExprNodeComma) { + || eastnode_type == kExprNodeSubscript) { break; + } else if (eastnode_type == kExprNodeColon) { + goto viml_pexpr_parse_invalid_colon; } else if (eastnode_lvl >= kEOpLvlTernaryValue) { // Do nothing } else if (eastnode_lvl > kEOpLvlComma) { diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 8d96c29db7..46e3328184 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -2700,6 +2700,42 @@ describe('Expressions parser', function() hl('Identifier', 'b', 1), hl('Curly', '}'), }) + check_parsing('{a : b : c}', 0, { + -- 01234567890 + -- 0 1 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Colon:0:6: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': c}', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('Dict', '{'), + hl('Identifier', 'a'), + hl('Colon', ':', 1), + hl('Identifier', 'b', 1), + hl('InvalidColon', ':', 1), + hl('Identifier', 'c', 1), + hl('Dict', '}'), + }) end) itp('works with ternary operator', function() check_parsing('a ? b : c', 0, { @@ -3348,6 +3384,43 @@ describe('Expressions parser', function() hl('TernaryColon', ':'), hl('Identifier', 'h'), }) + check_parsing('a ? b : c : d', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Colon:0:9: :', + children = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + err = { + arg = ': d', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('Ternary', '?', 1), + hl('Identifier', 'b', 1), + hl('TernaryColon', ':', 1), + hl('Identifier', 'c', 1), + hl('InvalidColon', ':', 1), + hl('Identifier', 'd', 1), + }) end) itp('works with comparison operators', function() check_parsing('a == b', 0, { From bd3a4166b25a64dbe406be09b3140955cf694477 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 8 Oct 2017 02:17:05 +0300 Subject: [PATCH 027/109] viml/parser/expressions: Add support for subscript and list literals --- src/nvim/viml/parser/expressions.c | 164 ++++++- test/unit/viml/expressions/parser_spec.lua | 533 +++++++++++++++++++++ 2 files changed, 675 insertions(+), 22 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index fffae8833d..0613cc66d5 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -905,6 +905,10 @@ static inline void viml_pexpr_debug_print_token( // NVimDict -> Delimiter // NVimCurly -> Delimiter // +// NVimList -> Delimiter +// NVimSubscript -> Delimiter +// NVimSubscriptColon -> NVimSubscript +// // NVimIdentifier -> Identifier // NVimIdentifierScope -> NVimIdentifier // NVimIdentifierScopeDelimiter -> NVimIdentifier @@ -945,6 +949,9 @@ static inline void viml_pexpr_debug_print_token( // NVimInvalidNumber -> NVimInvalidValue // NVimInvalidFloat -> NVimInvalidValue // NVimInvalidIdentifierKey -> NVimInvalidIdentifier +// NVimInvalidList -> NVimInvalidDelimiter +// NVimInvalidSubscript -> NVimInvalidDelimiter +// NVimInvalidSubscriptColon -> NVimInvalidSubscript /// Allocate a new node and set some of the values /// @@ -965,9 +972,10 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeOpMissing] = kEOpLvlMultiplication, [kExprNodeNested] = kEOpLvlParens, - // Note: it is kEOpLvlSubscript for “binary operator” itself, but + // Note: below nodes are kEOpLvlSubscript for “binary operator” itself, but // kEOpLvlParens when it comes to inside the parenthesis. [kExprNodeCall] = kEOpLvlParens, + [kExprNodeSubscript] = kEOpLvlParens, [kExprNodeUnknownFigure] = kEOpLvlParens, [kExprNodeLambda] = kEOpLvlParens, @@ -992,7 +1000,6 @@ static const ExprOpLvl node_type_to_op_lvl[] = { [kExprNodeUnaryPlus] = kEOpLvlUnary, [kExprNodeConcatOrSubscript] = kEOpLvlSubscript, - [kExprNodeSubscript] = kEOpLvlSubscript, [kExprNodeCurlyBracesIdentifier] = kEOpLvlComplexIdentifier, @@ -1010,6 +1017,7 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeNested] = kEOpAssNo, [kExprNodeCall] = kEOpAssNo, + [kExprNodeSubscript] = kEOpAssNo, [kExprNodeUnknownFigure] = kEOpAssLeft, [kExprNodeLambda] = kEOpAssNo, @@ -1042,7 +1050,6 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { [kExprNodeUnaryPlus] = kEOpAssNo, [kExprNodeConcatOrSubscript] = kEOpAssLeft, - [kExprNodeSubscript] = kEOpAssLeft, [kExprNodeCurlyBracesIdentifier] = kEOpAssLeft, @@ -1105,12 +1112,14 @@ static bool viml_pexpr_handle_bop(const ParserState *const pstate, ExprOpLvl top_node_lvl; ExprOpAssociativity top_node_ass; assert(kv_size(*ast_stack)); - const ExprOpLvl bop_node_lvl = (bop_node->type == kExprNodeCall + const ExprOpLvl bop_node_lvl = ((bop_node->type == kExprNodeCall + || bop_node->type == kExprNodeSubscript) ? kEOpLvlSubscript : node_lvl(*bop_node)); #ifndef NDEBUG const ExprOpAssociativity bop_node_ass = ( - bop_node->type == kExprNodeCall + (bop_node->type == kExprNodeCall + || bop_node->type == kExprNodeSubscript) ? kEOpAssLeft : node_ass(*bop_node)); #endif @@ -1214,8 +1223,8 @@ static inline ParserPosition shifted_pos(const ParserPosition pos, /// @param cur_token Token to set position from. #define POS_FROM_TOKEN(cur_node, cur_token) \ do { \ - cur_node->start = cur_token.start; \ - cur_node->len = cur_token.len; \ + (cur_node)->start = cur_token.start; \ + (cur_node)->len = cur_token.len; \ } while (0) /// Allocate new node and set its position from the current token @@ -1226,11 +1235,11 @@ static inline ParserPosition shifted_pos(const ParserPosition pos, /// @param typ Node type. #define NEW_NODE_WITH_CUR_POS(cur_node, typ) \ do { \ - cur_node = NEW_NODE(typ); \ - POS_FROM_TOKEN(cur_node, cur_token); \ + (cur_node) = NEW_NODE(typ); \ + POS_FROM_TOKEN((cur_node), cur_token); \ if (prev_token.type == kExprLexSpacing) { \ - cur_node->start = prev_token.start; \ - cur_node->len += prev_token.len; \ + (cur_node)->start = prev_token.start; \ + (cur_node)->len += prev_token.len; \ } \ } while (0) @@ -1280,9 +1289,8 @@ static inline ParserPosition shifted_pos(const ParserPosition pos, do { \ if (want_node == kENodeValue) { \ ERROR_FROM_TOKEN_AND_MSG(cur_token, (msg)); \ - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); \ - cur_node->len = 0; \ - *top_node_p = cur_node; \ + NEW_NODE_WITH_CUR_POS((*top_node_p), kExprNodeMissing); \ + (*top_node_p)->len = 0; \ want_node = kENodeOperator; \ } \ } while (0) @@ -1658,13 +1666,14 @@ viml_pexpr_parse_invalid_comma: HL_CUR_TOKEN(Comma); break; } +#define EXP_VAL_COLON "E15: Expected value, got colon: %.*s" case kExprLexColon: { - ADD_VALUE_IF_MISSING(_("E15: Expected value, got colon: %.*s")); if (kv_size(ast_stack) < 2) { goto viml_pexpr_parse_invalid_colon; } bool is_ternary = false; bool can_be_ternary = true; + bool is_subscript = false; for (size_t i = 1; i < kv_size(ast_stack); i++) { ExprASTNode *const *const eastnode_p = (ExprASTNode *const *)kv_Z(ast_stack, i); @@ -1683,6 +1692,7 @@ viml_pexpr_parse_invalid_comma: } is_ternary = true; (*eastnode_p)->data.ter.got_colon = true; + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); assert((*eastnode_p)->children != NULL); assert((*eastnode_p)->children->next == NULL); kvi_push(ast_stack, &(*eastnode_p)->children->next); @@ -1690,14 +1700,18 @@ viml_pexpr_parse_invalid_comma: } else if (eastnode_type == kExprNodeUnknownFigure) { SELECT_FIGURE_BRACE_TYPE(*eastnode_p, DictLiteral, Dict); break; - } else if (eastnode_type == kExprNodeDictLiteral - || eastnode_type == kExprNodeSubscript) { + } else if (eastnode_type == kExprNodeDictLiteral) { + break; + } else if (eastnode_type == kExprNodeSubscript) { + is_subscript = true; + can_be_ternary = false; + assert(!is_ternary); break; } else if (eastnode_type == kExprNodeColon) { goto viml_pexpr_parse_invalid_colon; } else if (eastnode_lvl >= kEOpLvlTernaryValue) { // Do nothing - } else if (eastnode_lvl > kEOpLvlComma) { + } else if (eastnode_lvl >= kEOpLvlComma) { can_be_ternary = false; } else { viml_pexpr_parse_invalid_colon: @@ -1711,16 +1725,122 @@ viml_pexpr_parse_invalid_colon: goto viml_pexpr_parse_invalid_colon; } } - if (is_ternary) { - HL_CUR_TOKEN(TernaryColon); - } else { + if (is_subscript) { + assert(kv_size(ast_stack) > 1); + // Colon immediately following subscript start: it is empty subscript + // part like a[:2]. + if (want_node == kENodeValue + && (*kv_Z(ast_stack, 1))->type == kExprNodeSubscript) { + NEW_NODE_WITH_CUR_POS(*top_node_p, kExprNodeMissing); + (*top_node_p)->len = 0; + want_node = kENodeOperator; + } else { + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); + } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); ADD_OP_NODE(cur_node); - HL_CUR_TOKEN(Colon); + HL_CUR_TOKEN(SubscriptColon); + } else { + ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); + if (is_ternary) { + HL_CUR_TOKEN(TernaryColon); + } else { + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(Colon); + } } want_node = kENodeValue; break; } +#undef EXP_VAL_COLON + case kExprLexBracket: { + if (cur_token.data.brc.closing) { + ExprASTNode **new_top_node_p = NULL; + // Always drop the topmost value: + // + // 1. When want_node != kENodeValue topmost item on stack is + // a *finished* left operand, which may as well be "{@a}" which + // needs not be finished again. + // 2. Otherwise it is pointing to NULL what nobody wants. + kv_drop(ast_stack, 1); + if (!kv_size(ast_stack)) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeListLiteral); + cur_node->len = 0; + if (want_node != kENodeValue) { + cur_node->children = *top_node_p; + } + *top_node_p = cur_node; + goto viml_pexpr_parse_bracket_closing_error; + } + if (want_node == kENodeValue) { + // It is OK to want value if + // + // 1. It is empty list literal, in which case top node will be + // ListLiteral. + // 2. It is list literal with trailing comma, in which case top node + // will be that comma. + // 3. It is subscript with colon, but without one of the values: + // e.g. "a[:]", "a[1:]", top node will be colon in this case. + if ((*kv_last(ast_stack))->type != kExprNodeListLiteral + && (*kv_last(ast_stack))->type != kExprNodeComma + && (*kv_last(ast_stack))->type != kExprNodeColon) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected value, got closing bracket: %.*s")); + } + } else { + if (!kv_size(ast_stack)) { + new_top_node_p = top_node_p; + goto viml_pexpr_parse_bracket_closing_error; + } + } + do { + new_top_node_p = kv_pop(ast_stack); + } while (kv_size(ast_stack) + && (new_top_node_p == NULL + || ((*new_top_node_p)->type != kExprNodeListLiteral + && (*new_top_node_p)->type != kExprNodeSubscript))); + ExprASTNode *new_top_node = *new_top_node_p; + switch (new_top_node->type) { + case kExprNodeListLiteral: { + HL_CUR_TOKEN(List); + break; + } + case kExprNodeSubscript: { + HL_CUR_TOKEN(Subscript); + break; + } + default: { +viml_pexpr_parse_bracket_closing_error: + assert(!kv_size(ast_stack)); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Unexpected closing figure brace: %.*s")); + HL_CUR_TOKEN(List); + break; + } + } + kvi_push(ast_stack, new_top_node_p); + want_node = kENodeOperator; + } else { + if (want_node == kENodeValue) { + // Value means list literal. + HL_CUR_TOKEN(List); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeListLiteral); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + want_node = kENodeValue; + } else { + if (prev_token.type == kExprLexSpacing) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeSubscript); + ADD_OP_NODE(cur_node); + HL_CUR_TOKEN(Subscript); + } + } + break; + } case kExprLexFigureBrace: { if (cur_token.data.brc.closing) { ExprASTNode **new_top_node_p = NULL; diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 46e3328184..41261cf7c9 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -91,6 +91,8 @@ make_enum_conv_tab(lib, { 'kExprNodeConcatOrSubscript', 'kExprNodeInteger', 'kExprNodeFloat', + 'kExprNodeSingleQuotedString', + 'kExprNodeDoubleQuotedString', }, 'kExprNode', function(ret) east_node_type_tab = ret end) local function conv_east_node_type(typ) @@ -204,6 +206,10 @@ describe('Expressions parser', function() local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) flags = flags or 0 + if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then + print(str, flags) + end + local pstate = new_pstate({str}) local east = lib.viml_pexpr_parse(pstate, flags) local ast = east2lua(pstate, east) @@ -4016,4 +4022,531 @@ describe('Expressions parser', function() hl('Identifier', 'b'), }) end) + itp('works with bracket subscripts', function() + check_parsing(':', 0, { + -- 0 + ast = { + { + 'Colon:0:0::', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ':', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('InvalidColon', ':'), + }) + check_parsing('a[]', 0, { + -- 012 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('Subscript', '['), + hl('InvalidSubscript', ']'), + }) + check_parsing('a[b:]', 0, { + -- 01234 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Subscript', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('Subscript', ']'), + }) + + check_parsing('a[b:c]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=c):0:2:b:c', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Subscript', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('Identifier', 'c'), + hl('Subscript', ']'), + }) + check_parsing('a[b : c]', 0, { + -- 01234567 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + 'PlainIdentifier(scope=0,ident=c):0:5: c', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Subscript', '['), + hl('Identifier', 'b'), + hl('SubscriptColon', ':', 1), + hl('Identifier', 'c', 1), + hl('Subscript', ']'), + }) + + check_parsing('a[: b]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:2::', + children = { + 'Missing:0:2:', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Subscript', '['), + hl('SubscriptColon', ':'), + hl('Identifier', 'b', 1), + hl('Subscript', ']'), + }) + + check_parsing('a[b :]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Subscript', '['), + hl('Identifier', 'b'), + hl('SubscriptColon', ':', 1), + hl('Subscript', ']'), + }) + check_parsing('a[b][c][d](e)(f)(g)', 0, { + -- 0123456789012345678 + -- 0 1 + ast = { + { + 'Call:0:16:(', + children = { + { + 'Call:0:13:(', + children = { + { + 'Call:0:10:(', + children = { + { + 'Subscript:0:7:[', + children = { + { + 'Subscript:0:4:[', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:11:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:14:f', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:17:g', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('Subscript', '['), + hl('Identifier', 'b'), + hl('Subscript', ']'), + hl('Subscript', '['), + hl('Identifier', 'c'), + hl('Subscript', ']'), + hl('Subscript', '['), + hl('Identifier', 'd'), + hl('Subscript', ']'), + hl('CallingParenthesis', '('), + hl('Identifier', 'e'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Identifier', 'f'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Identifier', 'g'), + hl('CallingParenthesis', ')'), + }) + check_parsing('{a}{b}{c}[d][e][f]', 0, { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Subscript:0:15:[', + children = { + { + 'Subscript:0:12:[', + children = { + { + 'Subscript:0:9:[', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + { + 'CurlyBracesIdentifier(-di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier(--i):0:3:{', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + }, + }, + { + 'CurlyBracesIdentifier(--i):0:6:{', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:10:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:13:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:16:f', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Identifier', 'a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('Identifier', 'b'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('Identifier', 'c'), + hl('Curly', '}'), + hl('Subscript', '['), + hl('Identifier', 'd'), + hl('Subscript', ']'), + hl('Subscript', '['), + hl('Identifier', 'e'), + hl('Subscript', ']'), + hl('Subscript', '['), + hl('Identifier', 'f'), + hl('Subscript', ']'), + }) + end) + itp('supports list literals', function() + check_parsing('[]', 0, { + -- 01 + ast = { + 'ListLiteral:0:0:[', + }, + }, { + hl('List', '['), + hl('List', ']'), + }) + + check_parsing('[a]', 0, { + -- 012 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('List', '['), + hl('Identifier', 'a'), + hl('List', ']'), + }) + + check_parsing('[a, b]', 0, { + -- 012345 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c]', 0, { + -- 012345678 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b', 1), + hl('Comma', ','), + hl('Identifier', 'c', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c, ]', 0, { + -- 01234567890 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b', 1), + hl('Comma', ','), + hl('Identifier', 'c', 1), + hl('Comma', ','), + hl('List', ']', 1), + }) + + check_parsing('[a : b, c : d]', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + { + 'Colon:0:9: :', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7: c', + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': b, c : d]', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('List', '['), + hl('Identifier', 'a'), + hl('InvalidColon', ':', 1), + hl('Identifier', 'b', 1), + hl('Comma', ','), + hl('Identifier', 'c', 1), + hl('InvalidColon', ':', 1), + hl('Identifier', 'd', 1), + hl('List', ']'), + }) + + check_parsing(']', 0, { + -- 0 + ast = { + 'ListLiteral:0:0:', + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidList', ']'), + }) + + check_parsing('a]', 0, { + -- 01 + ast = { + { + 'ListLiteral:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('InvalidList', ']'), + }) + + check_parsing('[] []', 0, { + -- 01234 + ast = { + { + 'OpMissing:0:2:', + children = { + 'ListLiteral:0:0:[', + 'ListLiteral:0:2: [', + }, + }, + }, + err = { + arg = '[]', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('InvalidSpacing', ' '), + hl('List', '['), + hl('List', ']'), + }) + + check_parsing('[][]', 0, { + -- 0123 + ast = { + { + 'Subscript:0:2:[', + children = { + 'ListLiteral:0:0:[', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('Subscript', '['), + hl('InvalidSubscript', ']'), + }) + end) end) From 6f22b5afad23ccc0390a6e2dd61d2e166ac6696a Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 8 Oct 2017 21:19:10 +0300 Subject: [PATCH 028/109] mbyte: Lint some functions which are to be copied for symbolic tests --- src/nvim/globals.h | 23 ----- src/nvim/mbyte.c | 248 +++++++++++++++++++++++++-------------------- 2 files changed, 137 insertions(+), 134 deletions(-) diff --git a/src/nvim/globals.h b/src/nvim/globals.h index 13ecafcbe3..86fff46737 100644 --- a/src/nvim/globals.h +++ b/src/nvim/globals.h @@ -725,29 +725,6 @@ EXTERN int vr_lines_changed INIT(= 0); /* #Lines changed by "gR" so far */ /// Encoding used when 'fencs' is set to "default" EXTERN char_u *fenc_default INIT(= NULL); -// To speed up BYTELEN(); keep a lookup table to quickly get the length in -// bytes of a UTF-8 character from the first byte of a UTF-8 string. Bytes -// which are illegal when used as the first byte have a 1. The NUL byte has -// length 1. -EXTERN char utf8len_tab[256] INIT(= { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, -}); - # if defined(USE_ICONV) && defined(DYNAMIC_ICONV) /* Pointers to functions and variables to be loaded at runtime */ EXTERN size_t (*iconv)(iconv_t cd, const char **inbuf, size_t *inbytesleft, diff --git a/src/nvim/mbyte.c b/src/nvim/mbyte.c index b24770a409..f65d7a6b13 100644 --- a/src/nvim/mbyte.c +++ b/src/nvim/mbyte.c @@ -72,19 +72,41 @@ struct interval { # include "unicode_tables.generated.h" #endif -/* - * Like utf8len_tab above, but using a zero for illegal lead bytes. - */ -const uint8_t utf8len_tab_zero[256] = -{ - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, +// To speed up BYTELEN(); keep a lookup table to quickly get the length in +// bytes of a UTF-8 character from the first byte of a UTF-8 string. Bytes +// which are illegal when used as the first byte have a 1. The NUL byte has +// length 1. +const uint8_t utf8len_tab[] = { + // ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9 ?A ?B ?C ?D ?E ?F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D? + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E? + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, // F? +}; + +// Like utf8len_tab above, but using a zero for illegal lead bytes. +const uint8_t utf8len_tab_zero[] = { + //1 2 3 4 5 6 7 8 9 A B C D E F 0 1 2 3 4 5 6 7 8 9 A B C D E F + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 2 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 4 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 6 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 8 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // A + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // C + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, // E }; /* @@ -528,45 +550,52 @@ int utf_off2cells(unsigned off, unsigned max_off) return (off + 1 < max_off && ScreenLines[off + 1] == 0) ? 2 : 1; } -/* - * Convert a UTF-8 byte sequence to a wide character. - * If the sequence is illegal or truncated by a NUL the first byte is - * returned. - * Does not include composing characters, of course. - */ -int utf_ptr2char(const char_u *p) +/// Convert a UTF-8 byte sequence to a wide character +/// +/// If the sequence is illegal or truncated by a NUL then the first byte is +/// returned. Does not include composing characters for obvious reasons. +/// +/// @param[in] p String to convert. +/// +/// @return Unicode codepoint or byte value. +int utf_ptr2char(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { - uint8_t len; - - if (p[0] < 0x80) /* be quick for ASCII */ + if (p[0] < 0x80) { // Be quick for ASCII. return p[0]; + } - len = utf8len_tab_zero[p[0]]; + const uint8_t len = utf8len_tab_zero[p[0]]; if (len > 1 && (p[1] & 0xc0) == 0x80) { - if (len == 2) + if (len == 2) { return ((p[0] & 0x1f) << 6) + (p[1] & 0x3f); + } if ((p[2] & 0xc0) == 0x80) { - if (len == 3) - return ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) - + (p[2] & 0x3f); + if (len == 3) { + return (((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + + (p[2] & 0x3f)); + } if ((p[3] & 0xc0) == 0x80) { - if (len == 4) - return ((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) - + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f); + if (len == 4) { + return (((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) + + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f)); + } if ((p[4] & 0xc0) == 0x80) { - if (len == 5) - return ((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) - + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) - + (p[4] & 0x3f); - if ((p[5] & 0xc0) == 0x80 && len == 6) - return ((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) - + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) - + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f); + if (len == 5) { + return (((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) + + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) + + (p[4] & 0x3f)); + } + if ((p[5] & 0xc0) == 0x80 && len == 6) { + return (((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) + + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) + + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f)); + } } } } } - /* Illegal value, just return the first byte */ + // Illegal value: just return the first byte. return p[0]; } @@ -767,23 +796,24 @@ int utfc_char2bytes(int off, char_u *buf) return len; } -/* - * Get the length of a UTF-8 byte sequence, not including any following - * composing characters. - * Returns 0 for "". - * Returns 1 for an illegal byte sequence. - */ -int utf_ptr2len(const char_u *p) +/// Get the length of a UTF-8 byte sequence representing a single codepoint +/// +/// @param[in] p UTF-8 string. +/// +/// @return Sequence length, 0 for empty string and 1 for non-UTF-8 byte +/// sequence. +int utf_ptr2len(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int len; - int i; - - if (*p == NUL) + if (*p == NUL) { return 0; - len = utf8len_tab[*p]; - for (i = 1; i < len; ++i) - if ((p[i] & 0xc0) != 0x80) + } + const int len = utf8len_tab[*p]; + for (int i = 1; i < len; i++) { + if ((p[i] & 0xc0) != 0x80) { return 1; + } + } return len; } @@ -824,38 +854,38 @@ int utf_ptr2len_len(const char_u *p, int size) return len; } -/* - * Return the number of bytes the UTF-8 encoding of the character at "p" takes. - * This includes following composing characters. - */ -int utfc_ptr2len(const char_u *p) +/// Return the number of bytes occupied by a UTF-8 character in a string +/// +/// This includes following composing characters. +int utfc_ptr2len(const char_u *const p) + FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int len; - int b0 = *p; - int prevlen; + uint8_t b0 = (uint8_t)(*p); - if (b0 == NUL) + if (b0 == NUL) { return 0; - if (b0 < 0x80 && p[1] < 0x80) /* be quick for ASCII */ + } + if (b0 < 0x80 && p[1] < 0x80) { // be quick for ASCII return 1; + } - /* Skip over first UTF-8 char, stopping at a NUL byte. */ - len = utf_ptr2len(p); + // Skip over first UTF-8 char, stopping at a NUL byte. + int len = utf_ptr2len(p); - /* Check for illegal byte. */ - if (len == 1 && b0 >= 0x80) + // Check for illegal byte. + if (len == 1 && b0 >= 0x80) { return 1; + } - /* - * Check for composing characters. We can handle only the first six, but - * skip all of them (otherwise the cursor would get stuck). - */ - prevlen = 0; - for (;; ) { - if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) + // Check for composing characters. We can handle only the first six, but + // skip all of them (otherwise the cursor would get stuck). + int prevlen = 0; + for (;;) { + if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) { return len; + } - /* Skip over composing char */ + // Skip over composing char. prevlen = len; len += utf_ptr2len(p + len); } @@ -913,23 +943,22 @@ int utfc_ptr2len_len(const char_u *p, int size) return len; } -/* - * Return the number of bytes the UTF-8 encoding of character "c" takes. - * This does not include composing characters. - */ -int utf_char2len(int c) +/// Determine how many bytes certain unicode codepoint will occupy +int utf_char2len(const int c) { - if (c < 0x80) + if (c < 0x80) { return 1; - if (c < 0x800) + } else if (c < 0x800) { return 2; - if (c < 0x10000) + } else if (c < 0x10000) { return 3; - if (c < 0x200000) + } else if (c < 0x200000) { return 4; - if (c < 0x4000000) + } else if (c < 0x4000000) { return 5; - return 6; + } else { + return 6; + } } /// Convert Unicode character to UTF-8 string @@ -937,46 +966,42 @@ int utf_char2len(int c) /// @param c character to convert to \p buf /// @param[out] buf UTF-8 string generated from \p c, does not add \0 /// @return Number of bytes (1-6). Does not include composing characters. -int utf_char2bytes(int c, char_u *const buf) +int utf_char2bytes(const int c, char_u *const buf) { - if (c < 0x80) { /* 7 bits */ + if (c < 0x80) { // 7 bits buf[0] = c; return 1; - } - if (c < 0x800) { /* 11 bits */ + } else if (c < 0x800) { // 11 bits buf[0] = 0xc0 + ((unsigned)c >> 6); buf[1] = 0x80 + (c & 0x3f); return 2; - } - if (c < 0x10000) { /* 16 bits */ + } else if (c < 0x10000) { // 16 bits buf[0] = 0xe0 + ((unsigned)c >> 12); buf[1] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[2] = 0x80 + (c & 0x3f); return 3; - } - if (c < 0x200000) { /* 21 bits */ + } else if (c < 0x200000) { // 21 bits buf[0] = 0xf0 + ((unsigned)c >> 18); buf[1] = 0x80 + (((unsigned)c >> 12) & 0x3f); buf[2] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[3] = 0x80 + (c & 0x3f); return 4; - } - if (c < 0x4000000) { /* 26 bits */ + } else if (c < 0x4000000) { // 26 bits buf[0] = 0xf8 + ((unsigned)c >> 24); buf[1] = 0x80 + (((unsigned)c >> 18) & 0x3f); buf[2] = 0x80 + (((unsigned)c >> 12) & 0x3f); buf[3] = 0x80 + (((unsigned)c >> 6) & 0x3f); buf[4] = 0x80 + (c & 0x3f); return 5; + } else { // 31 bits + buf[0] = 0xfc + ((unsigned)c >> 30); + buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[5] = 0x80 + (c & 0x3f); + return 6; } - /* 31 bits */ - buf[0] = 0xfc + ((unsigned)c >> 30); - buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); - buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); - buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); - buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); - buf[5] = 0x80 + (c & 0x3f); - return 6; } /* @@ -1513,14 +1538,15 @@ int utf_head_off(const char_u *base, const char_u *p) return (int)(p - q); } -/* - * Copy a character from "*fp" to "*tp" and advance the pointers. - */ -void mb_copy_char(const char_u **fp, char_u **tp) +/// Copy a character, advancing the pointers +/// +/// @param[in,out] fp Source of the character to copy. +/// @param[in,out] tp Destination to copy to. +void mb_copy_char(const char_u **const fp, char_u **const tp) { - int l = (*mb_ptr2len)(*fp); + const size_t l = (size_t)utfc_ptr2len(*fp); - memmove(*tp, *fp, (size_t)l); + memmove(*tp, *fp, l); *tp += l; *fp += l; } From e423cfe1948d75285dad2df151f53400f3b3be4e Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 8 Oct 2017 21:41:34 +0300 Subject: [PATCH 029/109] edit: Lint some functions which are to be copied for symbolic tests --- src/nvim/edit.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/nvim/edit.c b/src/nvim/edit.c index ca62679fab..e57598fc91 100644 --- a/src/nvim/edit.c +++ b/src/nvim/edit.c @@ -6074,27 +6074,30 @@ void free_last_insert(void) #endif -/* - * Add character "c" to buffer "s". Escape the special meaning of K_SPECIAL - * and CSI. Handle multi-byte characters. - * Returns a pointer to after the added bytes. - */ +/// Add character "c" to buffer "s" +/// +/// Escapes the special meaning of K_SPECIAL and CSI, handles multi-byte +/// characters. +/// +/// @param[in] c Character to add. +/// @param[out] s Buffer to add to. Must have at least MB_MAXBYTES + 1 bytes. +/// +/// @return Pointer to after the added bytes. char_u *add_char2buf(int c, char_u *s) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_WARN_UNUSED_RESULT { char_u temp[MB_MAXBYTES + 1]; - int i; - int len; - - len = (*mb_char2bytes)(c, temp); - for (i = 0; i < len; ++i) { + const int len = utf_char2bytes(c, temp); + for (int i = 0; i < len; ++i) { c = temp[i]; - /* Need to escape K_SPECIAL and CSI like in the typeahead buffer. */ + // Need to escape K_SPECIAL and CSI like in the typeahead buffer. if (c == K_SPECIAL) { *s++ = K_SPECIAL; *s++ = KS_SPECIAL; *s++ = KE_FILLER; - } else + } else { *s++ = c; + } } return s; } From c484613ce034cf9b10a4185621abdf8d82b570f8 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 8 Oct 2017 21:52:29 +0300 Subject: [PATCH 030/109] keymap: Lint some functions to be copied for symbolic tests --- src/nvim/keymap.c | 415 +++++++++++++++++++++++----------------------- 1 file changed, 206 insertions(+), 209 deletions(-) diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index 3d7ebb6382..f344d65c8d 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -24,12 +24,11 @@ * Some useful tables. */ -static struct modmasktable { - short mod_mask; /* Bit-mask for particular key modifier */ - short mod_flag; /* Bit(s) for particular key modifier */ - char_u name; /* Single letter name of modifier */ -} mod_mask_table[] = -{ +static const struct modmasktable { + short mod_mask; ///< Bit-mask for particular key modifier. + short mod_flag; ///< Bit(s) for particular key modifier. + char_u name; ///< Single letter name of modifier. +} mod_mask_table[] = { {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M'}, {MOD_MASK_META, MOD_MASK_META, (char_u)'T'}, {MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C'}, @@ -139,155 +138,154 @@ static char_u modifier_keys_table[] = NUL }; -static struct key_name_entry { - int key; /* Special key code or ascii value */ - char_u *name; /* Name of key */ -} key_names_table[] = -{ - {' ', (char_u *)"Space"}, - {TAB, (char_u *)"Tab"}, - {K_TAB, (char_u *)"Tab"}, - {NL, (char_u *)"NL"}, - {NL, (char_u *)"NewLine"}, /* Alternative name */ - {NL, (char_u *)"LineFeed"}, /* Alternative name */ - {NL, (char_u *)"LF"}, /* Alternative name */ - {CAR, (char_u *)"CR"}, - {CAR, (char_u *)"Return"}, /* Alternative name */ - {CAR, (char_u *)"Enter"}, /* Alternative name */ - {K_BS, (char_u *)"BS"}, - {K_BS, (char_u *)"BackSpace"}, /* Alternative name */ - {ESC, (char_u *)"Esc"}, - {CSI, (char_u *)"CSI"}, - {K_CSI, (char_u *)"xCSI"}, - {'|', (char_u *)"Bar"}, - {'\\', (char_u *)"Bslash"}, - {K_DEL, (char_u *)"Del"}, - {K_DEL, (char_u *)"Delete"}, /* Alternative name */ - {K_KDEL, (char_u *)"kDel"}, - {K_UP, (char_u *)"Up"}, - {K_DOWN, (char_u *)"Down"}, - {K_LEFT, (char_u *)"Left"}, - {K_RIGHT, (char_u *)"Right"}, - {K_XUP, (char_u *)"xUp"}, - {K_XDOWN, (char_u *)"xDown"}, - {K_XLEFT, (char_u *)"xLeft"}, - {K_XRIGHT, (char_u *)"xRight"}, +static const struct key_name_entry { + int key; ///< Special key code or ASCII value. + const char *name; ///< Name of the key +} key_names_table[] = { + {' ', "Space"}, + {TAB, "Tab"}, + {K_TAB, "Tab"}, + {NL, "NL"}, + {NL, "NewLine"}, // Alternative name + {NL, "LineFeed"}, // Alternative name + {NL, "LF"}, // Alternative name + {CAR, "CR"}, + {CAR, "Return"}, // Alternative name + {CAR, "Enter"}, // Alternative name + {K_BS, "BS"}, + {K_BS, "BackSpace"}, // Alternative name + {ESC, "Esc"}, + {CSI, "CSI"}, + {K_CSI, "xCSI"}, + {'|', "Bar"}, + {'\\', "Bslash"}, + {K_DEL, "Del"}, + {K_DEL, "Delete"}, // Alternative name + {K_KDEL, "kDel"}, + {K_UP, "Up"}, + {K_DOWN, "Down"}, + {K_LEFT, "Left"}, + {K_RIGHT, "Right"}, + {K_XUP, "xUp"}, + {K_XDOWN, "xDown"}, + {K_XLEFT, "xLeft"}, + {K_XRIGHT, "xRight"}, - {K_F1, (char_u *)"F1"}, - {K_F2, (char_u *)"F2"}, - {K_F3, (char_u *)"F3"}, - {K_F4, (char_u *)"F4"}, - {K_F5, (char_u *)"F5"}, - {K_F6, (char_u *)"F6"}, - {K_F7, (char_u *)"F7"}, - {K_F8, (char_u *)"F8"}, - {K_F9, (char_u *)"F9"}, - {K_F10, (char_u *)"F10"}, + {K_F1, "F1"}, + {K_F2, "F2"}, + {K_F3, "F3"}, + {K_F4, "F4"}, + {K_F5, "F5"}, + {K_F6, "F6"}, + {K_F7, "F7"}, + {K_F8, "F8"}, + {K_F9, "F9"}, + {K_F10, "F10"}, - {K_F11, (char_u *)"F11"}, - {K_F12, (char_u *)"F12"}, - {K_F13, (char_u *)"F13"}, - {K_F14, (char_u *)"F14"}, - {K_F15, (char_u *)"F15"}, - {K_F16, (char_u *)"F16"}, - {K_F17, (char_u *)"F17"}, - {K_F18, (char_u *)"F18"}, - {K_F19, (char_u *)"F19"}, - {K_F20, (char_u *)"F20"}, + {K_F11, "F11"}, + {K_F12, "F12"}, + {K_F13, "F13"}, + {K_F14, "F14"}, + {K_F15, "F15"}, + {K_F16, "F16"}, + {K_F17, "F17"}, + {K_F18, "F18"}, + {K_F19, "F19"}, + {K_F20, "F20"}, - {K_F21, (char_u *)"F21"}, - {K_F22, (char_u *)"F22"}, - {K_F23, (char_u *)"F23"}, - {K_F24, (char_u *)"F24"}, - {K_F25, (char_u *)"F25"}, - {K_F26, (char_u *)"F26"}, - {K_F27, (char_u *)"F27"}, - {K_F28, (char_u *)"F28"}, - {K_F29, (char_u *)"F29"}, - {K_F30, (char_u *)"F30"}, + {K_F21, "F21"}, + {K_F22, "F22"}, + {K_F23, "F23"}, + {K_F24, "F24"}, + {K_F25, "F25"}, + {K_F26, "F26"}, + {K_F27, "F27"}, + {K_F28, "F28"}, + {K_F29, "F29"}, + {K_F30, "F30"}, - {K_F31, (char_u *)"F31"}, - {K_F32, (char_u *)"F32"}, - {K_F33, (char_u *)"F33"}, - {K_F34, (char_u *)"F34"}, - {K_F35, (char_u *)"F35"}, - {K_F36, (char_u *)"F36"}, - {K_F37, (char_u *)"F37"}, + {K_F31, "F31"}, + {K_F32, "F32"}, + {K_F33, "F33"}, + {K_F34, "F34"}, + {K_F35, "F35"}, + {K_F36, "F36"}, + {K_F37, "F37"}, - {K_XF1, (char_u *)"xF1"}, - {K_XF2, (char_u *)"xF2"}, - {K_XF3, (char_u *)"xF3"}, - {K_XF4, (char_u *)"xF4"}, + {K_XF1, "xF1"}, + {K_XF2, "xF2"}, + {K_XF3, "xF3"}, + {K_XF4, "xF4"}, - {K_HELP, (char_u *)"Help"}, - {K_UNDO, (char_u *)"Undo"}, - {K_INS, (char_u *)"Insert"}, - {K_INS, (char_u *)"Ins"}, /* Alternative name */ - {K_KINS, (char_u *)"kInsert"}, - {K_HOME, (char_u *)"Home"}, - {K_KHOME, (char_u *)"kHome"}, - {K_XHOME, (char_u *)"xHome"}, - {K_ZHOME, (char_u *)"zHome"}, - {K_END, (char_u *)"End"}, - {K_KEND, (char_u *)"kEnd"}, - {K_XEND, (char_u *)"xEnd"}, - {K_ZEND, (char_u *)"zEnd"}, - {K_PAGEUP, (char_u *)"PageUp"}, - {K_PAGEDOWN, (char_u *)"PageDown"}, - {K_KPAGEUP, (char_u *)"kPageUp"}, - {K_KPAGEDOWN, (char_u *)"kPageDown"}, + {K_HELP, "Help"}, + {K_UNDO, "Undo"}, + {K_INS, "Insert"}, + {K_INS, "Ins"}, // Alternative name + {K_KINS, "kInsert"}, + {K_HOME, "Home"}, + {K_KHOME, "kHome"}, + {K_XHOME, "xHome"}, + {K_ZHOME, "zHome"}, + {K_END, "End"}, + {K_KEND, "kEnd"}, + {K_XEND, "xEnd"}, + {K_ZEND, "zEnd"}, + {K_PAGEUP, "PageUp"}, + {K_PAGEDOWN, "PageDown"}, + {K_KPAGEUP, "kPageUp"}, + {K_KPAGEDOWN, "kPageDown"}, - {K_KPLUS, (char_u *)"kPlus"}, - {K_KMINUS, (char_u *)"kMinus"}, - {K_KDIVIDE, (char_u *)"kDivide"}, - {K_KMULTIPLY, (char_u *)"kMultiply"}, - {K_KENTER, (char_u *)"kEnter"}, - {K_KPOINT, (char_u *)"kPoint"}, + {K_KPLUS, "kPlus"}, + {K_KMINUS, "kMinus"}, + {K_KDIVIDE, "kDivide"}, + {K_KMULTIPLY, "kMultiply"}, + {K_KENTER, "kEnter"}, + {K_KPOINT, "kPoint"}, - {K_K0, (char_u *)"k0"}, - {K_K1, (char_u *)"k1"}, - {K_K2, (char_u *)"k2"}, - {K_K3, (char_u *)"k3"}, - {K_K4, (char_u *)"k4"}, - {K_K5, (char_u *)"k5"}, - {K_K6, (char_u *)"k6"}, - {K_K7, (char_u *)"k7"}, - {K_K8, (char_u *)"k8"}, - {K_K9, (char_u *)"k9"}, + {K_K0, "k0"}, + {K_K1, "k1"}, + {K_K2, "k2"}, + {K_K3, "k3"}, + {K_K4, "k4"}, + {K_K5, "k5"}, + {K_K6, "k6"}, + {K_K7, "k7"}, + {K_K8, "k8"}, + {K_K9, "k9"}, - {'<', (char_u *)"lt"}, + {'<', "lt"}, - {K_MOUSE, (char_u *)"Mouse"}, - {K_LEFTMOUSE, (char_u *)"LeftMouse"}, - {K_LEFTMOUSE_NM, (char_u *)"LeftMouseNM"}, - {K_LEFTDRAG, (char_u *)"LeftDrag"}, - {K_LEFTRELEASE, (char_u *)"LeftRelease"}, - {K_LEFTRELEASE_NM, (char_u *)"LeftReleaseNM"}, - {K_MIDDLEMOUSE, (char_u *)"MiddleMouse"}, - {K_MIDDLEDRAG, (char_u *)"MiddleDrag"}, - {K_MIDDLERELEASE, (char_u *)"MiddleRelease"}, - {K_RIGHTMOUSE, (char_u *)"RightMouse"}, - {K_RIGHTDRAG, (char_u *)"RightDrag"}, - {K_RIGHTRELEASE, (char_u *)"RightRelease"}, - {K_MOUSEDOWN, (char_u *)"ScrollWheelUp"}, - {K_MOUSEUP, (char_u *)"ScrollWheelDown"}, - {K_MOUSELEFT, (char_u *)"ScrollWheelRight"}, - {K_MOUSERIGHT, (char_u *)"ScrollWheelLeft"}, - {K_MOUSEDOWN, (char_u *)"MouseDown"}, /* OBSOLETE: Use */ - {K_MOUSEUP, (char_u *)"MouseUp"}, /* ScrollWheelXXX instead */ - {K_X1MOUSE, (char_u *)"X1Mouse"}, - {K_X1DRAG, (char_u *)"X1Drag"}, - {K_X1RELEASE, (char_u *)"X1Release"}, - {K_X2MOUSE, (char_u *)"X2Mouse"}, - {K_X2DRAG, (char_u *)"X2Drag"}, - {K_X2RELEASE, (char_u *)"X2Release"}, - {K_DROP, (char_u *)"Drop"}, - {K_ZERO, (char_u *)"Nul"}, - {K_SNR, (char_u *)"SNR"}, - {K_PLUG, (char_u *)"Plug"}, - {K_PASTE, (char_u *)"Paste"}, - {K_FOCUSGAINED, (char_u *)"FocusGained"}, - {K_FOCUSLOST, (char_u *)"FocusLost"}, + {K_MOUSE, "Mouse"}, + {K_LEFTMOUSE, "LeftMouse"}, + {K_LEFTMOUSE_NM, "LeftMouseNM"}, + {K_LEFTDRAG, "LeftDrag"}, + {K_LEFTRELEASE, "LeftRelease"}, + {K_LEFTRELEASE_NM, "LeftReleaseNM"}, + {K_MIDDLEMOUSE, "MiddleMouse"}, + {K_MIDDLEDRAG, "MiddleDrag"}, + {K_MIDDLERELEASE, "MiddleRelease"}, + {K_RIGHTMOUSE, "RightMouse"}, + {K_RIGHTDRAG, "RightDrag"}, + {K_RIGHTRELEASE, "RightRelease"}, + {K_MOUSEDOWN, "ScrollWheelUp"}, + {K_MOUSEUP, "ScrollWheelDown"}, + {K_MOUSELEFT, "ScrollWheelRight"}, + {K_MOUSERIGHT, "ScrollWheelLeft"}, + {K_MOUSEDOWN, "MouseDown"}, // OBSOLETE: Use ScrollWheelXXX instead + {K_MOUSEUP, "MouseUp"}, // Same + {K_X1MOUSE, "X1Mouse"}, + {K_X1DRAG, "X1Drag"}, + {K_X1RELEASE, "X1Release"}, + {K_X2MOUSE, "X2Mouse"}, + {K_X2DRAG, "X2Drag"}, + {K_X2RELEASE, "X2Release"}, + {K_DROP, "Drop"}, + {K_ZERO, "Nul"}, + {K_SNR, "SNR"}, + {K_PLUG, "Plug"}, + {K_PASTE, "Paste"}, + {K_FOCUSGAINED, "FocusGained"}, + {K_FOCUSLOST, "FocusLost"}, {0, NULL} }; @@ -320,73 +318,73 @@ static struct mousetable { {0, 0, 0, 0}, }; -/* - * Return the modifier mask bit (MOD_MASK_*) which corresponds to the given - * modifier name ('S' for Shift, 'C' for Ctrl etc). - */ +/// Return the modifier mask bit (#MOD_MASK_*) corresponding to mod name +/// +/// E.g. 'S' for shift, 'C' for ctrl. int name_to_mod_mask(int c) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT { - int i; - c = TOUPPER_ASC(c); - for (i = 0; mod_mask_table[i].mod_mask != 0; i++) - if (c == mod_mask_table[i].name) + for (size_t i = 0; mod_mask_table[i].mod_mask != 0; i++) { + if (c == mod_mask_table[i].name) { return mod_mask_table[i].mod_flag; + } + } return 0; } -/* - * Check if if there is a special key code for "key" that includes the - * modifiers specified. - */ -int simplify_key(int key, int *modifiers) +/// Check if there is a special key code for "key" with specified modifiers +/// +/// @param[in] key Initial key code. +/// @param[in,out] modifiers Initial modifiers, is adjusted to have simplified +/// modifiers. +/// +/// @return Simplified key code. +int simplify_key(const int key, int *modifiers) + FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL { - int i; - int key0; - int key1; - if (*modifiers & (MOD_MASK_SHIFT | MOD_MASK_CTRL | MOD_MASK_ALT)) { - /* TAB is a special case */ + // TAB is a special case. if (key == TAB && (*modifiers & MOD_MASK_SHIFT)) { *modifiers &= ~MOD_MASK_SHIFT; return K_S_TAB; } - key0 = KEY2TERMCAP0(key); - key1 = KEY2TERMCAP1(key); - for (i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) + const int key0 = KEY2TERMCAP0(key); + const int key1 = KEY2TERMCAP1(key); + for (int i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) { if (key0 == modifier_keys_table[i + 3] && key1 == modifier_keys_table[i + 4] && (*modifiers & modifier_keys_table[i])) { *modifiers &= ~modifier_keys_table[i]; return TERMCAP2KEY(modifier_keys_table[i + 1], - modifier_keys_table[i + 2]); + modifier_keys_table[i + 2]); } + } } return key; } -/* - * Change to , to , etc. - */ -int handle_x_keys(int key) +/// Change to +int handle_x_keys(const int key) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT { switch (key) { - case K_XUP: return K_UP; - case K_XDOWN: return K_DOWN; - case K_XLEFT: return K_LEFT; - case K_XRIGHT: return K_RIGHT; - case K_XHOME: return K_HOME; - case K_ZHOME: return K_HOME; - case K_XEND: return K_END; - case K_ZEND: return K_END; - case K_XF1: return K_F1; - case K_XF2: return K_F2; - case K_XF3: return K_F3; - case K_XF4: return K_F4; - case K_S_XF1: return K_S_F1; - case K_S_XF2: return K_S_F2; - case K_S_XF3: return K_S_F3; - case K_S_XF4: return K_S_F4; + case K_XUP: return K_UP; + case K_XDOWN: return K_DOWN; + case K_XLEFT: return K_LEFT; + case K_XRIGHT: return K_RIGHT; + case K_XHOME: return K_HOME; + case K_ZHOME: return K_HOME; + case K_XEND: return K_END; + case K_ZEND: return K_END; + case K_XF1: return K_F1; + case K_XF2: return K_F2; + case K_XF3: return K_F3; + case K_XF4: return K_F4; + case K_S_XF1: return K_S_F1; + case K_S_XF2: return K_S_F2; + case K_S_XF3: return K_S_F3; + case K_S_XF4: return K_S_F4; } return key; } @@ -510,7 +508,7 @@ unsigned int trans_special(const char_u **srcp, const size_t src_len, return 0; } - /* Put the appropriate modifier in a string */ + // Put the appropriate modifier in a string. if (modifiers != 0) { dst[dlen++] = K_SPECIAL; dst[dlen++] = KS_MODIFIER; @@ -575,11 +573,7 @@ int find_special_key(const char_u **srcp, const size_t src_len, int *const modp, if (*bp == '-') { last_dash = bp; if (bp + 1 <= end) { - if (has_mbyte) { - l = mb_ptr2len_len(bp + 1, (int) (end - bp) + 1); - } else { - l = 1; - } + l = utfc_ptr2len_len(bp + 1, (int)(end - bp) + 1); // Anything accepted, like . // or are not special in strings as " is // the string delimiter. With a backslash it works: @@ -705,32 +699,35 @@ int find_special_key_in_table(int c) int i; for (i = 0; key_names_table[i].name != NULL; i++) - if (c == key_names_table[i].key) + if (c == (uint8_t)key_names_table[i].key) break; if (key_names_table[i].name == NULL) i = -1; return i; } -/* - * Find the special key with the given name (the given string does not have to - * end with NUL, the name is assumed to end before the first non-idchar). - * If the name starts with "t_" the next two characters are interpreted as a - * termcap name. - * Return the key code, or 0 if not found. - */ +/// Find the special key with the given name +/// +/// @param[in] name Name of the special. Does not have to end with NUL, it is +/// assumed to end before the first non-idchar. If name starts +/// with "t_" the next two characters are interpreted as +/// a termcap name. +/// +/// @return Key code or 0 if ton found. int get_special_key_code(const char_u *name) + FUNC_ATTR_NONNULL_ALL FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { - char_u *table_name; - int i, j; - - for (i = 0; key_names_table[i].name != NULL; i++) { - table_name = key_names_table[i].name; - for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) - if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) + for (int i = 0; key_names_table[i].name != NULL; i++) { + const char *const table_name = key_names_table[i].name; + int j; + for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) { + if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { break; - if (!vim_isIDc(name[j]) && table_name[j] == NUL) + } + } + if (!vim_isIDc(name[j]) && table_name[j] == NUL) { return key_names_table[i].key; + } } return 0; From af38cea133f5ebb67208cedd289e408cd1dad15a Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 8 Oct 2017 21:52:38 +0300 Subject: [PATCH 031/109] viml/parser/expressions: Add support for string parsing --- src/nvim/mbyte.h | 2 + src/nvim/viml/parser/expressions.c | 376 ++++++++- src/nvim/viml/parser/expressions.h | 7 + test/helpers.lua | 7 +- test/symbolic/klee/nvim/keymap.c | 540 ++++++++++++ test/symbolic/klee/nvim/mbyte.c | 193 ++++- test/symbolic/klee/viml_expressions_parser.c | 1 + test/unit/viml/expressions/parser_spec.lua | 825 +++++++++++++++++++ 8 files changed, 1938 insertions(+), 13 deletions(-) create mode 100644 test/symbolic/klee/nvim/keymap.c diff --git a/src/nvim/mbyte.h b/src/nvim/mbyte.h index bf6ccb13a5..fce600d0a9 100644 --- a/src/nvim/mbyte.h +++ b/src/nvim/mbyte.h @@ -73,6 +73,8 @@ typedef struct { extern const uint8_t utf8len_tab_zero[256]; +extern const uint8_t utf8len_tab[256]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "mbyte.h.generated.h" #endif diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 0613cc66d5..3f30fe2a0e 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -915,8 +915,6 @@ static inline void viml_pexpr_debug_print_token( // // NVimIdentifierKey -> Identifier // -// NVimFigureBrace -> NVimInternalError -// // NVimUnaryPlus -> NVimUnaryOperator // NVimBinaryPlus -> NVimBinaryOperator // NVimConcat -> NVimBinaryOperator @@ -929,6 +927,18 @@ static inline void viml_pexpr_debug_print_token( // NVimNestingParenthesis -> NVimParenthesis // NVimCallingParenthesis -> NVimParenthesis // +// NVimString -> String +// NVimStringSpecial -> SpecialChar +// NVimSingleQuote -> NVimString +// NVimSingleQuotedBody -> NVimString +// NVimSingleQuotedQuote -> NVimStringSpecial +// NVimDoubleQuote -> NVimString +// NVimDoubleQuotedBody -> NVimString +// NVimDoubleQuotedEscape -> NVimStringSpecial +// NVimDoubleQuotedUnknownEscape -> NVimInvalid +// +// " Note: NVimDoubleQuotedUnknownEscape is not actually invalid +// // NVimInvalidComma -> NVimInvalidDelimiter // NVimInvalidSpacing -> NVimInvalid // NVimInvalidTernary -> NVimInvalidOperator @@ -952,6 +962,19 @@ static inline void viml_pexpr_debug_print_token( // NVimInvalidList -> NVimInvalidDelimiter // NVimInvalidSubscript -> NVimInvalidDelimiter // NVimInvalidSubscriptColon -> NVimInvalidSubscript +// NVimInvalidString -> NVimInvalidValue +// NVimInvalidStringSpecial -> NVimInvalidString +// NVimInvalidSingleQuote -> NVimInvalidString +// NVimInvalidSingleQuotedBody -> NVimInvalidString +// NVimInvalidSingleQuotedQuote -> NVimInvalidStringSpecial +// NVimInvalidDoubleQuote -> NVimInvalidString +// NVimInvalidDoubleQuotedBody -> NVimInvalidString +// NVimInvalidDoubleQuotedEscape -> NVimInvalidStringSpecial +// NVimInvalidDoubleQuotedUnknownEscape -> NVimInvalid +// +// NVimFigureBrace -> NVimInternalError +// NVimInvalidSingleQuotedUnknownEscape -> NVimInternalError +// NVimSingleQuotedUnknownEscape -> NVimInternalError /// Allocate a new node and set some of the values /// @@ -1402,6 +1425,318 @@ static inline void east_set_error(const ParserState *const pstate, } \ } while (0) +/// Structure used to define “string shifts” necessary to map string +/// highlighting to actual strings. +typedef struct { + size_t start; ///< Where special character starts in original string. + size_t orig_len; ///< Length of orininal string (e.g. 4 for "\x80"). + size_t act_len; ///< Length of resulting character(s) (e.g. 1 for "\x80"). + bool escape_not_known; ///< True if escape sequence in original is not known. +} StringShift; + +/// Parse and highlight single- or double-quoted string +/// +/// Function is supposed to detect and highlight regular expressions (but does +/// not do now). +/// +/// @param[out] pstate Parser state which also contains a place where +/// highlighting is saved. +/// @param[out] node Node where string parsing results are saved. +/// @param[in] token Token to highlight. +/// @param[in] ast_stack Parser AST stack, used to detect whether current +/// string is a regex. +/// @param[in] is_invalid Whether currently processed token is not valid. +static void parse_quoted_string(ParserState *const pstate, + ExprASTNode *const node, + const LexExprToken token, + const ExprASTStack ast_stack, + const bool is_invalid) + FUNC_ATTR_NONNULL_ALL +{ + const ParserLine pline = pstate->reader.lines.items[token.start.line]; + const char *const s = pline.data + token.start.col; + const char *const e = s + token.len - token.data.str.closed; + const char *p = s + 1; + const bool is_double = (token.type == kExprLexDoubleQuotedString); + size_t size = token.len - token.data.str.closed - 1; + kvec_withinit_t(StringShift, 16) shifts; + kvi_init(shifts); + if (!is_double) { + viml_parser_highlight(pstate, token.start, 1, HL(SingleQuotedString)); + while (p < e) { + const char *const chunk_e = memchr(p, '\'', (size_t)(e - p)); + if (chunk_e == NULL) { + break; + } + size--; + p = chunk_e + 2; + if (pstate->colors) { + kvi_push(shifts, ((StringShift) { + .start = token.start.col + (size_t)(chunk_e - s), + .orig_len = 2, + .act_len = 1, + .escape_not_known = false, + })); + } + } + node->data.str.size = size; + if (size == 0) { + node->data.str.value = NULL; + } else { + char *v_p; + v_p = node->data.str.value = xmalloc(size); + p = s + 1; + while (p < e) { + const char *const chunk_e = memchr(p, '\'', (size_t)(e - p)); + if (chunk_e == NULL) { + memcpy(v_p, p, (size_t)(e - p)); + break; + } + memcpy(v_p, p, (size_t)(chunk_e - p)); + v_p += (size_t)(chunk_e - p) + 1; + v_p[-1] = '\''; + p = chunk_e + 2; + } + } + } else { + viml_parser_highlight(pstate, token.start, 1, HL(DoubleQuotedString)); + for (p = s + 1; p < e; p++) { + if (*p == '\\' && p + 1 < e) { + p++; + if (p + 1 == e) { + size--; + break; + } + switch (*p) { + // A "\" form occupies at least 4 characters, and produces up to + // 6 characters: reserve space for 2 extra, but do not compute actual + // length just now, it would be costy. + case '<': { + size += 2; + break; + } + // Hexadecimal, always single byte, but at least three bytes each. + case 'x': case 'X': { + size--; + if (ascii_isxdigit(p[1])) { + size--; + if (p + 2 < e && ascii_isxdigit(p[2])) { + size--; + } + } + break; + } + // Unicode + // + // \uF takes 1 byte which is 2 bytes less then escape sequence. + // \uFF: 2 bytes, 2 bytes less. + // \uFFF: 3 bytes, 2 bytes less. + // \uFFFF: 3 bytes, 3 bytes less. + // \UFFFFF: 4 bytes, 3 bytes less. + // \UFFFFFF: 5 bytes, 3 bytes less. + // \UFFFFFFF: 6 bytes, 3 bytes less. + // \U7FFFFFFF: 6 bytes, 4 bytes less. + case 'u': case 'U': { + const char *const esc_start = p; + size_t n = (*p == 'u' ? 4 : 8); + int nr = 0; + p++; + while (n-- && ascii_isxdigit(p[1])) { + p++; + nr = (nr << 4) + hex2nr(*p); + } + // Escape length: (esc_start - 1) points to "\\", esc_start to "u" + // or "U", p to the byte after last byte. So escape sequence + // occupies p - (esc_start - 1), but it stands for a utf_char2len + // bytes. + size -= (size_t)((p - (esc_start - 1)) - utf_char2len(nr)); + p--; + break; + } + // Octal, always single byte, but at least two bytes each. + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': { + size--; + p++; + if (*p >= '0' && *p <= '7') { + size--; + p++; + if (*p >= '0' && *p <= '7') { + size--; + p++; + } + } + break; + } + default: { + size--; + break; + } + } + } + } + if (size == 0) { + node->data.str.value = NULL; + node->data.str.size = 0; + } else { + char *v_p; + v_p = node->data.str.value = xmalloc(size); + p = s + 1; + while (p < e) { + const char *const chunk_e = memchr(p, '\\', (size_t)(e - p)); + if (chunk_e == NULL) { + memcpy(v_p, p, (size_t)(e - p)); + v_p += e - p; + break; + } + memcpy(v_p, p, (size_t)(chunk_e - p)); + v_p += (size_t)(chunk_e - p); + p = chunk_e + 1; + if (p == e) { + *v_p++ = '\\'; + break; + } + bool is_unknown = false; + const char *const v_p_start = v_p; + switch (*p) { +#define SINGLE_CHAR_ESC(ch, real_ch) \ + case ch: { \ + *v_p++ = real_ch; \ + p++; \ + break; \ + } + SINGLE_CHAR_ESC('b', BS) + SINGLE_CHAR_ESC('e', ESC) + SINGLE_CHAR_ESC('f', FF) + SINGLE_CHAR_ESC('n', NL) + SINGLE_CHAR_ESC('r', CAR) + SINGLE_CHAR_ESC('t', TAB) + SINGLE_CHAR_ESC('"', '"') + SINGLE_CHAR_ESC('\\', '\\') +#undef SINGLE_CHAR_ESC + + // Hexadecimal or unicode. + case 'X': case 'x': case 'u': case 'U': { + if (ascii_isxdigit(p[1])) { + size_t n; + int nr; + bool is_hex = (*p == 'x' || *p == 'X'); + + if (is_hex) { + n = 2; + } else if (*p == 'u') { + n = 4; + } else { + n = 8; + } + nr = 0; + while (n-- && ascii_isxdigit(p[1])) { + p++; + nr = (nr << 4) + hex2nr(*p); + } + p++; + if (is_hex) { + *v_p++ = (char)nr; + } else { + v_p += utf_char2bytes(nr, (char_u *)v_p); + } + } else { + is_unknown = true; + *v_p++ = *p; + p++; + } + break; + } + // Octal: "\1", "\12", "\123". + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': { + uint8_t ch = (uint8_t)(*p++ - '0'); + if (*p >= '0' && *p <= '7') { + ch = (uint8_t)((ch << 3) + *p++ - '0'); + if (*p >= '0' && *p <= '7') { + ch = (uint8_t)((ch << 3) + *p++ - '0'); + } + } + *v_p++ = (char)ch; + break; + } + // Special key, e.g.: "\" + case '<': { + const size_t special_len = ( + trans_special((const char_u **)&p, (size_t)(e - p), + (char_u *)v_p, true, true)); + if (special_len != 0) { + v_p += special_len; + } else { + is_unknown = true; + mb_copy_char((const char_u **)&p, (char_u **)&v_p); + } + break; + } + default: { + is_unknown = true; + mb_copy_char((const char_u **)&p, (char_u **)&v_p); + break; + } + } + if (pstate->colors) { + kvi_push(shifts, ((StringShift) { + .start = token.start.col + (size_t)(chunk_e - s), + .orig_len = (size_t)(p - chunk_e), + .act_len = (size_t)(v_p - (char *)v_p_start), + .escape_not_known = is_unknown, + })); + } + } + node->data.str.size = (size_t)(v_p - node->data.str.value); + } + } + if (pstate->colors) { + // TODO(ZyX-I): use ast_stack to determine and highlight regular expressions + // TODO(ZyX-I): use ast_stack to determine and highlight printf format str + // TODO(ZyX-I): use ast_stack to determine and highlight expression strings + size_t next_col = 1; + const char *const body_str = (is_double + ? HL(DoubleQuotedBody) + : HL(SingleQuotedBody)); + const char *const esc_str = (is_double + ? HL(DoubleQuotedEscape) + : HL(SingleQuotedQuote)); + const char *const ukn_esc_str = (is_double + ? HL(DoubleQuotedUnknownEscape) + : HL(SingleQuotedUnknownEscape)); + for (size_t i = 0; i < kv_size(shifts); i++) { + const StringShift cur_shift = kv_A(shifts, i); + if (cur_shift.start > next_col) { + viml_parser_highlight(pstate, shifted_pos(token.start, next_col), + cur_shift.start - next_col, + body_str); + } + viml_parser_highlight(pstate, shifted_pos(token.start, cur_shift.start), + cur_shift.orig_len, + (cur_shift.escape_not_known + ? ukn_esc_str + : esc_str)); + next_col = cur_shift.start + cur_shift.orig_len; + } + if (next_col < token.len - token.data.str.closed) { + viml_parser_highlight(pstate, shifted_pos(token.start, next_col), + token.len - token.data.str.closed - next_col, + body_str); + } + } + if (token.data.str.closed) { + if (is_double) { + viml_parser_highlight(pstate, shifted_pos(token.start, token.len - 1), + 1, HL(DoubleQuotedString)); + } else { + viml_parser_highlight(pstate, shifted_pos(token.start, token.len - 1), + 1, HL(SingleQuotedString)); + } + } + kvi_destroy(shifts); +} + /// Parse one VimL expression /// /// @param pstate Parser state. @@ -1714,12 +2049,7 @@ viml_pexpr_parse_invalid_comma: } else if (eastnode_lvl >= kEOpLvlComma) { can_be_ternary = false; } else { -viml_pexpr_parse_invalid_colon: - ERROR_FROM_TOKEN_AND_MSG( - cur_token, - _("E15: Colon outside of dictionary or ternary operator: " - "%.*s")); - break; + goto viml_pexpr_parse_invalid_colon; } if (i == kv_size(ast_stack) - 1) { goto viml_pexpr_parse_invalid_colon; @@ -1741,6 +2071,12 @@ viml_pexpr_parse_invalid_colon: ADD_OP_NODE(cur_node); HL_CUR_TOKEN(SubscriptColon); } else { + goto viml_pexpr_parse_valid_colon; +viml_pexpr_parse_invalid_colon: + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Colon outside of dictionary or ternary operator: %.*s")); +viml_pexpr_parse_valid_colon: ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); if (is_ternary) { @@ -2201,6 +2537,30 @@ viml_pexpr_parse_no_paren_closing_error: {} kvi_push(ast_stack, &ter_val_node->children); break; } + case kExprLexDoubleQuotedString: + case kExprLexSingleQuotedString: { + const bool is_double = (tok_type == kExprLexDoubleQuotedString); + if (!cur_token.data.str.closed) { + // It is weird, but Vim has two identical errors messages with + // different error numbers: "E114: Missing quote" and + // "E115: Missing quote". + ERROR_FROM_TOKEN_AND_MSG( + cur_token, (is_double + ? _("E114: Missing double quote: %.*s") + : _("E115: Missing single quote: %.*s"))); + } + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS( + cur_node, (is_double + ? kExprNodeDoubleQuotedString + : kExprNodeSingleQuotedString)); + *top_node_p = cur_node; + parse_quoted_string(pstate, cur_node, cur_token, ast_stack, is_invalid); + want_node = kENodeOperator; + break; + } } viml_pexpr_parse_cycle_end: prev_token = cur_token; diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 0d496c87ba..a09cdde4c0 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -195,6 +195,8 @@ typedef enum { kExprNodeConcatOrSubscript = 'S', kExprNodeInteger = '0', ///< Integral number. kExprNodeFloat = '1', ///< Floating-point number. + kExprNodeSingleQuotedString = '\'', + kExprNodeDoubleQuotedString = '"', } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -249,6 +251,11 @@ struct expr_ast_node { struct { float_T value; } flt; ///< For kExprNodeFloat. + struct { + char *value; + size_t size; + } str; ///< For kExprNodeSingleQuotedString and + ///< kExprNodeDoubleQuotedString. } data; }; diff --git a/test/helpers.lua b/test/helpers.lua index d5356416af..e7d8c185ba 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -330,9 +330,10 @@ format_luav = function(v, indent) end local ret = '' if type(v) == 'string' then - ret = '\'' .. tostring(v):gsub('[\'\\]', '\\%0'):gsub('[%z\1-\31]', function(match) - return SUBTBL[match:byte()] - end) .. '\'' + ret = tostring(v):gsub('[\'\\]', '\\%0'):gsub('[%z\1-\31]', function(match) + return SUBTBL[match:byte() + 1] + end) + ret = '\'' .. ret .. '\'' elseif type(v) == 'table' then local processed_keys = {} ret = '{' .. linesep diff --git a/test/symbolic/klee/nvim/keymap.c b/test/symbolic/klee/nvim/keymap.c new file mode 100644 index 0000000000..c59d8d4d6e --- /dev/null +++ b/test/symbolic/klee/nvim/keymap.c @@ -0,0 +1,540 @@ +#include + +#include "nvim/types.h" +#include "nvim/keymap.h" +#include "nvim/eval/typval.h" + +#define MOD_KEYS_ENTRY_SIZE 5 + +static char_u modifier_keys_table[] = +{ + MOD_MASK_SHIFT, '&', '9', '@', '1', + MOD_MASK_SHIFT, '&', '0', '@', '2', + MOD_MASK_SHIFT, '*', '1', '@', '4', + MOD_MASK_SHIFT, '*', '2', '@', '5', + MOD_MASK_SHIFT, '*', '3', '@', '6', + MOD_MASK_SHIFT, '*', '4', 'k', 'D', + MOD_MASK_SHIFT, '*', '5', 'k', 'L', + MOD_MASK_SHIFT, '*', '7', '@', '7', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_END, '@', '7', + MOD_MASK_SHIFT, '*', '9', '@', '9', + MOD_MASK_SHIFT, '*', '0', '@', '0', + MOD_MASK_SHIFT, '#', '1', '%', '1', + MOD_MASK_SHIFT, '#', '2', 'k', 'h', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_HOME, 'k', 'h', + MOD_MASK_SHIFT, '#', '3', 'k', 'I', + MOD_MASK_SHIFT, '#', '4', 'k', 'l', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_LEFT, 'k', 'l', + MOD_MASK_SHIFT, '%', 'a', '%', '3', + MOD_MASK_SHIFT, '%', 'b', '%', '4', + MOD_MASK_SHIFT, '%', 'c', '%', '5', + MOD_MASK_SHIFT, '%', 'd', '%', '7', + MOD_MASK_SHIFT, '%', 'e', '%', '8', + MOD_MASK_SHIFT, '%', 'f', '%', '9', + MOD_MASK_SHIFT, '%', 'g', '%', '0', + MOD_MASK_SHIFT, '%', 'h', '&', '3', + MOD_MASK_SHIFT, '%', 'i', 'k', 'r', + MOD_MASK_CTRL, KS_EXTRA, (int)KE_C_RIGHT, 'k', 'r', + MOD_MASK_SHIFT, '%', 'j', '&', '5', + MOD_MASK_SHIFT, '!', '1', '&', '6', + MOD_MASK_SHIFT, '!', '2', '&', '7', + MOD_MASK_SHIFT, '!', '3', '&', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_UP, 'k', 'u', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_DOWN, 'k', 'd', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF1, KS_EXTRA, (int)KE_XF1, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF2, KS_EXTRA, (int)KE_XF2, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF3, KS_EXTRA, (int)KE_XF3, + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_XF4, KS_EXTRA, (int)KE_XF4, + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F1, 'k', '1', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F2, 'k', '2', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F3, 'k', '3', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F4, 'k', '4', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F5, 'k', '5', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F6, 'k', '6', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F7, 'k', '7', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F8, 'k', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F9, 'k', '9', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F10, 'k', ';', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F11, 'F', '1', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F12, 'F', '2', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F13, 'F', '3', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F14, 'F', '4', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F15, 'F', '5', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F16, 'F', '6', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F17, 'F', '7', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F18, 'F', '8', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F19, 'F', '9', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F20, 'F', 'A', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F21, 'F', 'B', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F22, 'F', 'C', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F23, 'F', 'D', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F24, 'F', 'E', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F25, 'F', 'F', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F26, 'F', 'G', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F27, 'F', 'H', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F28, 'F', 'I', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F29, 'F', 'J', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F30, 'F', 'K', + + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F31, 'F', 'L', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F32, 'F', 'M', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F33, 'F', 'N', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F34, 'F', 'O', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F35, 'F', 'P', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F36, 'F', 'Q', + MOD_MASK_SHIFT, KS_EXTRA, (int)KE_S_F37, 'F', 'R', + + MOD_MASK_SHIFT, 'k', 'B', KS_EXTRA, (int)KE_TAB, + + NUL +}; + +int simplify_key(const int key, int *modifiers) +{ + if (*modifiers & (MOD_MASK_SHIFT | MOD_MASK_CTRL | MOD_MASK_ALT)) { + // TAB is a special case. + if (key == TAB && (*modifiers & MOD_MASK_SHIFT)) { + *modifiers &= ~MOD_MASK_SHIFT; + return K_S_TAB; + } + const int key0 = KEY2TERMCAP0(key); + const int key1 = KEY2TERMCAP1(key); + for (int i = 0; modifier_keys_table[i] != NUL; i += MOD_KEYS_ENTRY_SIZE) { + if (key0 == modifier_keys_table[i + 3] + && key1 == modifier_keys_table[i + 4] + && (*modifiers & modifier_keys_table[i])) { + *modifiers &= ~modifier_keys_table[i]; + return TERMCAP2KEY(modifier_keys_table[i + 1], + modifier_keys_table[i + 2]); + } + } + } + return key; +} + +int handle_x_keys(const int key) + FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + switch (key) { + case K_XUP: return K_UP; + case K_XDOWN: return K_DOWN; + case K_XLEFT: return K_LEFT; + case K_XRIGHT: return K_RIGHT; + case K_XHOME: return K_HOME; + case K_ZHOME: return K_HOME; + case K_XEND: return K_END; + case K_ZEND: return K_END; + case K_XF1: return K_F1; + case K_XF2: return K_F2; + case K_XF3: return K_F3; + case K_XF4: return K_F4; + case K_S_XF1: return K_S_F1; + case K_S_XF2: return K_S_F2; + case K_S_XF3: return K_S_F3; + case K_S_XF4: return K_S_F4; + } + return key; +} + +static const struct key_name_entry { + int key; ///< Special key code or ASCII value. + const char *name; ///< Name of the key +} key_names_table[] = { + {' ', "Space"}, + {TAB, "Tab"}, + {K_TAB, "Tab"}, + {NL, "NL"}, + {NL, "NewLine"}, // Alternative name + {NL, "LineFeed"}, // Alternative name + {NL, "LF"}, // Alternative name + {CAR, "CR"}, + {CAR, "Return"}, // Alternative name + {CAR, "Enter"}, // Alternative name + {K_BS, "BS"}, + {K_BS, "BackSpace"}, // Alternative name + {ESC, "Esc"}, + {CSI, "CSI"}, + {K_CSI, "xCSI"}, + {'|', "Bar"}, + {'\\', "Bslash"}, + {K_DEL, "Del"}, + {K_DEL, "Delete"}, // Alternative name + {K_KDEL, "kDel"}, + {K_UP, "Up"}, + {K_DOWN, "Down"}, + {K_LEFT, "Left"}, + {K_RIGHT, "Right"}, + {K_XUP, "xUp"}, + {K_XDOWN, "xDown"}, + {K_XLEFT, "xLeft"}, + {K_XRIGHT, "xRight"}, + + {K_F1, "F1"}, + {K_F2, "F2"}, + {K_F3, "F3"}, + {K_F4, "F4"}, + {K_F5, "F5"}, + {K_F6, "F6"}, + {K_F7, "F7"}, + {K_F8, "F8"}, + {K_F9, "F9"}, + {K_F10, "F10"}, + + {K_F11, "F11"}, + {K_F12, "F12"}, + {K_F13, "F13"}, + {K_F14, "F14"}, + {K_F15, "F15"}, + {K_F16, "F16"}, + {K_F17, "F17"}, + {K_F18, "F18"}, + {K_F19, "F19"}, + {K_F20, "F20"}, + + {K_F21, "F21"}, + {K_F22, "F22"}, + {K_F23, "F23"}, + {K_F24, "F24"}, + {K_F25, "F25"}, + {K_F26, "F26"}, + {K_F27, "F27"}, + {K_F28, "F28"}, + {K_F29, "F29"}, + {K_F30, "F30"}, + + {K_F31, "F31"}, + {K_F32, "F32"}, + {K_F33, "F33"}, + {K_F34, "F34"}, + {K_F35, "F35"}, + {K_F36, "F36"}, + {K_F37, "F37"}, + + {K_XF1, "xF1"}, + {K_XF2, "xF2"}, + {K_XF3, "xF3"}, + {K_XF4, "xF4"}, + + {K_HELP, "Help"}, + {K_UNDO, "Undo"}, + {K_INS, "Insert"}, + {K_INS, "Ins"}, // Alternative name + {K_KINS, "kInsert"}, + {K_HOME, "Home"}, + {K_KHOME, "kHome"}, + {K_XHOME, "xHome"}, + {K_ZHOME, "zHome"}, + {K_END, "End"}, + {K_KEND, "kEnd"}, + {K_XEND, "xEnd"}, + {K_ZEND, "zEnd"}, + {K_PAGEUP, "PageUp"}, + {K_PAGEDOWN, "PageDown"}, + {K_KPAGEUP, "kPageUp"}, + {K_KPAGEDOWN, "kPageDown"}, + + {K_KPLUS, "kPlus"}, + {K_KMINUS, "kMinus"}, + {K_KDIVIDE, "kDivide"}, + {K_KMULTIPLY, "kMultiply"}, + {K_KENTER, "kEnter"}, + {K_KPOINT, "kPoint"}, + + {K_K0, "k0"}, + {K_K1, "k1"}, + {K_K2, "k2"}, + {K_K3, "k3"}, + {K_K4, "k4"}, + {K_K5, "k5"}, + {K_K6, "k6"}, + {K_K7, "k7"}, + {K_K8, "k8"}, + {K_K9, "k9"}, + + {'<', "lt"}, + + {K_MOUSE, "Mouse"}, + {K_LEFTMOUSE, "LeftMouse"}, + {K_LEFTMOUSE_NM, "LeftMouseNM"}, + {K_LEFTDRAG, "LeftDrag"}, + {K_LEFTRELEASE, "LeftRelease"}, + {K_LEFTRELEASE_NM, "LeftReleaseNM"}, + {K_MIDDLEMOUSE, "MiddleMouse"}, + {K_MIDDLEDRAG, "MiddleDrag"}, + {K_MIDDLERELEASE, "MiddleRelease"}, + {K_RIGHTMOUSE, "RightMouse"}, + {K_RIGHTDRAG, "RightDrag"}, + {K_RIGHTRELEASE, "RightRelease"}, + {K_MOUSEDOWN, "ScrollWheelUp"}, + {K_MOUSEUP, "ScrollWheelDown"}, + {K_MOUSELEFT, "ScrollWheelRight"}, + {K_MOUSERIGHT, "ScrollWheelLeft"}, + {K_MOUSEDOWN, "MouseDown"}, // OBSOLETE: Use ScrollWheelXXX instead + {K_MOUSEUP, "MouseUp"}, // Same + {K_X1MOUSE, "X1Mouse"}, + {K_X1DRAG, "X1Drag"}, + {K_X1RELEASE, "X1Release"}, + {K_X2MOUSE, "X2Mouse"}, + {K_X2DRAG, "X2Drag"}, + {K_X2RELEASE, "X2Release"}, + {K_DROP, "Drop"}, + {K_ZERO, "Nul"}, + {K_SNR, "SNR"}, + {K_PLUG, "Plug"}, + {K_PASTE, "Paste"}, + {K_FOCUSGAINED, "FocusGained"}, + {K_FOCUSLOST, "FocusLost"}, + {0, NULL} +}; + +int get_special_key_code(const char_u *name) +{ + for (int i = 0; key_names_table[i].name != NULL; i++) { + const char *const table_name = key_names_table[i].name; + int j; + for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) { + if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { + break; + } + } + if (!vim_isIDc(name[j]) && table_name[j] == NUL) { + return key_names_table[i].key; + } + } + + return 0; +} + + +static const struct modmasktable { + short mod_mask; ///< Bit-mask for particular key modifier. + short mod_flag; ///< Bit(s) for particular key modifier. + char_u name; ///< Single letter name of modifier. +} mod_mask_table[] = { + {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M'}, + {MOD_MASK_META, MOD_MASK_META, (char_u)'T'}, + {MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C'}, + {MOD_MASK_SHIFT, MOD_MASK_SHIFT, (char_u)'S'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_2CLICK, (char_u)'2'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_3CLICK, (char_u)'3'}, + {MOD_MASK_MULTI_CLICK, MOD_MASK_4CLICK, (char_u)'4'}, + {MOD_MASK_CMD, MOD_MASK_CMD, (char_u)'D'}, + // 'A' must be the last one + {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'A'}, + {0, 0, NUL} +}; + +int name_to_mod_mask(int c) +{ + c = TOUPPER_ASC(c); + for (size_t i = 0; mod_mask_table[i].mod_mask != 0; i++) { + if (c == mod_mask_table[i].name) { + return mod_mask_table[i].mod_flag; + } + } + return 0; +} + +static int extract_modifiers(int key, int *modp) +{ + int modifiers = *modp; + + if (!(modifiers & MOD_MASK_CMD)) { // Command-key is special + if ((modifiers & MOD_MASK_SHIFT) && ASCII_ISALPHA(key)) { + key = TOUPPER_ASC(key); + modifiers &= ~MOD_MASK_SHIFT; + } + } + if ((modifiers & MOD_MASK_CTRL) + && ((key >= '?' && key <= '_') || ASCII_ISALPHA(key))) { + key = Ctrl_chr(key); + modifiers &= ~MOD_MASK_CTRL; + if (key == 0) { // is + key = K_ZERO; + } + } + + *modp = modifiers; + return key; +} + +int find_special_key(const char_u **srcp, const size_t src_len, int *const modp, + const bool keycode, const bool keep_x_key, + const bool in_string) +{ + const char_u *last_dash; + const char_u *end_of_name; + const char_u *src; + const char_u *bp; + const char_u *const end = *srcp + src_len - 1; + int modifiers; + int bit; + int key; + uvarnumber_T n; + int l; + + if (src_len == 0) { + return 0; + } + + src = *srcp; + if (src[0] != '<') { + return 0; + } + + // Find end of modifier list + last_dash = src; + for (bp = src + 1; bp <= end && (*bp == '-' || vim_isIDc(*bp)); bp++) { + if (*bp == '-') { + last_dash = bp; + if (bp + 1 <= end) { + l = utfc_ptr2len_len(bp + 1, (int)(end - bp) + 1); + // Anything accepted, like . + // or are not special in strings as " is + // the string delimiter. With a backslash it works: + if (end - bp > l && !(in_string && bp[1] == '"') && bp[2] == '>') { + bp += l; + } else if (end - bp > 2 && in_string && bp[1] == '\\' + && bp[2] == '"' && bp[3] == '>') { + bp += 2; + } + } + } + if (end - bp > 3 && bp[0] == 't' && bp[1] == '_') { + bp += 3; // skip t_xx, xx may be '-' or '>' + } else if (end - bp > 4 && STRNICMP(bp, "char-", 5) == 0) { + vim_str2nr(bp + 5, NULL, &l, STR2NR_ALL, NULL, NULL, 0); + bp += l + 5; + break; + } + } + + if (bp <= end && *bp == '>') { // found matching '>' + end_of_name = bp + 1; + + /* Which modifiers are given? */ + modifiers = 0x0; + for (bp = src + 1; bp < last_dash; bp++) { + if (*bp != '-') { + bit = name_to_mod_mask(*bp); + if (bit == 0x0) { + break; // Illegal modifier name + } + modifiers |= bit; + } + } + + // Legal modifier name. + if (bp >= last_dash) { + if (STRNICMP(last_dash + 1, "char-", 5) == 0 + && ascii_isdigit(last_dash[6])) { + // or or + vim_str2nr(last_dash + 6, NULL, NULL, STR2NR_ALL, NULL, &n, 0); + key = (int)n; + } else { + int off = 1; + + // Modifier with single letter, or special key name. + if (in_string && last_dash[1] == '\\' && last_dash[2] == '"') { + off = 2; + } + l = mb_ptr2len(last_dash + 1); + if (modifiers != 0 && last_dash[l + 1] == '>') { + key = PTR2CHAR(last_dash + off); + } else { + key = get_special_key_code(last_dash + off); + if (!keep_x_key) { + key = handle_x_keys(key); + } + } + } + + // get_special_key_code() may return NUL for invalid + // special key name. + if (key != NUL) { + // Only use a modifier when there is no special key code that + // includes the modifier. + key = simplify_key(key, &modifiers); + + if (!keycode) { + // don't want keycode, use single byte code + if (key == K_BS) { + key = BS; + } else if (key == K_DEL || key == K_KDEL) { + key = DEL; + } + } + + // Normal Key with modifier: + // Try to make a single byte code (except for Alt/Meta modifiers). + if (!IS_SPECIAL(key)) { + key = extract_modifiers(key, &modifiers); + } + + *modp = modifiers; + *srcp = end_of_name; + return key; + } + } + } + return 0; +} + +char_u *add_char2buf(int c, char_u *s) +{ + char_u temp[MB_MAXBYTES + 1]; + const int len = utf_char2bytes(c, temp); + for (int i = 0; i < len; ++i) { + c = temp[i]; + // Need to escape K_SPECIAL and CSI like in the typeahead buffer. + if (c == K_SPECIAL) { + *s++ = K_SPECIAL; + *s++ = KS_SPECIAL; + *s++ = KE_FILLER; + } else { + *s++ = c; + } + } + return s; +} + +unsigned int trans_special(const char_u **srcp, const size_t src_len, + char_u *const dst, const bool keycode, + const bool in_string) +{ + int modifiers = 0; + int key; + unsigned int dlen = 0; + + key = find_special_key(srcp, src_len, &modifiers, keycode, false, in_string); + if (key == 0) { + return 0; + } + + // Put the appropriate modifier in a string. + if (modifiers != 0) { + dst[dlen++] = K_SPECIAL; + dst[dlen++] = KS_MODIFIER; + dst[dlen++] = (char_u)modifiers; + } + + if (IS_SPECIAL(key)) { + dst[dlen++] = K_SPECIAL; + dst[dlen++] = (char_u)KEY2TERMCAP0(key); + dst[dlen++] = KEY2TERMCAP1(key); + } else if (has_mbyte && !keycode) { + dlen += (unsigned int)(*mb_char2bytes)(key, dst + dlen); + } else if (keycode) { + char_u *after = add_char2buf(key, dst + dlen); + assert(after >= dst && (uintmax_t)(after - dst) <= UINT_MAX); + dlen = (unsigned int)(after - dst); + } else { + dst[dlen++] = (char_u)key; + } + + return dlen; +} diff --git a/test/symbolic/klee/nvim/mbyte.c b/test/symbolic/klee/nvim/mbyte.c index 394d17b700..bfc191b1b7 100644 --- a/test/symbolic/klee/nvim/mbyte.c +++ b/test/symbolic/klee/nvim/mbyte.c @@ -1,9 +1,89 @@ #include +#include +#include +#include #include "nvim/types.h" #include "nvim/mbyte.h" #include "nvim/ascii.h" +const uint8_t utf8len_tab_zero[] = { + //1 2 3 4 5 6 7 8 9 A B C D E F 0 1 2 3 4 5 6 7 8 9 A B C D E F + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 2 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 4 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 6 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 8 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // A + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // C + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, // E +}; + +const uint8_t utf8len_tab[] = { + // ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9 ?A ?B ?C ?D ?E ?F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 8? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 9? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D? + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E? + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, // F? +}; + +int utf_ptr2char(const char_u *const p) +{ + if (p[0] < 0x80) { // Be quick for ASCII. + return p[0]; + } + + const uint8_t len = utf8len_tab_zero[p[0]]; + if (len > 1 && (p[1] & 0xc0) == 0x80) { + if (len == 2) { + return ((p[0] & 0x1f) << 6) + (p[1] & 0x3f); + } + if ((p[2] & 0xc0) == 0x80) { + if (len == 3) { + return (((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + + (p[2] & 0x3f)); + } + if ((p[3] & 0xc0) == 0x80) { + if (len == 4) { + return (((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12) + + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f)); + } + if ((p[4] & 0xc0) == 0x80) { + if (len == 5) { + return (((p[0] & 0x03) << 24) + ((p[1] & 0x3f) << 18) + + ((p[2] & 0x3f) << 12) + ((p[3] & 0x3f) << 6) + + (p[4] & 0x3f)); + } + if ((p[5] & 0xc0) == 0x80 && len == 6) { + return (((p[0] & 0x01) << 30) + ((p[1] & 0x3f) << 24) + + ((p[2] & 0x3f) << 18) + ((p[3] & 0x3f) << 12) + + ((p[4] & 0x3f) << 6) + (p[5] & 0x3f)); + } + } + } + } + } + // Illegal value: just return the first byte. + return p[0]; +} + +bool utf_composinglike(const char_u *p1, const char_u *p2) +{ + return false; +} + char_u *string_convert(const vimconv_T *conv, char_u *data, size_t *size) { return NULL; @@ -11,8 +91,117 @@ char_u *string_convert(const vimconv_T *conv, char_u *data, size_t *size) int utfc_ptr2len_len(const char_u *p, int size) { - if (size < 1 || *p == NUL) { + assert(false); + return 0; +} + +int utf_char2len(const int c) +{ + if (c < 0x80) { + return 1; + } else if (c < 0x800) { + return 2; + } else if (c < 0x10000) { + return 3; + } else if (c < 0x200000) { + return 4; + } else if (c < 0x4000000) { + return 5; + } else { + return 6; + } +} + +int utf_char2bytes(const int c, char_u *const buf) +{ + if (c < 0x80) { // 7 bits + buf[0] = c; + return 1; + } else if (c < 0x800) { // 11 bits + buf[0] = 0xc0 + ((unsigned)c >> 6); + buf[1] = 0x80 + (c & 0x3f); + return 2; + } else if (c < 0x10000) { // 16 bits + buf[0] = 0xe0 + ((unsigned)c >> 12); + buf[1] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[2] = 0x80 + (c & 0x3f); + return 3; + } else if (c < 0x200000) { // 21 bits + buf[0] = 0xf0 + ((unsigned)c >> 18); + buf[1] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[3] = 0x80 + (c & 0x3f); + return 4; + } else if (c < 0x4000000) { // 26 bits + buf[0] = 0xf8 + ((unsigned)c >> 24); + buf[1] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[4] = 0x80 + (c & 0x3f); + return 5; + } else { // 31 bits + buf[0] = 0xfc + ((unsigned)c >> 30); + buf[1] = 0x80 + (((unsigned)c >> 24) & 0x3f); + buf[2] = 0x80 + (((unsigned)c >> 18) & 0x3f); + buf[3] = 0x80 + (((unsigned)c >> 12) & 0x3f); + buf[4] = 0x80 + (((unsigned)c >> 6) & 0x3f); + buf[5] = 0x80 + (c & 0x3f); + return 6; + } +} + +int utf_ptr2len(const char_u *const p) +{ + if (*p == NUL) { return 0; } - return 1; + const int len = utf8len_tab[*p]; + for (int i = 1; i < len; i++) { + if ((p[i] & 0xc0) != 0x80) { + return 1; + } + } + return len; +} + +int utfc_ptr2len(const char_u *const p) +{ + uint8_t b0 = (uint8_t)(*p); + + if (b0 == NUL) { + return 0; + } + if (b0 < 0x80 && p[1] < 0x80) { // be quick for ASCII + return 1; + } + + // Skip over first UTF-8 char, stopping at a NUL byte. + int len = utf_ptr2len(p); + + // Check for illegal byte. + if (len == 1 && b0 >= 0x80) { + return 1; + } + + // Check for composing characters. We can handle only the first six, but + // skip all of them (otherwise the cursor would get stuck). + int prevlen = 0; + for (;;) { + if (p[len] < 0x80 || !UTF_COMPOSINGLIKE(p + prevlen, p + len)) { + return len; + } + + // Skip over composing char. + prevlen = len; + len += utf_ptr2len(p + len); + } +} + +void mb_copy_char(const char_u **fp, char_u **tp) +{ + const size_t l = utfc_ptr2len(*fp); + + memmove(*tp, *fp, (size_t)l); + *tp += l; + *fp += l; } diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index 5ad592b99f..e1cea2d990 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -18,6 +18,7 @@ #include "nvim/garray.c" #include "nvim/gettext.c" #include "nvim/viml/parser/expressions.c" +#include "nvim/keymap.c" #define INPUT_SIZE 50 diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 41261cf7c9..ed77a7cba4 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -142,6 +142,13 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) typ = typ .. ('(val=%u)'):format(tonumber(eastnode.data.num.value)) elseif typ == 'Float' then typ = typ .. ('(val=%e)'):format(tonumber(eastnode.data.flt.value)) + elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then + if eastnode.data.str.value == nil then + typ = typ .. '(val=NULL)' + else + local s = ffi.string(eastnode.data.str.value, eastnode.data.str.size) + typ = format_string('%s(val=%q)', typ, s) + end end ret_str = typ .. ':' .. ret_str local can_simplify = true @@ -4549,4 +4556,822 @@ describe('Expressions parser', function() hl('InvalidSubscript', ']'), }) end) + itp('works with strings', function() + check_parsing('\'abc\'', 0, { + -- 01234 + ast = { + 'SingleQuotedString(val="abc"):0:0:\'abc\'', + }, + }, { + hl('SingleQuotedString', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuotedString', '\''), + }) + check_parsing('"abc"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="abc"):0:0:"abc"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('\'\'', 0, { + -- 01 + ast = { + 'SingleQuotedString(val=NULL):0:0:\'\'', + }, + }, { + hl('SingleQuotedString', '\''), + hl('SingleQuotedString', '\''), + }) + check_parsing('""', 0, { + -- 01 + ast = { + 'DoubleQuotedString(val=NULL):0:0:""', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"', 0, { + -- 0 + ast = { + 'DoubleQuotedString(val=NULL):0:0:"', + }, + err = { + arg = '"', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + }) + check_parsing('\'', 0, { + -- 0 + ast = { + 'SingleQuotedString(val=NULL):0:0:\'', + }, + err = { + arg = '\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuotedString', '\''), + }) + check_parsing('"a', 0, { + -- 01 + ast = { + 'DoubleQuotedString(val="a"):0:0:"a', + }, + err = { + arg = '"a', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedBody', 'a'), + }) + check_parsing('\'a', 0, { + -- 01 + ast = { + 'SingleQuotedString(val="a"):0:0:\'a', + }, + err = { + arg = '\'a', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuotedString', '\''), + hl('InvalidSingleQuotedBody', 'a'), + }) + check_parsing('\'abc\'\'def\'', 0, { + -- 0123456789 + ast = { + 'SingleQuotedString(val="abc\'def"):0:0:\'abc\'\'def\'', + }, + }, { + hl('SingleQuotedString', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'def'), + hl('SingleQuotedString', '\''), + }) + check_parsing('\'abc\'\'', 0, { + -- 012345 + ast = { + 'SingleQuotedString(val="abc\'"):0:0:\'abc\'\'', + }, + err = { + arg = '\'abc\'\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuotedString', '\''), + hl('InvalidSingleQuotedBody', 'abc'), + hl('InvalidSingleQuotedQuote', '\'\''), + }) + check_parsing('\'\'\'\'\'\'\'\'', 0, { + -- 01234567 + ast = { + 'SingleQuotedString(val="\'\'\'"):0:0:\'\'\'\'\'\'\'\'', + }, + }, { + hl('SingleQuotedString', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedString', '\''), + }) + check_parsing('\'\'\'a\'\'\'\'bc\'', 0, { + -- 01234567890 + -- 0 1 + ast = { + 'SingleQuotedString(val="\'a\'\'bc"):0:0:\'\'\'a\'\'\'\'bc\'', + }, + }, { + hl('SingleQuotedString', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'a'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'bc'), + hl('SingleQuotedString', '\''), + }) + check_parsing('"\\"\\"\\"\\""', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\"\\"\\"\\""):0:0:"\\"\\"\\"\\""', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', 0, { + -- 0123456789012345678901234 + -- 0 1 2 + ast = { + 'DoubleQuotedString(val="abc\\"def\\"ghi\\"jkl\\"mno"):0:0:"abc\\"def\\"ghi\\"jkl\\"mno"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'def'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'ghi'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'jkl'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'mno'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"\\b\\e\\f\\r\\t\\\\"', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + [[DoubleQuotedString(val="\8\27\12\13\9\\"):0:0:"\b\e\f\r\t\\"]], + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\b'), + hl('DoubleQuotedEscape', '\\e'), + hl('DoubleQuotedEscape', '\\f'), + hl('DoubleQuotedEscape', '\\r'), + hl('DoubleQuotedEscape', '\\t'), + hl('DoubleQuotedEscape', '\\\\'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"\\n\n"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\\n\\\n"):0:0:"\\n\n"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\n'), + hl('DoubleQuotedBody', '\n'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"\\x00"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\x00"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"\\xFF"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\255"):0:0:"\\xFF"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\xFF'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"\\xF"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15"):0:0:"\\xF"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"\\u00AB"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="«"):0:0:"\\u00AB"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\u00AB'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"\\U000000AB"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="«"):0:0:"\\U000000AB"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U000000AB'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('"\\x"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="x"):0:0:"\\x"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\x', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="x"):0:0:"\\x', + }, + err = { + arg = '"\\x', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\x'), + }) + + check_parsing('"\\xF', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="\\15"):0:0:"\\xF', + }, + err = { + arg = '"\\xF', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedEscape', '\\xF'), + }) + + check_parsing('"\\u"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="u"):0:0:"\\u"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\u', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="u"):0:0:"\\u', + }, + err = { + arg = '"\\u', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\u'), + }) + + check_parsing('"\\U', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="U"):0:0:"\\U', + }, + err = { + arg = '"\\U', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + }) + + check_parsing('"\\U"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="U"):0:0:"\\U"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\xFX"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15X"):0:0:"\\xFX"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\XFX"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15X"):0:0:"\\XFX"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\XF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\xX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="xX"):0:0:"\\xX"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\XX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="XX"):0:0:"\\XX"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\X'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\uX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="uX"):0:0:"\\uX"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\UX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="UX"):0:0:"\\UX"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\x0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\x0X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\x0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\X0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\X0X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\X0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\u0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u0X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\u0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\x00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\x00X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\X00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\X00X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\u00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u00X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\u00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\u000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\u000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\u0000X"', 0, { + -- 012345678 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u0000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U0000X"', 0, { + -- 012345678 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U00000X"', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U00000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U000000X"', 0, { + -- 01234567890 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U000000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U0000000X"', 0, { + -- 012345678901 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U0000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U00000000X"', 0, { + -- 0123456789012 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\x000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\x000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\X000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\X000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\u00000X"', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\u00000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\U000000000X"', 0, { + -- 01234567890123 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\U000000000X"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\0"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\0"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\00"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\00"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\000"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\000"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\0000"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0000"):0:0:"\\0000"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '0'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\8"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="8"):0:0:"\\8"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\8'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\08"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\08"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\008"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\008"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\0008"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\0008"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\777"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\255"):0:0:"\\777"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\777'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\050"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\40"):0:0:"\\050"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\050'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\21"):0:0:"\\"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\<', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="<"):0:0:"\\<', + }, + err = { + arg = '"\\<', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\<'), + }) + + check_parsing('"\\<"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="<"):0:0:"\\<"', + }, + }, { + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedUnknownEscape', '\\<'), + hl('DoubleQuotedString', '"'), + }) + + check_parsing('"\\ Date: Mon, 9 Oct 2017 02:55:56 +0300 Subject: [PATCH 032/109] viml/parser/expressions: Finish parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note: formatc.lua was unable to swallow some newer additions to ExprASTNodeType (specifically `kExprNodeOr = '|'` and probably something else), so all `= …` were dropped: in any case they only were there in order to not bother updating viml_pexpr_debug_print_ast_node and since it is now known all nodes which will be present it is not much of an issue. --- src/nvim/viml/parser/expressions.c | 369 +++++++++++++------- src/nvim/viml/parser/expressions.h | 115 ++++-- src/nvim/viml/parser/parser.h | 2 +- test/symbolic/klee/viml_expressions_lexer.c | 1 + test/unit/viml/expressions/lexer_spec.lua | 6 +- test/unit/viml/expressions/parser_spec.lua | 19 + test/unit/viml/helpers.lua | 1 + 7 files changed, 355 insertions(+), 158 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 3f30fe2a0e..75fcb17bf6 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -361,11 +361,12 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) // Scope: `s:`, etc. } else if (ret.len == 1 && pline.size > 1 - && strchr("sgvbwtla", schar) != NULL + && memchr(EXPR_VAR_SCOPE_LIST, schar, + sizeof(EXPR_VAR_SCOPE_LIST)) != NULL && pline.data[ret.len] == ':' && !(flags & kELFlagForbidScope)) { ret.len++; - ret.data.var.scope = schar; + ret.data.var.scope = (ExprVarScope)schar; CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); ret.data.var.autoload = ( memchr(pline.data + 2, AUTOLOAD_CHAR, ret.len - 2) @@ -408,14 +409,13 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) ret.type = kExprLexOption; if (pline.size > 2 && pline.data[2] == ':' - && strchr("gl", pline.data[1]) != NULL) { + && memchr(EXPR_OPT_SCOPE_LIST, pline.data[1], + sizeof(EXPR_OPT_SCOPE_LIST)) != NULL) { ret.len += 2; - ret.data.opt.scope = (pline.data[1] == 'g' - ? kExprLexOptGlobal - : kExprLexOptLocal); + ret.data.opt.scope = (ExprOptScope)pline.data[1]; ret.data.opt.name = pline.data + 3; } else { - ret.data.opt.scope = kExprLexOptUnspecified; + ret.data.opt.scope = kExprOptScopeUnspecified; ret.data.opt.name = pline.data + 1; } const char *p = ret.data.opt.name; @@ -637,9 +637,9 @@ static const char *const eltkn_mul_type_tab[] = { }; static const char *const eltkn_opt_scope_tab[] = { - [kExprLexOptUnspecified] = "Unspecified", - [kExprLexOptGlobal] = "Global", - [kExprLexOptLocal] = "Local", + [kExprOptScopeUnspecified] = "Unspecified", + [kExprOptScopeGlobal] = "Global", + [kExprOptScopeLocal] = "Local", }; /// Represent `int` character as a string @@ -990,67 +990,25 @@ static inline ExprASTNode *viml_pexpr_new_node(const ExprASTNodeType type) return ret; } -static const ExprOpLvl node_type_to_op_lvl[] = { - [kExprNodeMissing] = kEOpLvlInvalid, - [kExprNodeOpMissing] = kEOpLvlMultiplication, +static struct { + ExprOpLvl lvl; + ExprOpAssociativity ass; +} node_type_to_node_props[] = { + [kExprNodeMissing] = { kEOpLvlInvalid, kEOpAssNo, }, + [kExprNodeOpMissing] = { kEOpLvlMultiplication, kEOpAssNo }, - [kExprNodeNested] = kEOpLvlParens, + [kExprNodeNested] = { kEOpLvlParens, kEOpAssNo }, // Note: below nodes are kEOpLvlSubscript for “binary operator” itself, but // kEOpLvlParens when it comes to inside the parenthesis. - [kExprNodeCall] = kEOpLvlParens, - [kExprNodeSubscript] = kEOpLvlParens, + [kExprNodeCall] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeSubscript] = { kEOpLvlParens, kEOpAssNo }, - [kExprNodeUnknownFigure] = kEOpLvlParens, - [kExprNodeLambda] = kEOpLvlParens, - [kExprNodeDictLiteral] = kEOpLvlParens, - [kExprNodeListLiteral] = kEOpLvlParens, + [kExprNodeUnknownFigure] = { kEOpLvlParens, kEOpAssLeft }, + [kExprNodeLambda] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeDictLiteral] = { kEOpLvlParens, kEOpAssNo }, + [kExprNodeListLiteral] = { kEOpLvlParens, kEOpAssNo }, - [kExprNodeArrow] = kEOpLvlArrow, - - [kExprNodeComma] = kEOpLvlComma, - - [kExprNodeColon] = kEOpLvlColon, - - [kExprNodeTernary] = kEOpLvlTernary, - - [kExprNodeTernaryValue] = kEOpLvlTernaryValue, - - [kExprNodeComparison] = kEOpLvlComparison, - - [kExprNodeBinaryPlus] = kEOpLvlAddition, - [kExprNodeConcat] = kEOpLvlAddition, - - [kExprNodeUnaryPlus] = kEOpLvlUnary, - - [kExprNodeConcatOrSubscript] = kEOpLvlSubscript, - - [kExprNodeCurlyBracesIdentifier] = kEOpLvlComplexIdentifier, - - [kExprNodeComplexIdentifier] = kEOpLvlValue, - [kExprNodePlainIdentifier] = kEOpLvlValue, - [kExprNodePlainKey] = kEOpLvlValue, - [kExprNodeRegister] = kEOpLvlValue, - [kExprNodeInteger] = kEOpLvlValue, - [kExprNodeFloat] = kEOpLvlValue, -}; - -static const ExprOpAssociativity node_type_to_op_ass[] = { - [kExprNodeMissing] = kEOpAssNo, - [kExprNodeOpMissing] = kEOpAssNo, - - [kExprNodeNested] = kEOpAssNo, - [kExprNodeCall] = kEOpAssNo, - [kExprNodeSubscript] = kEOpAssNo, - - [kExprNodeUnknownFigure] = kEOpAssLeft, - [kExprNodeLambda] = kEOpAssNo, - [kExprNodeDictLiteral] = kEOpAssNo, - [kExprNodeListLiteral] = kEOpAssNo, - - // Does not really matter. - [kExprNodeArrow] = kEOpAssNo, - - [kExprNodeColon] = kEOpAssNo, + [kExprNodeArrow] = { kEOpLvlArrow, kEOpAssNo }, // Right associativity for comma because this means easier access to arguments // list, etc: for "[a, b, c, d]" you can access "a" in one step if it is @@ -1059,29 +1017,48 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { // traverse all three comma() structures. And with comma operator (including // actual comma operator from C which is not present in VimL) nobody cares // about associativity, only about order of execution. - [kExprNodeComma] = kEOpAssRight, + [kExprNodeComma] = { kEOpLvlComma, kEOpAssRight }, - [kExprNodeTernary] = kEOpAssRight, + // Colons are not eligible for chaining, so nobody cares about associativity. + [kExprNodeColon] = { kEOpLvlColon, kEOpAssNo }, - [kExprNodeTernaryValue] = kEOpAssRight, + [kExprNodeTernary] = { kEOpLvlTernary, kEOpAssRight }, - [kExprNodeComparison] = kEOpAssRight, + [kExprNodeOr] = { kEOpLvlOr, kEOpAssLeft }, - [kExprNodeBinaryPlus] = kEOpAssLeft, - [kExprNodeConcat] = kEOpAssLeft, + [kExprNodeAnd] = { kEOpLvlAnd, kEOpAssLeft }, - [kExprNodeUnaryPlus] = kEOpAssNo, + [kExprNodeTernaryValue] = { kEOpLvlTernaryValue, kEOpAssRight }, - [kExprNodeConcatOrSubscript] = kEOpAssLeft, + [kExprNodeComparison] = { kEOpLvlComparison, kEOpAssRight }, - [kExprNodeCurlyBracesIdentifier] = kEOpAssLeft, + [kExprNodeBinaryPlus] = { kEOpLvlAddition, kEOpAssLeft }, + [kExprNodeBinaryMinus] = { kEOpLvlAddition, kEOpAssLeft }, + [kExprNodeConcat] = { kEOpLvlAddition, kEOpAssLeft }, - [kExprNodeComplexIdentifier] = kEOpAssLeft, - [kExprNodePlainIdentifier] = kEOpAssNo, - [kExprNodePlainKey] = kEOpAssNo, - [kExprNodeRegister] = kEOpAssNo, - [kExprNodeInteger] = kEOpAssNo, - [kExprNodeFloat] = kEOpAssNo, + [kExprNodeMultiplication] = { kEOpLvlMultiplication, kEOpAssLeft }, + [kExprNodeDivision] = { kEOpLvlMultiplication, kEOpAssLeft }, + [kExprNodeMod] = { kEOpLvlMultiplication, kEOpAssLeft }, + + [kExprNodeUnaryPlus] = { kEOpLvlUnary, kEOpAssNo }, + [kExprNodeUnaryMinus] = { kEOpLvlUnary, kEOpAssNo }, + [kExprNodeNot] = { kEOpLvlUnary, kEOpAssNo }, + + [kExprNodeConcatOrSubscript] = { kEOpLvlSubscript, kEOpAssLeft }, + + [kExprNodeCurlyBracesIdentifier] = { kEOpLvlComplexIdentifier, kEOpAssLeft }, + + [kExprNodeComplexIdentifier] = { kEOpLvlValue, kEOpAssLeft }, + + [kExprNodePlainIdentifier] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodePlainKey] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeRegister] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeInteger] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeFloat] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeDoubleQuotedString] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeSingleQuotedString] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeOption] = { kEOpLvlValue, kEOpAssNo }, + [kExprNodeEnvironment] = { kEOpLvlValue, kEOpAssNo }, }; /// Get AST node priority level @@ -1094,7 +1071,7 @@ static const ExprOpAssociativity node_type_to_op_ass[] = { static inline ExprOpLvl node_lvl(const ExprASTNode node) FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT { - return node_type_to_op_lvl[node.type]; + return node_type_to_node_props[node.type].lvl; } /// Get AST node associativity, to be used for operator nodes primary @@ -1107,7 +1084,7 @@ static inline ExprOpLvl node_lvl(const ExprASTNode node) static inline ExprOpAssociativity node_ass(const ExprASTNode node) FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT { - return node_type_to_op_ass[node.type]; + return node_type_to_node_props[node.type].ass; } /// Handle binary operator @@ -1837,7 +1814,7 @@ viml_pexpr_parse_process_token: is_concat_or_subscript && (cur_token.type == kExprLexPlainIdentifier ? (!cur_token.data.var.autoload - && cur_token.data.var.scope == 0) + && cur_token.data.var.scope == kExprVarScopeMissing) : (cur_token.type == kExprLexNumber)) && prev_token.type != kExprLexSpacing); if (is_concat_or_subscript && !node_is_key) { @@ -1856,7 +1833,7 @@ viml_pexpr_parse_process_token: && tok_type != kExprLexArrow) || (want_node == kENodeArgument && !(cur_token.type == kExprLexPlainIdentifier - && cur_token.data.var.scope == 0 + && cur_token.data.var.scope == kExprVarScopeMissing && !cur_token.data.var.autoload) && tok_type != kExprLexArrow)) { lambda_node->data.fig.type_guesses.allow_lambda = false; @@ -1885,6 +1862,8 @@ viml_pexpr_parse_process_token: || want_node == kENodeArgumentSeparator || want_node == kENodeArgument); switch (tok_type) { + case kExprLexMissing: + case kExprLexSpacing: case kExprLexEOC: { assert(false); } @@ -1894,31 +1873,111 @@ viml_pexpr_parse_process_token: goto viml_pexpr_parse_process_token; } case kExprLexRegister: { - if (want_node == kENodeValue) { - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeRegister); - cur_node->data.reg.name = cur_token.data.reg.name; - *top_node_p = cur_node; - want_node = kENodeOperator; - HL_CUR_TOKEN(Register); - } else { + if (want_node == kENodeOperator) { // Register in operator position: e.g. @a @a OP_MISSING; } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeRegister); + cur_node->data.reg.name = cur_token.data.reg.name; + *top_node_p = cur_node; + want_node = kENodeOperator; + HL_CUR_TOKEN(Register); break; } - case kExprLexPlus: { - if (want_node == kENodeValue) { - // Value level: assume unary plus - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnaryPlus); - *top_node_p = cur_node; - kvi_push(ast_stack, &cur_node->children); - HL_CUR_TOKEN(UnaryPlus); - } else { - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeBinaryPlus); - ADD_OP_NODE(cur_node); - HL_CUR_TOKEN(BinaryPlus); +#define SIMPLE_UB_OP(op) \ + case kExprLex##op: { \ + if (want_node == kENodeValue) { \ + /* Value level: assume unary operator. */ \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnary##op); \ + *top_node_p = cur_node; \ + kvi_push(ast_stack, &cur_node->children); \ + HL_CUR_TOKEN(Unary##op); \ + } else { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeBinary##op); \ + ADD_OP_NODE(cur_node); \ + HL_CUR_TOKEN(Binary##op); \ + } \ + want_node = kENodeValue; \ + break; \ + } + SIMPLE_UB_OP(Plus) + SIMPLE_UB_OP(Minus) +#undef SIMPLE_UB_OP +#define SIMPLE_B_OP(op, msg) \ + case kExprLex##op: { \ + ADD_VALUE_IF_MISSING(_("E15: Unexpected " msg ": %.*s")); \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNode##op); \ + HL_CUR_TOKEN(op); \ + ADD_OP_NODE(cur_node); \ + break; \ + } + SIMPLE_B_OP(Or, "or operator") + SIMPLE_B_OP(And, "and operator") +#undef SIMPLE_B_OP + case kExprLexMultiplication: { + ADD_VALUE_IF_MISSING( + _("E15: Unexpected multiplication-like operator: %.*s")); + switch (cur_token.data.mul.type) { +#define MUL_OP(lex_op_tail, node_op_tail) \ + case kExprLexMul##lex_op_tail: { \ + NEW_NODE_WITH_CUR_POS(cur_node, kExprNode##node_op_tail); \ + HL_CUR_TOKEN(node_op_tail); \ + break; \ + } + MUL_OP(Mul, Multiplication) + MUL_OP(Div, Division) + MUL_OP(Mod, Mod) +#undef MUL_OP } - want_node = kENodeValue; + ADD_OP_NODE(cur_node); + break; + } + case kExprLexOption: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOption); + cur_node->data.opt.ident = cur_token.data.opt.name; + cur_node->data.opt.ident_len = cur_token.data.opt.len; + cur_node->data.opt.scope = cur_token.data.opt.scope; + *top_node_p = cur_node; + want_node = kENodeOperator; + viml_parser_highlight(pstate, cur_token.start, 1, HL(OptionSigil)); + const size_t scope_shift = ( + cur_token.data.opt.scope == kExprOptScopeUnspecified ? 0 : 2); + if (scope_shift) { + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), 1, + HL(OptionScope)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 2), 1, + HL(OptionScopeDelimiter)); + } + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, scope_shift + 1), + cur_token.len - scope_shift + 1, HL(Option)); + break; + } + case kExprLexEnv: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeEnvironment); + cur_node->data.env.ident = pline.data + cur_token.start.col + 1; + cur_node->data.env.ident_len = cur_token.len - 1; + *top_node_p = cur_node; + want_node = kENodeOperator; + viml_parser_highlight(pstate, cur_token.start, 1, HL(EnvironmentSigil)); + viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), + cur_token.len - 1, HL(Environment)); + break; + } + case kExprLexNot: { + if (want_node == kENodeOperator) { + OP_MISSING; + } + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeNot); + *top_node_p = cur_node; + kvi_push(ast_stack, &cur_node->children); + HL_CUR_TOKEN(Not); break; } case kExprLexComparison: { @@ -2359,9 +2418,8 @@ viml_pexpr_parse_figure_brace_closing_error: ? kExprNodePlainKey : kExprNodePlainIdentifier)); cur_node->data.var.scope = cur_token.data.var.scope; - const size_t scope_shift = (cur_token.data.var.scope == 0 - ? 0 - : 2); + const size_t scope_shift = ( + cur_token.data.var.scope == kExprVarScopeMissing ? 0 : 2); cur_node->data.var.ident = (pline.data + cur_token.start.col + scope_shift); cur_node->data.var.ident_len = cur_token.len - scope_shift; @@ -2373,16 +2431,14 @@ viml_pexpr_parse_figure_brace_closing_error: viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), 1, HL(IdentifierScopeDelimiter)); } - if (scope_shift < cur_token.len) { - viml_parser_highlight(pstate, shifted_pos(cur_token.start, - scope_shift), - cur_token.len - scope_shift, - (node_is_key - ? HL(IdentifierKey) - : HL(Identifier))); - } + viml_parser_highlight(pstate, shifted_pos(cur_token.start, + scope_shift), + cur_token.len - scope_shift, + (node_is_key + ? HL(IdentifierKey) + : HL(Identifier))); } else { - if (cur_token.data.var.scope == 0) { + if (cur_token.data.var.scope == kExprVarScopeMissing) { ADD_IDENT( do { NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainIdentifier); @@ -2606,9 +2662,85 @@ viml_pexpr_parse_end: cur_node->start); break; } - case kExprNodeBinaryPlus: + case kExprNodeListLiteral: { + // For whatever reason "[1" yields "E696: Missing comma in list" error + // in Vim while "[1," yields E697. + east_set_error( + pstate, &ast.err, + _("E697: Missing end of List ']': %.*s"), + cur_node->start); + break; + } + case kExprNodeDictLiteral: { + // Same problem like with list literal with E722 (missing comma) vs + // E723, but additionally just "{" yields only E15. + east_set_error( + pstate, &ast.err, + _("E723: Missing end of Dictionary '}': %.*s"), + cur_node->start); + break; + } + case kExprNodeUnknownFigure: { + east_set_error( + pstate, &ast.err, + _("E15: Missing closing figure brace: %.*s"), + cur_node->start); + break; + } + case kExprNodeLambda: { + east_set_error( + pstate, &ast.err, + _("E15: Missing closing figure brace for lambda: %.*s"), + cur_node->start); + break; + } + case kExprNodeCurlyBracesIdentifier: { + // Until trailing "}" it is impossible to distinguish curly braces + // identifier and dictionary, so it must not appear in the stack like + // this. + assert(false); + } + case kExprNodeInteger: + case kExprNodeFloat: + case kExprNodeSingleQuotedString: + case kExprNodeDoubleQuotedString: + case kExprNodeOption: + case kExprNodeEnvironment: + case kExprNodeRegister: + case kExprNodePlainIdentifier: + case kExprNodePlainKey: { + // These are plain values and not containers, for them it should only + // be possible to show up in the topmost stack element, but it was + // unconditionally popped at the start. + assert(false); + } + case kExprNodeComma: + case kExprNodeColon: + case kExprNodeArrow: { + // It is actually only valid inside something else, but everything + // where one of the above is valid requires to be closed and thus is + // to be caught later. + break; + } + case kExprNodeConcatOrSubscript: + case kExprNodeComplexIdentifier: + case kExprNodeSubscript: { + // FIXME: Investigate whether above are OK to be present in the stack. + break; + } + case kExprNodeMod: + case kExprNodeDivision: + case kExprNodeMultiplication: + case kExprNodeNot: + case kExprNodeAnd: + case kExprNodeOr: + case kExprNodeConcat: + case kExprNodeComparison: + case kExprNodeUnaryMinus: case kExprNodeUnaryPlus: - case kExprNodeRegister: { + case kExprNodeBinaryMinus: + case kExprNodeTernary: + case kExprNodeBinaryPlus: { // It is OK to see these in the stack. break; } @@ -2621,7 +2753,6 @@ viml_pexpr_parse_end: } break; } - // TODO(ZyX-I): handle other values } } } diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index a09cdde4c0..0198852bed 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -61,6 +61,36 @@ typedef enum { kExprCmpIdentical, ///< `is` or `isnot` } ExprComparisonType; +/// All possible option scopes +typedef enum { + kExprOptScopeUnspecified = 0, + kExprOptScopeGlobal = 'g', + kExprOptScopeLocal = 'l', +} ExprOptScope; + +#define EXPR_OPT_SCOPE_LIST \ + ((char *)(char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) + +/// All possible variable scopes +typedef enum { + kExprVarScopeMissing = 0, + kExprVarScopeScript = 's', + kExprVarScopeGlobal = 'g', + kExprVarScopeVim = 'v', + kExprVarScopeBuffer = 'b', + kExprVarScopeWindow = 'w', + kExprVarScopeTabpage = 't', + kExprVarScopeLocal = 'l', + kExprVarScopeArguments = 'a', +} ExprVarScope; + +#define EXPR_VAR_SCOPE_LIST \ + ((char[]) { \ + kExprVarScopeScript, kExprVarScopeGlobal, kExprVarScopeVim, \ + kExprVarScopeBuffer, kExprVarScopeWindow, kExprVarScopeTabpage, \ + kExprVarScopeLocal, kExprVarScopeBuffer, kExprVarScopeArguments, \ + }) + /// Lexer token typedef struct { ParserPosition start; @@ -96,15 +126,11 @@ typedef struct { struct { const char *name; ///< Option name start. size_t len; ///< Option name length. - enum { - kExprLexOptUnspecified = 0, - kExprLexOptGlobal = 1, - kExprLexOptLocal = 2, - } scope; ///< Option scope: &l:, &g: or not specified. + ExprOptScope scope; ///< Option scope: &l:, &g: or not specified. } opt; ///< Option properties. struct { - int scope; ///< Scope character or 0 if not present. + ExprVarScope scope; ///< Scope character or 0 if not present. bool autoload; ///< Has autoload characters. } var; ///< For kExprLexPlainIdentifier @@ -150,53 +176,63 @@ typedef enum { /// Expression AST node type typedef enum { - kExprNodeMissing = 'X', - kExprNodeOpMissing = '_', - kExprNodeTernary = '?', ///< Ternary operator. - kExprNodeTernaryValue = 'C', ///< Ternary operator, colon. - kExprNodeRegister = '@', ///< Register. - kExprNodeSubscript = 's', ///< Subscript. - kExprNodeListLiteral = 'l', ///< List literal. - kExprNodeUnaryPlus = 'p', - kExprNodeBinaryPlus = '+', - kExprNodeNested = 'e', ///< Nested parenthesised expression. - kExprNodeCall = 'c', ///< Function call. + kExprNodeMissing = 0, + kExprNodeOpMissing, + kExprNodeTernary, ///< Ternary operator. + kExprNodeTernaryValue, ///< Ternary operator, colon. + kExprNodeRegister, ///< Register. + kExprNodeSubscript, ///< Subscript. + kExprNodeListLiteral, ///< List literal. + kExprNodeUnaryPlus, + kExprNodeBinaryPlus, + kExprNodeNested, ///< Nested parenthesised expression. + kExprNodeCall, ///< Function call. /// Plain identifier: simple variable/function name /// /// Looks like "string", "g:Foo", etc: consists from a single /// kExprLexPlainIdentifier token. - kExprNodePlainIdentifier = 'i', + kExprNodePlainIdentifier, /// Plain dictionary key, for use with kExprNodeConcatOrSubscript - kExprNodePlainKey = 'k', + kExprNodePlainKey, /// Complex identifier: variable/function name with curly braces - kExprNodeComplexIdentifier = 'I', + kExprNodeComplexIdentifier, /// Figure brace expression which is not yet known /// /// May resolve to any of kExprNodeDictLiteral, kExprNodeLambda or /// kExprNodeCurlyBracesIdentifier. - kExprNodeUnknownFigure = '{', - kExprNodeLambda = '\\', ///< Lambda. - kExprNodeDictLiteral = 'd', ///< Dictionary literal. - kExprNodeCurlyBracesIdentifier= '}', ///< Part of the curly braces name. - kExprNodeComma = ',', ///< Comma “operator”. - kExprNodeColon = ':', ///< Colon “operator”. - kExprNodeArrow = '>', ///< Arrow “operator”. - kExprNodeComparison = '=', ///< Various comparison operators. + kExprNodeUnknownFigure, + kExprNodeLambda, ///< Lambda. + kExprNodeDictLiteral, ///< Dictionary literal. + kExprNodeCurlyBracesIdentifier, ///< Part of the curly braces name. + kExprNodeComma, ///< Comma “operator”. + kExprNodeColon, ///< Colon “operator”. + kExprNodeArrow, ///< Arrow “operator”. + kExprNodeComparison, ///< Various comparison operators. /// Concat operator /// /// To be only used in cases when it is known for sure it is not a subscript. - kExprNodeConcat = '.', + kExprNodeConcat, /// Concat or subscript operator /// /// For cases when it is not obvious whether expression is a concat or /// a subscript. May only have either number or plain identifier as the second /// child. To make it easier to avoid curly braces in place of /// kExprNodePlainIdentifier node kExprNodePlainKey is used. - kExprNodeConcatOrSubscript = 'S', - kExprNodeInteger = '0', ///< Integral number. - kExprNodeFloat = '1', ///< Floating-point number. - kExprNodeSingleQuotedString = '\'', - kExprNodeDoubleQuotedString = '"', + kExprNodeConcatOrSubscript, + kExprNodeInteger, ///< Integral number. + kExprNodeFloat, ///< Floating-point number. + kExprNodeSingleQuotedString, + kExprNodeDoubleQuotedString, + kExprNodeOr, + kExprNodeAnd, + kExprNodeUnaryMinus, + kExprNodeBinaryMinus, + kExprNodeNot, + kExprNodeMultiplication, + kExprNodeDivision, + kExprNodeMod, + kExprNodeOption, + kExprNodeEnvironment, } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -230,7 +266,7 @@ struct expr_ast_node { size_t opening_hl_idx; } fig; ///< For kExprNodeUnknownFigure. struct { - int scope; ///< Scope character or 0 if not present. + ExprVarScope scope; ///< Scope character or 0 if not present. /// Actual identifier without scope. /// /// Points to inside parser reader state. @@ -256,6 +292,15 @@ struct expr_ast_node { size_t size; } str; ///< For kExprNodeSingleQuotedString and ///< kExprNodeDoubleQuotedString. + struct { + const char *ident; ///< Option name start. + size_t ident_len; ///< Option name length. + ExprOptScope scope; ///< Option scope: &l:, &g: or not specified. + } opt; ///< For kExprNodeOption. + struct { + const char *ident; ///< Environment variable name start. + size_t ident_len; ///< Environment variable name length. + } env; ///< For kExprNodeEnvironment. } data; }; diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h index a17edac403..10ced57977 100644 --- a/src/nvim/viml/parser/parser.h +++ b/src/nvim/viml/parser/parser.h @@ -173,7 +173,7 @@ static inline void viml_parser_highlight(ParserState *const pstate, const size_t len, const char *const group) { - if (pstate->colors == NULL) { + if (pstate->colors == NULL || len == 0) { return; } // TODO(ZyX-I): May do some assert() sanitizing here. diff --git a/test/symbolic/klee/viml_expressions_lexer.c b/test/symbolic/klee/viml_expressions_lexer.c index cddc1cb2f1..ee7dc312e9 100644 --- a/test/symbolic/klee/viml_expressions_lexer.c +++ b/test/symbolic/klee/viml_expressions_lexer.c @@ -17,6 +17,7 @@ #include "nvim/charset.c" #include "nvim/garray.c" #include "nvim/gettext.c" +#include "nvim/keymap.c" #include "nvim/viml/parser/expressions.c" #define INPUT_SIZE 7 diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index f180d8ceff..674b1b37db 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -62,9 +62,9 @@ child_call_once(function() } eltkn_opt_scope_tab = { - [tonumber(lib.kExprLexOptUnspecified)] = 'Unspecified', - [tonumber(lib.kExprLexOptGlobal)] = 'Global', - [tonumber(lib.kExprLexOptLocal)] = 'Local', + [tonumber(lib.kExprOptScopeUnspecified)] = 'Unspecified', + [tonumber(lib.kExprOptScopeGlobal)] = 'Global', + [tonumber(lib.kExprOptScopeLocal)] = 'Local', } end) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index ed77a7cba4..5041708a3e 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -93,6 +93,16 @@ make_enum_conv_tab(lib, { 'kExprNodeFloat', 'kExprNodeSingleQuotedString', 'kExprNodeDoubleQuotedString', + 'kExprNodeOr', + 'kExprNodeAnd', + 'kExprNodeUnaryMinus', + 'kExprNodeBinaryMinus', + 'kExprNodeNot', + 'kExprNodeMultiplication', + 'kExprNodeDivision', + 'kExprNodeMod', + 'kExprNodeOption', + 'kExprNodeEnvironment', }, 'kExprNode', function(ret) east_node_type_tab = ret end) local function conv_east_node_type(typ) @@ -149,6 +159,15 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) local s = ffi.string(eastnode.data.str.value, eastnode.data.str.size) typ = format_string('%s(val=%q)', typ, s) end + elseif typ == 'Option' then + typ = ('%s(scope=%s,ident=%s)'):format( + typ, + tostring(intchar2lua(eastnode.data.opt.scope)), + ffi.string(eastnode.data.opt.ident, eastnode.data.opt.ident_len)) + elseif typ == 'Environment' then + typ = ('%s(ident=%s)'):format( + typ, + ffi.string(eastnode.data.env.ident, eastnode.data.env.ident_len)) end ret_str = typ .. ':' .. ret_str local can_simplify = true diff --git a/test/unit/viml/helpers.lua b/test/unit/viml/helpers.lua index 0b92be2654..c965cacb29 100644 --- a/test/unit/viml/helpers.lua +++ b/test/unit/viml/helpers.lua @@ -5,6 +5,7 @@ local cimport = helpers.cimport local kvi_new = helpers.kvi_new local kvi_init = helpers.kvi_init local conv_enum = helpers.conv_enum +local child_call_once = helpers.child_call_once local make_enum_conv_tab = helpers.make_enum_conv_tab local lib = cimport('./src/nvim/viml/parser/expressions.h') From 8178ba2871bb427e03419a2f68c0fb119e44e717 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 14 Oct 2017 00:26:52 +0300 Subject: [PATCH 033/109] =?UTF-8?q?klee:=20Fix=20some=20errors=20made=20in?= =?UTF-8?q?=20=E2=80=A6parser.c?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/symbolic/klee/viml_expressions_parser.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index e1cea2d990..a4fbdb6bf4 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -1,5 +1,4 @@ #ifdef USE_KLEE -# error UNFINISHED # include #else # include @@ -47,10 +46,10 @@ int main(const int argc, const char *const *const argv, #ifdef USE_KLEE klee_make_symbolic(input, sizeof(input), "input"); klee_make_symbolic(&shift, sizeof(shift), "shift"); - klee_make_symbolic(&flags, sizeof{flags}, "flags"); + klee_make_symbolic(&flags, sizeof(flags), "flags"); klee_assume(shift < INPUT_SIZE); klee_assume( - flags <= kExprFlagsMulti|kExprFlagsDisallowEOC|kExprFlagsPrintError); + flags <= (kExprFlagsMulti|kExprFlagsDisallowEOC|kExprFlagsPrintError)); #endif ParserLine plines[] = { @@ -93,5 +92,6 @@ int main(const int argc, const char *const *const argv, assert(ast.root != NULL || plines[0].size == 0); assert(ast.root != NULL || ast.err.msg); + // FIXME: check for AST recursiveness // FIXME: free memory and assert no memory leaks } From c286155bfa53c828ebe5479fd81a544740a92403 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 19:06:41 +0300 Subject: [PATCH 034/109] viml/parser/expressions: Create tests for latest additions --- src/nvim/viml/parser/expressions.c | 25 +- test/helpers.lua | 7 + test/symbolic/klee/nvim/mbyte.c | 63 +- test/unit/viml/expressions/parser_spec.lua | 1430 ++++++++++++++++++++ 4 files changed, 1519 insertions(+), 6 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 75fcb17bf6..8928179349 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1937,9 +1937,22 @@ viml_pexpr_parse_process_token: OP_MISSING; } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeOption); - cur_node->data.opt.ident = cur_token.data.opt.name; - cur_node->data.opt.ident_len = cur_token.data.opt.len; - cur_node->data.opt.scope = cur_token.data.opt.scope; + if (cur_token.type == kExprLexInvalid) { + assert(cur_token.len == 1 + || (cur_token.len == 3 + && pline.data[cur_token.start.col + 2] == ':')); + cur_node->data.opt.ident = ( + pline.data + cur_token.start.col + cur_token.len); + cur_node->data.opt.ident_len = 0; + cur_node->data.opt.scope = ( + cur_token.len == 3 + ? (ExprOptScope)pline.data[cur_token.start.col + 1] + : kExprOptScopeUnspecified); + } else { + cur_node->data.opt.ident = cur_token.data.opt.name; + cur_node->data.opt.ident_len = cur_token.data.opt.len; + cur_node->data.opt.scope = cur_token.data.opt.scope; + } *top_node_p = cur_node; want_node = kENodeOperator; viml_parser_highlight(pstate, cur_token.start, 1, HL(OptionSigil)); @@ -1953,7 +1966,7 @@ viml_pexpr_parse_process_token: } viml_parser_highlight( pstate, shifted_pos(cur_token.start, scope_shift + 1), - cur_token.len - scope_shift + 1, HL(Option)); + cur_token.len - (scope_shift + 1), HL(Option)); break; } case kExprLexEnv: { @@ -1963,6 +1976,10 @@ viml_pexpr_parse_process_token: NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeEnvironment); cur_node->data.env.ident = pline.data + cur_token.start.col + 1; cur_node->data.env.ident_len = cur_token.len - 1; + if (cur_node->data.env.ident_len == 0) { + ERROR_FROM_TOKEN_AND_MSG(cur_token, + _("E15: Environment variable name missing")); + } *top_node_p = cur_node; want_node = kENodeOperator; viml_parser_highlight(pstate, cur_token.start, 1, HL(EnvironmentSigil)); diff --git a/test/helpers.lua b/test/helpers.lua index e7d8c185ba..6a42963d7f 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -352,7 +352,14 @@ format_luav = function(v, indent) end end ret = ret .. indent .. '}' + elseif type(v) == 'number' then + if v % 1 == 0 then + ret = ('%d'):format(v) + else + ret = ('%e'):format(v) + end else + print(type(v)) -- Not implemented yet assert(false) end diff --git a/test/symbolic/klee/nvim/mbyte.c b/test/symbolic/klee/nvim/mbyte.c index bfc191b1b7..f98a531206 100644 --- a/test/symbolic/klee/nvim/mbyte.c +++ b/test/symbolic/klee/nvim/mbyte.c @@ -89,10 +89,69 @@ char_u *string_convert(const vimconv_T *conv, char_u *data, size_t *size) return NULL; } +int utf_ptr2len_len(const char_u *p, int size) +{ + int len; + int i; + int m; + + len = utf8len_tab[*p]; + if (len == 1) + return 1; /* NUL, ascii or illegal lead byte */ + if (len > size) + m = size; /* incomplete byte sequence. */ + else + m = len; + for (i = 1; i < m; ++i) + if ((p[i] & 0xc0) != 0x80) + return 1; + return len; +} + int utfc_ptr2len_len(const char_u *p, int size) { - assert(false); - return 0; + int len; + int prevlen; + + if (size < 1 || *p == NUL) + return 0; + if (p[0] < 0x80 && (size == 1 || p[1] < 0x80)) /* be quick for ASCII */ + return 1; + + /* Skip over first UTF-8 char, stopping at a NUL byte. */ + len = utf_ptr2len_len(p, size); + + /* Check for illegal byte and incomplete byte sequence. */ + if ((len == 1 && p[0] >= 0x80) || len > size) + return 1; + + /* + * Check for composing characters. We can handle only the first six, but + * skip all of them (otherwise the cursor would get stuck). + */ + prevlen = 0; + while (len < size) { + int len_next_char; + + if (p[len] < 0x80) + break; + + /* + * Next character length should not go beyond size to ensure that + * UTF_COMPOSINGLIKE(...) does not read beyond size. + */ + len_next_char = utf_ptr2len_len(p + len, size - len); + if (len_next_char > size - len) + break; + + if (!UTF_COMPOSINGLIKE(p + prevlen, p + len)) + break; + + /* Skip over composing char */ + prevlen = len; + len += len_next_char; + } + return len; } int utf_char2len(const int c) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 5041708a3e..d95eaca79b 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -1453,6 +1453,24 @@ describe('Expressions parser', function() hl('CallingParenthesis', '(', 1), hl('CallingParenthesis', ')'), }) + check_parsing('{@a', 0, { + -- 012 + ast = { + { + 'UnknownFigure(-di):0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '{@a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('Register', '@a'), + }) end) itp('works with lambdas and dictionaries', function() check_parsing('{}', 0, { @@ -2768,6 +2786,182 @@ describe('Expressions parser', function() hl('Identifier', 'c', 1), hl('Dict', '}'), }) + check_parsing('{', 0, { + -- 0 + ast = { + 'UnknownFigure(\\di):0:0:{', + }, + err = { + arg = '{', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + }) + check_parsing('{a', 0, { + -- 01 + ast = { + { + 'UnknownFigure(\\di):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + err = { + arg = '{a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('Identifier', 'a'), + }) + check_parsing('{a,b', 0, { + -- 0123 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '{a,b', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + }) + check_parsing('{a,b->', 0, { + -- 012345 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + 'Arrow:0:4:->', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('Arrow', '->'), + }) + check_parsing('{a,b->c', 0, { + -- 0123456 + ast = { + { + 'Lambda(\\di):0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + }, + }, + }, + }, + }, + err = { + arg = '{a,b->c', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Identifier', 'a'), + hl('Comma', ','), + hl('Identifier', 'b'), + hl('Arrow', '->'), + hl('Identifier', 'c'), + }) + check_parsing('{a : b', 0, { + -- 012345 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + err = { + arg = '{a : b', + msg = 'E723: Missing end of Dictionary \'}\': %.*s', + }, + }, { + hl('Dict', '{'), + hl('Identifier', 'a'), + hl('Colon', ':', 1), + hl('Identifier', 'b', 1), + }) + check_parsing('{a : b,', 0, { + -- 0123456 + ast = { + { + 'DictLiteral(-di):0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Dict', '{'), + hl('Identifier', 'a'), + hl('Colon', ':', 1), + hl('Identifier', 'b', 1), + hl('Comma', ','), + }) end) itp('works with ternary operator', function() check_parsing('a ? b : c', 0, { @@ -4574,6 +4768,38 @@ describe('Expressions parser', function() hl('Subscript', '['), hl('InvalidSubscript', ']'), }) + + check_parsing('[', 0, { + -- 0 + ast = { + 'ListLiteral:0:0:[', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('List', '['), + }) + + check_parsing('[1', 0, { + -- 01 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'Integer(val=1):0:1:1', + }, + }, + }, + err = { + arg = '[1', + msg = 'E697: Missing end of List \']\': %.*s', + }, + }, { + hl('List', '['), + hl('Number', '1'), + }) end) itp('works with strings', function() check_parsing('\'abc\'', 0, { @@ -5393,4 +5619,1208 @@ describe('Expressions parser', function() hl('DoubleQuotedString', '"'), }) end) + itp('works with multiplication-like operators', function() + check_parsing('2+2*2', 0, { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:3:*', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Multiplication', '*'), + hl('Number', '2'), + }) + + check_parsing('2+2*', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:3:*', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Multiplication', '*'), + }) + + check_parsing('2+*2', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Multiplication:0:2:*', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '*2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidMultiplication', '*'), + hl('Number', '2'), + }) + + check_parsing('2+2/2', 0, { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:3:/', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Division', '/'), + hl('Number', '2'), + }) + + check_parsing('2+2/', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:3:/', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Division', '/'), + }) + + check_parsing('2+/2', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Division:0:2:/', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '/2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidDivision', '/'), + hl('Number', '2'), + }) + + check_parsing('2+2%2', 0, { + -- 01234 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:3:%', + children = { + 'Integer(val=2):0:2:2', + 'Integer(val=2):0:4:2', + }, + }, + }, + }, + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Mod', '%'), + hl('Number', '2'), + }) + + check_parsing('2+2%', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:3:%', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('Number', '2'), + hl('Mod', '%'), + }) + + check_parsing('2+%2', 0, { + -- 0123 + ast = { + { + 'BinaryPlus:0:1:+', + children = { + 'Integer(val=2):0:0:2', + { + 'Mod:0:2:%', + children = { + 'Missing:0:2:', + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + err = { + arg = '%2', + msg = 'E15: Unexpected multiplication-like operator: %.*s', + }, + }, { + hl('Number', '2'), + hl('BinaryPlus', '+'), + hl('InvalidMod', '%'), + hl('Number', '2'), + }) + end) + itp('works with -', function() + check_parsing('@a', 0, { + ast = { + 'Register(name=a):0:0:@a', + }, + }, { + hl('Register', '@a'), + }) + check_parsing('-@a', 0, { + ast = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + }) + check_parsing('@a-@b', 0, { + ast = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('@a-@b-@c', 0, { + ast = { + { + 'BinaryMinus:0:5:-', + children = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + 'Register(name=c):0:6:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + hl('BinaryMinus', '-'), + hl('Register', '@c'), + }) + check_parsing('-@a-@b', 0, { + ast = { + { + 'BinaryMinus:0:3:-', + children = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:4:@b', + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('-@a--@b', 0, { + ast = { + { + 'BinaryMinus:0:3:-', + children = { + { + 'UnaryMinus:0:0:-', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'UnaryMinus:0:4:-', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryMinus', '-'), + hl('Register', '@a'), + hl('BinaryMinus', '-'), + hl('UnaryMinus', '-'), + hl('Register', '@b'), + }) + check_parsing('@a@b', 0, { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:2:@b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidRegister', '@b'), + }) + check_parsing(' @a \t @b', 0, { + ast = { + { + 'OpMissing:0:3:', + children = { + 'Register(name=a):0:0: @a', + 'Register(name=b):0:3: \t @b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a', 1), + hl('InvalidSpacing', ' \t '), + hl('Register', '@b'), + }) + check_parsing('-', 0, { + ast = { + 'UnaryMinus:0:0:-', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryMinus', '-'), + }) + check_parsing(' -', 0, { + ast = { + 'UnaryMinus:0:0: -', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryMinus', '-', 1), + }) + check_parsing('@a- ', 0, { + ast = { + { + 'BinaryMinus:0:2:-', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Register', '@a'), + hl('BinaryMinus', '-'), + }) + end) + itp('works with logical operators', function() + check_parsing('a && b || c && d', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Or:0:6: ||', + children = { + { + 'And:0:1: &&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + { + 'And:0:11: &&', + children = { + 'PlainIdentifier(scope=0,ident=c):0:9: c', + 'PlainIdentifier(scope=0,ident=d):0:14: d', + }, + }, + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('And', '&&', 1), + hl('Identifier', 'b', 1), + hl('Or', '||', 1), + hl('Identifier', 'c', 1), + hl('And', '&&', 1), + hl('Identifier', 'd', 1), + }) + + check_parsing('&& a', 0, { + -- 0123 + ast = { + { + 'And:0:0:&&', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=a):0:2: a', + }, + }, + }, + err = { + arg = '&& a', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('InvalidAnd', '&&'), + hl('Identifier', 'a', 1), + }) + + check_parsing('|| a', 0, { + -- 0123 + ast = { + { + 'Or:0:0:||', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=a):0:2: a', + }, + }, + }, + err = { + arg = '|| a', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('InvalidOr', '||'), + hl('Identifier', 'a', 1), + }) + + check_parsing('a||', 0, { + -- 012 + ast = { + { + 'Or:0:1:||', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('Or', '||'), + }) + + check_parsing('a&&', 0, { + -- 012 + ast = { + { + 'And:0:1:&&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Identifier', 'a'), + hl('And', '&&'), + }) + + check_parsing('(&&)', 0, { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:1:&&', + children = { + 'Missing:0:1:', + 'Missing:0:3:', + }, + }, + }, + }, + }, + err = { + arg = '&&)', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidAnd', '&&'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(||)', 0, { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:1:||', + children = { + 'Missing:0:1:', + 'Missing:0:3:', + }, + }, + }, + }, + }, + err = { + arg = '||)', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOr', '||'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(a||)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:2:||', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Identifier', 'a'), + hl('Or', '||'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(a&&)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:2:&&', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Identifier', 'a'), + hl('And', '&&'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(&&a)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'And:0:1:&&', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:3:a', + }, + }, + }, + }, + }, + err = { + arg = '&&a)', + msg = 'E15: Unexpected and operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidAnd', '&&'), + hl('Identifier', 'a'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(||a)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Or:0:1:||', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:3:a', + }, + }, + }, + }, + }, + err = { + arg = '||a)', + msg = 'E15: Unexpected or operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOr', '||'), + hl('Identifier', 'a'), + hl('NestingParenthesis', ')'), + }) + end) + itp('works with &opt', function() + check_parsing('&', 0, { + -- 0 + ast = { + 'Option(scope=0,ident=):0:0:&', + }, + err = { + arg = '&', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + }) + + check_parsing('&opt', 0, { + -- 0123 + ast = { + 'Option(scope=0,ident=opt):0:0:&opt', + }, + }, { + hl('OptionSigil', '&'), + hl('Option', 'opt'), + }) + + check_parsing('&l:opt', 0, { + -- 012345 + ast = { + 'Option(scope=l,ident=opt):0:0:&l:opt', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionScope', 'l'), + hl('OptionScopeDelimiter', ':'), + hl('Option', 'opt'), + }) + + check_parsing('&g:opt', 0, { + -- 012345 + ast = { + 'Option(scope=g,ident=opt):0:0:&g:opt', + }, + }, { + hl('OptionSigil', '&'), + hl('OptionScope', 'g'), + hl('OptionScopeDelimiter', ':'), + hl('Option', 'opt'), + }) + + check_parsing('&s:opt', 0, { + -- 012345 + ast = { + { + 'Colon:0:2::', + children = { + 'Option(scope=0,ident=s):0:0:&s', + 'PlainIdentifier(scope=0,ident=opt):0:3:opt', + }, + }, + }, + err = { + arg = ':opt', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('OptionSigil', '&'), + hl('Option', 's'), + hl('InvalidColon', ':'), + hl('Identifier', 'opt'), + }) + + check_parsing('& ', 0, { + -- 01 + ast = { + 'Option(scope=0,ident=):0:0:&', + }, + err = { + arg = '& ', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + }) + + check_parsing('&-', 0, { + -- 01 + ast = { + { + 'BinaryMinus:0:1:-', + children = { + 'Option(scope=0,ident=):0:0:&', + }, + }, + }, + err = { + arg = '&-', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('InvalidOptionSigil', '&'), + hl('BinaryMinus', '-'), + }) + + check_parsing('&A', 0, { + -- 01 + ast = { + 'Option(scope=0,ident=A):0:0:&A', + }, + }, { + hl('OptionSigil', '&'), + hl('Option', 'A'), + }) + + check_parsing('&xxx_yyy', 0, { + -- 01234567 + ast = { + { + 'OpMissing:0:4:', + children = { + 'Option(scope=0,ident=xxx):0:0:&xxx', + 'PlainIdentifier(scope=0,ident=_yyy):0:4:_yyy', + }, + }, + }, + err = { + arg = '_yyy', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('OptionSigil', '&'), + hl('Option', 'xxx'), + hl('InvalidIdentifier', '_yyy'), + }) + + check_parsing('(1+&)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Integer(val=1):0:1:1', + 'Option(scope=0,ident=):0:3:&', + }, + }, + }, + }, + }, + err = { + arg = '&)', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('BinaryPlus', '+'), + hl('InvalidOptionSigil', '&'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(&+1)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Option(scope=0,ident=):0:1:&', + 'Integer(val=1):0:3:1', + }, + }, + }, + }, + }, + err = { + arg = '&+1)', + msg = 'E112: Option name missing: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidOptionSigil', '&'), + hl('BinaryPlus', '+'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + end) + itp('works with $ENV', function() + check_parsing('$', 0, { + -- 0 + ast = { + 'Environment(ident=):0:0:$', + }, + err = { + arg = '$', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('InvalidEnvironmentSigil', '$'), + }) + + check_parsing('$g:A', 0, { + -- 0123 + ast = { + { + 'Colon:0:2::', + children = { + 'Environment(ident=g):0:0:$g', + 'PlainIdentifier(scope=0,ident=A):0:3:A', + }, + }, + }, + err = { + arg = ':A', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('Environment', 'g'), + hl('InvalidColon', ':'), + hl('Identifier', 'A'), + }) + + check_parsing('$A', 0, { + -- 01 + ast = { + 'Environment(ident=A):0:0:$A', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('Environment', 'A'), + }) + + check_parsing('$ABC', 0, { + -- 0123 + ast = { + 'Environment(ident=ABC):0:0:$ABC', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('Environment', 'ABC'), + }) + + check_parsing('(1+$)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Integer(val=1):0:1:1', + 'Environment(ident=):0:3:$', + }, + }, + }, + }, + }, + err = { + arg = '$)', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('BinaryPlus', '+'), + hl('InvalidEnvironmentSigil', '$'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('($+1)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Environment(ident=):0:1:$', + 'Integer(val=1):0:3:1', + }, + }, + }, + }, + }, + err = { + arg = '$+1)', + msg = 'E15: Environment variable name missing', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidEnvironmentSigil', '$'), + hl('BinaryPlus', '+'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('$_ABC', 0, { + -- 01234 + ast = { + 'Environment(ident=_ABC):0:0:$_ABC', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('Environment', '_ABC'), + }) + + check_parsing('$_', 0, { + -- 01 + ast = { + 'Environment(ident=_):0:0:$_', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('Environment', '_'), + }) + + check_parsing('$ABC_DEF', 0, { + -- 01234567 + ast = { + 'Environment(ident=ABC_DEF):0:0:$ABC_DEF', + }, + }, { + hl('EnvironmentSigil', '$'), + hl('Environment', 'ABC_DEF'), + }) + end) + itp('works with unary !', function() + check_parsing('!', 0, { + -- 0 + ast = { + 'Not:0:0:!', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Not', '!'), + }) + + check_parsing('!!', 0, { + -- 01 + ast = { + { + 'Not:0:0:!', + children = { + 'Not:0:1:!', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Not', '!'), + hl('Not', '!'), + }) + + check_parsing('!!1', 0, { + -- 012 + ast = { + { + 'Not:0:0:!', + children = { + { + 'Not:0:1:!', + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + }, + }, { + hl('Not', '!'), + hl('Not', '!'), + hl('Number', '1'), + }) + + check_parsing('!1', 0, { + -- 01 + ast = { + { + 'Not:0:0:!', + children = { + 'Integer(val=1):0:1:1', + }, + }, + }, + }, { + hl('Not', '!'), + hl('Number', '1'), + }) + + check_parsing('(!1)', 0, { + -- 0123 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Not:0:1:!', + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Not', '!'), + hl('Number', '1'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(!)', 0, { + -- 012 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'Not:0:1:!', + children = { + 'Missing:0:2:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Not', '!'), + hl('InvalidNestingParenthesis', ')'), + }) + + check_parsing('(1!2)', 0, { + -- 01234 + ast = { + { + 'Nested:0:0:(', + children = { + { + 'OpMissing:0:2:', + children = { + 'Integer(val=1):0:1:1', + { + 'Not:0:2:!', + children = { + 'Integer(val=2):0:3:2', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '!2)', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Number', '1'), + hl('InvalidNot', '!'), + hl('Number', '2'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('1!2', 0, { + -- 012 + ast = { + { + 'OpMissing:0:1:', + children = { + 'Integer(val=1):0:0:1', + { + 'Not:0:1:!', + children = { + 'Integer(val=2):0:2:2', + }, + }, + }, + }, + }, + err = { + arg = '!2', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Number', '1'), + hl('InvalidNot', '!'), + hl('Number', '2'), + }) + end) end) From 206f7ae76a4a06c6bac10402649e515dd3fb2365 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 19:18:17 +0300 Subject: [PATCH 035/109] unittests: Test some edge cases --- test/unit/viml/expressions/parser_spec.lua | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index d95eaca79b..407114ff33 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -4241,6 +4241,40 @@ describe('Expressions parser', function() hl('UnaryPlus', '+', 1), hl('Identifier', 'b'), }) + + check_parsing('a. b', 0, { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2: b', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ConcatOrSubscript', '.'), + hl('Identifier', 'b', 1), + }) + + check_parsing('a. 1', 0, { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2: 1', + }, + }, + }, + }, { + hl('Identifier', 'a'), + hl('ConcatOrSubscript', '.'), + hl('Number', '1', 1), + }) end) itp('works with bracket subscripts', function() check_parsing(':', 0, { @@ -6823,4 +6857,14 @@ describe('Expressions parser', function() hl('Number', '2'), }) end) + itp('works (KLEE tests)', function() + check_parsing('\0002&A:\000', 0, { + ast = nil, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + }) + end) end) From 6c19cbef2611c389da6f3e06d8c8eb635d065774 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 20:05:35 +0300 Subject: [PATCH 036/109] viml/parser/expressions,tests: Add AST freeing, with sanity checks --- src/nvim/viml/parser/expressions.c | 239 ++++++++++++++++--- test/helpers.lua | 2 + test/symbolic/klee/viml_expressions_parser.c | 4 +- test/unit/viml/expressions/parser_spec.lua | 1 + 4 files changed, 210 insertions(+), 36 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 8928179349..2f7ec6bcca 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -642,37 +642,6 @@ static const char *const eltkn_opt_scope_tab[] = { [kExprOptScopeLocal] = "Local", }; -/// Represent `int` character as a string -/// -/// Converts -/// - ASCII digits into '{digit}' -/// - ASCII printable characters into a single-character strings -/// - everything else to numbers. -/// -/// @param[in] ch Character to convert. -/// -/// @return Converted string, stored in a static buffer (overriden after each -/// call). -static const char *intchar2str(const int ch) - FUNC_ATTR_WARN_UNUSED_RESULT -{ - static char buf[sizeof(int) * 3 + 1]; - if (' ' <= ch && ch < 0x7f) { - if (ascii_isdigit(ch)) { - buf[0] = '\''; - buf[1] = (char)ch; - buf[2] = '\''; - buf[3] = NUL; - } else { - buf[0] = (char)ch; - buf[1] = NUL; - } - } else { - snprintf(buf, sizeof(buf), "%i", ch); - } - return buf; -} - /// Represent token as a string /// /// Intended for testing and debugging purposes. @@ -756,6 +725,78 @@ viml_pexpr_repr_token_end: return ret; } +static const char *const east_node_type_tab[] = { + [kExprNodeMissing] = "Missing", + [kExprNodeOpMissing] = "OpMissing", + [kExprNodeTernary] = "Ternary", + [kExprNodeTernaryValue] = "TernaryValue", + [kExprNodeRegister] = "Register", + [kExprNodeSubscript] = "Subscript", + [kExprNodeListLiteral] = "ListLiteral", + [kExprNodeUnaryPlus] = "UnaryPlus", + [kExprNodeBinaryPlus] = "BinaryPlus", + [kExprNodeNested] = "Nested", + [kExprNodeCall] = "Call", + [kExprNodePlainIdentifier] = "PlainIdentifier", + [kExprNodePlainKey] = "PlainKey", + [kExprNodeComplexIdentifier] = "ComplexIdentifier", + [kExprNodeUnknownFigure] = "UnknownFigure", + [kExprNodeLambda] = "Lambda", + [kExprNodeDictLiteral] = "DictLiteral", + [kExprNodeCurlyBracesIdentifier] = "CurlyBracesIdentifier", + [kExprNodeComma] = "Comma", + [kExprNodeColon] = "Colon", + [kExprNodeArrow] = "Arrow", + [kExprNodeComparison] = "Comparison", + [kExprNodeConcat] = "Concat", + [kExprNodeConcatOrSubscript] = "ConcatOrSubscript", + [kExprNodeInteger] = "Integer", + [kExprNodeFloat] = "Float", + [kExprNodeSingleQuotedString] = "SingleQuotedString", + [kExprNodeDoubleQuotedString] = "DoubleQuotedString", + [kExprNodeOr] = "Or", + [kExprNodeAnd] = "And", + [kExprNodeUnaryMinus] = "UnaryMinus", + [kExprNodeBinaryMinus] = "BinaryMinus", + [kExprNodeNot] = "Not", + [kExprNodeMultiplication] = "Multiplication", + [kExprNodeDivision] = "Division", + [kExprNodeMod] = "Mod", + [kExprNodeOption] = "Option", + [kExprNodeEnvironment] = "Environment", +}; + +/// Represent `int` character as a string +/// +/// Converts +/// - ASCII digits into '{digit}' +/// - ASCII printable characters into a single-character strings +/// - everything else to numbers. +/// +/// @param[in] ch Character to convert. +/// +/// @return Converted string, stored in a static buffer (overriden after each +/// call). +static const char *intchar2str(const int ch) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + static char buf[sizeof(int) * 3 + 1]; + if (' ' <= ch && ch < 0x7f) { + if (ascii_isdigit(ch)) { + buf[0] = '\''; + buf[1] = (char)ch; + buf[2] = '\''; + buf[3] = NUL; + } else { + buf[0] = (char)ch; + buf[1] = NUL; + } + } else { + snprintf(buf, sizeof(buf), "%i", ch); + } + return buf; +} + #ifdef UNIT_TESTING #include @@ -767,9 +808,9 @@ static inline void viml_pexpr_debug_print_ast_node( if (*eastnode_p == NULL) { fprintf(stderr, "%s %p : NULL\n", prefix, (void *)eastnode_p); } else { - fprintf(stderr, "%s %p : %p : %c : %zu:%zu:%zu\n", + fprintf(stderr, "%s %p : %p : %s : %zu:%zu:%zu\n", prefix, (void *)eastnode_p, (void *)(*eastnode_p), - (*eastnode_p)->type, (*eastnode_p)->start.line, + east_node_type_tab[(*eastnode_p)->type], (*eastnode_p)->start.line, (*eastnode_p)->start.col, (*eastnode_p)->len); } } @@ -800,11 +841,141 @@ static inline void viml_pexpr_debug_print_token( #define PSTACK_P(msg) \ viml_pexpr_debug_print_ast_stack(ast_stack, #msg) #define PNODE_P(eastnode_p, msg) \ - viml_pexpr_debug_print_ast_node((const ExprASTNode *const *)ast_stack, #msg) + viml_pexpr_debug_print_ast_node((const ExprASTNode *const *)eastnode_p, \ + (#msg)) #define PTOKEN(tkn) \ viml_pexpr_debug_print_token(pstate, tkn) #endif +#ifndef NDEBUG +static const uint8_t node_maxchildren[] = { + [kExprNodeMissing] = 0, + [kExprNodeOpMissing] = 2, + [kExprNodeTernary] = 2, + [kExprNodeTernaryValue] = 2, + [kExprNodeRegister] = 0, + [kExprNodeSubscript] = 2, + [kExprNodeListLiteral] = 1, + [kExprNodeUnaryPlus] = 1, + [kExprNodeBinaryPlus] = 2, + [kExprNodeNested] = 1, + [kExprNodeCall] = 2, + [kExprNodePlainIdentifier] = 0, + [kExprNodePlainKey] = 0, + [kExprNodeComplexIdentifier] = 2, + [kExprNodeUnknownFigure] = 1, + [kExprNodeLambda] = 2, + [kExprNodeDictLiteral] = 1, + [kExprNodeCurlyBracesIdentifier] = 1, + [kExprNodeComma] = 2, + [kExprNodeColon] = 2, + [kExprNodeArrow] = 2, + [kExprNodeComparison] = 2, + [kExprNodeConcat] = 2, + [kExprNodeConcatOrSubscript] = 2, + [kExprNodeInteger] = 0, + [kExprNodeFloat] = 0, + [kExprNodeSingleQuotedString] = 0, + [kExprNodeDoubleQuotedString] = 0, + [kExprNodeOr] = 2, + [kExprNodeAnd] = 2, + [kExprNodeUnaryMinus] = 1, + [kExprNodeBinaryMinus] = 2, + [kExprNodeNot] = 1, + [kExprNodeMultiplication] = 2, + [kExprNodeDivision] = 2, + [kExprNodeMod] = 2, + [kExprNodeOption] = 0, + [kExprNodeEnvironment] = 0, +}; +#endif + +/// Free memory occupied by AST +/// +/// @param ast AST stack to free. +void viml_pexpr_free_ast(ExprAST ast) +{ + ExprASTStack ast_stack; + kvi_init(ast_stack); + kvi_push(ast_stack, &ast.root); + while (kv_size(ast_stack)) { + ExprASTNode **const cur_node = kv_last(ast_stack); +#ifndef NDEBUG + // Explicitly check for AST recursiveness. + for (size_t i = 0 ; i < kv_size(ast_stack) - 1 ; i++) { + assert(*kv_A(ast_stack, i) != *cur_node); + } +#endif + if (*cur_node == NULL) { + assert(kv_size(ast_stack) == 1); + kv_drop(ast_stack, 1); + } else if ((*cur_node)->children != NULL) { +#ifndef NDEBUG + const uint8_t maxchildren = node_maxchildren[(*cur_node)->type]; + assert(maxchildren > 0); + assert(maxchildren <= 2); + assert(maxchildren == 1 + ? (*cur_node)->children->next == NULL + : ((*cur_node)->children->next == NULL + || (*cur_node)->children->next->next == NULL)); +#endif + kvi_push(ast_stack, &(*cur_node)->children); + } else if ((*cur_node)->next != NULL) { + kvi_push(ast_stack, &(*cur_node)->next); + } else if (*cur_node != NULL) { + kv_drop(ast_stack, 1); + switch ((*cur_node)->type) { + case kExprNodeDoubleQuotedString: + case kExprNodeSingleQuotedString: { + xfree((*cur_node)->data.str.value); + break; + } + case kExprNodeMissing: + case kExprNodeOpMissing: + case kExprNodeTernary: + case kExprNodeTernaryValue: + case kExprNodeRegister: + case kExprNodeSubscript: + case kExprNodeListLiteral: + case kExprNodeUnaryPlus: + case kExprNodeBinaryPlus: + case kExprNodeNested: + case kExprNodeCall: + case kExprNodePlainIdentifier: + case kExprNodePlainKey: + case kExprNodeComplexIdentifier: + case kExprNodeUnknownFigure: + case kExprNodeLambda: + case kExprNodeDictLiteral: + case kExprNodeCurlyBracesIdentifier: + case kExprNodeComma: + case kExprNodeColon: + case kExprNodeArrow: + case kExprNodeComparison: + case kExprNodeConcat: + case kExprNodeConcatOrSubscript: + case kExprNodeInteger: + case kExprNodeFloat: + case kExprNodeOr: + case kExprNodeAnd: + case kExprNodeUnaryMinus: + case kExprNodeBinaryMinus: + case kExprNodeNot: + case kExprNodeMultiplication: + case kExprNodeDivision: + case kExprNodeMod: + case kExprNodeOption: + case kExprNodeEnvironment: { + break; + } + } + xfree(*cur_node); + *cur_node = NULL; + } + } + kvi_destroy(ast_stack); +} + // start = s ternary_expr s EOC // ternary_expr = binop_expr // ( s Question s ternary_expr s Colon s ternary_expr s )? diff --git a/test/helpers.lua b/test/helpers.lua index 6a42963d7f..83e78ba059 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -358,6 +358,8 @@ format_luav = function(v, indent) else ret = ('%e'):format(v) end + elseif type(v) == 'nil' then + ret = 'nil' else print(type(v)) -- Not implemented yet diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index a4fbdb6bf4..c0cedceb21 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -92,6 +92,6 @@ int main(const int argc, const char *const *const argv, assert(ast.root != NULL || plines[0].size == 0); assert(ast.root != NULL || ast.err.msg); - // FIXME: check for AST recursiveness - // FIXME: free memory and assert no memory leaks + viml_pexpr_free_ast(ast); + assert(allocated_memory == 0); } diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 407114ff33..9624ced022 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -253,6 +253,7 @@ describe('Expressions parser', function() end eq(exp_highlighting, hls) end + lib.viml_pexpr_free_ast(east) end local function hl(group, str, shift) return function(next_col) From 57bb3346d95f325d4894d41c88d3a1434de2d2df Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 20:43:16 +0300 Subject: [PATCH 037/109] viml/parser/expressions: Update some comments and add another check --- src/nvim/viml/parser/expressions.c | 104 +++++++++++------------------ 1 file changed, 38 insertions(+), 66 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 2f7ec6bcca..370034943e 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -976,59 +976,6 @@ void viml_pexpr_free_ast(ExprAST ast) kvi_destroy(ast_stack); } -// start = s ternary_expr s EOC -// ternary_expr = binop_expr -// ( s Question s ternary_expr s Colon s ternary_expr s )? -// binop_expr = unaryop_expr ( binop unaryop_expr )? -// unaryop_expr = ( unaryop )? subscript_expr -// subscript_expr = subscript_expr subscript -// | value_expr -// subscript = Bracket('[') s ternary_expr s Bracket(']') -// | s Parenthesis('(') call_args Parenthesis(')') -// | Dot ( PlainIdentifier | Number )+ -// # Note: `s` before Parenthesis('(') is only valid if preceding subscript_expr -// # is PlainIdentifier -// value_expr = ( float | Number -// | DoubleQuotedString | SingleQuotedString -// | paren_expr -// | list_literal -// | lambda_literal -// | dict_literal -// | Environment -// | Option -// | Register -// | var ) -// float = Number Dot Number ( PlainIdentifier('e') ( Plus | Minus )? Number )? -// # Note: `1.2.3` is concat and not float. `"abc".2.3` is also concat without -// # floats. -// paren_expr = Parenthesis('(') s ternary_expr s Parenthesis(')') -// list_literal = Bracket('[') s -// ( ternary_expr s Comma s )* -// ternary_expr? s -// Bracket(']') -// dict_literal = FigureBrace('{') s -// ( ternary_expr s Colon s ternary_expr s Comma s )* -// ( ternary_expr s Colon s ternary_expr s )? -// FigureBrace('}') -// lambda_literal = FigureBrace('{') s -// ( PlainIdentifier s Comma s )* -// PlainIdentifier s -// Arrow s -// ternary_expr s -// FigureBrace('}') -// var = varchunk+ -// varchunk = PlainIdentifier -// | Comparison("is" | "is#" | "isnot" | "isnot#") -// | FigureBrace('{') s ternary_expr s FigureBrace('}') -// call_args = ( s ternary_expr s Comma s )* s ternary_expr? s -// binop = s ( Plus | Minus | Dot -// | Comparison -// | Multiplication -// | Or -// | And ) s -// unaryop = s ( Not | Plus | Minus ) s -// s = Spacing? -// // Binary operator precedence and associativity: // // Operator | Precedence | Associativity @@ -1885,6 +1832,14 @@ static void parse_quoted_string(ParserState *const pstate, kvi_destroy(shifts); } +/// Additional flags to pass to lexer depending on want_node +static const int want_node_to_lexer_flags[] = { + [kENodeValue] = kELFlagIsNotCmp, + [kENodeOperator] = kELFlagForbidScope, + [kENodeArgument] = kELFlagIsNotCmp, + [kENodeArgumentSeparator] = kELFlagForbidScope, +}; + /// Parse one VimL expression /// /// @param pstate Parser state. @@ -1902,26 +1857,25 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) }, .root = NULL, }; + // Expression stack contains current branch in AST tree: that is + // - Stack item 0 contains root of the tree, i.e. &ast->root. + // - Stack item i points to the previous stack items’ last child. + // + // When parser expects “value” node that is something like identifier or "[" + // (list start) last stack item contains NULL. Otherwise last stack item is + // supposed to contain last “finished” value: e.g. "1" or "+(1, 1)" (node + // representing "1+1"). + // + // Both kENodeValue and kENodeArgument stand for “value” nodes. ExprASTStack ast_stack; kvi_init(ast_stack); kvi_push(ast_stack, &ast.root); - // Expressions stack: - // 1. *last is NULL if want_node is kExprLexValue. Indicates where expression - // is to be put. - // 2. *last is not NULL otherwise, indicates current expression to be used as - // an operator argument. ExprASTWantedNode want_node = kENodeValue; LexExprToken prev_token = { .type = kExprLexMissing }; bool highlighted_prev_spacing = false; // Lambda node, valid when parsing lambda arguments only. ExprASTNode *lambda_node = NULL; do { - const int want_node_to_lexer_flags[] = { - [kENodeValue] = kELFlagIsNotCmp, - [kENodeOperator] = kELFlagForbidScope, - [kENodeArgument] = kELFlagIsNotCmp, - [kENodeArgumentSeparator] = kELFlagForbidScope, - }; const bool is_concat_or_subscript = ( want_node == kENodeValue && kv_size(ast_stack) > 1 @@ -1965,9 +1919,27 @@ viml_pexpr_parse_process_token: } const ParserLine pline = pstate->reader.lines.items[cur_token.start.line]; ExprASTNode **const top_node_p = kv_last(ast_stack); + assert(kv_size(ast_stack) >= 1); ExprASTNode *cur_node = NULL; - assert((want_node == kENodeValue || want_node == kENodeArgument) - == (*top_node_p == NULL)); +#ifndef NDEBUG + const bool want_value = ( + want_node == kENodeValue || want_node == kENodeArgument); + assert(want_value == (*top_node_p == NULL)); + assert(kv_A(ast_stack, 0) == &ast.root); + // Check that stack item i + 1 points to stack items’ i *last* child. + for (size_t i = 0; i + 1 < kv_size(ast_stack); i++) { + const bool item_null = (want_value && i + 2 == kv_size(ast_stack)); + assert((&(*kv_A(ast_stack, i))->children == kv_A(ast_stack, i + 1) + && (item_null + ? (*kv_A(ast_stack, i))->children == NULL + : (*kv_A(ast_stack, i))->children->next == NULL)) + || ((&(*kv_A(ast_stack, i))->children->next + == kv_A(ast_stack, i + 1)) + && (item_null + ? (*kv_A(ast_stack, i))->children->next == NULL + : (*kv_A(ast_stack, i))->children->next->next == NULL))); + } +#endif // Note: in Vim whether expression "cond?d.a:2" is valid depends both on // "cond" and whether "d" is a dictionary: expression is valid if condition // is true and "d" is a dictionary (with "a" key or it will complain about From bc386c48829c431fe5b17592c82eb4099621953a Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 21:09:08 +0300 Subject: [PATCH 038/109] charset: Fix out-of-bounds array access It is incorrect to *first* access ptr[2] and *then* check whether maxlen allows it. --- src/nvim/charset.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/nvim/charset.c b/src/nvim/charset.c index 403ef65c4f..1147b78c9a 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -1621,6 +1621,7 @@ bool vim_isblankline(char_u *lbuf) void vim_str2nr(const char_u *const start, int *const prep, int *const len, const int what, varnumber_T *const nptr, uvarnumber_T *const unptr, const int maxlen) + FUNC_ATTR_NONNULL_ARG(1) { const char_u *ptr = start; int pre = 0; // default is decimal @@ -1633,20 +1634,21 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } // Recognize hex, octal and bin. - if ((ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9') - && (maxlen == 0 || maxlen > 1)) { + if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) + && (maxlen == 0 || maxlen > 1) + && (ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9')) { pre = ptr[1]; if ((what & STR2NR_HEX) + && (maxlen == 0 || maxlen > 2) && ((pre == 'X') || (pre == 'x')) - && ascii_isxdigit(ptr[2]) - && (maxlen == 0 || maxlen > 2)) { + && ascii_isxdigit(ptr[2])) { // hexadecimal ptr += 2; } else if ((what & STR2NR_BIN) + && (maxlen == 0 || maxlen > 2) && ((pre == 'B') || (pre == 'b')) - && ascii_isbdigit(ptr[2]) - && (maxlen == 0 || maxlen > 2)) { + && ascii_isbdigit(ptr[2])) { // binary ptr += 2; } else { @@ -1675,7 +1677,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. int n = 1; - if ((pre == 'B') || (pre == 'b') || what == STR2NR_BIN + STR2NR_FORCE) { + if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { // bin if (pre != 0) { n += 2; // skip over "0b" @@ -1692,7 +1694,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, break; } } - } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) { + } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { // octal while ('0' <= *ptr && *ptr <= '7') { // avoid ubsan error for overflow @@ -1706,8 +1708,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, break; } } - } else if ((pre == 'X') || (pre == 'x') - || what == STR2NR_HEX + STR2NR_FORCE) { + } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { // hex if (pre != 0) { n += 2; // skip over "0x" From 3aa2c0d63ae488e302a89fdcdd650404cb2670fd Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 21:11:00 +0300 Subject: [PATCH 039/109] viml/parser/expressions,klee: Fix some problems found by KLEE run --- src/nvim/viml/parser/expressions.h | 2 +- test/symbolic/klee/nvim/charset.c | 20 ++++++++++---------- test/symbolic/klee/run.sh | 2 +- test/symbolic/klee/viml_expressions_parser.c | 2 -- test/unit/viml/expressions/parser_spec.lua | 8 ++++++++ 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 0198852bed..025f0f766e 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -69,7 +69,7 @@ typedef enum { } ExprOptScope; #define EXPR_OPT_SCOPE_LIST \ - ((char *)(char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) + ((char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) /// All possible variable scopes typedef enum { diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c index 409d7d443c..59fe6a1430 100644 --- a/test/symbolic/klee/nvim/charset.c +++ b/test/symbolic/klee/nvim/charset.c @@ -38,20 +38,21 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } // Recognize hex, octal and bin. - if ((ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9') - && (maxlen == 0 || maxlen > 1)) { + if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) + && (maxlen == 0 || maxlen > 1) + && (ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9')) { pre = ptr[1]; if ((what & STR2NR_HEX) + && (maxlen == 0 || maxlen > 2) && ((pre == 'X') || (pre == 'x')) - && ascii_isxdigit(ptr[2]) - && (maxlen == 0 || maxlen > 2)) { + && ascii_isxdigit(ptr[2])) { // hexadecimal ptr += 2; } else if ((what & STR2NR_BIN) + && (maxlen == 0 || maxlen > 2) && ((pre == 'B') || (pre == 'b')) - && ascii_isbdigit(ptr[2]) - && (maxlen == 0 || maxlen > 2)) { + && ascii_isbdigit(ptr[2])) { // binary ptr += 2; } else { @@ -80,7 +81,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. int n = 1; - if ((pre == 'B') || (pre == 'b') || what == STR2NR_BIN + STR2NR_FORCE) { + if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { // bin if (pre != 0) { n += 2; // skip over "0b" @@ -97,7 +98,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, break; } } - } else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) { + } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { // octal while ('0' <= *ptr && *ptr <= '7') { // avoid ubsan error for overflow @@ -111,8 +112,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, break; } } - } else if ((pre == 'X') || (pre == 'x') - || what == STR2NR_HEX + STR2NR_FORCE) { + } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { // hex if (pre != 0) { n += 2; // skip over "0x" diff --git a/test/symbolic/klee/run.sh b/test/symbolic/klee/run.sh index 388903c234..c143cd0624 100755 --- a/test/symbolic/klee/run.sh +++ b/test/symbolic/klee/run.sh @@ -46,7 +46,7 @@ main() { line1="$line1 '--output-dir=$KLEE_OUT_DIR' '$KLEE_BIN_DIR/a.bc'" local line2="for t in '$KLEE_OUT_DIR'/*.err" line2="$line2 ; do ktest-tool --write-ints" - line2="$line2 \"\$(printf '%s' \"\$t\" | sed -e 's@.[^/]*\$@.out@')\"" + line2="$line2 \"\$(printf '%s' \"\$t\" | sed -e 's@\\.[^/]*\$@.ktest@')\"" line2="$line2 ; done" printf '%s\n%s\n' "$line1" "$line2" | \ docker run \ diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index c0cedceb21..ed280adb22 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -89,8 +89,6 @@ int main(const int argc, const char *const *const argv, kvi_init(pstate.reader.lines); const ExprAST ast = viml_pexpr_parse(&pstate, flags); - assert(ast.root != NULL - || plines[0].size == 0); assert(ast.root != NULL || ast.err.msg); viml_pexpr_free_ast(ast); assert(allocated_memory == 0); diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 9624ced022..22263af8e2 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -6867,5 +6867,13 @@ describe('Expressions parser', function() }, }, { }) + check_parsing('0', 0, { + -- 0 + ast = { + 'Integer(val=0):0:0:0', + }, + }, { + hl('Number', '0'), + }) end) end) From 4ccaf861107f8c737ba5b749a69c75d7f097d1f1 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 21:22:49 +0300 Subject: [PATCH 040/109] keymap: Readd figure braces disappeared when resolving conflicts --- src/nvim/keymap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index a44b663cc6..55ac835663 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -719,8 +719,8 @@ int get_special_key_code(const char_u *name) for (int i = 0; key_names_table[i].name != NULL; i++) { const char *const table_name = key_names_table[i].name; int j; - for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) - if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) + for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) { + if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { break; } } From 2cb95bd9378d3c45f2eea527683546825e16f7d3 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 15 Oct 2017 21:39:01 +0300 Subject: [PATCH 041/109] viml/parser/expressions: Define east_node_type_tab only when needed --- src/nvim/viml/parser/expressions.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 370034943e..4801d66988 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -725,6 +725,7 @@ viml_pexpr_repr_token_end: return ret; } +#ifdef UNIT_TESTING static const char *const east_node_type_tab[] = { [kExprNodeMissing] = "Missing", [kExprNodeOpMissing] = "OpMissing", @@ -765,6 +766,7 @@ static const char *const east_node_type_tab[] = { [kExprNodeOption] = "Option", [kExprNodeEnvironment] = "Environment", }; +#endif /// Represent `int` character as a string /// From 1a3635304b80b48625bcd9d48f4f38778b42e4af Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 00:07:32 +0300 Subject: [PATCH 042/109] charset: Avoid overflow in vim_str2nr --- src/nvim/charset.c | 60 ++++++++-------------- test/symbolic/klee/nvim/charset.c | 60 ++++++++-------------- test/unit/viml/expressions/lexer_spec.lua | 3 ++ test/unit/viml/expressions/parser_spec.lua | 10 +++- 4 files changed, 53 insertions(+), 80 deletions(-) diff --git a/src/nvim/charset.c b/src/nvim/charset.c index f824717fcd..7b307b8160 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -1620,13 +1620,16 @@ bool vim_isblankline(char_u *lbuf) /// @param maxlen Max length of string to check. void vim_str2nr(const char_u *const start, int *const prep, int *const len, const int what, varnumber_T *const nptr, - uvarnumber_T *const unptr, const int maxlen) + uvarnumber_T *const unptr, int maxlen) FUNC_ATTR_NONNULL_ARG(1) { - const char_u *ptr = start; + const char *ptr = (const char *)start; + if (maxlen == 0) { + maxlen = (int)strlen(ptr); + } + const char *const e = ptr + maxlen; int pre = 0; // default is decimal bool negative = false; - uvarnumber_T un = 0; if (ptr[0] == '-') { negative = true; @@ -1635,19 +1638,19 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // Recognize hex, octal and bin. if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) - && (maxlen == 0 || maxlen > 1) - && (ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9')) { + && maxlen > 1 + && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { pre = ptr[1]; if ((what & STR2NR_HEX) - && (maxlen == 0 || maxlen > 2) - && ((pre == 'X') || (pre == 'x')) + && maxlen > 2 + && (pre == 'X' || pre == 'x') && ascii_isxdigit(ptr[2])) { // hexadecimal ptr += 2; } else if ((what & STR2NR_BIN) - && (maxlen == 0 || maxlen > 2) - && ((pre == 'B') || (pre == 'b')) + && maxlen > 2 + && (pre == 'B' || pre == 'b') && ascii_isbdigit(ptr[2])) { // binary ptr += 2; @@ -1657,32 +1660,26 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, if (what & STR2NR_OCT) { // Don't interpret "0", "08" or "0129" as octal. - for (int n = 1; ascii_isdigit(ptr[n]); ++n) { - if (ptr[n] > '7') { + for (int i = 1; i < maxlen && ascii_isdigit(ptr[i]); i++) { + if (ptr[i] > '7') { // can't be octal pre = 0; break; } - if (ptr[n] >= '0') { + if (ptr[i] >= '0') { // assume octal pre = '0'; } - if (n == maxlen) { - break; - } } } } } // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. - int n = 1; + uvarnumber_T un = 0; if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { // bin - if (pre != 0) { - n += 2; // skip over "0b" - } - while ('0' <= *ptr && *ptr <= '1') { + while (ptr < e && '0' <= *ptr && *ptr <= '1') { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 2) { un = 2 * un + (uvarnumber_T)(*ptr - '0'); @@ -1690,13 +1687,10 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, un = UVARNUMBER_MAX; } ptr++; - if (n++ == maxlen) { - break; - } } } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { // octal - while ('0' <= *ptr && *ptr <= '7') { + while (ptr < e && '0' <= *ptr && *ptr <= '7') { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 8) { un = 8 * un + (uvarnumber_T)(*ptr - '0'); @@ -1704,16 +1698,10 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, un = UVARNUMBER_MAX; } ptr++; - if (n++ == maxlen) { - break; - } } } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { // hex - if (pre != 0) { - n += 2; // skip over "0x" - } - while (ascii_isxdigit(*ptr)) { + while (ptr < e && ascii_isxdigit(*ptr)) { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 16) { un = 16 * un + (uvarnumber_T)hex2nr(*ptr); @@ -1721,13 +1709,10 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, un = UVARNUMBER_MAX; } ptr++; - if (n++ == maxlen) { - break; - } } } else { // decimal - while (ascii_isdigit(*ptr)) { + while (ptr < e && ascii_isdigit(*ptr)) { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 10) { un = 10 * un + (uvarnumber_T)(*ptr - '0'); @@ -1735,9 +1720,6 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, un = UVARNUMBER_MAX; } ptr++; - if (n++ == maxlen) { - break; - } } } @@ -1746,7 +1728,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } if (len != NULL) { - *len = (int)(ptr - start); + *len = (int)(ptr - (const char *)start); } if (nptr != NULL) { diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c index 59fe6a1430..7dcf6868bf 100644 --- a/test/symbolic/klee/nvim/charset.c +++ b/test/symbolic/klee/nvim/charset.c @@ -25,12 +25,15 @@ int hex2nr(int c) void vim_str2nr(const char_u *const start, int *const prep, int *const len, const int what, varnumber_T *const nptr, - uvarnumber_T *const unptr, const int maxlen) + uvarnumber_T *const unptr, int maxlen) { - const char_u *ptr = start; + const char *ptr = (const char *)start; + if (maxlen == 0) { + maxlen = (int)strlen(ptr); + } + const char *const e = ptr + maxlen; int pre = 0; // default is decimal bool negative = false; - uvarnumber_T un = 0; if (ptr[0] == '-') { negative = true; @@ -39,19 +42,19 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // Recognize hex, octal and bin. if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) - && (maxlen == 0 || maxlen > 1) - && (ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9')) { + && maxlen > 1 + && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { pre = ptr[1]; if ((what & STR2NR_HEX) - && (maxlen == 0 || maxlen > 2) - && ((pre == 'X') || (pre == 'x')) + && maxlen > 2 + && (pre == 'X' || pre == 'x') && ascii_isxdigit(ptr[2])) { // hexadecimal ptr += 2; } else if ((what & STR2NR_BIN) - && (maxlen == 0 || maxlen > 2) - && ((pre == 'B') || (pre == 'b')) + && maxlen > 2 + && (pre == 'B' || pre == 'b') && ascii_isbdigit(ptr[2])) { // binary ptr += 2; @@ -61,32 +64,26 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, if (what & STR2NR_OCT) { // Don't interpret "0", "08" or "0129" as octal. - for (int n = 1; ascii_isdigit(ptr[n]); ++n) { - if (ptr[n] > '7') { + for (int i = 1; i < maxlen && ascii_isdigit(ptr[i]); i++) { + if (ptr[i] > '7') { // can't be octal pre = 0; break; } - if (ptr[n] >= '0') { + if (ptr[i] >= '0') { // assume octal pre = '0'; } - if (n == maxlen) { - break; - } } } } } // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. - int n = 1; + uvarnumber_T un = 0; if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { // bin - if (pre != 0) { - n += 2; // skip over "0b" - } - while ('0' <= *ptr && *ptr <= '1') { + while (ptr < e && '0' <= *ptr && *ptr <= '1') { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 2) { un = 2 * un + (uvarnumber_T)(*ptr - '0'); @@ -94,13 +91,10 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, un = UVARNUMBER_MAX; } ptr++; - if (n++ == maxlen) { - break; - } } } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { // octal - while ('0' <= *ptr && *ptr <= '7') { + while (ptr < e && '0' <= *ptr && *ptr <= '7') { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 8) { un = 8 * un + (uvarnumber_T)(*ptr - '0'); @@ -108,16 +102,10 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, un = UVARNUMBER_MAX; } ptr++; - if (n++ == maxlen) { - break; - } } } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { // hex - if (pre != 0) { - n += 2; // skip over "0x" - } - while (ascii_isxdigit(*ptr)) { + while (ptr < e && ascii_isxdigit(*ptr)) { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 16) { un = 16 * un + (uvarnumber_T)hex2nr(*ptr); @@ -125,13 +113,10 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, un = UVARNUMBER_MAX; } ptr++; - if (n++ == maxlen) { - break; - } } } else { // decimal - while (ascii_isdigit(*ptr)) { + while (ptr < e && ascii_isdigit(*ptr)) { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 10) { un = 10 * un + (uvarnumber_T)(*ptr - '0'); @@ -139,9 +124,6 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, un = UVARNUMBER_MAX; } ptr++; - if (n++ == maxlen) { - break; - } } } @@ -150,7 +132,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } if (len != NULL) { - *len = (int)(ptr - start); + *len = (int)(ptr - (const char *)start); } if (nptr != NULL) { diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index 674b1b37db..5910468017 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -292,6 +292,9 @@ describe('Expressions lexer', function() simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'}) simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'}) simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'}) + simple_test({{data='00', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'}) + simple_test({{data='009', size=2}}, 'Number', 2, {data={is_float=false, base=8, val=0}, str='00'}) + simple_test({{data='01', size=1}}, 'Number', 1, {data={is_float=false, base=10, val=0}, str='0'}) end local function regular_scope_tests() diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 22263af8e2..36af875bd2 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -6867,13 +6867,19 @@ describe('Expressions parser', function() }, }, { }) - check_parsing('0', 0, { - -- 0 + check_parsing({data='01', size=1}, 0, { ast = { 'Integer(val=0):0:0:0', }, }, { hl('Number', '0'), }) + check_parsing({data='001', size=2}, 0, { + ast = { + 'Integer(val=0):0:0:00', + }, + }, { + hl('Number', '00'), + }) end) end) From 5e92ee6565233c56e7a33b12ef6a61f05eae91aa Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 00:19:02 +0300 Subject: [PATCH 043/109] charset: Do not call strlen() from vim_str2nr --- src/nvim/charset.c | 25 ++++++++++++------------- test/symbolic/klee/nvim/charset.c | 25 ++++++++++++------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/nvim/charset.c b/src/nvim/charset.c index 7b307b8160..a7ffa4bc00 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -1620,14 +1620,12 @@ bool vim_isblankline(char_u *lbuf) /// @param maxlen Max length of string to check. void vim_str2nr(const char_u *const start, int *const prep, int *const len, const int what, varnumber_T *const nptr, - uvarnumber_T *const unptr, int maxlen) + uvarnumber_T *const unptr, const int maxlen) FUNC_ATTR_NONNULL_ARG(1) { const char *ptr = (const char *)start; - if (maxlen == 0) { - maxlen = (int)strlen(ptr); - } - const char *const e = ptr + maxlen; +#define STRING_ENDED(ptr) \ + (!(maxlen == 0 || (int)((ptr) - (const char *)start) < maxlen)) int pre = 0; // default is decimal bool negative = false; @@ -1638,18 +1636,18 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // Recognize hex, octal and bin. if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) - && maxlen > 1 + && !STRING_ENDED(ptr + 1) && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { pre = ptr[1]; if ((what & STR2NR_HEX) - && maxlen > 2 + && !STRING_ENDED(ptr + 2) && (pre == 'X' || pre == 'x') && ascii_isxdigit(ptr[2])) { // hexadecimal ptr += 2; } else if ((what & STR2NR_BIN) - && maxlen > 2 + && !STRING_ENDED(ptr + 2) && (pre == 'B' || pre == 'b') && ascii_isbdigit(ptr[2])) { // binary @@ -1660,7 +1658,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, if (what & STR2NR_OCT) { // Don't interpret "0", "08" or "0129" as octal. - for (int i = 1; i < maxlen && ascii_isdigit(ptr[i]); i++) { + for (int i = 1; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { if (ptr[i] > '7') { // can't be octal pre = 0; @@ -1679,7 +1677,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, uvarnumber_T un = 0; if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { // bin - while (ptr < e && '0' <= *ptr && *ptr <= '1') { + while (!STRING_ENDED(ptr) && '0' <= *ptr && *ptr <= '1') { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 2) { un = 2 * un + (uvarnumber_T)(*ptr - '0'); @@ -1690,7 +1688,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { // octal - while (ptr < e && '0' <= *ptr && *ptr <= '7') { + while (!STRING_ENDED(ptr) && '0' <= *ptr && *ptr <= '7') { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 8) { un = 8 * un + (uvarnumber_T)(*ptr - '0'); @@ -1701,7 +1699,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { // hex - while (ptr < e && ascii_isxdigit(*ptr)) { + while (!STRING_ENDED(ptr) && ascii_isxdigit(*ptr)) { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 16) { un = 16 * un + (uvarnumber_T)hex2nr(*ptr); @@ -1712,7 +1710,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } } else { // decimal - while (ptr < e && ascii_isdigit(*ptr)) { + while (!STRING_ENDED(ptr) && ascii_isdigit(*ptr)) { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 10) { un = 10 * un + (uvarnumber_T)(*ptr - '0'); @@ -1750,6 +1748,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, if (unptr != NULL) { *unptr = un; } +#undef STRING_ENDED } /// Return the value of a single hex character. diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c index 7dcf6868bf..c584bd72ef 100644 --- a/test/symbolic/klee/nvim/charset.c +++ b/test/symbolic/klee/nvim/charset.c @@ -25,13 +25,11 @@ int hex2nr(int c) void vim_str2nr(const char_u *const start, int *const prep, int *const len, const int what, varnumber_T *const nptr, - uvarnumber_T *const unptr, int maxlen) + uvarnumber_T *const unptr, const int maxlen) { const char *ptr = (const char *)start; - if (maxlen == 0) { - maxlen = (int)strlen(ptr); - } - const char *const e = ptr + maxlen; +#define STRING_ENDED(ptr) \ + (!(maxlen == 0 || (int)((ptr) - (const char *)start) < maxlen)) int pre = 0; // default is decimal bool negative = false; @@ -42,18 +40,18 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // Recognize hex, octal and bin. if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) - && maxlen > 1 + && !STRING_ENDED(ptr + 1) && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { pre = ptr[1]; if ((what & STR2NR_HEX) - && maxlen > 2 + && !STRING_ENDED(ptr + 2) && (pre == 'X' || pre == 'x') && ascii_isxdigit(ptr[2])) { // hexadecimal ptr += 2; } else if ((what & STR2NR_BIN) - && maxlen > 2 + && !STRING_ENDED(ptr + 2) && (pre == 'B' || pre == 'b') && ascii_isbdigit(ptr[2])) { // binary @@ -64,7 +62,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, if (what & STR2NR_OCT) { // Don't interpret "0", "08" or "0129" as octal. - for (int i = 1; i < maxlen && ascii_isdigit(ptr[i]); i++) { + for (int i = 1; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { if (ptr[i] > '7') { // can't be octal pre = 0; @@ -83,7 +81,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, uvarnumber_T un = 0; if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { // bin - while (ptr < e && '0' <= *ptr && *ptr <= '1') { + while (!STRING_ENDED(ptr) && '0' <= *ptr && *ptr <= '1') { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 2) { un = 2 * un + (uvarnumber_T)(*ptr - '0'); @@ -94,7 +92,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { // octal - while (ptr < e && '0' <= *ptr && *ptr <= '7') { + while (!STRING_ENDED(ptr) && '0' <= *ptr && *ptr <= '7') { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 8) { un = 8 * un + (uvarnumber_T)(*ptr - '0'); @@ -105,7 +103,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { // hex - while (ptr < e && ascii_isxdigit(*ptr)) { + while (!STRING_ENDED(ptr) && ascii_isxdigit(*ptr)) { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 16) { un = 16 * un + (uvarnumber_T)hex2nr(*ptr); @@ -116,7 +114,7 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, } } else { // decimal - while (ptr < e && ascii_isdigit(*ptr)) { + while (!STRING_ENDED(ptr) && ascii_isdigit(*ptr)) { // avoid ubsan error for overflow if (un < UVARNUMBER_MAX / 10) { un = 10 * un + (uvarnumber_T)(*ptr - '0'); @@ -154,4 +152,5 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, if (unptr != NULL) { *unptr = un; } +#undef STRING_ENDED } From fe81380bf5d4d161187998088aa9cff948b7c891 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 00:30:55 +0300 Subject: [PATCH 044/109] viml/parser/expressions: Highlight prefix separately from number Should make accidental octals more visible. --- src/nvim/viml/parser/expressions.c | 17 +++- test/unit/viml/expressions/parser_spec.lua | 113 ++++++++++++++++++++- 2 files changed, 128 insertions(+), 2 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 4801d66988..35f4385f33 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1042,6 +1042,7 @@ void viml_pexpr_free_ast(ExprAST ast) // // NVimRegister -> SpecialChar // NVimNumber -> Number +// NVimNumberPrefix -> SpecialChar // NVimFloat -> NVimNumber // // NVimNestingParenthesis -> NVimParenthesis @@ -1842,6 +1843,14 @@ static const int want_node_to_lexer_flags[] = { [kENodeArgumentSeparator] = kELFlagForbidScope, }; +/// Number of characters to highlight as NumberPrefix depending on the base +static const uint8_t base_to_prefix_length[] = { + [2] = 2, + [8] = 1, + [10] = 0, + [16] = 2, +}; + /// Parse one VimL expression /// /// @param pstate Parser state. @@ -2632,7 +2641,13 @@ viml_pexpr_parse_figure_brace_closing_error: } else { NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeInteger); cur_node->data.num.value = cur_token.data.num.val.integer; - HL_CUR_TOKEN(Number); + const uint8_t prefix_length = base_to_prefix_length[ + cur_token.data.num.base]; + viml_parser_highlight(pstate, cur_token.start, prefix_length, + HL(NumberPrefix)); + viml_parser_highlight( + pstate, shifted_pos(cur_token.start, prefix_length), + cur_token.len - prefix_length, HL(Number)); } want_node = kENodeOperator; *top_node_p = cur_node; diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 36af875bd2..54fd54aea8 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -6858,6 +6858,116 @@ describe('Expressions parser', function() hl('Number', '2'), }) end) + itp('highlights numbers with prefix', function() + check_parsing('0xABCDEF', 0, { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0xABCDEF', + }, + }, { + hl('NumberPrefix', '0x'), + hl('Number', 'ABCDEF'), + }) + + check_parsing('0Xabcdef', 0, { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0Xabcdef', + }, + }, { + hl('NumberPrefix', '0X'), + hl('Number', 'abcdef'), + }) + + check_parsing('0XABCDEF', 0, { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0XABCDEF', + }, + }, { + hl('NumberPrefix', '0X'), + hl('Number', 'ABCDEF'), + }) + + check_parsing('0xabcdef', 0, { + -- 01234567 + ast = { + 'Integer(val=11259375):0:0:0xabcdef', + }, + }, { + hl('NumberPrefix', '0x'), + hl('Number', 'abcdef'), + }) + + check_parsing('0b001', 0, { + -- 01234 + ast = { + 'Integer(val=1):0:0:0b001', + }, + }, { + hl('NumberPrefix', '0b'), + hl('Number', '001'), + }) + + check_parsing('0B001', 0, { + -- 01234 + ast = { + 'Integer(val=1):0:0:0B001', + }, + }, { + hl('NumberPrefix', '0B'), + hl('Number', '001'), + }) + + check_parsing('0B00', 0, { + -- 0123 + ast = { + 'Integer(val=0):0:0:0B00', + }, + }, { + hl('NumberPrefix', '0B'), + hl('Number', '00'), + }) + + check_parsing('00', 0, { + -- 01 + ast = { + 'Integer(val=0):0:0:00', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '0'), + }) + + check_parsing('001', 0, { + -- 012 + ast = { + 'Integer(val=1):0:0:001', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '01'), + }) + + check_parsing('01', 0, { + -- 01 + ast = { + 'Integer(val=1):0:0:01', + }, + }, { + hl('NumberPrefix', '0'), + hl('Number', '1'), + }) + + check_parsing('1', 0, { + -- 0 + ast = { + 'Integer(val=1):0:0:1', + }, + }, { + hl('Number', '1'), + }) + end) itp('works (KLEE tests)', function() check_parsing('\0002&A:\000', 0, { ast = nil, @@ -6879,7 +6989,8 @@ describe('Expressions parser', function() 'Integer(val=0):0:0:00', }, }, { - hl('Number', '00'), + hl('NumberPrefix', '0'), + hl('Number', '0'), }) end) end) From ed253b5fe6515840fe6dd9df83855a0316de8bad Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 00:39:48 +0300 Subject: [PATCH 045/109] klee: Include colors in test --- src/nvim/viml/parser/parser.h | 5 +++-- test/symbolic/klee/viml_expressions_parser.c | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h index 10ced57977..fbc5ba5f07 100644 --- a/src/nvim/viml/parser/parser.h +++ b/src/nvim/viml/parser/parser.h @@ -176,8 +176,9 @@ static inline void viml_parser_highlight(ParserState *const pstate, if (pstate->colors == NULL || len == 0) { return; } - // TODO(ZyX-I): May do some assert() sanitizing here. - // TODO(ZyX-I): May join chunks. + assert(kv_size(*pstate->colors) == 0 + || kv_Z(*pstate->colors, 0).start.line < start.line + || kv_Z(*pstate->colors, 0).end_col <= start.col); kvi_push(*pstate->colors, ((ParserHighlightChunk) { .start = start, .end_col = start.col + len, diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index ed280adb22..8f015ae9a7 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -75,6 +75,9 @@ int main(const int argc, const char *const *const argv, #endif ParserLine *cur_pline = &plines[0]; + ParserHighlight colors; + kvi_init(colors); + ParserState pstate = { .reader = { .get_line = simple_get_line, @@ -83,13 +86,18 @@ int main(const int argc, const char *const *const argv, .conv.vc_type = CONV_NONE, }, .pos = { 0, 0 }, - .colors = NULL, + .colors = &colors, .can_continuate = false, }; kvi_init(pstate.reader.lines); const ExprAST ast = viml_pexpr_parse(&pstate, flags); assert(ast.root != NULL || ast.err.msg); + // Can’t possibly have more highlight tokens then there are bytes in string. + assert(kv_size(colors) <= INPUT_SIZE - shift); + kvi_destroy(colors); + // Not destroying pstate.reader.lines because there is no way it could exceed + // its limits in the current circumstances. viml_pexpr_free_ast(ast); assert(allocated_memory == 0); } From 15043e93b681ffdf1142d24aa918997e2e63a4b1 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 00:41:41 +0300 Subject: [PATCH 046/109] klee: Update key_name_entry table --- src/nvim/keymap.c | 3 +- test/symbolic/klee/nvim/keymap.c | 272 +++++++++++++++---------------- 2 files changed, 136 insertions(+), 139 deletions(-) diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index 55ac835663..4f120e66f7 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -141,8 +141,7 @@ static char_u modifier_keys_table[] = static const struct key_name_entry { int key; // Special key code or ascii value const char *name; // Name of key -} key_names_table[] = -{ +} key_names_table[] = { { ' ', "Space" }, { TAB, "Tab" }, { K_TAB, "Tab" }, diff --git a/test/symbolic/klee/nvim/keymap.c b/test/symbolic/klee/nvim/keymap.c index c59d8d4d6e..ff5e46e75b 100644 --- a/test/symbolic/klee/nvim/keymap.c +++ b/test/symbolic/klee/nvim/keymap.c @@ -141,154 +141,152 @@ int handle_x_keys(const int key) } static const struct key_name_entry { - int key; ///< Special key code or ASCII value. - const char *name; ///< Name of the key + int key; // Special key code or ascii value + const char *name; // Name of key } key_names_table[] = { - {' ', "Space"}, - {TAB, "Tab"}, - {K_TAB, "Tab"}, - {NL, "NL"}, - {NL, "NewLine"}, // Alternative name - {NL, "LineFeed"}, // Alternative name - {NL, "LF"}, // Alternative name - {CAR, "CR"}, - {CAR, "Return"}, // Alternative name - {CAR, "Enter"}, // Alternative name - {K_BS, "BS"}, - {K_BS, "BackSpace"}, // Alternative name - {ESC, "Esc"}, - {CSI, "CSI"}, - {K_CSI, "xCSI"}, - {'|', "Bar"}, - {'\\', "Bslash"}, - {K_DEL, "Del"}, - {K_DEL, "Delete"}, // Alternative name - {K_KDEL, "kDel"}, - {K_UP, "Up"}, - {K_DOWN, "Down"}, - {K_LEFT, "Left"}, - {K_RIGHT, "Right"}, - {K_XUP, "xUp"}, - {K_XDOWN, "xDown"}, - {K_XLEFT, "xLeft"}, - {K_XRIGHT, "xRight"}, + { ' ', "Space" }, + { TAB, "Tab" }, + { K_TAB, "Tab" }, + { NL, "NL" }, + { NL, "NewLine" }, // Alternative name + { NL, "LineFeed" }, // Alternative name + { NL, "LF" }, // Alternative name + { CAR, "CR" }, + { CAR, "Return" }, // Alternative name + { CAR, "Enter" }, // Alternative name + { K_BS, "BS" }, + { K_BS, "BackSpace" }, // Alternative name + { ESC, "Esc" }, + { CSI, "CSI" }, + { K_CSI, "xCSI" }, + { '|', "Bar" }, + { '\\', "Bslash" }, + { K_DEL, "Del" }, + { K_DEL, "Delete" }, // Alternative name + { K_KDEL, "kDel" }, + { K_UP, "Up" }, + { K_DOWN, "Down" }, + { K_LEFT, "Left" }, + { K_RIGHT, "Right" }, + { K_XUP, "xUp" }, + { K_XDOWN, "xDown" }, + { K_XLEFT, "xLeft" }, + { K_XRIGHT, "xRight" }, - {K_F1, "F1"}, - {K_F2, "F2"}, - {K_F3, "F3"}, - {K_F4, "F4"}, - {K_F5, "F5"}, - {K_F6, "F6"}, - {K_F7, "F7"}, - {K_F8, "F8"}, - {K_F9, "F9"}, - {K_F10, "F10"}, + { K_F1, "F1" }, + { K_F2, "F2" }, + { K_F3, "F3" }, + { K_F4, "F4" }, + { K_F5, "F5" }, + { K_F6, "F6" }, + { K_F7, "F7" }, + { K_F8, "F8" }, + { K_F9, "F9" }, + { K_F10, "F10" }, - {K_F11, "F11"}, - {K_F12, "F12"}, - {K_F13, "F13"}, - {K_F14, "F14"}, - {K_F15, "F15"}, - {K_F16, "F16"}, - {K_F17, "F17"}, - {K_F18, "F18"}, - {K_F19, "F19"}, - {K_F20, "F20"}, + { K_F11, "F11" }, + { K_F12, "F12" }, + { K_F13, "F13" }, + { K_F14, "F14" }, + { K_F15, "F15" }, + { K_F16, "F16" }, + { K_F17, "F17" }, + { K_F18, "F18" }, + { K_F19, "F19" }, + { K_F20, "F20" }, - {K_F21, "F21"}, - {K_F22, "F22"}, - {K_F23, "F23"}, - {K_F24, "F24"}, - {K_F25, "F25"}, - {K_F26, "F26"}, - {K_F27, "F27"}, - {K_F28, "F28"}, - {K_F29, "F29"}, - {K_F30, "F30"}, + { K_F21, "F21" }, + { K_F22, "F22" }, + { K_F23, "F23" }, + { K_F24, "F24" }, + { K_F25, "F25" }, + { K_F26, "F26" }, + { K_F27, "F27" }, + { K_F28, "F28" }, + { K_F29, "F29" }, + { K_F30, "F30" }, - {K_F31, "F31"}, - {K_F32, "F32"}, - {K_F33, "F33"}, - {K_F34, "F34"}, - {K_F35, "F35"}, - {K_F36, "F36"}, - {K_F37, "F37"}, + { K_F31, "F31" }, + { K_F32, "F32" }, + { K_F33, "F33" }, + { K_F34, "F34" }, + { K_F35, "F35" }, + { K_F36, "F36" }, + { K_F37, "F37" }, - {K_XF1, "xF1"}, - {K_XF2, "xF2"}, - {K_XF3, "xF3"}, - {K_XF4, "xF4"}, + { K_XF1, "xF1" }, + { K_XF2, "xF2" }, + { K_XF3, "xF3" }, + { K_XF4, "xF4" }, - {K_HELP, "Help"}, - {K_UNDO, "Undo"}, - {K_INS, "Insert"}, - {K_INS, "Ins"}, // Alternative name - {K_KINS, "kInsert"}, - {K_HOME, "Home"}, - {K_KHOME, "kHome"}, - {K_XHOME, "xHome"}, - {K_ZHOME, "zHome"}, - {K_END, "End"}, - {K_KEND, "kEnd"}, - {K_XEND, "xEnd"}, - {K_ZEND, "zEnd"}, - {K_PAGEUP, "PageUp"}, - {K_PAGEDOWN, "PageDown"}, - {K_KPAGEUP, "kPageUp"}, - {K_KPAGEDOWN, "kPageDown"}, + { K_HELP, "Help" }, + { K_UNDO, "Undo" }, + { K_INS, "Insert" }, + { K_INS, "Ins" }, // Alternative name + { K_KINS, "kInsert" }, + { K_HOME, "Home" }, + { K_KHOME, "kHome" }, + { K_XHOME, "xHome" }, + { K_ZHOME, "zHome" }, + { K_END, "End" }, + { K_KEND, "kEnd" }, + { K_XEND, "xEnd" }, + { K_ZEND, "zEnd" }, + { K_PAGEUP, "PageUp" }, + { K_PAGEDOWN, "PageDown" }, + { K_KPAGEUP, "kPageUp" }, + { K_KPAGEDOWN, "kPageDown" }, - {K_KPLUS, "kPlus"}, - {K_KMINUS, "kMinus"}, - {K_KDIVIDE, "kDivide"}, - {K_KMULTIPLY, "kMultiply"}, - {K_KENTER, "kEnter"}, - {K_KPOINT, "kPoint"}, + { K_KPLUS, "kPlus" }, + { K_KMINUS, "kMinus" }, + { K_KDIVIDE, "kDivide" }, + { K_KMULTIPLY, "kMultiply" }, + { K_KENTER, "kEnter" }, + { K_KPOINT, "kPoint" }, - {K_K0, "k0"}, - {K_K1, "k1"}, - {K_K2, "k2"}, - {K_K3, "k3"}, - {K_K4, "k4"}, - {K_K5, "k5"}, - {K_K6, "k6"}, - {K_K7, "k7"}, - {K_K8, "k8"}, - {K_K9, "k9"}, + { K_K0, "k0" }, + { K_K1, "k1" }, + { K_K2, "k2" }, + { K_K3, "k3" }, + { K_K4, "k4" }, + { K_K5, "k5" }, + { K_K6, "k6" }, + { K_K7, "k7" }, + { K_K8, "k8" }, + { K_K9, "k9" }, - {'<', "lt"}, + { '<', "lt" }, - {K_MOUSE, "Mouse"}, - {K_LEFTMOUSE, "LeftMouse"}, - {K_LEFTMOUSE_NM, "LeftMouseNM"}, - {K_LEFTDRAG, "LeftDrag"}, - {K_LEFTRELEASE, "LeftRelease"}, - {K_LEFTRELEASE_NM, "LeftReleaseNM"}, - {K_MIDDLEMOUSE, "MiddleMouse"}, - {K_MIDDLEDRAG, "MiddleDrag"}, - {K_MIDDLERELEASE, "MiddleRelease"}, - {K_RIGHTMOUSE, "RightMouse"}, - {K_RIGHTDRAG, "RightDrag"}, - {K_RIGHTRELEASE, "RightRelease"}, - {K_MOUSEDOWN, "ScrollWheelUp"}, - {K_MOUSEUP, "ScrollWheelDown"}, - {K_MOUSELEFT, "ScrollWheelRight"}, - {K_MOUSERIGHT, "ScrollWheelLeft"}, - {K_MOUSEDOWN, "MouseDown"}, // OBSOLETE: Use ScrollWheelXXX instead - {K_MOUSEUP, "MouseUp"}, // Same - {K_X1MOUSE, "X1Mouse"}, - {K_X1DRAG, "X1Drag"}, - {K_X1RELEASE, "X1Release"}, - {K_X2MOUSE, "X2Mouse"}, - {K_X2DRAG, "X2Drag"}, - {K_X2RELEASE, "X2Release"}, - {K_DROP, "Drop"}, - {K_ZERO, "Nul"}, - {K_SNR, "SNR"}, - {K_PLUG, "Plug"}, - {K_PASTE, "Paste"}, - {K_FOCUSGAINED, "FocusGained"}, - {K_FOCUSLOST, "FocusLost"}, - {0, NULL} + { K_MOUSE, "Mouse" }, + { K_LEFTMOUSE, "LeftMouse" }, + { K_LEFTMOUSE_NM, "LeftMouseNM" }, + { K_LEFTDRAG, "LeftDrag" }, + { K_LEFTRELEASE, "LeftRelease" }, + { K_LEFTRELEASE_NM, "LeftReleaseNM" }, + { K_MIDDLEMOUSE, "MiddleMouse" }, + { K_MIDDLEDRAG, "MiddleDrag" }, + { K_MIDDLERELEASE, "MiddleRelease" }, + { K_RIGHTMOUSE, "RightMouse" }, + { K_RIGHTDRAG, "RightDrag" }, + { K_RIGHTRELEASE, "RightRelease" }, + { K_MOUSEDOWN, "ScrollWheelUp" }, + { K_MOUSEUP, "ScrollWheelDown" }, + { K_MOUSELEFT, "ScrollWheelRight" }, + { K_MOUSERIGHT, "ScrollWheelLeft" }, + { K_MOUSEDOWN, "MouseDown" }, // OBSOLETE: Use + { K_MOUSEUP, "MouseUp" }, // ScrollWheelXXX instead + { K_X1MOUSE, "X1Mouse" }, + { K_X1DRAG, "X1Drag" }, + { K_X1RELEASE, "X1Release" }, + { K_X2MOUSE, "X2Mouse" }, + { K_X2DRAG, "X2Drag" }, + { K_X2RELEASE, "X2Release" }, + { K_DROP, "Drop" }, + { K_ZERO, "Nul" }, + { K_SNR, "SNR" }, + { K_PLUG, "Plug" }, + { K_PASTE, "Paste" }, + { 0, NULL } }; int get_special_key_code(const char_u *name) From a535d68380e472a661478cd6a65aad243ebd6a82 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 01:00:58 +0300 Subject: [PATCH 047/109] keymap: Remove incorrect cast --- src/nvim/keymap.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index 4f120e66f7..6ed54464e8 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -696,11 +696,14 @@ int find_special_key_in_table(int c) { int i; - for (i = 0; key_names_table[i].name != NULL; i++) - if (c == (uint8_t)key_names_table[i].key) + for (i = 0; key_names_table[i].name != NULL; i++) { + if (c == key_names_table[i].key) { break; - if (key_names_table[i].name == NULL) + } + } + if (key_names_table[i].name == NULL) { i = -1; + } return i; } From 248493f155e42186440c7d081b27ffe760b67b9e Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 03:03:34 +0300 Subject: [PATCH 048/109] test/unit/formatc: Fix parsing of most recent viml_parser_highlight --- test/unit/formatc.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/unit/formatc.lua b/test/unit/formatc.lua index e288081960..2fd37c599a 100644 --- a/test/unit/formatc.lua +++ b/test/unit/formatc.lua @@ -65,11 +65,12 @@ local tokens = P { "tokens"; identifier = Ct(C(R("az","AZ","__") * R("09","az","AZ","__")^0) * Cc"identifier"), -- Single character in a string - string_char = R("az","AZ","09") + S"$%^&*()_-+={[}]:;@~#<,>.!?/ \t" + (P"\\" * S[[ntvbrfa\?'"0x]]), + sstring_char = R("\001&","([","]\255") + (P"\\" * S[[ntvbrfa\?'"0x]]), + dstring_char = R("\001!","#[","]\255") + (P"\\" * S[[ntvbrfa\?'"0x]]), -- String literal - string = Ct(C(P"'" * (V"string_char" + P'"')^0 * P"'" + - P'"' * (V"string_char" + P"'")^0 * P'"') * Cc"string"), + string = Ct(C(P"'" * (V"sstring_char" + P'"')^0 * P"'" + + P'"' * (V"dstring_char" + P"'")^0 * P'"') * Cc"string"), -- Operator operator = Ct(C(P">>=" + P"<<=" + P"..." + From 4c8ed65b608df06b4c72b641f4ecc86985295633 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 03:04:22 +0300 Subject: [PATCH 049/109] viml/parser/expressions: Fix memory leak when processing ternary --- src/nvim/viml/parser/expressions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 35f4385f33..876fbc8d37 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -2308,10 +2308,10 @@ viml_pexpr_parse_invalid_colon: _("E15: Colon outside of dictionary or ternary operator: %.*s")); viml_pexpr_parse_valid_colon: ADD_VALUE_IF_MISSING(_(EXP_VAL_COLON)); - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); if (is_ternary) { HL_CUR_TOKEN(TernaryColon); } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeColon); ADD_OP_NODE(cur_node); HL_CUR_TOKEN(Colon); } From c03dc13bb74205d15a83ce3bd6ecb6b76b869878 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 03:05:27 +0300 Subject: [PATCH 050/109] klee: Fix possible assertion error No idea how it did not happen to hit me yet. --- test/symbolic/klee/nvim/memory.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/symbolic/klee/nvim/memory.c b/test/symbolic/klee/nvim/memory.c index 1f9cdce6c0..df422cea3e 100644 --- a/test/symbolic/klee/nvim/memory.c +++ b/test/symbolic/klee/nvim/memory.c @@ -35,6 +35,9 @@ void *xmalloc(const size_t size) void xfree(void *const p) { + if (p == NULL) { + return; + } RINGBUF_FORALL(&arecs, AllocRecord, arec) { if (arec->ptr == p) { allocated_memory -= arec->size; From 252a76db80dd846f9ccac4d7001697c12c009826 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 03:06:34 +0300 Subject: [PATCH 051/109] unittests: Free everything and check for memory leaks Also improves error reporting. --- test/unit/helpers.lua | 11 ++++- test/unit/viml/expressions/parser_spec.lua | 56 ++++++++++++++-------- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index d3d14a5ca2..68ce9eed62 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -315,7 +315,7 @@ local function alloc_log_new() eq(exp, self.log) self:clear() end - function log:clear_tmp_allocs() + function log:clear_tmp_allocs(clear_null_frees) local toremove = {} local allocs = {} for i, v in ipairs(self.log) do @@ -327,6 +327,8 @@ local function alloc_log_new() if v.func == 'free' then toremove[#toremove + 1] = i end + elseif clear_null_frees and v.args[1] == self.null then + toremove[#toremove + 1] = i end if v.func == 'realloc' then allocs[tostring(v.ret)] = i @@ -779,6 +781,12 @@ local function kvi_init(kvi) return kvi end +local function kvi_destroy(kvi) + if kvi.items ~= kvi.init_array then + lib.xfree(kvi.items) + end +end + local function kvi_new(ct) return kvi_init(ffi.new(ct)) end @@ -830,6 +838,7 @@ local module = { sc = sc, conv_enum = conv_enum, array_size = array_sive, + kvi_destroy = kvi_destroy, kvi_size = kvi_size, kvi_init = kvi_init, kvi_new = kvi_new, diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 54fd54aea8..5f5924c630 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -5,6 +5,8 @@ local viml_helpers = require('test.unit.viml.helpers') local make_enum_conv_tab = helpers.make_enum_conv_tab local child_call_once = helpers.child_call_once +local alloc_log_new = helpers.alloc_log_new +local kvi_destroy = helpers.kvi_destroy local conv_enum = helpers.conv_enum local ptr2key = helpers.ptr2key local cimport = helpers.cimport @@ -23,6 +25,8 @@ local format_luav = global_helpers.format_luav local lib = cimport('./src/nvim/viml/parser/expressions.h') +local alloc_log = alloc_log_new() + local function format_check(expr, flags, ast, hls) -- That forces specific order. print( format_string('\ncheck_parsing(%r, %u, {', expr, flags)) @@ -230,30 +234,40 @@ end) describe('Expressions parser', function() local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) - flags = flags or 0 + local err, msg = pcall(function() + flags = flags or 0 - if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then - print(str, flags) - end - - local pstate = new_pstate({str}) - local east = lib.viml_pexpr_parse(pstate, flags) - local ast = east2lua(pstate, east) - local hls = phl2lua(pstate) - if exp_ast == nil then - format_check(str, flags, ast, hls) - return - end - eq(exp_ast, ast) - if exp_highlighting_fs then - local exp_highlighting = {} - local next_col = 0 - for i, h in ipairs(exp_highlighting_fs) do - exp_highlighting[i], next_col = h(next_col) + if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then + print(str, flags) end - eq(exp_highlighting, hls) + alloc_log:check({}) + + local pstate = new_pstate({str}) + local east = lib.viml_pexpr_parse(pstate, flags) + local ast = east2lua(pstate, east) + local hls = phl2lua(pstate) + if exp_ast == nil then + format_check(str, flags, ast, hls) + return + end + eq(exp_ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exp_highlighting_fs) do + exp_highlighting[i], next_col = h(next_col) + end + eq(exp_highlighting, hls) + end + lib.viml_pexpr_free_ast(east) + kvi_destroy(pstate.colors) + alloc_log:clear_tmp_allocs(true) + alloc_log:check({}) + end) + if not err then + msg = format_string('Error while processing test (%r, %u):\n%s', str, flags, msg) + error(msg) end - lib.viml_pexpr_free_ast(east) end local function hl(group, str, shift) return function(next_col) From 8e856ebcd0357c8782d6825a233a5c800475f396 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 08:51:26 +0300 Subject: [PATCH 052/109] klee: Add run.sh --help and run.sh -s --- test/symbolic/klee/run.sh | 51 ++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/test/symbolic/klee/run.sh b/test/symbolic/klee/run.sh index c143cd0624..0234a935b5 100755 --- a/test/symbolic/klee/run.sh +++ b/test/symbolic/klee/run.sh @@ -10,13 +10,43 @@ KLEE_TEST_DIR="$PROJECT_SOURCE_DIR/test/symbolic/klee" KLEE_BIN_DIR="$PROJECT_BINARY_DIR/klee" KLEE_OUT_DIR="$KLEE_BIN_DIR/out" +help() { + echo "Usage:" + echo + echo " $0 -c fname" + echo " $0 fname" + echo " $0 -s" + echo + echo "First form compiles executable out of test/symbolic/klee/{fname}.c." + echo "Compiled executable is placed into build/klee/{fname}. Must first" + echo "successfully compile Neovim in order to generate declarations." + echo + echo "Second form runs KLEE in a docker container using file " + echo "test/symbolic/klee/{fname.c}. Bitcode is placed into build/klee/a.bc," + echo "results are placed into build/klee/out/. The latter is first deleted if" + echo "it exists." + echo + echo "Third form runs ktest-tool which prints errors found by KLEE via " + echo "the same container used to run KLEE." +} + main() { local compile= - if test "$1" = "-c" ; then + local print_errs= + if test "$1" = "--help" ; then + help + return + fi + if test "$1" = "-s" ; then + print_errs=1 + shift + elif test "$1" = "-c" ; then compile=1 shift fi - local test="$1" ; shift + if test -z "$print_errs" ; then + local test="$1" ; shift + fi local includes= includes="$includes -I$KLEE_TEST_DIR" @@ -36,14 +66,17 @@ main() { test -d "$KLEE_BIN_DIR" || mkdir -p "$KLEE_BIN_DIR" if test -z "$compile" ; then - test -d "$KLEE_OUT_DIR" && rm -r "$KLEE_OUT_DIR" local line1='cd /image' - line1="$line1 && $(echo clang \ - $includes $defines \ - -o "$KLEE_BIN_DIR/a.bc" -emit-llvm -g -c \ - "$KLEE_TEST_DIR/$test.c")" - line1="$line1 && klee --libc=uclibc --posix-runtime " - line1="$line1 '--output-dir=$KLEE_OUT_DIR' '$KLEE_BIN_DIR/a.bc'" + if test -z "$print_errs" ; then + test -d "$KLEE_OUT_DIR" && rm -r "$KLEE_OUT_DIR" + + line1="$line1 && $(echo clang \ + $includes $defines \ + -o "$KLEE_BIN_DIR/a.bc" -emit-llvm -g -c \ + "$KLEE_TEST_DIR/$test.c")" + line1="$line1 && klee --libc=uclibc --posix-runtime " + line1="$line1 '--output-dir=$KLEE_OUT_DIR' '$KLEE_BIN_DIR/a.bc'" + fi local line2="for t in '$KLEE_OUT_DIR'/*.err" line2="$line2 ; do ktest-tool --write-ints" line2="$line2 \"\$(printf '%s' \"\$t\" | sed -e 's@\\.[^/]*\$@.ktest@')\"" From c9f511d24a64da135bef4b9874c7bec04d9330e4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 09:06:05 +0300 Subject: [PATCH 053/109] viml/parser/expressions: Remove unused flag --- src/nvim/viml/parser/expressions.h | 4 ---- test/symbolic/klee/viml_expressions_parser.c | 7 +++---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 025f0f766e..d783518b3a 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -314,10 +314,6 @@ enum { /// When parsing expressions input by user bar is assumed to be a binary /// operator and other two are spacings. kExprFlagsDisallowEOC = (1 << 1), - /// Print errors when encountered - /// - /// Without the flag they are only taken into account when parsing. - kExprFlagsPrintError = (1 << 2), // WARNING: whenever you add a new flag, alter klee_assume() statement in // viml_expressions_parser.c. } ExprParserFlags; diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index 8f015ae9a7..9448937430 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -35,7 +35,7 @@ int main(const int argc, const char *const *const argv, { char input[INPUT_SIZE]; uint8_t shift; - int flags; + unsigned flags; const bool peek = false; avoid_optimizing_out = argc; @@ -48,8 +48,7 @@ int main(const int argc, const char *const *const argv, klee_make_symbolic(&shift, sizeof(shift), "shift"); klee_make_symbolic(&flags, sizeof(flags), "flags"); klee_assume(shift < INPUT_SIZE); - klee_assume( - flags <= (kExprFlagsMulti|kExprFlagsDisallowEOC|kExprFlagsPrintError)); + klee_assume(flags <= (kExprFlagsMulti|kExprFlagsDisallowEOC)); #endif ParserLine plines[] = { @@ -91,7 +90,7 @@ int main(const int argc, const char *const *const argv, }; kvi_init(pstate.reader.lines); - const ExprAST ast = viml_pexpr_parse(&pstate, flags); + const ExprAST ast = viml_pexpr_parse(&pstate, (int)flags); assert(ast.root != NULL || ast.err.msg); // Can’t possibly have more highlight tokens then there are bytes in string. assert(kv_size(colors) <= INPUT_SIZE - shift); From 895793fc820e04ea2d6bdaa90c6643c4dce2f0e7 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 16 Oct 2017 09:14:02 +0300 Subject: [PATCH 054/109] viml/parser/expressions: Add some casts --- src/nvim/viml/parser/expressions.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 876fbc8d37..69817bf24f 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -698,8 +698,8 @@ const char *viml_pexpr_repr_token(const ParserState *const pstate, (int)token.data.num.is_float, (int)token.data.num.base, (double)(token.data.num.is_float - ? token.data.num.val.floating - : token.data.num.val.integer)) + ? (double)token.data.num.val.floating + : (double)token.data.num.val.integer)) TKNARGS(kExprLexInvalid, "(msg=%s)", token.data.err.msg) default: { // No additional arguments. From 47938e1e22816381f26e8882eacd6e7e8baf37fd Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 19 Oct 2017 10:48:05 +0300 Subject: [PATCH 055/109] viml/parser/expressions: Fix some errors spotted by KLEE Not all of them are fixed yet though. --- src/nvim/viml/parser/expressions.c | 58 ++++++--- test/unit/viml/expressions/parser_spec.lua | 139 +++++++++++++++++++++ 2 files changed, 178 insertions(+), 19 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 69817bf24f..da22cf4cdb 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1091,7 +1091,7 @@ void viml_pexpr_free_ast(ExprAST ast) // NVimInvalidDoubleQuote -> NVimInvalidString // NVimInvalidDoubleQuotedBody -> NVimInvalidString // NVimInvalidDoubleQuotedEscape -> NVimInvalidStringSpecial -// NVimInvalidDoubleQuotedUnknownEscape -> NVimInvalid +// NVimInvalidDoubleQuotedUnknownEscape -> NVimInvalidDoubleQuotedEscape // // NVimFigureBrace -> NVimInternalError // NVimInvalidSingleQuotedUnknownEscape -> NVimInternalError @@ -1313,7 +1313,7 @@ static bool viml_pexpr_handle_bop(const ParserState *const pstate, /// ParserPosition literal based on ParserPosition pos with columns shifted /// -/// Function does not check whether remaining position is valid. +/// Function does not check whether resulting position is valid. /// /// @param[in] pos Position to shift. /// @param[in] shift Number of bytes to shift. @@ -1326,6 +1326,21 @@ static inline ParserPosition shifted_pos(const ParserPosition pos, return (ParserPosition) { .line = pos.line, .col = pos.col + shift }; } +/// ParserPosition literal based on ParserPosition pos with specified column +/// +/// Function does not check whether remaining position is valid. +/// +/// @param[in] pos Position to adjust. +/// @param[in] new_col New column. +/// +/// @return Shifted position. +static inline ParserPosition recol_pos(const ParserPosition pos, + const size_t new_col) + FUNC_ATTR_CONST FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT +{ + return (ParserPosition) { .line = pos.line, .col = new_col }; +} + /// Get highlight group name #define HL(g) (is_invalid ? "NVimInvalid" #g : "NVim" #g) @@ -1639,7 +1654,7 @@ static void parse_quoted_string(ParserState *const pstate, size_t n = (*p == 'u' ? 4 : 8); int nr = 0; p++; - while (n-- && ascii_isxdigit(p[1])) { + while (p + 1 < e && n-- && ascii_isxdigit(p[1])) { p++; nr = (nr << 4) + hex2nr(*p); } @@ -1659,7 +1674,7 @@ static void parse_quoted_string(ParserState *const pstate, if (*p >= '0' && *p <= '7') { size--; p++; - if (*p >= '0' && *p <= '7') { + if (p < e && *p >= '0' && *p <= '7') { size--; p++; } @@ -1715,7 +1730,7 @@ static void parse_quoted_string(ParserState *const pstate, // Hexadecimal or unicode. case 'X': case 'x': case 'u': case 'U': { - if (ascii_isxdigit(p[1])) { + if (p + 1 < e && ascii_isxdigit(p[1])) { size_t n; int nr; bool is_hex = (*p == 'x' || *p == 'X'); @@ -1728,7 +1743,7 @@ static void parse_quoted_string(ParserState *const pstate, n = 8; } nr = 0; - while (n-- && ascii_isxdigit(p[1])) { + while (p + 1 < e && n-- && ascii_isxdigit(p[1])) { p++; nr = (nr << 4) + hex2nr(*p); } @@ -1749,9 +1764,9 @@ static void parse_quoted_string(ParserState *const pstate, case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': { uint8_t ch = (uint8_t)(*p++ - '0'); - if (*p >= '0' && *p <= '7') { + if (p < e && *p >= '0' && *p <= '7') { ch = (uint8_t)((ch << 3) + *p++ - '0'); - if (*p >= '0' && *p <= '7') { + if (p < e && *p >= '0' && *p <= '7') { ch = (uint8_t)((ch << 3) + *p++ - '0'); } } @@ -1793,7 +1808,7 @@ static void parse_quoted_string(ParserState *const pstate, // TODO(ZyX-I): use ast_stack to determine and highlight regular expressions // TODO(ZyX-I): use ast_stack to determine and highlight printf format str // TODO(ZyX-I): use ast_stack to determine and highlight expression strings - size_t next_col = 1; + size_t next_col = token.start.col + 1; const char *const body_str = (is_double ? HL(DoubleQuotedBody) : HL(SingleQuotedBody)); @@ -1806,20 +1821,23 @@ static void parse_quoted_string(ParserState *const pstate, for (size_t i = 0; i < kv_size(shifts); i++) { const StringShift cur_shift = kv_A(shifts, i); if (cur_shift.start > next_col) { - viml_parser_highlight(pstate, shifted_pos(token.start, next_col), + viml_parser_highlight(pstate, recol_pos(token.start, next_col), cur_shift.start - next_col, body_str); } - viml_parser_highlight(pstate, shifted_pos(token.start, cur_shift.start), + viml_parser_highlight(pstate, recol_pos(token.start, cur_shift.start), cur_shift.orig_len, (cur_shift.escape_not_known ? ukn_esc_str : esc_str)); next_col = cur_shift.start + cur_shift.orig_len; } - if (next_col < token.len - token.data.str.closed) { - viml_parser_highlight(pstate, shifted_pos(token.start, next_col), - token.len - token.data.str.closed - next_col, + if (next_col - token.start.col < token.len - token.data.str.closed) { + viml_parser_highlight(pstate, recol_pos(token.start, next_col), + (token.start.col + + token.len + - token.data.str.closed + - next_col), body_str); } } @@ -2580,6 +2598,9 @@ viml_pexpr_parse_figure_brace_closing_error: break; } case kExprLexPlainIdentifier: { + const ExprVarScope scope = (cur_token.type == kExprLexInvalid + ? kExprVarScopeMissing + : cur_token.data.var.scope); if (want_node == kENodeValue || want_node == kENodeArgument) { want_node = (want_node == kENodeArgument ? kENodeArgumentSeparator @@ -2588,9 +2609,8 @@ viml_pexpr_parse_figure_brace_closing_error: (node_is_key ? kExprNodePlainKey : kExprNodePlainIdentifier)); - cur_node->data.var.scope = cur_token.data.var.scope; - const size_t scope_shift = ( - cur_token.data.var.scope == kExprVarScopeMissing ? 0 : 2); + cur_node->data.var.scope = scope; + const size_t scope_shift = (scope == kExprVarScopeMissing ? 0 : 2); cur_node->data.var.ident = (pline.data + cur_token.start.col + scope_shift); cur_node->data.var.ident_len = cur_token.len - scope_shift; @@ -2609,11 +2629,11 @@ viml_pexpr_parse_figure_brace_closing_error: ? HL(IdentifierKey) : HL(Identifier))); } else { - if (cur_token.data.var.scope == kExprVarScopeMissing) { + if (scope == kExprVarScopeMissing) { ADD_IDENT( do { NEW_NODE_WITH_CUR_POS(cur_node, kExprNodePlainIdentifier); - cur_node->data.var.scope = cur_token.data.var.scope; + cur_node->data.var.scope = scope; cur_node->data.var.ident = pline.data + cur_token.start.col; cur_node->data.var.ident_len = cur_token.len; want_node = kENodeOperator; diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 5f5924c630..b81a3c0f82 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -7006,5 +7006,144 @@ describe('Expressions parser', function() hl('NumberPrefix', '0'), hl('Number', '0'), }) + check_parsing('"\\U\\', 0, { + -- 0123 + ast = { + [[DoubleQuotedString(val="U\\"):0:0:"\U\]], + }, + err = { + arg = '"\\U\\', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + hl('InvalidDoubleQuotedBody', '\\'), + }) + check_parsing('"\\U', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="U"):0:0:"\\U', + }, + err = { + arg = '"\\U', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + }) + check_parsing('|"\\U\\', 2, { + -- 01234 + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + 'DoubleQuotedString(val="U\\\\"):0:1:"\\U\\', + }, + }, + }, + err = { + arg = '|"\\U\\', + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, { + hl('InvalidOr', '|'), + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + hl('InvalidDoubleQuotedBody', '\\'), + }) + check_parsing('|"\\e"', 2, { + -- 01234 + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + 'DoubleQuotedString(val="\\27"):0:1:"\\e"', + }, + }, + }, + err = { + arg = '|"\\e"', + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, { + hl('InvalidOr', '|'), + hl('DoubleQuotedString', '"'), + hl('DoubleQuotedEscape', '\\e'), + hl('DoubleQuotedString', '"'), + }) + check_parsing('|\029', 2, { + -- 01 + ast = { + { + 'Or:0:0:|', + children = { + 'Missing:0:0:', + 'PlainIdentifier(scope=0,ident=\029):0:1:\029', + }, + }, + }, + err = { + arg = '|\029', + msg = 'E15: Unexpected EOC character: %.*s', + }, + }, { + hl('InvalidOr', '|'), + hl('InvalidIdentifier', '\029'), + }) + check_parsing('"\\<', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="<"):0:0:"\\<', + }, + err = { + arg = '"\\<', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\<'), + }) + check_parsing('"\\1', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="\\1"):0:0:"\\1', + }, + err = { + arg = '"\\1', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuotedEscape', '\\1'), + }) + check_parsing('}l') + check_parsing(':?\000\000\000\000\000\000\000', 0, { + ast = { + { + 'Colon:0:0::', + children = { + 'Missing:0:0:', + { + 'Ternary:0:1:?', + children = { + 'Missing:0:1:', + 'TernaryValue:0:1:?', + }, + }, + }, + }, + }, + err = { + arg = ':?', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('InvalidColon', ':'), + hl('InvalidTernary', '?'), + }) end) end) From b574e95850c064d4e746a7373aff1a3d7bd4de27 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 01:17:00 +0300 Subject: [PATCH 056/109] charset: Some more refactoring of vim_str2nr --- src/nvim/charset.c | 78 ++++++++++++++++++---------------------------- src/nvim/version.c | 2 ++ 2 files changed, 32 insertions(+), 48 deletions(-) diff --git a/src/nvim/charset.c b/src/nvim/charset.c index a7ffa4bc00..c895d65eb7 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -1656,18 +1656,19 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // decimal or octal, default is decimal pre = 0; - if (what & STR2NR_OCT) { - // Don't interpret "0", "08" or "0129" as octal. - for (int i = 1; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { + if (what & STR2NR_OCT + && !STRING_ENDED(ptr + 1) + && ('0' <= ptr[1] && ptr[1] <= '7')) { + // Assume octal now: what we already know is that string starts with + // zero and some octal digit. + pre = '0'; + // Don’t interpret "0", "008" or "0129" as octal. + for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { if (ptr[i] > '7') { - // can't be octal + // Can’t be octal. pre = 0; break; } - if (ptr[i] >= '0') { - // assume octal - pre = '0'; - } } } } @@ -1675,51 +1676,32 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. uvarnumber_T un = 0; +#define PARSE_NUMBER(base, cond, conv) \ + do { \ + while (!STRING_ENDED(ptr) && (cond)) { \ + /* avoid ubsan error for overflow */ \ + if (un < UVARNUMBER_MAX / base) { \ + un = base * un + (uvarnumber_T)(conv); \ + } else { \ + un = UVARNUMBER_MAX; \ + } \ + ptr++; \ + } \ + } while (0) if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { - // bin - while (!STRING_ENDED(ptr) && '0' <= *ptr && *ptr <= '1') { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 2) { - un = 2 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - } + // Binary number. + PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { - // octal - while (!STRING_ENDED(ptr) && '0' <= *ptr && *ptr <= '7') { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 8) { - un = 8 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - } + // Octal number. + PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { - // hex - while (!STRING_ENDED(ptr) && ascii_isxdigit(*ptr)) { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 16) { - un = 16 * un + (uvarnumber_T)hex2nr(*ptr); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - } + // Hexadecimal number. + PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); } else { - // decimal - while (!STRING_ENDED(ptr) && ascii_isdigit(*ptr)) { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 10) { - un = 10 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - } + // Decimal number. + PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); } +#undef PARSE_NUMBER if (prep != NULL) { *prep = pre; diff --git a/src/nvim/version.c b/src/nvim/version.c index 7e59f3327a..3382e394f8 100644 --- a/src/nvim/version.c +++ b/src/nvim/version.c @@ -77,6 +77,8 @@ static char *features[] = { // clang-format off static const int included_patches[] = { + 1229, + 1230, // 1026, 1025, 1024, From 568cf73c90af2966ee091f2180905a8cf9582064 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 01:29:48 +0300 Subject: [PATCH 057/109] viml/parser/expressions: Fix last error found by KLEE --- src/nvim/viml/parser/expressions.c | 1 + test/unit/viml/expressions/parser_spec.lua | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index da22cf4cdb..b413d56592 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -2445,6 +2445,7 @@ viml_pexpr_parse_bracket_closing_error: cur_node->children = *top_node_p; } *top_node_p = cur_node; + new_top_node_p = top_node_p; goto viml_pexpr_parse_figure_brace_closing_error; } if (want_node == kENodeValue) { diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index b81a3c0f82..f3d0790e68 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -7120,7 +7120,25 @@ describe('Expressions parser', function() hl('InvalidDoubleQuotedString', '"'), hl('InvalidDoubleQuotedEscape', '\\1'), }) - check_parsing('}l') + check_parsing('}l', 0, { + -- 01 + ast = { + { + 'OpMissing:0:1:', + children = { + 'UnknownFigure(---):0:0:', + 'PlainIdentifier(scope=0,ident=l):0:1:l', + }, + }, + }, + err = { + arg = '}l', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidFigureBrace', '}'), + hl('InvalidIdentifier', 'l'), + }) check_parsing(':?\000\000\000\000\000\000\000', 0, { ast = { { From c202f17c8d8cb2140aa1b21b2e2d2ab3925a7812 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 01:31:31 +0300 Subject: [PATCH 058/109] unittests: Avoid alloc log checking errors when printing tests --- test/unit/viml/expressions/parser_spec.lua | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index f3d0790e68..125a658f7b 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -248,16 +248,16 @@ describe('Expressions parser', function() local hls = phl2lua(pstate) if exp_ast == nil then format_check(str, flags, ast, hls) - return - end - eq(exp_ast, ast) - if exp_highlighting_fs then - local exp_highlighting = {} - local next_col = 0 - for i, h in ipairs(exp_highlighting_fs) do - exp_highlighting[i], next_col = h(next_col) + else + eq(exp_ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exp_highlighting_fs) do + exp_highlighting[i], next_col = h(next_col) + end + eq(exp_highlighting, hls) end - eq(exp_highlighting, hls) end lib.viml_pexpr_free_ast(east) kvi_destroy(pstate.colors) From 06bdc9ed839eedbead34d58214927d3c0cebff58 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 01:40:55 +0300 Subject: [PATCH 059/109] klee: Update vim_str2nr in mock as well --- test/symbolic/klee/nvim/charset.c | 78 ++++++++++++------------------- 1 file changed, 30 insertions(+), 48 deletions(-) diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c index c584bd72ef..f3a218949f 100644 --- a/test/symbolic/klee/nvim/charset.c +++ b/test/symbolic/klee/nvim/charset.c @@ -60,18 +60,19 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // decimal or octal, default is decimal pre = 0; - if (what & STR2NR_OCT) { - // Don't interpret "0", "08" or "0129" as octal. - for (int i = 1; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { + if (what & STR2NR_OCT + && !STRING_ENDED(ptr + 1) + && ('0' <= ptr[1] && ptr[1] <= '7')) { + // Assume octal now: what we already know is that string starts with + // zero and some octal digit. + pre = '0'; + // Don’t interpret "0", "008" or "0129" as octal. + for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { if (ptr[i] > '7') { - // can't be octal + // Can’t be octal. pre = 0; break; } - if (ptr[i] >= '0') { - // assume octal - pre = '0'; - } } } } @@ -79,51 +80,32 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. uvarnumber_T un = 0; +#define PARSE_NUMBER(base, cond, conv) \ + do { \ + while (!STRING_ENDED(ptr) && (cond)) { \ + /* avoid ubsan error for overflow */ \ + if (un < UVARNUMBER_MAX / base) { \ + un = base * un + (uvarnumber_T)(conv); \ + } else { \ + un = UVARNUMBER_MAX; \ + } \ + ptr++; \ + } \ + } while (0) if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { - // bin - while (!STRING_ENDED(ptr) && '0' <= *ptr && *ptr <= '1') { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 2) { - un = 2 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - } + // Binary number. + PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { - // octal - while (!STRING_ENDED(ptr) && '0' <= *ptr && *ptr <= '7') { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 8) { - un = 8 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - } + // Octal number. + PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { - // hex - while (!STRING_ENDED(ptr) && ascii_isxdigit(*ptr)) { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 16) { - un = 16 * un + (uvarnumber_T)hex2nr(*ptr); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - } + // Hexadecimal number. + PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); } else { - // decimal - while (!STRING_ENDED(ptr) && ascii_isdigit(*ptr)) { - // avoid ubsan error for overflow - if (un < UVARNUMBER_MAX / 10) { - un = 10 * un + (uvarnumber_T)(*ptr - '0'); - } else { - un = UVARNUMBER_MAX; - } - ptr++; - } + // Decimal number. + PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); } +#undef PARSE_NUMBER if (prep != NULL) { *prep = pre; From b935a12dab17c3887db9c5fd7c90b34b2c51170f Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 16:32:13 +0300 Subject: [PATCH 060/109] ex_getln: Make use of new parser to color expressions Retires g:Nvim_color_expr callback. --- src/nvim/ex_getln.c | 67 +++++++++++++++++-- src/nvim/mbyte.c | 4 +- src/nvim/mbyte.h | 6 ++ src/nvim/viml/parser/parser.c | 13 ++++ src/nvim/viml/parser/parser.h | 55 +++++++++++++++ test/functional/ui/cmdline_highlight_spec.lua | 26 ++++--- 6 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 src/nvim/viml/parser/parser.c diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c index 54e5bcb9ff..386e9e81aa 100644 --- a/src/nvim/ex_getln.c +++ b/src/nvim/ex_getln.c @@ -66,6 +66,8 @@ #include "nvim/lib/kvec.h" #include "nvim/api/private/helpers.h" #include "nvim/highlight_defs.h" +#include "nvim/viml/parser/parser.h" +#include "nvim/viml/parser/expressions.h" /* * Variables shared between getcmdline(), redrawcmdline() and others. @@ -2341,6 +2343,62 @@ void free_cmdline_buf(void) enum { MAX_CB_ERRORS = 1 }; +/// Color expression cmdline using built-in expressions parser +/// +/// @param[in] colored_ccline Command-line to color. +/// @param[out] ret_ccline_colors What should be colored. +/// +/// Always colors the whole cmdline. +static void color_expr_cmdline(const CmdlineInfo *const colored_ccline, + ColoredCmdline *const ret_ccline_colors) + FUNC_ATTR_NONNULL_ALL +{ + ParserLine plines[] = { + { + .data = (const char *)colored_ccline->cmdbuff, + .size = STRLEN(colored_ccline->cmdbuff), + .allocated = false, + }, + { NULL, 0, false }, + }; + ParserLine *plines_p = plines; + ParserHighlight colors; + kvi_init(colors); + ParserState pstate; + viml_parser_init( + &pstate, parser_simple_get_line, &plines_p, &colors); + ExprAST east = viml_pexpr_parse(&pstate, kExprFlagsDisallowEOC); + viml_pexpr_free_ast(east); + viml_parser_destroy(&pstate); + kv_resize(ret_ccline_colors->colors, kv_size(colors)); + size_t prev_end = 0; + for (size_t i = 0 ; i < kv_size(colors) ; i++) { + const ParserHighlightChunk chunk = kv_A(colors, i); + if (chunk.start.col != prev_end) { + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = prev_end, + .end = chunk.start.col, + .attr = 0, + })); + } + const int id = syn_name2id((const char_u *)chunk.group); + const int attr = (id == 0 ? 0 : syn_id2attr(id)); + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = chunk.start.col, + .end = chunk.end_col, + .attr = attr, + })); + prev_end = chunk.end_col; + } + if (prev_end < (size_t)colored_ccline->cmdlen) { + kv_push(ret_ccline_colors->colors, ((CmdlineColorChunk) { + .start = prev_end, + .end = (size_t)colored_ccline->cmdlen, + .attr = 0, + })); + } +} + /// Color command-line /// /// Should use built-in command parser or user-specified one. Currently only the @@ -2422,13 +2480,8 @@ static bool color_cmdline(const CmdlineInfo *const colored_ccline, tl_ret = try_leave(&tstate, &err); can_free_cb = true; } else if (colored_ccline->cmdfirstc == '=') { - try_enter(&tstate); - err_errmsg = N_( - "E5409: Unable to get g:Nvim_color_expr callback: %s"); - dgc_ret = tv_dict_get_callback(&globvardict, S_LEN("Nvim_color_expr"), - &color_cb); - tl_ret = try_leave(&tstate, &err); - can_free_cb = true; + color_expr_cmdline(colored_ccline, ret_ccline_colors); + can_free_cb = false; } if (!tl_ret || !dgc_ret) { goto color_cmdline_error; diff --git a/src/nvim/mbyte.c b/src/nvim/mbyte.c index f65d7a6b13..843007b97b 100644 --- a/src/nvim/mbyte.c +++ b/src/nvim/mbyte.c @@ -2288,9 +2288,7 @@ int convert_setup_ext(vimconv_T *vcp, char_u *from, bool from_unicode_is_utf8, if (vcp->vc_type == CONV_ICONV && vcp->vc_fd != (iconv_t)-1) iconv_close(vcp->vc_fd); # endif - vcp->vc_type = CONV_NONE; - vcp->vc_factor = 1; - vcp->vc_fail = false; + *vcp = (vimconv_T)MBYTE_NONE_CONV; /* No conversion when one of the names is empty or they are equal. */ if (from == NULL || *from == NUL || to == NULL || *to == NUL diff --git a/src/nvim/mbyte.h b/src/nvim/mbyte.h index fce600d0a9..a5ce1b0a15 100644 --- a/src/nvim/mbyte.h +++ b/src/nvim/mbyte.h @@ -60,6 +60,12 @@ typedef enum { CONV_ICONV = 5, } ConvFlags; +#define MBYTE_NONE_CONV { \ + .vc_type = CONV_NONE, \ + .vc_factor = 1, \ + .vc_fail = false, \ +} + /// Structure used for string conversions typedef struct { int vc_type; ///< Zero or more ConvFlags. diff --git a/src/nvim/viml/parser/parser.c b/src/nvim/viml/parser/parser.c new file mode 100644 index 0000000000..08d8846018 --- /dev/null +++ b/src/nvim/viml/parser/parser.c @@ -0,0 +1,13 @@ +#include "nvim/viml/parser/parser.h" + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/parser.c.generated.h" +#endif + + +void parser_simple_get_line(void *cookie, ParserLine *ret_pline) +{ + ParserLine **plines_p = (ParserLine **)cookie; + *ret_pline = **plines_p; + (*plines_p)++; +} diff --git a/src/nvim/viml/parser/parser.h b/src/nvim/viml/parser/parser.h index fbc5ba5f07..7ac49709d8 100644 --- a/src/nvim/viml/parser/parser.h +++ b/src/nvim/viml/parser/parser.h @@ -8,6 +8,7 @@ #include "nvim/lib/kvec.h" #include "nvim/func_attr.h" #include "nvim/mbyte.h" +#include "nvim/memory.h" /// One parsed line typedef struct { @@ -80,6 +81,56 @@ typedef struct { bool can_continuate; } ParserState; +static inline void viml_parser_init( + ParserState *const ret_pstate, + const ParserLineGetter get_line, void *const cookie, + ParserHighlight *const colors) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ARG(1, 2); + +/// Initialize a new parser state instance +/// +/// @param[out] ret_pstate Parser state to initialize. +/// @param[in] get_line Line getter function. +/// @param[in] cookie Argument for the get_line function. +/// @param[in] colors Where to save highlighting. May be NULL if it is not +/// needed. +static inline void viml_parser_init( + ParserState *const ret_pstate, + const ParserLineGetter get_line, void *const cookie, + ParserHighlight *const colors) +{ + *ret_pstate = (ParserState) { + .reader = { + .get_line = get_line, + .cookie = cookie, + .conv = MBYTE_NONE_CONV, + }, + .pos = { 0, 0 }, + .colors = colors, + .can_continuate = false, + }; + kvi_init(ret_pstate->reader.lines); + kvi_init(ret_pstate->stack); +} + +static inline void viml_parser_destroy(ParserState *const pstate) + REAL_FATTR_NONNULL_ALL REAL_FATTR_ALWAYS_INLINE; + +/// Free all memory allocated by the parser on heap +/// +/// @param pstate Parser state to free. +static inline void viml_parser_destroy(ParserState *const pstate) +{ + for (size_t i = 0; i < kv_size(pstate->reader.lines); i++) { + ParserLine pline = kv_A(pstate->reader.lines, i); + if (pline.allocated) { + xfree((void *)pline.data); + } + } + kvi_destroy(pstate->reader.lines); + kvi_destroy(pstate->stack); +} + static inline void viml_preader_get_line(ParserInputReader *const preader, ParserLine *const ret_pline) REAL_FATTR_NONNULL_ALL; @@ -186,4 +237,8 @@ static inline void viml_parser_highlight(ParserState *const pstate, })); } +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "viml/parser/parser.h.generated.h" +#endif + #endif // NVIM_VIML_PARSER_PARSER_H diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua index d87ce72599..60a4a815e7 100644 --- a/test/functional/ui/cmdline_highlight_spec.lua +++ b/test/functional/ui/cmdline_highlight_spec.lua @@ -144,7 +144,9 @@ before_each(function() EOB={bold = true, foreground = Screen.colors.Blue1}, ERR={foreground = Screen.colors.Grey100, background = Screen.colors.Red}, SK={foreground = Screen.colors.Blue}, - PE={bold = true, foreground = Screen.colors.SeaGreen4} + PE={bold = true, foreground = Screen.colors.SeaGreen4}, + NUM={foreground = Screen.colors.Blue2}, + NPAR={foreground = Screen.colors.Yellow}, }) end) @@ -863,7 +865,10 @@ describe('Ex commands coloring support', function() end) describe('Expressions coloring support', function() it('works', function() - meths.set_var('Nvim_color_expr', 'RainBowParens') + meths.command('hi clear NVimNumber') + meths.command('hi clear NVimNestingParenthesis') + meths.command('hi NVimNumber guifg=Blue2') + meths.command('hi NVimNestingParenthesis guifg=Yellow') feed(':echo =(((1)))') screen:expect([[ | @@ -873,21 +878,24 @@ describe('Expressions coloring support', function() {EOB:~ }| {EOB:~ }| {EOB:~ }| - ={RBP1:(}{RBP2:(}{RBP3:(}1{RBP3:)}{RBP2:)}{RBP1:)}^ | + ={NPAR:(((}{NUM:1}{NPAR:)))}^ | ]]) end) - it('errors out when failing to get callback', function() + it('does not use Nvim_color_expr', function() meths.set_var('Nvim_color_expr', 42) + -- Used to error out due to failing to get callback. + meths.command('hi clear NVimNumber') + meths.command('hi NVimNumber guifg=Blue2') feed(':=1') screen:expect([[ + | {EOB:~ }| {EOB:~ }| {EOB:~ }| - = | - {ERR:E5409: Unable to get g:Nvim_color_expr c}| - {ERR:allback: Vim:E6000: Argument is not a fu}| - {ERR:nction or function name} | - =1^ | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + ={NUM:1}^ | ]]) end) end) From 1be29dc5acc071591bd776890fc3b61aba1488f9 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 16:54:50 +0300 Subject: [PATCH 061/109] gen_declarations: Do not generate line numbers by default --- src/nvim/generators/gen_declarations.lua | 43 +++++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/src/nvim/generators/gen_declarations.lua b/src/nvim/generators/gen_declarations.lua index e999e53e4a..0c73376ba0 100755 --- a/src/nvim/generators/gen_declarations.lua +++ b/src/nvim/generators/gen_declarations.lua @@ -164,9 +164,40 @@ local pattern = concat( ) if fname == '--help' then - print'Usage:' - print() - print' gendeclarations.lua definitions.c static.h non-static.h preprocessor.i' + print([[ +Usage: + + gendeclarations.lua definitions.c static.h non-static.h definitions.i + +Generates declarations for a C file defitions.c, putting declarations for +static functions into static.h and declarations for non-static functions into +non-static.h. File `definitions.i' should contain an already preprocessed +version of defintions.c and it is the only one which is actually parsed, +definitions.c is needed only to determine functions from which file out of all +functions found in definitions.i are needed. + +Additionally uses the following environment variables: + + NVIM_GEN_DECLARATIONS_LINE_NUMBERS: + If set to 1 then all generated declarations receive a comment with file + name and line number after the declaration. This may be useful for + debugging gen_declarations script, but not much beyound that with + configured development environment (i.e. with ctags/cscope/finding + definitions with clang/etc). + + WARNING: setting this to 1 will cause extensive rebuilds: declarations + generator script will not regenerate non-static.h file if its + contents did not change, but including line numbers will make + contents actually change. + + With contents changed timestamp of the file is regenerated even + when no real changes were made (e.g. a few lines were added to + a function which is not at the bottom of the file). + + With changed timestamp build system will assume that header + changed, triggering rebuilds of all C files which depend on the + "changed" header. +]]) os.exit() end @@ -249,8 +280,10 @@ while init ~= nil do declaration = declaration:gsub(' $', '') declaration = declaration:gsub('^ ', '') declaration = declaration .. ';' - declaration = declaration .. (' // %s/%s:%u'):format( - curdir, curfile, declline) + if os.getenv('NVIM_GEN_DECLARATIONS_LINE_NUMBERS') == '1' then + declaration = declaration .. (' // %s/%s:%u'):format( + curdir, curfile, declline) + end declaration = declaration .. '\n' if declaration:sub(1, 6) == 'static' then static = static .. declaration From 22d161a5dd1c519f998916f45d61be92662fbb44 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 20:11:44 +0300 Subject: [PATCH 062/109] api/vim: Add nvim_parse_expression function --- src/nvim/api/vim.c | 66 ++++++++++++++++++++++++++++++ src/nvim/viml/parser/expressions.c | 4 +- src/nvim/viml/parser/expressions.h | 3 ++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 98f4410347..9daa2fb398 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -886,6 +886,72 @@ theend: return rv; } +/// Parse a VimL expression +/// +/// @param[in] expr Expression to parse. Is always treated as a single line. +/// @param[in] flags Flags: "m" if multiple expressions in a row are allowed +/// (only the first one will be parsed), "E" if EOC tokens +/// are not allowed (determines whether they will stop +/// parsing process or be recognized as an operator/space, +/// though also yielding an error). +/// +/// Use only "m" to parse like for "=", only "E" to +/// parse like for ":echo", empty string for ":let". +/// +/// @return AST: top-level dectionary holds keys +/// +/// "error": Dictionary with error, present only if parser saw some +/// error. Contains the following keys: +/// +/// "message": String, error message in printf format, translated. +/// Must contain exactly one "%.*s". +/// "arg": String, error message argument. +/// +/// "ast": actual AST, a dictionary with the following keys: +/// +/// "type": node type, one of the value names from ExprASTNodeType +/// stringified without "kExprNode" prefix. +/// "start": a pair [line, column] describing where node is “started” +/// where "line" is always 0 (will not be 0 if you will be +/// using nvim_parse_viml() on e.g. ":let", but that is not +/// present yet). Both elements are Integers. +/// "len": “length” of the node. This and "start" are there for +/// debugging purposes primary (debugging parser and providing +/// debug information). +/// "children": a list of nodes described in top/"ast". There always +/// is zero, one or two children, key will contain an +/// empty array if node can have children, but has no and +/// will not be present at all if node can’t have any +/// children. Maximum number of children may be found in +/// node_maxchildren array. +/// +/// Local values (present only for certain nodes): +/// +/// "scope": a single Integer, specifies scope for "Option" and +/// "PlainIdentifier" nodes. For "Option" it is one of +/// ExprOptScope values, for "PlainIdentifier" it is one of +/// ExprVarScope values. +/// "ident": identifier (without scope, if any), present for "Option", +/// "PlainIdentifier", "PlainKey" and "Environment" nodes. +/// "name": Integer, register name (one character) or -1. Only present +/// for "Register" nodes. +/// "cmp_type": String, comparison type, one of the value names from +/// ExprComparisonType, stringified without "kExprCmp" +/// prefix. Only present for "Comparison" nodes. +/// "ccs_strategy": String, case comparison strategy, one of the +/// value names from ExprCaseCompareStrategy, +/// stringified without "kCCStrategy" prefix. Only +/// present for "Comparison" nodes. +/// "ivalue": Integer, integer value for "Integer" nodes. +/// "fvalue": Float, floating-point value for "Float" nodes. +/// "svalue": String, value for "SingleQuotedString" and +/// "DoubleQuotedString" nodes. +Dictionary nvim_parse_expression(String expr, String flags, Error *err) + FUNC_API_SINCE(4) +{ + return (Dictionary)ARRAY_DICT_INIT; +} + /// Writes a message to vim output or error buffer. The string is split /// and flushed after each newline. Incomplete lines are kept for writing diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index b413d56592..4e8a9b8523 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -849,8 +849,7 @@ static inline void viml_pexpr_debug_print_token( viml_pexpr_debug_print_token(pstate, tkn) #endif -#ifndef NDEBUG -static const uint8_t node_maxchildren[] = { +const uint8_t node_maxchildren[] = { [kExprNodeMissing] = 0, [kExprNodeOpMissing] = 2, [kExprNodeTernary] = 2, @@ -890,7 +889,6 @@ static const uint8_t node_maxchildren[] = { [kExprNodeOption] = 0, [kExprNodeEnvironment] = 0, }; -#endif /// Free memory occupied by AST /// diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index d783518b3a..d00d4855f3 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -338,6 +338,9 @@ typedef struct { ExprASTNode *root; } ExprAST; +/// Array mapping ExprASTNodeType to maximum amount of children node may have +extern const uint8_t node_maxchildren[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.h.generated.h" #endif From 748f3ad5bbf9706dddddeea5df693221d6ae5e94 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 21:30:06 +0300 Subject: [PATCH 063/109] syntax,viml/expressions/parser: Create defaults for expr highlighting --- src/nvim/ex_docmd.c | 5 +- src/nvim/syntax.c | 597 +++++++++++++-------- src/nvim/viml/parser/expressions.c | 115 +--- test/unit/viml/expressions/parser_spec.lua | 530 +++++++++--------- 4 files changed, 666 insertions(+), 581 deletions(-) diff --git a/src/nvim/ex_docmd.c b/src/nvim/ex_docmd.c index 3130747e08..664ab69792 100644 --- a/src/nvim/ex_docmd.c +++ b/src/nvim/ex_docmd.c @@ -5911,9 +5911,10 @@ static void ex_colorscheme(exarg_T *eap) static void ex_highlight(exarg_T *eap) { - if (*eap->arg == NUL && eap->cmd[2] == '!') + if (*eap->arg == NUL && eap->cmd[2] == '!') { MSG(_("Greetings, Vim user!")); - do_highlight(eap->arg, eap->forceit, FALSE); + } + do_highlight((const char *)eap->arg, eap->forceit, false); } diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index 70bda42d83..b418df77a4 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -5929,8 +5929,7 @@ static void syntime_report(void) // // When making changes here, also change runtime/colors/default.vim! -static char *highlight_init_both[] = -{ +static const char *highlight_init_both[] = { "Conceal ctermbg=DarkGrey ctermfg=LightGrey guibg=DarkGrey guifg=LightGrey", "Cursor guibg=fg guifg=bg", "lCursor guibg=fg guifg=bg", @@ -5954,8 +5953,7 @@ static char *highlight_init_both[] = NULL }; -static char *highlight_init_light[] = -{ +static const char *highlight_init_light[] = { "ColorColumn ctermbg=LightRed guibg=LightRed", "CursorColumn ctermbg=LightGrey guibg=Grey90", "CursorLine cterm=underline guibg=Grey90", @@ -5988,8 +5986,7 @@ static char *highlight_init_light[] = NULL }; -static char *highlight_init_dark[] = -{ +static const char *highlight_init_dark[] = { "ColorColumn ctermbg=DarkRed guibg=DarkRed", "CursorColumn ctermbg=DarkGrey guibg=Grey40", "CursorLine cterm=underline guibg=Grey40", @@ -6022,6 +6019,157 @@ static char *highlight_init_dark[] = NULL }; +static const char *highlight_init_cmdline[] = { + // NVimInternalError should appear only when highlighter has a bug. + "NVimInternalError ctermfg=Red ctermbg=Red guifg=Red guibg=Red", + + // Highlight groups (links) used by parser: + + "default link NVimOperator Operator", + + "default link NVimUnaryOperator NVimOperator", + "default link NVimUnaryPlus NVimUnaryOperator", + + "default link NVimBinaryOperator NVimOperator", + "default link NVimComparison NVimBinaryOperator", + "default link NVimComparisonModifier NVimComparison", + "default link NVimBinaryPlus NVimBinaryOperator", + "default link NVimConcat NVimBinaryOperator", + "default link NVimConcatOrSubscript NVimConcat", + + "default link NVimTernary NVimOperator", + "default link NVimTernaryColon NVimTernary", + + "default link NVimParenthesis Delimiter", + "default link NVimLambda NVimParenthesis", + "default link NVimNestingParenthesis NVimParenthesis", + "default link NVimCallingParenthesis NVimParenthesis", + + "default link NVimSubscript NVimParenthesis", + "default link NVimSubscriptBracket NVimSubscript", + "default link NVimSubscriptColon NVimSubscript", + "default link NVimSubscriptColon NVimSubscript", + "default link NVimCurly NVimSubscript", + + "default link NVimContainer NVimParenthesis", + "default link NVimDict NVimContainer", + "default link NVimList NVimContainer", + + "default link NVimIdentifier Identifier", + "default link NVimIdentifierScope NVimIdentifier", + "default link NVimIdentifierScopeDelimiter NVimIdentifier", + "default link NVimIdentifierName NVimIdentifier", + "default link NVimIdentifierKey NVimIdentifier", + + "default link NVimColon Delimiter", + "default link NVimComma Delimiter", + "default link NVimArrow Delimiter", + + "default link NVimRegister SpecialChar", + "default link NVimNumber Number", + "default link NVimFloat NVimNumber", + "default link NVimNumberPrefix SpecialChar", + + "default link NVimString String", + "default link NVimStringBody NVimString", + "default link NVimStringQuote NVimString", + "default link NVimStringSpecial SpecialChar", + + "default link NVimSingleQuote NVimStringQuote", + "default link NVimSingleQuotedBody NVimStringBody", + "default link NVimSingleQuotedQuote NVimStringSpecial", + + "default link NVimDoubleQuote NVimStringQuote", + "default link NVimDoubleQuotedBody NVimStringBody", + "default link NVimDoubleQuotedEscape NVimStringSpecial", + // Not actually invalid, but we highlight user that he is doing something + // wrong. + "default link NVimDoubleQuotedUnknownEscape NVimInvalidValue", + + "default link NVimFigureBrace NVimInternalError", + "default link NVimSingleQuotedUnknownEscape NVimInternalError", + + // NVimInvalid groups: + + "default link NVimInvalidSingleQuotedUnknownEscape NVimInternalError", + + "default link NVimInvalid Error", + + "default link NVimInvalidOperator NVimInvalid", + + "default link NVimInvalidUnaryOperator NVimInvalidOperator", + "default link NVimInvalidUnaryPlus NVimInvalidUnaryOperator", + + "default link NVimInvalidBinaryOperator NVimInvalidOperator", + "default link NVimInvalidComparison NVimInvalidBinaryOperator", + "default link NVimInvalidComparisonModifier NVimInvalidComparison", + "default link NVimInvalidBinaryPlus NVimInvalidBinaryOperator", + "default link NVimInvalidConcat NVimInvalidBinaryOperator", + "default link NVimInvalidConcatOrSubscript NVimInvalidConcat", + + "default link NVimInvalidTernary NVimInvalidOperator", + "default link NVimInvalidTernaryColon NVimInvalidTernary", + + "default link NVimInvalidDelimiter NVimInvalid", + + "default link NVimInvalidParenthesis NVimInvalidDelimiter", + "default link NVimInvalidLambda NVimInvalidParenthesis", + "default link NVimInvalidNestingParenthesis NVimInvalidParenthesis", + "default link NVimInvalidCallingParenthesis NVimInvalidParenthesis", + + "default link NVimInvalidSubscript NVimInvalidParenthesis", + "default link NVimInvalidSubscriptBracket NVimInvalidSubscript", + "default link NVimInvalidSubscriptColon NVimInvalidSubscript", + "default link NVimInvalidSubscriptColon NVimInvalidSubscript", + "default link NVimInvalidCurly NVimInvalidSubscript", + + "default link NVimInvalidContainer NVimInvalidParenthesis", + "default link NVimInvalidDict NVimInvalidContainer", + "default link NVimInvalidList NVimInvalidContainer", + + "default link NVimInvalidIdentifier Identifier", + "default link NVimInvalidIdentifierScope NVimIdentifier", + "default link NVimInvalidIdentifierScopeDelimiter NVimIdentifier", + "default link NVimInvalidIdentifierName NVimIdentifier", + "default link NVimInvalidIdentifierKey NVimIdentifier", + + "default link NVimInvalidColon NVimInvalidDelimiter", + "default link NVimInvalidComma NVimInvalidDelimiter", + "default link NVimInvalidArrow NVimInvalidDelimiter", + + "default link NVimInvalidValue NVimInvalid", + + "default link NVimInvalidRegister NVimInvalidValue", + "default link NVimInvalidNumber NVimInvalidValue", + "default link NVimInvalidFloat NVimInvalidNumber", + "default link NVimInvalidNumberPrefix NVimInvalidNumber", + + // Invalid string bodies and specials are still highlighted as valid ones to + // minimize the red area. + "default link NVimInvalidString NVimInvalidValue", + "default link NVimInvalidStringBody NVimString", + "default link NVimInvalidStringQuote NVimInvalidString", + "default link NVimInvalidStringSpecial NVimStringSpecial", + + "default link NVimInvalidSingleQuote NVimInvalidStringQuote", + "default link NVimInvalidSingleQuotedBody NVimInvalidStringBody", + "default link NVimInvalidSingleQuotedQuote NVimInvalidStringSpecial", + + "default link NVimInvalidDoubleQuote NVimInvalidStringQuote", + "default link NVimInvalidDoubleQuotedBody NVimInvalidStringBody", + "default link NVimInvalidDoubleQuotedEscape NVimInvalidStringSpecial", + "default link NVimInvalidDoubleQuotedUnknownEscape NVimInvalidValue", + + "default link NVimInvalidFigureBrace NVimInternalError", +}; + +/// Create default links for NVim* highlight groups used for cmdline coloring +void syn_init_cmdline_highlight(bool reset, bool init) +{ + for (size_t i = 0 ; i < ARRAY_SIZE(highlight_init_cmdline) ; i++) { + do_highlight(highlight_init_cmdline[i], reset, init); + } +} /// Load colors from a file if "g:colors_name" is set, otherwise load builtin /// colors @@ -6032,7 +6180,6 @@ void init_highlight(int both, int reset) { int i; - char **pp; static int had_both = FALSE; // Try finding the color scheme file. Used when a color file was loaded @@ -6054,9 +6201,9 @@ init_highlight(int both, int reset) */ if (both) { had_both = TRUE; - pp = highlight_init_both; + const char *const *const pp = highlight_init_both; for (i = 0; pp[i] != NULL; i++) { - do_highlight((char_u *)pp[i], reset, true); + do_highlight(pp[i], reset, true); } } else if (!had_both) { // Don't do anything before the call with both == TRUE from main(). @@ -6065,10 +6212,11 @@ init_highlight(int both, int reset) return; } - pp = (*p_bg == 'l') ? highlight_init_light : highlight_init_dark; - + const char *const *const pp = ((*p_bg == 'l') + ? highlight_init_light + : highlight_init_dark); for (i = 0; pp[i] != NULL; i++) { - do_highlight((char_u *)pp[i], reset, true); + do_highlight(pp[i], reset, true); } /* Reverse looks ugly, but grey may not work for 8 colors. Thus let it @@ -6078,15 +6226,13 @@ init_highlight(int both, int reset) * Clear the attributes, needed when changing the t_Co value. */ if (t_colors > 8) { do_highlight( - (char_u *)(*p_bg == 'l' - ? "Visual cterm=NONE ctermbg=LightGrey" - : "Visual cterm=NONE ctermbg=DarkGrey"), false, - true); + (*p_bg == 'l' + ? "Visual cterm=NONE ctermbg=LightGrey" + : "Visual cterm=NONE ctermbg=DarkGrey"), false, true); } else { - do_highlight((char_u *)"Visual cterm=reverse ctermbg=NONE", - FALSE, TRUE); + do_highlight("Visual cterm=reverse ctermbg=NONE", false, true); if (*p_bg == 'l') - do_highlight((char_u *)"Search ctermfg=black", FALSE, TRUE); + do_highlight("Search ctermfg=black", false, true); } /* @@ -6102,6 +6248,10 @@ init_highlight(int both, int reset) (void)source_runtime((char_u *)"syntax/syncolor.vim", DIP_ALL); recursive--; } + // Without syncolor.vim it is going to screw everything over by defining + // cleared highlight groups by creating links to non-existent groups. This + // effectively prevents ":highlight default" from working properly. + syn_init_cmdline_highlight(reset, true); } } @@ -6137,17 +6287,22 @@ int load_colors(char_u *name) } -/// Handle the ":highlight .." command. -/// When using ":hi clear" this is called recursively for each group with -/// "forceit" and "init" both TRUE. -/// @param init TRUE when called for initializing -void -do_highlight(char_u *line, int forceit, int init) { - char_u *name_end; - char_u *linep; - char_u *key_start; - char_u *arg_start; - char_u *key = NULL, *arg = NULL; +/// Handle ":highlight" command +/// +/// When using ":highlight clear" this is called recursively for each group with +/// forceit and init being both true. +/// +/// @param[in] line Command arguments. +/// @param[in] forceit True when bang is given, allows to link group even if +/// it has its own settings. +/// @param[in] init True when initializing. +void do_highlight(const char *line, const bool forceit, const bool init) + FUNC_ATTR_NONNULL_ALL +{ + const char *name_end; + const char *linep; + const char *key_start; + const char *arg_start; long i; int off; int len; @@ -6161,94 +6316,87 @@ do_highlight(char_u *line, int forceit, int init) { int color; bool is_normal_group = false; // "Normal" group - /* - * If no argument, list current highlighting. - */ - if (ends_excmd(*line)) { + // If no argument, list current highlighting. + if (ends_excmd((uint8_t)(*line))) { for (int i = 1; i <= highlight_ga.ga_len && !got_int; i++) { - // todo(vim): only call when the group has attributes set + // TODO(brammool): only call when the group has attributes set highlight_list_one(i); } return; } - /* - * Isolate the name. - */ - name_end = skiptowhite(line); - linep = skipwhite(name_end); + // Isolate the name. + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); - /* - * Check for "default" argument. - */ - if (STRNCMP(line, "default", name_end - line) == 0) { - dodefault = TRUE; + // Check for "default" argument. + if (strncmp(line, "default", name_end - line) == 0) { + dodefault = true; line = linep; - name_end = skiptowhite(line); - linep = skipwhite(name_end); + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); } - /* - * Check for "clear" or "link" argument. - */ - if (STRNCMP(line, "clear", name_end - line) == 0) - doclear = TRUE; - if (STRNCMP(line, "link", name_end - line) == 0) - dolink = TRUE; + // Check for "clear" or "link" argument. + if (strncmp(line, "clear", name_end - line) == 0) { + doclear = true; + } else if (strncmp(line, "link", name_end - line) == 0) { + dolink = true; + } - /* - * ":highlight {group-name}": list highlighting for one group. - */ - if (!doclear && !dolink && ends_excmd(*linep)) { - id = syn_namen2id(line, (int)(name_end - line)); - if (id == 0) - EMSG2(_("E411: highlight group not found: %s"), line); - else + // ":highlight {group-name}": list highlighting for one group. + if (!doclear && !dolink && ends_excmd((uint8_t)(*linep))) { + id = syn_namen2id((const char_u *)line, (int)(name_end - line)); + if (id == 0) { + emsgf(_("E411: highlight group not found: %s"), line); + } else { highlight_list_one(id); + } return; } - /* - * Handle ":highlight link {from} {to}" command. - */ + // Handle ":highlight link {from} {to}" command. if (dolink) { - char_u *from_start = linep; - char_u *from_end; - char_u *to_start; - char_u *to_end; + const char *from_start = linep; + const char *from_end; + const char *to_start; + const char *to_end; int from_id; int to_id; - from_end = skiptowhite(from_start); - to_start = skipwhite(from_end); - to_end = skiptowhite(to_start); + from_end = (const char *)skiptowhite((const char_u *)from_start); + to_start = (const char *)skipwhite((const char_u *)from_end); + to_end = (const char *)skiptowhite((const char_u *)to_start); - if (ends_excmd(*from_start) || ends_excmd(*to_start)) { - EMSG2(_("E412: Not enough arguments: \":highlight link %s\""), - from_start); + if (ends_excmd((uint8_t)(*from_start)) + || ends_excmd((uint8_t)(*to_start))) { + emsgf(_("E412: Not enough arguments: \":highlight link %s\""), + from_start); return; } - if (!ends_excmd(*skipwhite(to_end))) { - EMSG2(_("E413: Too many arguments: \":highlight link %s\""), from_start); + if (!ends_excmd(*skipwhite((const char_u *)to_end))) { + emsgf(_("E413: Too many arguments: \":highlight link %s\""), from_start); return; } - from_id = syn_check_group(from_start, (int)(from_end - from_start)); - if (STRNCMP(to_start, "NONE", 4) == 0) + from_id = syn_check_group((const char_u *)from_start, + (int)(from_end - from_start)); + if (strncmp(to_start, "NONE", 4) == 0) { to_id = 0; - else - to_id = syn_check_group(to_start, (int)(to_end - to_start)); + } else { + to_id = syn_check_group((const char_u *)to_start, + (int)(to_end - to_start)); + } if (from_id > 0 && (!init || HL_TABLE()[from_id - 1].sg_set == 0)) { - /* - * Don't allow a link when there already is some highlighting - * for the group, unless '!' is used - */ + // Don't allow a link when there already is some highlighting + // for the group, unless '!' is used if (to_id > 0 && !forceit && !init && hl_has_settings(from_id - 1, dodefault)) { - if (sourcing_name == NULL && !dodefault) + if (sourcing_name == NULL && !dodefault) { EMSG(_("E414: group has settings, highlight link ignored")); + } } else { if (!init) HL_TABLE()[from_id - 1].sg_set |= SG_LINK; @@ -6258,43 +6406,38 @@ do_highlight(char_u *line, int forceit, int init) { } } - /* Only call highlight_changed() once, after sourcing a syntax file */ - need_highlight_changed = TRUE; + // Only call highlight_changed() once, after sourcing a syntax file. + need_highlight_changed = true; return; } if (doclear) { - /* - * ":highlight clear [group]" command. - */ + // ":highlight clear [group]" command. line = linep; - if (ends_excmd(*line)) { + if (ends_excmd((uint8_t)(*line))) { do_unlet(S_LEN("colors_name"), true); restore_cterm_colors(); - /* - * Clear all default highlight groups and load the defaults. - */ + // Clear all default highlight groups and load the defaults. for (int idx = 0; idx < highlight_ga.ga_len; ++idx) { highlight_clear(idx); } - init_highlight(TRUE, TRUE); + init_highlight(true, true); highlight_changed(); redraw_later_clear(); return; } - name_end = skiptowhite(line); - linep = skipwhite(name_end); + name_end = (const char *)skiptowhite((const char_u *)line); + linep = (const char *)skipwhite((const char_u *)name_end); } - /* - * Find the group name in the table. If it does not exist yet, add it. - */ - id = syn_check_group(line, (int)(name_end - line)); - if (id == 0) /* failed (out of memory) */ + // Find the group name in the table. If it does not exist yet, add it. + id = syn_check_group((const char_u *)line, (int)(name_end - line)); + if (id == 0) { // Failed (out of memory). return; - idx = id - 1; /* index is ID minus one */ + } + idx = id - 1; // Index is ID minus one. // Return if "default" was used and the group already has settings if (dodefault && hl_has_settings(idx, true)) { @@ -6303,19 +6446,21 @@ do_highlight(char_u *line, int forceit, int init) { is_normal_group = (STRCMP(HL_TABLE()[idx].sg_name_u, "NORMAL") == 0); - /* Clear the highlighting for ":hi clear {group}" and ":hi clear". */ + // Clear the highlighting for ":hi clear {group}" and ":hi clear". if (doclear || (forceit && init)) { highlight_clear(idx); if (!doclear) HL_TABLE()[idx].sg_set = 0; } + char *key = NULL; + char *arg = NULL; if (!doclear) { - while (!ends_excmd(*linep)) { + while (!ends_excmd((uint8_t)(*linep))) { key_start = linep; if (*linep == '=') { - EMSG2(_("E415: unexpected equal sign: %s"), key_start); - error = TRUE; + emsgf(_("E415: unexpected equal sign: %s"), key_start); + error = true; break; } @@ -6325,61 +6470,58 @@ do_highlight(char_u *line, int forceit, int init) { linep++; } xfree(key); - key = vim_strnsave_up(key_start, (int)(linep - key_start)); - linep = skipwhite(linep); + key = (char *)vim_strnsave_up((const char_u *)key_start, + (int)(linep - key_start)); + linep = (const char *)skipwhite((const char_u *)linep); - if (STRCMP(key, "NONE") == 0) { + if (strcmp(key, "NONE") == 0) { if (!init || HL_TABLE()[idx].sg_set == 0) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_CTERM+SG_GUI; + } highlight_clear(idx); } continue; } - /* - * Check for the equal sign. - */ + // Check for the equal sign. if (*linep != '=') { - EMSG2(_("E416: missing equal sign: %s"), key_start); - error = TRUE; + emsgf(_("E416: missing equal sign: %s"), key_start); + error = true; break; } - ++linep; + linep++; - /* - * Isolate the argument. - */ - linep = skipwhite(linep); - if (*linep == '\'') { /* guifg='color name' */ + // Isolate the argument. + linep = (const char *)skipwhite((const char_u *)linep); + if (*linep == '\'') { // guifg='color name' arg_start = ++linep; - linep = vim_strchr(linep, '\''); + linep = strchr(linep, '\''); if (linep == NULL) { - EMSG2(_(e_invarg2), key_start); - error = TRUE; + emsgf(_(e_invarg2), key_start); + error = true; break; } } else { arg_start = linep; - linep = skiptowhite(linep); + linep = (const char *)skiptowhite((const char_u *)linep); } if (linep == arg_start) { - EMSG2(_("E417: missing argument: %s"), key_start); - error = TRUE; + emsgf(_("E417: missing argument: %s"), key_start); + error = true; break; } xfree(arg); - arg = vim_strnsave(arg_start, (int)(linep - arg_start)); + arg = xstrndup(arg_start, (size_t)(linep - arg_start)); - if (*linep == '\'') - ++linep; + if (*linep == '\'') { + linep++; + } - /* - * Store the argument. - */ - if ( STRCMP(key, "TERM") == 0 - || STRCMP(key, "CTERM") == 0 - || STRCMP(key, "GUI") == 0) { + // Store the argument. + if (strcmp(key, "TERM") == 0 + || strcmp(key, "CTERM") == 0 + || strcmp(key, "GUI") == 0) { attr = 0; off = 0; while (arg[off] != NUL) { @@ -6392,26 +6534,30 @@ do_highlight(char_u *line, int forceit, int init) { } } if (i < 0) { - EMSG2(_("E418: Illegal value: %s"), arg); - error = TRUE; + emsgf(_("E418: Illegal value: %s"), arg); + error = true; break; } - if (arg[off] == ',') /* another one follows */ - ++off; + if (arg[off] == ',') { // Another one follows. + off++; + } } - if (error) + if (error) { break; + } if (*key == 'C') { if (!init || !(HL_TABLE()[idx].sg_set & SG_CTERM)) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_CTERM; + } HL_TABLE()[idx].sg_cterm = attr; HL_TABLE()[idx].sg_cterm_bold = FALSE; } } else if (*key == 'G') { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { - if (!init) + if (!init) { HL_TABLE()[idx].sg_set |= SG_GUI; + } HL_TABLE()[idx].sg_gui = attr; } } @@ -6430,14 +6576,14 @@ do_highlight(char_u *line, int forceit, int init) { HL_TABLE()[idx].sg_cterm_bold = FALSE; } - if (ascii_isdigit(*arg)) + if (ascii_isdigit(*arg)) { color = atoi((char *)arg); - else if (STRICMP(arg, "fg") == 0) { - if (cterm_normal_fg_color) + } else if (STRICMP(arg, "fg") == 0) { + if (cterm_normal_fg_color) { color = cterm_normal_fg_color - 1; - else { + } else { EMSG(_("E419: FG color unknown")); - error = TRUE; + error = true; break; } } else if (STRICMP(arg, "bg") == 0) { @@ -6445,70 +6591,85 @@ do_highlight(char_u *line, int forceit, int init) { color = cterm_normal_bg_color - 1; else { EMSG(_("E420: BG color unknown")); - error = TRUE; + error = true; break; } } else { - static char *(color_names[28]) = { + static const char *color_names[] = { "Black", "DarkBlue", "DarkGreen", "DarkCyan", "DarkRed", "DarkMagenta", "Brown", "DarkYellow", "Gray", "Grey", "LightGray", "LightGrey", "DarkGray", "DarkGrey", "Blue", "LightBlue", "Green", "LightGreen", "Cyan", "LightCyan", "Red", "LightRed", "Magenta", - "LightMagenta", "Yellow", "LightYellow", "White", "NONE" + "LightMagenta", "Yellow", "LightYellow", "White", + "NONE" + }; + static const int color_numbers_16[] = { + 0, 1, 2, 3, + 4, 5, 6, 6, + 7, 7, + 7, 7, 8, 8, + 9, 9, 10, 10, + 11, 11, 12, 12, 13, + 13, 14, 14, 15, + -1 + }; + // For xterm with 88 colors: + static int color_numbers_88[] = { + 0, 4, 2, 6, + 1, 5, 32, 72, + 84, 84, + 7, 7, 82, 82, + 12, 43, 10, 61, + 14, 63, 9, 74, 13, + 75, 11, 78, 15, + -1 + }; + // For xterm with 256 colors: + static int color_numbers_256[] = { + 0, 4, 2, 6, + 1, 5, 130, 130, + 248, 248, + 7, 7, 242, 242, + 12, 81, 10, 121, + 14, 159, 9, 224, 13, + 225, 11, 229, 15, + -1 + }; + // For terminals with less than 16 colors: + static int color_numbers_8[28] = { + 0, 4, 2, 6, + 1, 5, 3, 3, + 7, 7, + 7, 7, 0+8, 0+8, + 4+8, 4+8, 2+8, 2+8, + 6+8, 6+8, 1+8, 1+8, 5+8, + 5+8, 3+8, 3+8, 7+8, + -1 }; - static int color_numbers_16[28] = {0, 1, 2, 3, - 4, 5, 6, 6, - 7, 7, - 7, 7, 8, 8, - 9, 9, 10, 10, - 11, 11, 12, 12, 13, - 13, 14, 14, 15, -1}; - /* for xterm with 88 colors... */ - static int color_numbers_88[28] = {0, 4, 2, 6, - 1, 5, 32, 72, - 84, 84, - 7, 7, 82, 82, - 12, 43, 10, 61, - 14, 63, 9, 74, 13, - 75, 11, 78, 15, -1}; - /* for xterm with 256 colors... */ - static int color_numbers_256[28] = {0, 4, 2, 6, - 1, 5, 130, 130, - 248, 248, - 7, 7, 242, 242, - 12, 81, 10, 121, - 14, 159, 9, 224, 13, - 225, 11, 229, 15, -1}; - /* for terminals with less than 16 colors... */ - static int color_numbers_8[28] = {0, 4, 2, 6, - 1, 5, 3, 3, - 7, 7, - 7, 7, 0+8, 0+8, - 4+8, 4+8, 2+8, 2+8, - 6+8, 6+8, 1+8, 1+8, 5+8, - 5+8, 3+8, 3+8, 7+8, -1}; - /* reduce calls to STRICMP a bit, it can be slow */ + // Reduce calls to STRICMP a bit, it can be slow. off = TOUPPER_ASC(*arg); - for (i = ARRAY_SIZE(color_names); --i >= 0; ) + for (i = ARRAY_SIZE(color_names); --i >= 0; ) { if (off == color_names[i][0] - && STRICMP(arg + 1, color_names[i] + 1) == 0) + && STRICMP(arg + 1, color_names[i] + 1) == 0) { break; + } + } if (i < 0) { - EMSG2(_( + emsgf(_( "E421: Color name or number not recognized: %s"), key_start); - error = TRUE; + error = true; break; } - /* Use the _16 table to check if its a valid color name. */ + // Use the _16 table to check if its a valid color name. color = color_numbers_16[i]; if (color >= 0) { if (t_colors == 8) { - /* t_Co is 8: use the 8 colors table */ + // t_Co is 8: use the 8 colors table. color = color_numbers_8[i]; if (key[5] == 'F') { /* set/reset bold attribute to get light foreground @@ -6532,16 +6693,14 @@ do_highlight(char_u *line, int forceit, int init) { } } } - /* Add one to the argument, to avoid zero. Zero is used for - * "NONE", then "color" is -1. */ + // Add one to the argument, to avoid zero. Zero is used for + // "NONE", then "color" is -1. if (key[5] == 'F') { HL_TABLE()[idx].sg_cterm_fg = color + 1; if (is_normal_group) { cterm_normal_fg_color = color + 1; cterm_normal_fg_bold = (HL_TABLE()[idx].sg_cterm & HL_BOLD); - { - must_redraw = CLEAR; - } + must_redraw = CLEAR; } } else { HL_TABLE()[idx].sg_cterm_bg = color + 1; @@ -6565,15 +6724,15 @@ do_highlight(char_u *line, int forceit, int init) { } } } - } else if (STRCMP(key, "GUIFG") == 0) { + } else if (strcmp(key, "GUIFG") == 0) { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { if (!init) HL_TABLE()[idx].sg_set |= SG_GUI; xfree(HL_TABLE()[idx].sg_rgb_fg_name); - if (STRCMP(arg, "NONE")) { - HL_TABLE()[idx].sg_rgb_fg_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_fg = name_to_color(arg); + if (strcmp(arg, "NONE")) { + HL_TABLE()[idx].sg_rgb_fg_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_fg = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_fg_name = NULL; HL_TABLE()[idx].sg_rgb_fg = -1; @@ -6590,8 +6749,8 @@ do_highlight(char_u *line, int forceit, int init) { xfree(HL_TABLE()[idx].sg_rgb_bg_name); if (STRCMP(arg, "NONE") != 0) { - HL_TABLE()[idx].sg_rgb_bg_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_bg = name_to_color(arg); + HL_TABLE()[idx].sg_rgb_bg_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_bg = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_bg_name = NULL; HL_TABLE()[idx].sg_rgb_bg = -1; @@ -6601,15 +6760,15 @@ do_highlight(char_u *line, int forceit, int init) { if (is_normal_group) { normal_bg = HL_TABLE()[idx].sg_rgb_bg; } - } else if (STRCMP(key, "GUISP") == 0) { + } else if (strcmp(key, "GUISP") == 0) { if (!init || !(HL_TABLE()[idx].sg_set & SG_GUI)) { if (!init) HL_TABLE()[idx].sg_set |= SG_GUI; xfree(HL_TABLE()[idx].sg_rgb_sp_name); - if (STRCMP(arg, "NONE") != 0) { - HL_TABLE()[idx].sg_rgb_sp_name = (uint8_t *)xstrdup((char *)arg); - HL_TABLE()[idx].sg_rgb_sp = name_to_color(arg); + if (strcmp(arg, "NONE") != 0) { + HL_TABLE()[idx].sg_rgb_sp_name = (char_u *)xstrdup((char *)arg); + HL_TABLE()[idx].sg_rgb_sp = name_to_color((const char_u *)arg); } else { HL_TABLE()[idx].sg_rgb_sp_name = NULL; HL_TABLE()[idx].sg_rgb_sp = -1; @@ -6619,31 +6778,25 @@ do_highlight(char_u *line, int forceit, int init) { if (is_normal_group) { normal_sp = HL_TABLE()[idx].sg_rgb_sp; } - } else if (STRCMP(key, "START") == 0 || STRCMP(key, "STOP") == 0) { + } else if (strcmp(key, "START") == 0 || strcmp(key, "STOP") == 0) { // Ignored for now } else { - EMSG2(_("E423: Illegal argument: %s"), key_start); - error = TRUE; + emsgf(_("E423: Illegal argument: %s"), key_start); + error = true; break; } - /* - * When highlighting has been given for a group, don't link it. - */ + // When highlighting has been given for a group, don't link it. if (!init || !(HL_TABLE()[idx].sg_set & SG_LINK)) { HL_TABLE()[idx].sg_link = 0; } - /* - * Continue with next argument. - */ - linep = skipwhite(linep); + // Continue with next argument. + linep = (const char *)skipwhite((const char_u *)linep); } } - /* - * If there is an error, and it's a new entry, remove it from the table. - */ + // If there is an error, and it's a new entry, remove it from the table. if (error && idx == highlight_ga.ga_len) { syn_unadd_group(); } else { @@ -7201,7 +7354,7 @@ char_u *syn_id2name(int id) /* * Like syn_name2id(), but take a pointer + length argument. */ -int syn_namen2id(char_u *linep, int len) +int syn_namen2id(const char_u *linep, int len) { char_u *name = vim_strnsave(linep, len); int id = syn_name2id(name); @@ -7217,7 +7370,7 @@ int syn_namen2id(char_u *linep, int len) /// @param len length of \p pp /// /// @return 0 for failure else the id of the group -int syn_check_group(char_u *pp, int len) +int syn_check_group(const char_u *pp, int len) { char_u *name = vim_strnsave(pp, len); int id = syn_name2id(name); @@ -8220,7 +8373,7 @@ color_name_table_T color_name_table[] = { /// /// @param[in] name string value to convert to RGB /// return the hex value or -1 if could not find a correct value -RgbValue name_to_color(const uint8_t *name) +RgbValue name_to_color(const char_u *name) { if (name[0] == '#' && isxdigit(name[1]) && isxdigit(name[2]) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 4e8a9b8523..615a59573f 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -993,107 +993,6 @@ void viml_pexpr_free_ast(ExprAST ast) // > ># >? <= <=# <=? // < <# = >=# >=? // is is# is? isnot isnot# isnot? -// -// Used highlighting groups and assumed linkage: -// -// NVimInternalError -> highlight as fg:red/bg:red -// -// NVimInvalid -> Error -// NVimInvalidValue -> NVimInvalid -// NVimInvalidOperator -> NVimInvalid -// NVimInvalidDelimiter -> NVimInvalid -// -// NVimOperator -> Operator -// NVimUnaryOperator -> NVimOperator -// NVimBinaryOperator -> NVimOperator -// -// NVimComparisonOperator -> NVimBinaryOperator -// NVimComparisonOperatorModifier -> NVimComparisonOperator -// -// NVimTernary -> NVimOperator -// NVimTernaryColon -> NVimTernary -// -// NVimParenthesis -> Delimiter -// -// NVimColon -> Delimiter -// NVimComma -> Delimiter -// NVimArrow -> Delimiter -// -// NVimLambda -> Delimiter -// NVimDict -> Delimiter -// NVimCurly -> Delimiter -// -// NVimList -> Delimiter -// NVimSubscript -> Delimiter -// NVimSubscriptColon -> NVimSubscript -// -// NVimIdentifier -> Identifier -// NVimIdentifierScope -> NVimIdentifier -// NVimIdentifierScopeDelimiter -> NVimIdentifier -// -// NVimIdentifierKey -> Identifier -// -// NVimUnaryPlus -> NVimUnaryOperator -// NVimBinaryPlus -> NVimBinaryOperator -// NVimConcat -> NVimBinaryOperator -// NVimConcatOrSubscript -> NVimConcat -// -// NVimRegister -> SpecialChar -// NVimNumber -> Number -// NVimNumberPrefix -> SpecialChar -// NVimFloat -> NVimNumber -// -// NVimNestingParenthesis -> NVimParenthesis -// NVimCallingParenthesis -> NVimParenthesis -// -// NVimString -> String -// NVimStringSpecial -> SpecialChar -// NVimSingleQuote -> NVimString -// NVimSingleQuotedBody -> NVimString -// NVimSingleQuotedQuote -> NVimStringSpecial -// NVimDoubleQuote -> NVimString -// NVimDoubleQuotedBody -> NVimString -// NVimDoubleQuotedEscape -> NVimStringSpecial -// NVimDoubleQuotedUnknownEscape -> NVimInvalid -// -// " Note: NVimDoubleQuotedUnknownEscape is not actually invalid -// -// NVimInvalidComma -> NVimInvalidDelimiter -// NVimInvalidSpacing -> NVimInvalid -// NVimInvalidTernary -> NVimInvalidOperator -// NVimInvalidTernaryColon -> NVimInvalidTernary -// NVimInvalidRegister -> NVimInvalidValue -// NVimInvalidClosingBracket -> NVimInvalidDelimiter -// NVimInvalidSpacing -> NVimInvalid -// NVimInvalidArrow -> NVimInvalidDelimiter -// NVimInvalidLambda -> NVimInvalidDelimiter -// NVimInvalidDict -> NVimInvalidDelimiter -// NVimInvalidCurly -> NVimInvalidDelimiter -// NVimInvalidFigureBrace -> NVimInvalidDelimiter -// NVimInvalidIdentifier -> NVimInvalidValue -// NVimInvalidIdentifierScope -> NVimInvalidValue -// NVimInvalidIdentifierScopeDelimiter -> NVimInvalidValue -// NVimInvalidComparisonOperator -> NVimInvalidOperator -// NVimInvalidComparisonOperatorModifier -> NVimInvalidComparisonOperator -// NVimInvalidNumber -> NVimInvalidValue -// NVimInvalidFloat -> NVimInvalidValue -// NVimInvalidIdentifierKey -> NVimInvalidIdentifier -// NVimInvalidList -> NVimInvalidDelimiter -// NVimInvalidSubscript -> NVimInvalidDelimiter -// NVimInvalidSubscriptColon -> NVimInvalidSubscript -// NVimInvalidString -> NVimInvalidValue -// NVimInvalidStringSpecial -> NVimInvalidString -// NVimInvalidSingleQuote -> NVimInvalidString -// NVimInvalidSingleQuotedBody -> NVimInvalidString -// NVimInvalidSingleQuotedQuote -> NVimInvalidStringSpecial -// NVimInvalidDoubleQuote -> NVimInvalidString -// NVimInvalidDoubleQuotedBody -> NVimInvalidString -// NVimInvalidDoubleQuotedEscape -> NVimInvalidStringSpecial -// NVimInvalidDoubleQuotedUnknownEscape -> NVimInvalidDoubleQuotedEscape -// -// NVimFigureBrace -> NVimInternalError -// NVimInvalidSingleQuotedUnknownEscape -> NVimInternalError -// NVimSingleQuotedUnknownEscape -> NVimInternalError /// Allocate a new node and set some of the values /// @@ -2183,12 +2082,12 @@ viml_pexpr_parse_process_token: ADD_OP_NODE(cur_node); if (cur_token.data.cmp.ccs != kCCStrategyUseOption) { viml_parser_highlight(pstate, cur_token.start, cur_token.len - 1, - HL(ComparisonOperator)); + HL(Comparison)); viml_parser_highlight( pstate, shifted_pos(cur_token.start, cur_token.len - 1), 1, - HL(ComparisonOperatorModifier)); + HL(ComparisonModifier)); } else { - HL_CUR_TOKEN(ComparisonOperator); + HL_CUR_TOKEN(Comparison); } want_node = kENodeValue; break; @@ -2390,7 +2289,7 @@ viml_pexpr_parse_valid_colon: break; } case kExprNodeSubscript: { - HL_CUR_TOKEN(Subscript); + HL_CUR_TOKEN(SubscriptBracket); break; } default: { @@ -2418,7 +2317,7 @@ viml_pexpr_parse_bracket_closing_error: } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeSubscript); ADD_OP_NODE(cur_node); - HL_CUR_TOKEN(Subscript); + HL_CUR_TOKEN(SubscriptBracket); } } break; @@ -2626,7 +2525,7 @@ viml_pexpr_parse_figure_brace_closing_error: cur_token.len - scope_shift, (node_is_key ? HL(IdentifierKey) - : HL(Identifier))); + : HL(IdentifierName))); } else { if (scope == kExprVarScopeMissing) { ADD_IDENT( @@ -2637,7 +2536,7 @@ viml_pexpr_parse_figure_brace_closing_error: cur_node->data.var.ident_len = cur_token.len; want_node = kENodeOperator; } while (0), - Identifier); + IdentifierName); } else { OP_MISSING; } diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 125a658f7b..454fbad236 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -1183,7 +1183,7 @@ describe('Expressions parser', function() 'PlainIdentifier(scope=0,ident=var):0:0:var', }, }, { - hl('Identifier', 'var'), + hl('IdentifierName', 'var'), }) check_parsing('g:var', 0, { ast = { @@ -1192,7 +1192,7 @@ describe('Expressions parser', function() }, { hl('IdentifierScope', 'g'), hl('IdentifierScopeDelimiter', ':'), - hl('Identifier', 'var'), + hl('IdentifierName', 'var'), }) check_parsing('g:', 0, { ast = { @@ -1214,7 +1214,7 @@ describe('Expressions parser', function() }, }, { hl('Curly', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Curly', '}'), }) check_parsing('{a:b}', 0, { @@ -1231,7 +1231,7 @@ describe('Expressions parser', function() hl('Curly', '{'), hl('IdentifierScope', 'a'), hl('IdentifierScopeDelimiter', ':'), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Curly', '}'), }) check_parsing('{a:@b}', 0, { @@ -1347,7 +1347,7 @@ describe('Expressions parser', function() hl('Curly', '{'), hl('Register', '@a'), hl('Curly', '}'), - hl('Identifier', '_test'), + hl('IdentifierName', '_test'), }) check_parsing('g:{@a}_test', 0, { -- 01234567890 @@ -1377,7 +1377,7 @@ describe('Expressions parser', function() hl('Curly', '{'), hl('Register', '@a'), hl('Curly', '}'), - hl('Identifier', '_test'), + hl('IdentifierName', '_test'), }) check_parsing('g:{@a}_test()', 0, { -- 0123456789012 @@ -1412,7 +1412,7 @@ describe('Expressions parser', function() hl('Curly', '{'), hl('Register', '@a'), hl('Curly', '}'), - hl('Identifier', '_test'), + hl('IdentifierName', '_test'), hl('CallingParenthesis', '('), hl('CallingParenthesis', ')'), }) @@ -1563,7 +1563,7 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Arrow', '->'), hl('Register', '@a'), hl('Lambda', '}'), @@ -1592,9 +1592,9 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Arrow', '->'), hl('Register', '@a'), hl('Lambda', '}'), @@ -1629,11 +1629,11 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Comma', ','), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), hl('Arrow', '->'), hl('Register', '@a'), hl('Lambda', '}'), @@ -1674,13 +1674,13 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Comma', ','), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), hl('Comma', ','), - hl('Identifier', 'd'), + hl('IdentifierName', 'd'), hl('Arrow', '->'), hl('Register', '@a'), hl('Lambda', '}'), @@ -1726,13 +1726,13 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Comma', ','), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), hl('Comma', ','), - hl('Identifier', 'd'), + hl('IdentifierName', 'd'), hl('Comma', ','), hl('Arrow', '->'), hl('Register', '@a'), @@ -1797,19 +1797,19 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Arrow', '->'), hl('Lambda', '{'), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), hl('Comma', ','), - hl('Identifier', 'd'), + hl('IdentifierName', 'd'), hl('Arrow', '->'), hl('Lambda', '{'), - hl('Identifier', 'e'), + hl('IdentifierName', 'e'), hl('Comma', ','), - hl('Identifier', 'f'), + hl('IdentifierName', 'f'), hl('Arrow', '->'), hl('Register', '@a'), hl('Lambda', '}'), @@ -1850,13 +1850,13 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Arrow', '->'), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), hl('InvalidComma', ','), - hl('Identifier', 'd'), + hl('IdentifierName', 'd'), hl('Lambda', '}'), }) check_parsing('a,b,c,d', 0, { @@ -1887,13 +1887,13 @@ describe('Expressions parser', function() msg = 'E15: Comma outside of call, lambda or literal: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('InvalidComma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('InvalidComma', ','), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), hl('InvalidComma', ','), - hl('Identifier', 'd'), + hl('IdentifierName', 'd'), }) check_parsing('a,b,c,d,', 0, { -- 0123456789 @@ -1928,13 +1928,13 @@ describe('Expressions parser', function() msg = 'E15: Comma outside of call, lambda or literal: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('InvalidComma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('InvalidComma', ','), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), hl('InvalidComma', ','), - hl('Identifier', 'd'), + hl('IdentifierName', 'd'), hl('InvalidComma', ','), }) check_parsing(',', 0, { @@ -1983,7 +1983,7 @@ describe('Expressions parser', function() }, { hl('Curly', '{'), hl('InvalidComma', ','), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('InvalidArrow', '->'), hl('Register', '@a'), hl('Curly', '}'), @@ -2041,9 +2041,9 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('InvalidLambda', '}'), }) check_parsing('{a,}', 0, { @@ -2067,7 +2067,7 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), hl('InvalidLambda', '}'), }) @@ -2343,9 +2343,9 @@ describe('Expressions parser', function() hl('Curly', '{'), hl('NestingParenthesis', '('), hl('Lambda', '{'), - hl('Identifier', 'f'), + hl('IdentifierName', 'f'), hl('Arrow', '->', 1), - hl('Identifier', 'g', 1), + hl('IdentifierName', 'g', 1), hl('Lambda', '}'), hl('NestingParenthesis', ')'), hl('CallingParenthesis', '('), @@ -2387,11 +2387,11 @@ describe('Expressions parser', function() hl('IdentifierScope', 'a'), hl('IdentifierScopeDelimiter', ':'), hl('Curly', '{'), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('CallingParenthesis', '('), hl('CallingParenthesis', ')'), hl('Curly', '}'), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), }) check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', 0, { -- 01234567890123456789012345678901234567890123456 @@ -2478,9 +2478,9 @@ describe('Expressions parser', function() hl('IdentifierScopeDelimiter', ':'), hl('Curly', '{'), hl('Lambda', '{'), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Comma', ','), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('Arrow', '->', 1), hl('Register', '@d', 1), hl('BinaryPlus', '+', 1), @@ -2488,9 +2488,9 @@ describe('Expressions parser', function() hl('BinaryPlus', '+', 1), hl('NestingParenthesis', '(', 1), hl('Lambda', '{'), - hl('Identifier', 'f'), + hl('IdentifierName', 'f'), hl('Arrow', '->', 1), - hl('Identifier', 'g', 1), + hl('IdentifierName', 'g', 1), hl('Lambda', '}'), hl('NestingParenthesis', ')'), hl('CallingParenthesis', '('), @@ -2501,7 +2501,7 @@ describe('Expressions parser', function() hl('Register', '@i'), hl('CallingParenthesis', ')'), hl('Curly', '}'), - hl('Identifier', 'j'), + hl('IdentifierName', 'j'), }) check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', 0, { -- 01234567890123456789012345678901234567 @@ -2635,13 +2635,13 @@ describe('Expressions parser', function() msg = 'E15: Arrow outside of lambda: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('InvalidArrow', '->', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('InvalidArrow', '->', 1), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('InvalidArrow', '->', 1), - hl('Identifier', 'd', 1), + hl('IdentifierName', 'd', 1), }) check_parsing('{a -> b -> c}', 0, { -- 0123456789012 @@ -2672,11 +2672,11 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Arrow', '->', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('InvalidArrow', '->', 1), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('Lambda', '}'), }) check_parsing('{a: -> b}', 0, { @@ -2704,7 +2704,7 @@ describe('Expressions parser', function() hl('IdentifierScope', 'a'), hl('IdentifierScopeDelimiter', ':'), hl('InvalidArrow', '->', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('Curly', '}'), }) @@ -2732,9 +2732,9 @@ describe('Expressions parser', function() hl('Curly', '{'), hl('IdentifierScope', 'a'), hl('IdentifierScopeDelimiter', ':'), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('InvalidArrow', '->', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('Curly', '}'), }) @@ -2760,9 +2760,9 @@ describe('Expressions parser', function() }, }, { hl('Curly', '{'), - hl('Identifier', 'a#b'), + hl('IdentifierName', 'a#b'), hl('InvalidArrow', '->', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('Curly', '}'), }) check_parsing('{a : b : c}', 0, { @@ -2794,11 +2794,11 @@ describe('Expressions parser', function() }, }, { hl('Dict', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Colon', ':', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('InvalidColon', ':', 1), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('Dict', '}'), }) check_parsing('{', 0, { @@ -2829,7 +2829,7 @@ describe('Expressions parser', function() }, }, { hl('FigureBrace', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), }) check_parsing('{a,b', 0, { -- 0123 @@ -2853,9 +2853,9 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), }) check_parsing('{a,b->', 0, { -- 012345 @@ -2880,9 +2880,9 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Arrow', '->'), }) check_parsing('{a,b->c', 0, { @@ -2913,11 +2913,11 @@ describe('Expressions parser', function() }, }, { hl('Lambda', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Arrow', '->'), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), }) check_parsing('{a : b', 0, { -- 012345 @@ -2941,9 +2941,9 @@ describe('Expressions parser', function() }, }, { hl('Dict', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Colon', ':', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), }) check_parsing('{a : b,', 0, { -- 0123456 @@ -2972,9 +2972,9 @@ describe('Expressions parser', function() }, }, { hl('Dict', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Colon', ':', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('Comma', ','), }) end) @@ -2997,11 +2997,11 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Ternary', '?', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('TernaryColon', ':', 1), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), }) check_parsing('@a?@b?@c:@d:@e', 0, { -- 01234567890123 @@ -3271,9 +3271,9 @@ describe('Expressions parser', function() msg = 'E109: Missing \':\' after \'?\': %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Ternary', '?'), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), }) check_parsing('a?b:', 0, { -- 0123 @@ -3296,7 +3296,7 @@ describe('Expressions parser', function() msg = 'E109: Missing \':\' after \'?\': %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Ternary', '?'), hl('IdentifierScope', 'b'), hl('IdentifierScopeDelimiter', ':'), @@ -3320,12 +3320,12 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Ternary', '?'), hl('IdentifierScope', 'b'), hl('IdentifierScopeDelimiter', ':'), hl('TernaryColon', ':'), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), }) check_parsing('a?b :', 0, { @@ -3349,9 +3349,9 @@ describe('Expressions parser', function() msg = 'E15: Expected value, got EOC: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Ternary', '?'), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('TernaryColon', ':', 1), }) @@ -3615,15 +3615,15 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Ternary', '?'), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Curly', '{'), - hl('Identifier', 'cdef'), + hl('IdentifierName', 'cdef'), hl('Curly', '}'), - hl('Identifier', 'g'), + hl('IdentifierName', 'g'), hl('TernaryColon', ':'), - hl('Identifier', 'h'), + hl('IdentifierName', 'h'), }) check_parsing('a ? b : c : d', 0, { -- 0123456789012 @@ -3654,13 +3654,13 @@ describe('Expressions parser', function() msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Ternary', '?', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('TernaryColon', ':', 1), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('InvalidColon', ':', 1), - hl('Identifier', 'd', 1), + hl('IdentifierName', 'd', 1), }) end) itp('works with comparison operators', function() @@ -3676,9 +3676,9 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '==', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('IdentifierName', 'b', 1), }) check_parsing('a ==? b', 0, { @@ -3693,10 +3693,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '==', 1), - hl('ComparisonOperatorModifier', '?'), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b', 1), }) check_parsing('a ==# b', 0, { @@ -3711,10 +3711,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '==', 1), - hl('ComparisonOperatorModifier', '#'), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), }) check_parsing('a !=# b', 0, { @@ -3729,10 +3729,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '!=', 1), - hl('ComparisonOperatorModifier', '#'), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '!=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), }) check_parsing('a <=# b', 0, { @@ -3747,10 +3747,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '<=', 1), - hl('ComparisonOperatorModifier', '#'), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '<=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), }) check_parsing('a >=# b', 0, { @@ -3765,10 +3765,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '>=', 1), - hl('ComparisonOperatorModifier', '#'), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '>=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), }) check_parsing('a ># b', 0, { @@ -3783,10 +3783,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '>', 1), - hl('ComparisonOperatorModifier', '#'), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '>', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), }) check_parsing('a <# b', 0, { @@ -3801,10 +3801,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '<', 1), - hl('ComparisonOperatorModifier', '#'), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), }) check_parsing('a is#b', 0, { @@ -3819,10 +3819,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', 'is', 1), - hl('ComparisonOperatorModifier', '#'), - hl('Identifier', 'b'), + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b'), }) check_parsing('a is?b', 0, { @@ -3837,10 +3837,10 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', 'is', 1), - hl('ComparisonOperatorModifier', '?'), - hl('Identifier', 'b'), + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b'), }) check_parsing('a isnot b', 0, { @@ -3855,9 +3855,9 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', 'isnot', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'a'), + hl('Comparison', 'isnot', 1), + hl('IdentifierName', 'b', 1), }) check_parsing('a < b < c', 0, { @@ -3882,12 +3882,43 @@ describe('Expressions parser', function() msg = 'E15: Operator is not associative: %.*s', }, }, { - hl('Identifier', 'a'), - hl('ComparisonOperator', '<', 1), - hl('Identifier', 'b', 1), - hl('InvalidComparisonOperator', '<', 1), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('IdentifierName', 'c', 1), }) + + check_parsing('a < b <# c', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:5: <#', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + err = { + arg = ' <# c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('InvalidComparisonModifier', '#'), + hl('IdentifierName', 'c', 1), + }) + check_parsing('a += b', 0, { -- 012345 ast = { @@ -3910,10 +3941,10 @@ describe('Expressions parser', function() msg = 'E15: Expected == or =~: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('BinaryPlus', '+', 1), - hl('InvalidComparisonOperator', '='), - hl('Identifier', 'b', 1), + hl('InvalidComparison', '='), + hl('IdentifierName', 'b', 1), }) check_parsing('a + b == c + d', 0, { -- 01234567890123 @@ -3940,13 +3971,13 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('BinaryPlus', '+', 1), - hl('Identifier', 'b', 1), - hl('ComparisonOperator', '==', 1), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'b', 1), + hl('Comparison', '==', 1), + hl('IdentifierName', 'c', 1), hl('BinaryPlus', '+', 1), - hl('Identifier', 'd', 1), + hl('IdentifierName', 'd', 1), }) check_parsing('+ a == + b', 0, { -- 0123456789 @@ -3971,10 +4002,10 @@ describe('Expressions parser', function() }, }, { hl('UnaryPlus', '+'), - hl('Identifier', 'a', 1), - hl('ComparisonOperator', '==', 1), + hl('IdentifierName', 'a', 1), + hl('Comparison', '==', 1), hl('UnaryPlus', '+', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), }) end) itp('works with concat/subscript', function() @@ -4011,7 +4042,7 @@ describe('Expressions parser', function() msg = 'E15: Expected value, got EOC: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('ConcatOrSubscript', '.'), }) @@ -4027,7 +4058,7 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('ConcatOrSubscript', '.'), hl('IdentifierKey', 'b'), }) @@ -4084,7 +4115,7 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Concat', '.', 1), hl('Number', '1', 1), hl('ConcatOrSubscript', '.'), @@ -4116,7 +4147,7 @@ describe('Expressions parser', function() hl('BinaryPlus', '+', 1), hl('Float', '1.2', 1), hl('Concat', '.', 1), - hl('Identifier', 'a', 1), + hl('IdentifierName', 'a', 1), }) check_parsing('1.3e-5 + a . 1.2', 0, { @@ -4146,7 +4177,7 @@ describe('Expressions parser', function() }, { hl('Float', '1.3e-5'), hl('BinaryPlus', '+', 1), - hl('Identifier', 'a', 1), + hl('IdentifierName', 'a', 1), hl('Concat', '.', 1), hl('Number', '1', 1), hl('ConcatOrSubscript', '.'), @@ -4196,7 +4227,7 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('ConcatOrSubscript', '.'), hl('IdentifierKey', '1'), hl('ConcatOrSubscript', '.'), @@ -4221,7 +4252,7 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Concat', '.', 1), hl('Number', '1', 1), hl('ConcatOrSubscript', '.'), @@ -4251,10 +4282,10 @@ describe('Expressions parser', function() }, }, { hl('UnaryPlus', '+'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Concat', '.', 1), hl('UnaryPlus', '+', 1), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), }) check_parsing('a. b', 0, { @@ -4269,9 +4300,9 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('ConcatOrSubscript', '.'), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), }) check_parsing('a. 1', 0, { @@ -4286,7 +4317,7 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('ConcatOrSubscript', '.'), hl('Number', '1', 1), }) @@ -4324,9 +4355,9 @@ describe('Expressions parser', function() msg = 'E15: Expected value, got closing bracket: %.*s', }, }, { - hl('Identifier', 'a'), - hl('Subscript', '['), - hl('InvalidSubscript', ']'), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), }) check_parsing('a[b:]', 0, { -- 01234 @@ -4340,11 +4371,11 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('Subscript', '['), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), hl('IdentifierScope', 'b'), hl('IdentifierScopeDelimiter', ':'), - hl('Subscript', ']'), + hl('SubscriptBracket', ']'), }) check_parsing('a[b:c]', 0, { @@ -4359,12 +4390,12 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('Subscript', '['), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), hl('IdentifierScope', 'b'), hl('IdentifierScopeDelimiter', ':'), - hl('Identifier', 'c'), - hl('Subscript', ']'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), }) check_parsing('a[b : c]', 0, { -- 01234567 @@ -4384,12 +4415,12 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('Subscript', '['), - hl('Identifier', 'b'), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), hl('SubscriptColon', ':', 1), - hl('Identifier', 'c', 1), - hl('Subscript', ']'), + hl('IdentifierName', 'c', 1), + hl('SubscriptBracket', ']'), }) check_parsing('a[: b]', 0, { @@ -4410,11 +4441,11 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('Subscript', '['), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), hl('SubscriptColon', ':'), - hl('Identifier', 'b', 1), - hl('Subscript', ']'), + hl('IdentifierName', 'b', 1), + hl('SubscriptBracket', ']'), }) check_parsing('a[b :]', 0, { @@ -4434,11 +4465,11 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('Subscript', '['), - hl('Identifier', 'b'), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), hl('SubscriptColon', ':', 1), - hl('Subscript', ']'), + hl('SubscriptBracket', ']'), }) check_parsing('a[b][c][d](e)(f)(g)', 0, { -- 0123456789012345678 @@ -4483,24 +4514,24 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), - hl('Subscript', '['), - hl('Identifier', 'b'), - hl('Subscript', ']'), - hl('Subscript', '['), - hl('Identifier', 'c'), - hl('Subscript', ']'), - hl('Subscript', '['), - hl('Identifier', 'd'), - hl('Subscript', ']'), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), hl('CallingParenthesis', '('), - hl('Identifier', 'e'), + hl('IdentifierName', 'e'), hl('CallingParenthesis', ')'), hl('CallingParenthesis', '('), - hl('Identifier', 'f'), + hl('IdentifierName', 'f'), hl('CallingParenthesis', ')'), hl('CallingParenthesis', '('), - hl('Identifier', 'g'), + hl('IdentifierName', 'g'), hl('CallingParenthesis', ')'), }) check_parsing('{a}{b}{c}[d][e][f]', 0, { @@ -4556,23 +4587,23 @@ describe('Expressions parser', function() }, }, { hl('Curly', '{'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Curly', '}'), hl('Curly', '{'), - hl('Identifier', 'b'), + hl('IdentifierName', 'b'), hl('Curly', '}'), hl('Curly', '{'), - hl('Identifier', 'c'), + hl('IdentifierName', 'c'), hl('Curly', '}'), - hl('Subscript', '['), - hl('Identifier', 'd'), - hl('Subscript', ']'), - hl('Subscript', '['), - hl('Identifier', 'e'), - hl('Subscript', ']'), - hl('Subscript', '['), - hl('Identifier', 'f'), - hl('Subscript', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'e'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'f'), + hl('SubscriptBracket', ']'), }) end) itp('supports list literals', function() @@ -4598,7 +4629,7 @@ describe('Expressions parser', function() }, }, { hl('List', '['), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('List', ']'), }) @@ -4620,9 +4651,9 @@ describe('Expressions parser', function() }, }, { hl('List', '['), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('List', ']'), }) @@ -4650,11 +4681,11 @@ describe('Expressions parser', function() }, }, { hl('List', '['), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('Comma', ','), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('List', ']'), }) @@ -4688,11 +4719,11 @@ describe('Expressions parser', function() }, }, { hl('List', '['), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Comma', ','), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('Comma', ','), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('Comma', ','), hl('List', ']', 1), }) @@ -4732,13 +4763,13 @@ describe('Expressions parser', function() }, }, { hl('List', '['), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('InvalidColon', ':', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('Comma', ','), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('InvalidColon', ':', 1), - hl('Identifier', 'd', 1), + hl('IdentifierName', 'd', 1), hl('List', ']'), }) @@ -4770,7 +4801,7 @@ describe('Expressions parser', function() msg = 'E15: Unexpected closing figure brace: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('InvalidList', ']'), }) @@ -4814,8 +4845,8 @@ describe('Expressions parser', function() }, { hl('List', '['), hl('List', ']'), - hl('Subscript', '['), - hl('InvalidSubscript', ']'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), }) check_parsing('[', 0, { @@ -6119,13 +6150,13 @@ describe('Expressions parser', function() }, }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('And', '&&', 1), - hl('Identifier', 'b', 1), + hl('IdentifierName', 'b', 1), hl('Or', '||', 1), - hl('Identifier', 'c', 1), + hl('IdentifierName', 'c', 1), hl('And', '&&', 1), - hl('Identifier', 'd', 1), + hl('IdentifierName', 'd', 1), }) check_parsing('&& a', 0, { @@ -6145,7 +6176,7 @@ describe('Expressions parser', function() }, }, { hl('InvalidAnd', '&&'), - hl('Identifier', 'a', 1), + hl('IdentifierName', 'a', 1), }) check_parsing('|| a', 0, { @@ -6165,7 +6196,7 @@ describe('Expressions parser', function() }, }, { hl('InvalidOr', '||'), - hl('Identifier', 'a', 1), + hl('IdentifierName', 'a', 1), }) check_parsing('a||', 0, { @@ -6183,7 +6214,7 @@ describe('Expressions parser', function() msg = 'E15: Expected value, got EOC: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Or', '||'), }) @@ -6202,7 +6233,7 @@ describe('Expressions parser', function() msg = 'E15: Expected value, got EOC: %.*s', }, }, { - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('And', '&&'), }) @@ -6280,7 +6311,7 @@ describe('Expressions parser', function() }, }, { hl('NestingParenthesis', '('), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('Or', '||'), hl('InvalidNestingParenthesis', ')'), }) @@ -6307,7 +6338,7 @@ describe('Expressions parser', function() }, }, { hl('NestingParenthesis', '('), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('And', '&&'), hl('InvalidNestingParenthesis', ')'), }) @@ -6335,7 +6366,7 @@ describe('Expressions parser', function() }, { hl('NestingParenthesis', '('), hl('InvalidAnd', '&&'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('NestingParenthesis', ')'), }) @@ -6362,7 +6393,7 @@ describe('Expressions parser', function() }, { hl('NestingParenthesis', '('), hl('InvalidOr', '||'), - hl('Identifier', 'a'), + hl('IdentifierName', 'a'), hl('NestingParenthesis', ')'), }) end) @@ -6433,7 +6464,7 @@ describe('Expressions parser', function() hl('OptionSigil', '&'), hl('Option', 's'), hl('InvalidColon', ':'), - hl('Identifier', 'opt'), + hl('IdentifierName', 'opt'), }) check_parsing('& ', 0, { @@ -6496,7 +6527,7 @@ describe('Expressions parser', function() }, { hl('OptionSigil', '&'), hl('Option', 'xxx'), - hl('InvalidIdentifier', '_yyy'), + hl('InvalidIdentifierName', '_yyy'), }) check_parsing('(1+&)', 0, { @@ -6588,7 +6619,7 @@ describe('Expressions parser', function() hl('EnvironmentSigil', '$'), hl('Environment', 'g'), hl('InvalidColon', ':'), - hl('Identifier', 'A'), + hl('IdentifierName', 'A'), }) check_parsing('$A', 0, { @@ -7092,7 +7123,7 @@ describe('Expressions parser', function() }, }, { hl('InvalidOr', '|'), - hl('InvalidIdentifier', '\029'), + hl('InvalidIdentifierName', '\029'), }) check_parsing('"\\<', 0, { -- 012 @@ -7137,7 +7168,7 @@ describe('Expressions parser', function() }, }, { hl('InvalidFigureBrace', '}'), - hl('InvalidIdentifier', 'l'), + hl('InvalidIdentifierName', 'l'), }) check_parsing(':?\000\000\000\000\000\000\000', 0, { ast = { @@ -7164,4 +7195,5 @@ describe('Expressions parser', function() hl('InvalidTernary', '?'), }) end) + -- FIXME: check flag effects end) From b91cb18c3688a4a936c14484af57de05ca113641 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 21:42:37 +0300 Subject: [PATCH 064/109] syntax: Adjust position and arguments of syn_init_cmdline_highlight This way it works both after `nvim -u NORC` and after that and `colorscheme wombat256mod`. Removed the comment because I do not actually know why it works here with these arguments and not in previous position with previous arguments. --- src/nvim/syntax.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index b418df77a4..f8a62423ce 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -6248,11 +6248,8 @@ init_highlight(int both, int reset) (void)source_runtime((char_u *)"syntax/syncolor.vim", DIP_ALL); recursive--; } - // Without syncolor.vim it is going to screw everything over by defining - // cleared highlight groups by creating links to non-existent groups. This - // effectively prevents ":highlight default" from working properly. - syn_init_cmdline_highlight(reset, true); } + syn_init_cmdline_highlight(false, false); } /* From 538af1c90a4ac9928f60e97338869e516def4956 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 29 Oct 2017 22:02:19 +0300 Subject: [PATCH 065/109] syntax,viml/parser/expressions: Add missing highlight groups Also adjusts some names. --- src/nvim/syntax.c | 39 ++- src/nvim/viml/parser/expressions.c | 12 +- test/unit/viml/expressions/parser_spec.lua | 290 ++++++++++----------- 3 files changed, 188 insertions(+), 153 deletions(-) diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index f8a62423ce..d787790bc3 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -6029,6 +6029,8 @@ static const char *highlight_init_cmdline[] = { "default link NVimUnaryOperator NVimOperator", "default link NVimUnaryPlus NVimUnaryOperator", + "default link NVimUnaryMinus NVimUnaryOperator", + "default link NVimNot NVimUnaryOperator", "default link NVimBinaryOperator NVimOperator", "default link NVimComparison NVimBinaryOperator", @@ -6036,6 +6038,11 @@ static const char *highlight_init_cmdline[] = { "default link NVimBinaryPlus NVimBinaryOperator", "default link NVimConcat NVimBinaryOperator", "default link NVimConcatOrSubscript NVimConcat", + "default link NVimOr NVimBinaryOperator", + "default link NVimAnd NVimBinaryOperator", + "default link NVimMultiplication NVimBinaryOperator", + "default link NVimDivision NVimBinaryOperator", + "default link NVimMod NVimBinaryOperator", "default link NVimTernary NVimOperator", "default link NVimTernaryColon NVimTernary", @@ -6068,7 +6075,15 @@ static const char *highlight_init_cmdline[] = { "default link NVimRegister SpecialChar", "default link NVimNumber Number", "default link NVimFloat NVimNumber", - "default link NVimNumberPrefix SpecialChar", + "default link NVimNumberPrefix Type", + + "default link NVimOptionSigil Type", + "default link NVimOptionName NVimIdentifier", + "default link NVimOptionScope NVimIdentifierScope", + "default link NVimOptionScopeDelimiter NVimIdentifierScopeDelimiter", + + "default link NVimEnvironmentSigil NVimOptionSigil", + "default link NVimEnvironmentName NVimIdentifier", "default link NVimString String", "default link NVimStringBody NVimString", @@ -6089,6 +6104,8 @@ static const char *highlight_init_cmdline[] = { "default link NVimFigureBrace NVimInternalError", "default link NVimSingleQuotedUnknownEscape NVimInternalError", + "default link NVimSpacing Normal", + // NVimInvalid groups: "default link NVimInvalidSingleQuotedUnknownEscape NVimInternalError", @@ -6099,6 +6116,8 @@ static const char *highlight_init_cmdline[] = { "default link NVimInvalidUnaryOperator NVimInvalidOperator", "default link NVimInvalidUnaryPlus NVimInvalidUnaryOperator", + "default link NVimInvalidUnaryMinus NVimInvalidUnaryOperator", + "default link NVimInvalidNot NVimInvalidUnaryOperator", "default link NVimInvalidBinaryOperator NVimInvalidOperator", "default link NVimInvalidComparison NVimInvalidBinaryOperator", @@ -6106,6 +6125,11 @@ static const char *highlight_init_cmdline[] = { "default link NVimInvalidBinaryPlus NVimInvalidBinaryOperator", "default link NVimInvalidConcat NVimInvalidBinaryOperator", "default link NVimInvalidConcatOrSubscript NVimInvalidConcat", + "default link NVimInvalidOr NVimInvalidBinaryOperator", + "default link NVimInvalidAnd NVimInvalidBinaryOperator", + "default link NVimInvalidMultiplication NVimInvalidBinaryOperator", + "default link NVimInvalidDivision NVimInvalidBinaryOperator", + "default link NVimInvalidMod NVimInvalidBinaryOperator", "default link NVimInvalidTernary NVimInvalidOperator", "default link NVimInvalidTernaryColon NVimInvalidTernary", @@ -6144,6 +6168,15 @@ static const char *highlight_init_cmdline[] = { "default link NVimInvalidFloat NVimInvalidNumber", "default link NVimInvalidNumberPrefix NVimInvalidNumber", + "default link NVimInvalidOptionSigil NVimInvalidIdentifier", + "default link NVimInvalidOptionName NVimInvalidIdentifier", + "default link NVimInvalidOptionScope NVimInvalidIdentifierScope", + "default link NVimInvalidOptionScopeDelimiter " + "NVimInvalidIdentifierScopeDelimiter", + + "default link NVimInvalidEnvironmentSigil NVimInvalidOptionSigil", + "default link NVimInvalidEnvironmentName NVimInvalidIdentifier", + // Invalid string bodies and specials are still highlighted as valid ones to // minimize the red area. "default link NVimInvalidString NVimInvalidValue", @@ -6160,7 +6193,9 @@ static const char *highlight_init_cmdline[] = { "default link NVimInvalidDoubleQuotedEscape NVimInvalidStringSpecial", "default link NVimInvalidDoubleQuotedUnknownEscape NVimInvalidValue", - "default link NVimInvalidFigureBrace NVimInternalError", + "default link NVimInvalidFigureBrace NVimInvalidDelimiter", + + "default link NVimInvalidSpacing ErrorMsg", }; /// Create default links for NVim* highlight groups used for cmdline coloring diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 615a59573f..f5bc547d54 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1472,7 +1472,7 @@ static void parse_quoted_string(ParserState *const pstate, kvec_withinit_t(StringShift, 16) shifts; kvi_init(shifts); if (!is_double) { - viml_parser_highlight(pstate, token.start, 1, HL(SingleQuotedString)); + viml_parser_highlight(pstate, token.start, 1, HL(SingleQuote)); while (p < e) { const char *const chunk_e = memchr(p, '\'', (size_t)(e - p)); if (chunk_e == NULL) { @@ -1509,7 +1509,7 @@ static void parse_quoted_string(ParserState *const pstate, } } } else { - viml_parser_highlight(pstate, token.start, 1, HL(DoubleQuotedString)); + viml_parser_highlight(pstate, token.start, 1, HL(DoubleQuote)); for (p = s + 1; p < e; p++) { if (*p == '\\' && p + 1 < e) { p++; @@ -1741,10 +1741,10 @@ static void parse_quoted_string(ParserState *const pstate, if (token.data.str.closed) { if (is_double) { viml_parser_highlight(pstate, shifted_pos(token.start, token.len - 1), - 1, HL(DoubleQuotedString)); + 1, HL(DoubleQuote)); } else { viml_parser_highlight(pstate, shifted_pos(token.start, token.len - 1), - 1, HL(SingleQuotedString)); + 1, HL(SingleQuote)); } } kvi_destroy(shifts); @@ -2035,7 +2035,7 @@ viml_pexpr_parse_process_token: } viml_parser_highlight( pstate, shifted_pos(cur_token.start, scope_shift + 1), - cur_token.len - (scope_shift + 1), HL(Option)); + cur_token.len - (scope_shift + 1), HL(OptionName)); break; } case kExprLexEnv: { @@ -2053,7 +2053,7 @@ viml_pexpr_parse_process_token: want_node = kENodeOperator; viml_parser_highlight(pstate, cur_token.start, 1, HL(EnvironmentSigil)); viml_parser_highlight(pstate, shifted_pos(cur_token.start, 1), - cur_token.len - 1, HL(Environment)); + cur_token.len - 1, HL(EnvironmentName)); break; } case kExprLexNot: { diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 454fbad236..019fe69046 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -4888,9 +4888,9 @@ describe('Expressions parser', function() 'SingleQuotedString(val="abc"):0:0:\'abc\'', }, }, { - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), hl('SingleQuotedBody', 'abc'), - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), }) check_parsing('"abc"', 0, { -- 01234 @@ -4898,9 +4898,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="abc"):0:0:"abc"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedBody', 'abc'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('\'\'', 0, { -- 01 @@ -4908,8 +4908,8 @@ describe('Expressions parser', function() 'SingleQuotedString(val=NULL):0:0:\'\'', }, }, { - hl('SingleQuotedString', '\''), - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), + hl('SingleQuote', '\''), }) check_parsing('""', 0, { -- 01 @@ -4917,8 +4917,8 @@ describe('Expressions parser', function() 'DoubleQuotedString(val=NULL):0:0:""', }, }, { - hl('DoubleQuotedString', '"'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"', 0, { -- 0 @@ -4930,7 +4930,7 @@ describe('Expressions parser', function() msg = 'E114: Missing double quote: %.*s', }, }, { - hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuote', '"'), }) check_parsing('\'', 0, { -- 0 @@ -4942,7 +4942,7 @@ describe('Expressions parser', function() msg = 'E115: Missing single quote: %.*s', }, }, { - hl('InvalidSingleQuotedString', '\''), + hl('InvalidSingleQuote', '\''), }) check_parsing('"a', 0, { -- 01 @@ -4954,7 +4954,7 @@ describe('Expressions parser', function() msg = 'E114: Missing double quote: %.*s', }, }, { - hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuote', '"'), hl('InvalidDoubleQuotedBody', 'a'), }) check_parsing('\'a', 0, { @@ -4967,7 +4967,7 @@ describe('Expressions parser', function() msg = 'E115: Missing single quote: %.*s', }, }, { - hl('InvalidSingleQuotedString', '\''), + hl('InvalidSingleQuote', '\''), hl('InvalidSingleQuotedBody', 'a'), }) check_parsing('\'abc\'\'def\'', 0, { @@ -4976,11 +4976,11 @@ describe('Expressions parser', function() 'SingleQuotedString(val="abc\'def"):0:0:\'abc\'\'def\'', }, }, { - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), hl('SingleQuotedBody', 'abc'), hl('SingleQuotedQuote', '\'\''), hl('SingleQuotedBody', 'def'), - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), }) check_parsing('\'abc\'\'', 0, { -- 012345 @@ -4992,7 +4992,7 @@ describe('Expressions parser', function() msg = 'E115: Missing single quote: %.*s', }, }, { - hl('InvalidSingleQuotedString', '\''), + hl('InvalidSingleQuote', '\''), hl('InvalidSingleQuotedBody', 'abc'), hl('InvalidSingleQuotedQuote', '\'\''), }) @@ -5002,11 +5002,11 @@ describe('Expressions parser', function() 'SingleQuotedString(val="\'\'\'"):0:0:\'\'\'\'\'\'\'\'', }, }, { - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), hl('SingleQuotedQuote', '\'\''), hl('SingleQuotedQuote', '\'\''), hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), }) check_parsing('\'\'\'a\'\'\'\'bc\'', 0, { -- 01234567890 @@ -5015,13 +5015,13 @@ describe('Expressions parser', function() 'SingleQuotedString(val="\'a\'\'bc"):0:0:\'\'\'a\'\'\'\'bc\'', }, }, { - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), hl('SingleQuotedQuote', '\'\''), hl('SingleQuotedBody', 'a'), hl('SingleQuotedQuote', '\'\''), hl('SingleQuotedQuote', '\'\''), hl('SingleQuotedBody', 'bc'), - hl('SingleQuotedString', '\''), + hl('SingleQuote', '\''), }) check_parsing('"\\"\\"\\"\\""', 0, { -- 0123456789 @@ -5029,12 +5029,12 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\"\\"\\"\\""):0:0:"\\"\\"\\"\\""', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\"'), hl('DoubleQuotedEscape', '\\"'), hl('DoubleQuotedEscape', '\\"'), hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', 0, { -- 0123456789012345678901234 @@ -5043,7 +5043,7 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="abc\\"def\\"ghi\\"jkl\\"mno"):0:0:"abc\\"def\\"ghi\\"jkl\\"mno"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedBody', 'abc'), hl('DoubleQuotedEscape', '\\"'), hl('DoubleQuotedBody', 'def'), @@ -5053,7 +5053,7 @@ describe('Expressions parser', function() hl('DoubleQuotedBody', 'jkl'), hl('DoubleQuotedEscape', '\\"'), hl('DoubleQuotedBody', 'mno'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\b\\e\\f\\r\\t\\\\"', 0, { -- 0123456789012345 @@ -5062,14 +5062,14 @@ describe('Expressions parser', function() [[DoubleQuotedString(val="\8\27\12\13\9\\"):0:0:"\b\e\f\r\t\\"]], }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\b'), hl('DoubleQuotedEscape', '\\e'), hl('DoubleQuotedEscape', '\\f'), hl('DoubleQuotedEscape', '\\r'), hl('DoubleQuotedEscape', '\\t'), hl('DoubleQuotedEscape', '\\\\'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\n\n"', 0, { -- 01234 @@ -5077,10 +5077,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\\n\\\n"):0:0:"\\n\n"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\n'), hl('DoubleQuotedBody', '\n'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\x00"', 0, { -- 012345 @@ -5088,9 +5088,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0"):0:0:"\\x00"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\x00'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\xFF"', 0, { -- 012345 @@ -5098,9 +5098,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\255"):0:0:"\\xFF"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\xFF'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\xF"', 0, { -- 012345 @@ -5108,9 +5108,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\15"):0:0:"\\xF"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\xF'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\u00AB"', 0, { -- 01234567 @@ -5118,9 +5118,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="«"):0:0:"\\u00AB"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\u00AB'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U000000AB"', 0, { -- 01234567 @@ -5128,9 +5128,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="«"):0:0:"\\U000000AB"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U000000AB'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\x"', 0, { -- 0123 @@ -5138,9 +5138,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="x"):0:0:"\\x"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\x'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\x', 0, { @@ -5153,7 +5153,7 @@ describe('Expressions parser', function() msg = 'E114: Missing double quote: %.*s', }, }, { - hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuote', '"'), hl('InvalidDoubleQuotedUnknownEscape', '\\x'), }) @@ -5167,7 +5167,7 @@ describe('Expressions parser', function() msg = 'E114: Missing double quote: %.*s', }, }, { - hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuote', '"'), hl('InvalidDoubleQuotedEscape', '\\xF'), }) @@ -5177,9 +5177,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="u"):0:0:"\\u"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\u'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\u', 0, { @@ -5192,7 +5192,7 @@ describe('Expressions parser', function() msg = 'E114: Missing double quote: %.*s', }, }, { - hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuote', '"'), hl('InvalidDoubleQuotedUnknownEscape', '\\u'), }) @@ -5206,7 +5206,7 @@ describe('Expressions parser', function() msg = 'E114: Missing double quote: %.*s', }, }, { - hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuote', '"'), hl('InvalidDoubleQuotedUnknownEscape', '\\U'), }) @@ -5216,9 +5216,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="U"):0:0:"\\U"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\U'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\xFX"', 0, { @@ -5227,10 +5227,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\15X"):0:0:"\\xFX"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\xF'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\XFX"', 0, { @@ -5239,10 +5239,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\15X"):0:0:"\\XFX"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\XF'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\xX"', 0, { @@ -5251,10 +5251,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="xX"):0:0:"\\xX"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\x'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\XX"', 0, { @@ -5263,10 +5263,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="XX"):0:0:"\\XX"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\X'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\uX"', 0, { @@ -5275,10 +5275,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="uX"):0:0:"\\uX"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\u'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\UX"', 0, { @@ -5287,10 +5287,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="UX"):0:0:"\\UX"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\U'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\x0X"', 0, { @@ -5299,10 +5299,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\x0X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\x0'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\X0X"', 0, { @@ -5311,10 +5311,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\X0X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\X0'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\u0X"', 0, { @@ -5323,10 +5323,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\u0X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\u0'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U0X"', 0, { @@ -5335,10 +5335,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\U0X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U0'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\x00X"', 0, { @@ -5347,10 +5347,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\x00X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\x00'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\X00X"', 0, { @@ -5359,10 +5359,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\X00X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\X00'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\u00X"', 0, { @@ -5371,10 +5371,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\u00X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\u00'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U00X"', 0, { @@ -5383,10 +5383,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\U00X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U00'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\u000X"', 0, { @@ -5395,10 +5395,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\u000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\u000'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U000X"', 0, { @@ -5407,10 +5407,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\U000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U000'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\u0000X"', 0, { @@ -5419,10 +5419,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\u0000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\u0000'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U0000X"', 0, { @@ -5431,10 +5431,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U0000'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U00000X"', 0, { @@ -5443,10 +5443,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U00000'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U000000X"', 0, { @@ -5456,10 +5456,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\U000000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U000000'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U0000000X"', 0, { @@ -5469,10 +5469,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U0000000'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U00000000X"', 0, { @@ -5482,10 +5482,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U00000000'), hl('DoubleQuotedBody', 'X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\x000X"', 0, { @@ -5494,10 +5494,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0000X"):0:0:"\\x000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\x00'), hl('DoubleQuotedBody', '0X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\X000X"', 0, { @@ -5506,10 +5506,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0000X"):0:0:"\\X000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\X00'), hl('DoubleQuotedBody', '0X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\u00000X"', 0, { @@ -5518,10 +5518,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0000X"):0:0:"\\u00000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\u0000'), hl('DoubleQuotedBody', '0X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\U000000000X"', 0, { @@ -5531,10 +5531,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0000X"):0:0:"\\U000000000X"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\U00000000'), hl('DoubleQuotedBody', '0X'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\0"', 0, { @@ -5543,9 +5543,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0"):0:0:"\\0"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\0'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\00"', 0, { @@ -5554,9 +5554,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0"):0:0:"\\00"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\00'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\000"', 0, { @@ -5565,9 +5565,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0"):0:0:"\\000"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\000'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\0000"', 0, { @@ -5576,10 +5576,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0000"):0:0:"\\0000"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\000'), hl('DoubleQuotedBody', '0'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\8"', 0, { @@ -5588,9 +5588,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="8"):0:0:"\\8"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\8'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\08"', 0, { @@ -5599,10 +5599,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0008"):0:0:"\\08"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\0'), hl('DoubleQuotedBody', '8'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\008"', 0, { @@ -5611,10 +5611,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0008"):0:0:"\\008"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\00'), hl('DoubleQuotedBody', '8'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\0008"', 0, { @@ -5623,10 +5623,10 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\0008"):0:0:"\\0008"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\000'), hl('DoubleQuotedBody', '8'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\777"', 0, { @@ -5635,9 +5635,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\255"):0:0:"\\777"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\777'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\050"', 0, { @@ -5646,9 +5646,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\40"):0:0:"\\050"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\050'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\"', 0, { @@ -5657,9 +5657,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="\\21"):0:0:"\\"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedEscape', '\\'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\<', 0, { @@ -5672,7 +5672,7 @@ describe('Expressions parser', function() msg = 'E114: Missing double quote: %.*s', }, }, { - hl('InvalidDoubleQuotedString', '"'), + hl('InvalidDoubleQuote', '"'), hl('InvalidDoubleQuotedUnknownEscape', '\\<'), }) @@ -5682,9 +5682,9 @@ describe('Expressions parser', function() 'DoubleQuotedString(val="<"):0:0:"\\<"', }, }, { - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), hl('DoubleQuotedUnknownEscape', '\\<'), - hl('DoubleQuotedString', '"'), + hl('DoubleQuote', '"'), }) check_parsing('"\\ Date: Mon, 30 Oct 2017 01:32:10 +0300 Subject: [PATCH 066/109] *: Fix linter errors Big function in expressions.c may be refactored, if I ever catch the idea how to split it right. --- src/nvim/edit.c | 2 +- src/nvim/keymap.c | 24 ++++++++++++------------ src/nvim/mbyte.c | 26 +++++++++++++++++--------- src/nvim/syntax.c | 25 ++++++++++++------------- src/nvim/viml/parser/expressions.c | 8 ++++---- src/nvim/viml/parser/expressions.h | 2 +- 6 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/nvim/edit.c b/src/nvim/edit.c index 28722a4d10..859f98d2ad 100644 --- a/src/nvim/edit.c +++ b/src/nvim/edit.c @@ -6080,7 +6080,7 @@ char_u *add_char2buf(int c, char_u *s) { char_u temp[MB_MAXBYTES + 1]; const int len = utf_char2bytes(c, temp); - for (int i = 0; i < len; ++i) { + for (int i = 0; i < len; i++) { c = temp[i]; // Need to escape K_SPECIAL and CSI like in the typeahead buffer. if (c == K_SPECIAL) { diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index 6ed54464e8..0c8e47b02e 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -25,21 +25,21 @@ */ static const struct modmasktable { - short mod_mask; ///< Bit-mask for particular key modifier. - short mod_flag; ///< Bit(s) for particular key modifier. + uint16_t mod_mask; ///< Bit-mask for particular key modifier. + uint16_t mod_flag; ///< Bit(s) for particular key modifier. char_u name; ///< Single letter name of modifier. } mod_mask_table[] = { - {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M'}, - {MOD_MASK_META, MOD_MASK_META, (char_u)'T'}, - {MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C'}, - {MOD_MASK_SHIFT, MOD_MASK_SHIFT, (char_u)'S'}, - {MOD_MASK_MULTI_CLICK, MOD_MASK_2CLICK, (char_u)'2'}, - {MOD_MASK_MULTI_CLICK, MOD_MASK_3CLICK, (char_u)'3'}, - {MOD_MASK_MULTI_CLICK, MOD_MASK_4CLICK, (char_u)'4'}, - {MOD_MASK_CMD, MOD_MASK_CMD, (char_u)'D'}, + { MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'M' }, + { MOD_MASK_META, MOD_MASK_META, (char_u)'T' }, + { MOD_MASK_CTRL, MOD_MASK_CTRL, (char_u)'C' }, + { MOD_MASK_SHIFT, MOD_MASK_SHIFT, (char_u)'S' }, + { MOD_MASK_MULTI_CLICK, MOD_MASK_2CLICK, (char_u)'2' }, + { MOD_MASK_MULTI_CLICK, MOD_MASK_3CLICK, (char_u)'3' }, + { MOD_MASK_MULTI_CLICK, MOD_MASK_4CLICK, (char_u)'4' }, + { MOD_MASK_CMD, MOD_MASK_CMD, (char_u)'D' }, // 'A' must be the last one - {MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'A'}, - {0, 0, NUL} + { MOD_MASK_ALT, MOD_MASK_ALT, (char_u)'A' }, + { 0, 0, NUL } }; /* diff --git a/src/nvim/mbyte.c b/src/nvim/mbyte.c index 843007b97b..008bce6df6 100644 --- a/src/nvim/mbyte.c +++ b/src/nvim/mbyte.c @@ -98,15 +98,23 @@ const uint8_t utf8len_tab[] = { // Like utf8len_tab above, but using a zero for illegal lead bytes. const uint8_t utf8len_tab_zero[] = { - //1 2 3 4 5 6 7 8 9 A B C D E F 0 1 2 3 4 5 6 7 8 9 A B C D E F - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0 - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 2 - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 4 - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 6 - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 8 - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // A - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // C - 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0, // E + // ?1 ?2 ?3 ?4 ?5 ?6 ?7 ?8 ?9 ?A ?B ?C ?D ?E ?F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 3? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 4? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 5? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 6? + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 7? + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8? + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9? + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // A? + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // B? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C? + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D? + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E? + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0, // F? }; /* diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index d787790bc3..e0bf74567d 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -5930,7 +5930,8 @@ static void syntime_report(void) // When making changes here, also change runtime/colors/default.vim! static const char *highlight_init_both[] = { - "Conceal ctermbg=DarkGrey ctermfg=LightGrey guibg=DarkGrey guifg=LightGrey", + "Conceal " + "ctermbg=DarkGrey ctermfg=LightGrey guibg=DarkGrey guifg=LightGrey", "Cursor guibg=fg guifg=bg", "lCursor guibg=fg guifg=bg", "DiffText cterm=bold ctermbg=Red gui=bold guibg=Red", @@ -6211,11 +6212,9 @@ void syn_init_cmdline_highlight(bool reset, bool init) /// /// @param both include groups where 'bg' doesn't matter /// @param reset clear groups first -void -init_highlight(int both, int reset) +void init_highlight(bool both, bool reset) { - int i; - static int had_both = FALSE; + static int had_both = false; // Try finding the color scheme file. Used when a color file was loaded // and 'background' or 't_Co' is changed. @@ -6235,9 +6234,9 @@ init_highlight(int both, int reset) * Didn't use a color file, use the compiled-in colors. */ if (both) { - had_both = TRUE; + had_both = true; const char *const *const pp = highlight_init_both; - for (i = 0; pp[i] != NULL; i++) { + for (size_t i = 0; pp[i] != NULL; i++) { do_highlight(pp[i], reset, true); } } else if (!had_both) { @@ -6250,7 +6249,7 @@ init_highlight(int both, int reset) const char *const *const pp = ((*p_bg == 'l') ? highlight_init_light : highlight_init_dark); - for (i = 0; pp[i] != NULL; i++) { + for (size_t i = 0; pp[i] != NULL; i++) { do_highlight(pp[i], reset, true); } @@ -6266,8 +6265,9 @@ init_highlight(int both, int reset) : "Visual cterm=NONE ctermbg=DarkGrey"), false, true); } else { do_highlight("Visual cterm=reverse ctermbg=NONE", false, true); - if (*p_bg == 'l') + if (*p_bg == 'l') { do_highlight("Search ctermfg=black", false, true); + } } /* @@ -6452,7 +6452,7 @@ void do_highlight(const char *line, const bool forceit, const bool init) restore_cterm_colors(); // Clear all default highlight groups and load the defaults. - for (int idx = 0; idx < highlight_ga.ga_len; ++idx) { + for (int idx = 0; idx < highlight_ga.ga_len; idx++) { highlight_clear(idx); } init_highlight(true, true); @@ -6690,9 +6690,8 @@ void do_highlight(const char *line, const bool forceit, const bool init) } } if (i < 0) { - emsgf(_( - "E421: Color name or number not recognized: %s"), - key_start); + emsgf(_("E421: Color name or number not recognized: %s"), + key_start); error = true; break; } diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index f5bc547d54..fc184f56f5 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1814,8 +1814,8 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) || ((*kv_Z(ast_stack, 1))->type != kExprNodeConcat && ((*kv_Z(ast_stack, 1))->type != kExprNodeConcatOrSubscript)))) - ? kELFlagAllowFloat - : 0)); + ? kELFlagAllowFloat + : 0)); LexExprToken cur_token = viml_pexpr_next_token( pstate, want_node_to_lexer_flags[want_node] | lexer_additional_flags); if (cur_token.type == kExprLexEOC) { @@ -1876,7 +1876,7 @@ viml_pexpr_parse_process_token: // time. // // Here example will always contain a concat with "a:2" sucking colon, - // making expression invalid both because there is no longer a spare colon + // making expression invalid both because there is no longer a spare colon // for ternary and because concatenating dictionary with anything is not // valid. There are more cases when this will make a difference though. const bool node_is_key = ( @@ -2853,7 +2853,7 @@ viml_pexpr_parse_end: } kvi_destroy(ast_stack); return ast; -} +} // NOLINT(readability/fn_size) #undef NEW_NODE #undef HL diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index d00d4855f3..668c2a4c84 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -189,7 +189,7 @@ typedef enum { kExprNodeCall, ///< Function call. /// Plain identifier: simple variable/function name /// - /// Looks like "string", "g:Foo", etc: consists from a single + /// Looks like "string", "g:Foo", etc: consists from a single /// kExprLexPlainIdentifier token. kExprNodePlainIdentifier, /// Plain dictionary key, for use with kExprNodeConcatOrSubscript From 0356dbbb36ffa7934c35e9dbfc6667f9118c244f Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 30 Oct 2017 01:38:02 +0300 Subject: [PATCH 067/109] ex_getln: Fix variable name which is wrong after the merge --- src/nvim/ex_getln.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c index 785038f5d6..f64efe08a1 100644 --- a/src/nvim/ex_getln.c +++ b/src/nvim/ex_getln.c @@ -2500,7 +2500,7 @@ static bool color_cmdline(CmdlineInfo *colored_ccline) tl_ret = try_leave(&tstate, &err); can_free_cb = true; } else if (colored_ccline->cmdfirstc == '=') { - color_expr_cmdline(colored_ccline, ret_ccline_colors); + color_expr_cmdline(colored_ccline, ccline_colors); can_free_cb = false; } if (!tl_ret || !dgc_ret) { From 3ecb95298ffd9ef6ee681876f2d32553fd222b96 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 30 Oct 2017 01:48:32 +0300 Subject: [PATCH 068/109] tests: Fix testlint errors --- test/functional/ui/cmdline_highlight_spec.lua | 4 ++++ test/helpers.lua | 4 ++-- test/unit/helpers.lua | 6 +++--- test/unit/viml/expressions/lexer_spec.lua | 3 +-- test/unit/viml/expressions/parser_spec.lua | 9 ++------- test/unit/viml/helpers.lua | 6 ------ 6 files changed, 12 insertions(+), 20 deletions(-) diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua index 60a4a815e7..b16b4a5602 100644 --- a/test/functional/ui/cmdline_highlight_spec.lua +++ b/test/functional/ui/cmdline_highlight_spec.lua @@ -898,4 +898,8 @@ describe('Expressions coloring support', function() ={NUM:1}^ | ]]) end) + -- FIXME: Test expr coloring when using -u NORC and -u NONE. + -- FIXME: Test all highlight groups, using long expression. + -- FIXME: Test different ways of triggering expression highlighting (:=, + -- i=, :e, "=). end) diff --git a/test/helpers.lua b/test/helpers.lua index 83e78ba059..10f2f80191 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -376,8 +376,8 @@ local function format_string(fmt, ...) return args[i] end local ret = fmt:gsub('%%[0-9*]*%.?[0-9*]*[cdEefgGiouXxqsr%%]', function(match) - local subfmt = match:gsub('%*', function(match) - return getarg() + local subfmt = match:gsub('%*', function() + return tostring(getarg()) end) local arg = nil if subfmt:sub(-1) ~= '%' then diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index 68ce9eed62..96aa505739 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -791,7 +791,7 @@ local function kvi_new(ct) return kvi_init(ffi.new(ct)) end -local function make_enum_conv_tab(lib, values, skip_pref, set_cb) +local function make_enum_conv_tab(m, values, skip_pref, set_cb) child_call_once(function() local ret = {} for _, v in ipairs(values) do @@ -799,7 +799,7 @@ local function make_enum_conv_tab(lib, values, skip_pref, set_cb) if v:sub(1, #skip_pref) == skip_pref then str_v = v:sub(#skip_pref + 1) end - ret[tonumber(lib[v])] = str_v + ret[tonumber(m[v])] = str_v end set_cb(ret) end) @@ -837,7 +837,7 @@ local module = { child_cleanup_once = child_cleanup_once, sc = sc, conv_enum = conv_enum, - array_size = array_sive, + array_size = array_size, kvi_destroy = kvi_destroy, kvi_size = kvi_size, kvi_init = kvi_init, diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index 5910468017..d4ec870a4e 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -10,7 +10,6 @@ local ffi = helpers.ffi local eq = helpers.eq local conv_ccs = viml_helpers.conv_ccs -local pline2lua = viml_helpers.pline2lua local new_pstate = viml_helpers.new_pstate local intchar2lua = viml_helpers.intchar2lua local conv_cmp_type = viml_helpers.conv_cmp_type @@ -183,7 +182,7 @@ describe('Expressions lexer', function() end local function simple_test(pstate_arg, exp_type, exp_len, exp) local pstate = new_pstate(pstate_arg) - local exp = shallowcopy(exp) + exp = shallowcopy(exp) exp.type = exp_type exp.len = exp_len or #(pstate_arg[0]) exp.start = { col = 0, line = 0 } diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 019fe69046..df69c60dd0 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -14,7 +14,6 @@ local ffi = helpers.ffi local eq = helpers.eq local conv_ccs = viml_helpers.conv_ccs -local pline2lua = viml_helpers.pline2lua local new_pstate = viml_helpers.new_pstate local intchar2lua = viml_helpers.intchar2lua local conv_cmp_type = viml_helpers.conv_cmp_type @@ -174,10 +173,7 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) ffi.string(eastnode.data.env.ident, eastnode.data.env.ident_len)) end ret_str = typ .. ':' .. ret_str - local can_simplify = true - for k, v in pairs(ret) do - can_simplify = false - end + local can_simplify = not ret.children if can_simplify then ret = ret_str else @@ -218,12 +214,11 @@ local function phl2lua(pstate) pstate, chunk.start, chunk.end_col - chunk.start.col, { group = ffi.string(chunk.group), }) - chunk_str = ('%s:%u:%u:%s'):format( + ret[i + 1] = ('%s:%u:%u:%s'):format( chunk_tbl.group, chunk_tbl.start.line, chunk_tbl.start.col, chunk_tbl.str) - ret[i + 1] = chunk_str end return ret end diff --git a/test/unit/viml/helpers.lua b/test/unit/viml/helpers.lua index c965cacb29..70949e8278 100644 --- a/test/unit/viml/helpers.lua +++ b/test/unit/viml/helpers.lua @@ -5,7 +5,6 @@ local cimport = helpers.cimport local kvi_new = helpers.kvi_new local kvi_init = helpers.kvi_init local conv_enum = helpers.conv_enum -local child_call_once = helpers.child_call_once local make_enum_conv_tab = helpers.make_enum_conv_tab local lib = cimport('./src/nvim/viml/parser/expressions.h') @@ -33,11 +32,6 @@ local function new_pstate(strings) ret_pline.size = size ret_pline.allocated = false end - local pline_init = { - data = nil, - size = 0, - allocated = false, - } local state = { reader = { get_line = get_line, From d98199de9c2969d430ba489187ffa02d8c489dea Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 2 Nov 2017 10:44:20 +0300 Subject: [PATCH 069/109] charset: Refactor vim_str2nr --- src/nvim/charset.c | 135 +++++++++++++++++--------- test/symbolic/klee/nvim/charset.c | 129 +++++++++++++++--------- test/unit/charset/vim_str2nr_spec.lua | 1 + 3 files changed, 173 insertions(+), 92 deletions(-) create mode 100644 test/unit/charset/vim_str2nr_spec.lua diff --git a/src/nvim/charset.c b/src/nvim/charset.c index c895d65eb7..5aebf90194 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -1611,8 +1611,9 @@ bool vim_isblankline(char_u *lbuf) /// If maxlen > 0, check at a maximum maxlen chars. /// /// @param start -/// @param prep Returns type of number 0 = decimal, 'x' or 'X' is hex, -/// '0' = octal, 'b' or 'B' is bin +/// @param prep Returns guessed type of number 0 = decimal, 'x' or 'X' is +/// hexadecimal, '0' = octal, 'b' or 'B' is binary. When using +/// STR2NR_FORCE is always zero. /// @param len Returns the detected length of number. /// @param what Recognizes what number passed. /// @param nptr Returns the signed result. @@ -1627,55 +1628,84 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, #define STRING_ENDED(ptr) \ (!(maxlen == 0 || (int)((ptr) - (const char *)start) < maxlen)) int pre = 0; // default is decimal - bool negative = false; + const bool negative = (ptr[0] == '-'); + uvarnumber_T un = 0; - if (ptr[0] == '-') { - negative = true; + if (negative) { ptr++; } - // Recognize hex, octal and bin. - if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) - && !STRING_ENDED(ptr + 1) - && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { + if (what & STR2NR_FORCE) { + // When forcing main consideration is skipping the prefix. Octal and decimal + // numbers have no prefixes to skip. pre is not set. + switch ((unsigned)what & (~(unsigned)STR2NR_FORCE)) { + case STR2NR_HEX: { + if (!STRING_ENDED(ptr + 2) + && ptr[0] == '0' + && (ptr[1] == 'x' || ptr[1] == 'X') + && ascii_isxdigit(ptr[2])) { + ptr += 2; + } + goto vim_str2nr_hex; + } + case STR2NR_BIN: { + if (!STRING_ENDED(ptr + 2) + && ptr[0] == '0' + && (ptr[1] == 'b' || ptr[1] == 'B') + && ascii_isbdigit(ptr[2])) { + ptr += 2; + } + goto vim_str2nr_bin; + } + case STR2NR_OCT: { + goto vim_str2nr_oct; + } + case 0: { + goto vim_str2nr_dec; + } + default: { + assert(false); + } + } + } else if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) + && !STRING_ENDED(ptr + 1) + && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { pre = ptr[1]; - + // Detect hexadecimal: 0x or 0X follwed by hex digit if ((what & STR2NR_HEX) && !STRING_ENDED(ptr + 2) && (pre == 'X' || pre == 'x') && ascii_isxdigit(ptr[2])) { - // hexadecimal ptr += 2; - } else if ((what & STR2NR_BIN) - && !STRING_ENDED(ptr + 2) - && (pre == 'B' || pre == 'b') - && ascii_isbdigit(ptr[2])) { - // binary + goto vim_str2nr_hex; + } + // Detect binary: 0b or 0B follwed by 0 or 1 + if ((what & STR2NR_BIN) + && !STRING_ENDED(ptr + 2) + && (pre == 'B' || pre == 'b') + && ascii_isbdigit(ptr[2])) { ptr += 2; - } else { - // decimal or octal, default is decimal - pre = 0; - - if (what & STR2NR_OCT - && !STRING_ENDED(ptr + 1) - && ('0' <= ptr[1] && ptr[1] <= '7')) { - // Assume octal now: what we already know is that string starts with - // zero and some octal digit. - pre = '0'; - // Don’t interpret "0", "008" or "0129" as octal. - for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { - if (ptr[i] > '7') { - // Can’t be octal. - pre = 0; - break; - } - } + goto vim_str2nr_bin; + } + // Detect octal number: zero followed by octal digits without '8' or '9' + pre = 0; + if (!(what & STR2NR_OCT) + || !('0' <= ptr[1] && ptr[1] <= '7')) { + goto vim_str2nr_dec; + } + for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { + if (ptr[i] > '7') { + goto vim_str2nr_dec; } } + pre = '0'; + goto vim_str2nr_oct; + } else { + goto vim_str2nr_dec; } // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. - uvarnumber_T un = 0; + assert(false); // Should’ve used goto earlier. #define PARSE_NUMBER(base, cond, conv) \ do { \ while (!STRING_ENDED(ptr) && (cond)) { \ @@ -1688,18 +1718,29 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, ptr++; \ } \ } while (0) - if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { - // Binary number. - PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); - } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { - // Octal number. - PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); - } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { - // Hexadecimal number. - PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); - } else { - // Decimal number. - PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); + switch (pre) { + case 'b': + case 'B': { +vim_str2nr_bin: + PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); + break; + } + case '0': { +vim_str2nr_oct: + PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); + break; + } + case 0: { +vim_str2nr_dec: + PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); + break; + } + case 'x': + case 'X': { +vim_str2nr_hex: + PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); + break; + } } #undef PARSE_NUMBER diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c index f3a218949f..77f690f08d 100644 --- a/test/symbolic/klee/nvim/charset.c +++ b/test/symbolic/klee/nvim/charset.c @@ -31,55 +31,83 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, #define STRING_ENDED(ptr) \ (!(maxlen == 0 || (int)((ptr) - (const char *)start) < maxlen)) int pre = 0; // default is decimal - bool negative = false; + const bool negative = (ptr[0] == '-'); + uvarnumber_T un = 0; - if (ptr[0] == '-') { - negative = true; + if (negative) { ptr++; } - // Recognize hex, octal and bin. - if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) - && !STRING_ENDED(ptr + 1) - && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { + if (what & STR2NR_FORCE) { + // When forcing main consideration is skipping the prefix. Octal and decimal + // numbers have no prefixes to skip. pre is not set. + switch ((unsigned)what & (~(unsigned)STR2NR_FORCE)) { + case STR2NR_HEX: { + if (!STRING_ENDED(ptr + 2) + && ptr[0] == '0' + && (ptr[1] == 'x' || ptr[1] == 'X') + && ascii_isxdigit(ptr[2])) { + ptr += 2; + } + goto vim_str2nr_hex; + } + case STR2NR_BIN: { + if (!STRING_ENDED(ptr + 2) + && ptr[0] == '0' + && (ptr[1] == 'b' || ptr[1] == 'B') + && ascii_isbdigit(ptr[2])) { + ptr += 2; + } + goto vim_str2nr_bin; + } + case STR2NR_OCT: { + goto vim_str2nr_oct; + } + case 0: { + goto vim_str2nr_dec; + } + default: { + assert(false); + } + } + } else if ((what & (STR2NR_HEX|STR2NR_OCT|STR2NR_BIN)) + && !STRING_ENDED(ptr + 1) + && ptr[0] == '0' && ptr[1] != '8' && ptr[1] != '9') { pre = ptr[1]; - + // Detect hexadecimal: 0x or 0X follwed by hex digit if ((what & STR2NR_HEX) && !STRING_ENDED(ptr + 2) && (pre == 'X' || pre == 'x') && ascii_isxdigit(ptr[2])) { - // hexadecimal ptr += 2; - } else if ((what & STR2NR_BIN) - && !STRING_ENDED(ptr + 2) - && (pre == 'B' || pre == 'b') - && ascii_isbdigit(ptr[2])) { - // binary + goto vim_str2nr_hex; + } + // Detect binary: 0b or 0B follwed by 0 or 1 + if ((what & STR2NR_BIN) + && !STRING_ENDED(ptr + 2) + && (pre == 'B' || pre == 'b') + && ascii_isbdigit(ptr[2])) { ptr += 2; - } else { - // decimal or octal, default is decimal - pre = 0; - - if (what & STR2NR_OCT - && !STRING_ENDED(ptr + 1) - && ('0' <= ptr[1] && ptr[1] <= '7')) { - // Assume octal now: what we already know is that string starts with - // zero and some octal digit. - pre = '0'; - // Don’t interpret "0", "008" or "0129" as octal. - for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { - if (ptr[i] > '7') { - // Can’t be octal. - pre = 0; - break; - } - } + goto vim_str2nr_bin; + } + // Detect octal number: zero followed by octal digits without '8' or '9' + pre = 0; + if (!(what & STR2NR_OCT)) { + goto vim_str2nr_dec; + } + for (int i = 2; !STRING_ENDED(ptr + i) && ascii_isdigit(ptr[i]); i++) { + if (ptr[i] > '7') { + goto vim_str2nr_dec; } } + pre = '0'; + goto vim_str2nr_oct; + } else { + goto vim_str2nr_dec; } // Do the string-to-numeric conversion "manually" to avoid sscanf quirks. - uvarnumber_T un = 0; + assert(false); // Should’ve used goto earlier. #define PARSE_NUMBER(base, cond, conv) \ do { \ while (!STRING_ENDED(ptr) && (cond)) { \ @@ -92,18 +120,29 @@ void vim_str2nr(const char_u *const start, int *const prep, int *const len, ptr++; \ } \ } while (0) - if (pre == 'B' || pre == 'b' || what == (STR2NR_BIN|STR2NR_FORCE)) { - // Binary number. - PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); - } else if (pre == '0' || what == (STR2NR_OCT|STR2NR_FORCE)) { - // Octal number. - PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); - } else if (pre == 'X' || pre == 'x' || what == (STR2NR_HEX|STR2NR_FORCE)) { - // Hexadecimal number. - PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); - } else { - // Decimal number. - PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); + switch (pre) { + case 'b': + case 'B': { +vim_str2nr_bin: + PARSE_NUMBER(2, (*ptr == '0' || *ptr == '1'), (*ptr - '0')); + break; + } + case '0': { +vim_str2nr_oct: + PARSE_NUMBER(8, ('0' <= *ptr && *ptr <= '7'), (*ptr - '0')); + break; + } + case 0: { +vim_str2nr_dec: + PARSE_NUMBER(10, (ascii_isdigit(*ptr)), (*ptr - '0')); + break; + } + case 'x': + case 'X': { +vim_str2nr_hex: + PARSE_NUMBER(16, (ascii_isxdigit(*ptr)), (hex2nr(*ptr))); + break; + } } #undef PARSE_NUMBER diff --git a/test/unit/charset/vim_str2nr_spec.lua b/test/unit/charset/vim_str2nr_spec.lua new file mode 100644 index 0000000000..cfbc77bc2a --- /dev/null +++ b/test/unit/charset/vim_str2nr_spec.lua @@ -0,0 +1 @@ +-- FIXME From b9d5aea073521f3278bea257be306b7ac2b8d83c Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 3 Nov 2017 11:38:59 +0300 Subject: [PATCH 070/109] api/vim: Create part of nvim_parse_expression function --- src/nvim/api/vim.c | 106 ++++++++++++++++++++++++++++++- test/functional/api/vim_spec.lua | 5 ++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 9daa2fb398..446a049897 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -33,6 +33,8 @@ #include "nvim/syntax.h" #include "nvim/getchar.h" #include "nvim/os/input.h" +#include "nvim/viml/parser/expressions.h" +#include "nvim/viml/parser/parser.h" #define LINE_BUFFER_SIZE 4096 @@ -897,6 +899,12 @@ theend: /// /// Use only "m" to parse like for "=", only "E" to /// parse like for ":echo", empty string for ":let". +/// @param[in] highlight If true, return value will also include "highlight" +/// key containing array of 4-tuples (arrays) (Integer, +/// Integer, Integer, String), where first three numbers +/// define the highlighted region and represent line, +/// starting column and ending column (latter exclusive: +/// one should highlight region [start_col, end_col)). /// /// @return AST: top-level dectionary holds keys /// @@ -946,9 +954,105 @@ theend: /// "fvalue": Float, floating-point value for "Float" nodes. /// "svalue": String, value for "SingleQuotedString" and /// "DoubleQuotedString" nodes. -Dictionary nvim_parse_expression(String expr, String flags, Error *err) +Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, + Error *err) FUNC_API_SINCE(4) { + int pflags = 0; + for (size_t i = 0 ; i < flags.size ; i++) { + switch (flags.data[i]) { + case 'm': { pflags |= kExprFlagsMulti; break; } + case 'E': { pflags |= kExprFlagsDisallowEOC; break; } + default: { + api_set_error(err, kErrorTypeValidation, "Invalid flag: '%c' (%u)", + flags.data[i], (unsigned)flags.data[i]); + return (Dictionary)ARRAY_DICT_INIT; + } + } + } + ParserLine plines[] = { + { + .data = expr.data, + .size = expr.size, + .allocated = false, + }, + { NULL, 0, false }, + }; + ParserLine *plines_p = plines; + ParserHighlight colors; + kvi_init(colors); + ParserHighlight *const colors_p = (highlight ? &colors : NULL); + ParserState pstate; + viml_parser_init( + &pstate, parser_simple_get_line, &plines_p, colors_p); + ExprAST east = viml_pexpr_parse(&pstate, pflags); + + const size_t dict_size = ( + 1 // "ast" + + (size_t)(east.err.arg != NULL) // "error" + + (size_t)highlight // "highlight" + ); + Dictionary ret = { + .items = xmalloc(dict_size * sizeof(ret.items[0])), + .size = 0, + .capacity = dict_size, + }; + ret.items[ret.size++] = (KeyValuePair) { + .key = STATIC_CSTR_AS_STRING("ast"), + .value = NIL, + }; + if (east.err.arg != NULL) { + Dictionary err_dict = { + .items = xmalloc(2 * sizeof(err_dict.items[0])), + .size = 2, + .capacity = 2, + }; + err_dict.items[0] = (KeyValuePair) { + .key = STATIC_CSTR_AS_STRING("message"), + .value = STRING_OBJ(cstr_to_string(east.err.arg)), + }; + err_dict.items[1] = (KeyValuePair) { + .key = STATIC_CSTR_AS_STRING("arg"), + .value = STRING_OBJ(((String) { + .data = xmemdup(east.err.arg, (size_t)east.err.arg_len), + .size = (size_t)east.err.arg_len, + })), + }; + ret.items[ret.size++] = (KeyValuePair) { + .key = STATIC_CSTR_AS_STRING("error"), + .value = DICTIONARY_OBJ(err_dict), + }; + } + if (highlight) { + Array hl = (Array) { + .items = xmalloc(kv_size(colors) * sizeof(hl.items[0])), + .capacity = kv_size(colors), + .size = kv_size(colors), + }; + for (size_t i = 0 ; i < kv_size(colors) ; i++) { + const ParserHighlightChunk chunk = kv_A(colors, i); + Array chunk_arr = (Array) { + .items = xmalloc(4), + .capacity = 4, + .size = 4, + }; + chunk_arr.items[0] = INTEGER_OBJ((Integer)chunk.start.line); + chunk_arr.items[1] = INTEGER_OBJ((Integer)chunk.start.col); + chunk_arr.items[2] = INTEGER_OBJ((Integer)chunk.end_col); + chunk_arr.items[3] = STRING_OBJ(cstr_to_string(chunk.group)); + hl.items[i] = ARRAY_OBJ(chunk_arr); + } + ret.items[ret.size++] = (KeyValuePair) { + .key = STATIC_CSTR_AS_STRING("highlight"), + .value = ARRAY_OBJ(hl), + }; + } + + // FIXME: populate AST + + assert(ret.size == ret.capacity); + viml_pexpr_free_ast(east); + viml_parser_destroy(&pstate); return (Dictionary)ARRAY_DICT_INIT; } diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index b849304d45..6b44698638 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -710,4 +710,9 @@ describe('api', function() ok(err:match(': Wrong type for argument 1, expecting String') ~= nil) end) + describe('nvim_parse_expression', function() + -- FIXME + -- FIXME Test error + end) + end) From 07ec709141886c6db4f944665e07a36ef7302eb4 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 5 Nov 2017 01:33:44 +0300 Subject: [PATCH 071/109] vim/api: Actually dump AST, fix some bugs in nvim_parse_expression --- src/nvim/api/vim.c | 264 +++++++++++++++++++++++++++-- src/nvim/viml/parser/expressions.c | 10 +- src/nvim/viml/parser/expressions.h | 9 + 3 files changed, 262 insertions(+), 21 deletions(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 446a049897..1aab87010e 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -888,6 +888,13 @@ theend: return rv; } +typedef struct { + ExprASTNode **node_p; + Object *ret_node_p; +} ExprASTConvStackItem; + +typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack; + /// Parse a VimL expression /// /// @param[in] expr Expression to parse. Is always treated as a single line. @@ -915,7 +922,8 @@ theend: /// Must contain exactly one "%.*s". /// "arg": String, error message argument. /// -/// "ast": actual AST, a dictionary with the following keys: +/// "ast": actual AST, either nil or a dictionary with the following +/// keys: /// /// "type": node type, one of the value names from ExprASTNodeType /// stringified without "kExprNode" prefix. @@ -927,11 +935,9 @@ theend: /// debugging purposes primary (debugging parser and providing /// debug information). /// "children": a list of nodes described in top/"ast". There always -/// is zero, one or two children, key will contain an -/// empty array if node can have children, but has no and -/// will not be present at all if node can’t have any -/// children. Maximum number of children may be found in -/// node_maxchildren array. +/// is zero, one or two children, key will not be present +/// if node has no children. Maximum number of children +/// may be found in node_maxchildren array. /// /// Local values (present only for certain nodes): /// @@ -950,6 +956,8 @@ theend: /// value names from ExprCaseCompareStrategy, /// stringified without "kCCStrategy" prefix. Only /// present for "Comparison" nodes. +/// "invert": Boolean, true if result of comparison needs to be +/// inverted. Only present for "Comparison" nodes. /// "ivalue": Integer, integer value for "Integer" nodes. /// "fvalue": Float, floating-point value for "Float" nodes. /// "svalue": String, value for "SingleQuotedString" and @@ -998,7 +1006,7 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, .capacity = dict_size, }; ret.items[ret.size++] = (KeyValuePair) { - .key = STATIC_CSTR_AS_STRING("ast"), + .key = STATIC_CSTR_TO_STRING("ast"), .value = NIL, }; if (east.err.arg != NULL) { @@ -1008,18 +1016,18 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, .capacity = 2, }; err_dict.items[0] = (KeyValuePair) { - .key = STATIC_CSTR_AS_STRING("message"), + .key = STATIC_CSTR_TO_STRING("message"), .value = STRING_OBJ(cstr_to_string(east.err.arg)), }; err_dict.items[1] = (KeyValuePair) { - .key = STATIC_CSTR_AS_STRING("arg"), + .key = STATIC_CSTR_TO_STRING("arg"), .value = STRING_OBJ(((String) { - .data = xmemdup(east.err.arg, (size_t)east.err.arg_len), + .data = xmemdupz(east.err.arg, (size_t)east.err.arg_len), .size = (size_t)east.err.arg_len, })), }; ret.items[ret.size++] = (KeyValuePair) { - .key = STATIC_CSTR_AS_STRING("error"), + .key = STATIC_CSTR_TO_STRING("error"), .value = DICTIONARY_OBJ(err_dict), }; } @@ -1032,7 +1040,7 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, for (size_t i = 0 ; i < kv_size(colors) ; i++) { const ParserHighlightChunk chunk = kv_A(colors, i); Array chunk_arr = (Array) { - .items = xmalloc(4), + .items = xmalloc(4 * sizeof(chunk_arr.items[0])), .capacity = 4, .size = 4, }; @@ -1043,17 +1051,243 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, hl.items[i] = ARRAY_OBJ(chunk_arr); } ret.items[ret.size++] = (KeyValuePair) { - .key = STATIC_CSTR_AS_STRING("highlight"), + .key = STATIC_CSTR_TO_STRING("highlight"), .value = ARRAY_OBJ(hl), }; } - // FIXME: populate AST + // Walk over the AST, freeing nodes in process. + ExprASTConvStack ast_conv_stack; + kvi_init(ast_conv_stack); + kvi_push(ast_conv_stack, ((ExprASTConvStackItem) { + .node_p = &east.root, + .ret_node_p = &ret.items[0].value, + })); + while (kv_size(ast_conv_stack)) { + ExprASTConvStackItem cur_item = kv_last(ast_conv_stack); + if (*cur_item.node_p == NULL) { + assert(kv_size(ast_conv_stack) == 1); + kv_drop(ast_conv_stack, 1); + } else { + ExprASTNode *const node = *cur_item.node_p; + if (cur_item.ret_node_p->type == kObjectTypeNil) { + const size_t ret_node_items_size = (size_t)( + 3 // "type", "start" and "len" + + (node->children != NULL) // "children" + + (node->type == kExprNodeOption + || node->type == kExprNodePlainIdentifier) // "scope" + + (node->type == kExprNodeOption + || node->type == kExprNodePlainIdentifier + || node->type == kExprNodePlainKey + || node->type == kExprNodeEnvironment) // "ident" + + (node->type == kExprNodeRegister) // "name" + + (3 // "cmp_type", "ccs_strategy", "invert" + * (node->type == kExprNodeComparison)) + + (node->type == kExprNodeInteger) // "ivalue" + + (node->type == kExprNodeFloat) // "fvalue" + + (node->type == kExprNodeDoubleQuotedString + || node->type == kExprNodeSingleQuotedString) // "svalue" + + 0); + Dictionary ret_node = { + .items = xmalloc(ret_node_items_size * sizeof(ret_node.items[0])), + .capacity = ret_node_items_size, + .size = 0, + }; + *cur_item.ret_node_p = DICTIONARY_OBJ(ret_node); + } + Dictionary *ret_node = &cur_item.ret_node_p->data.dictionary; + if (node->children != NULL) { + const size_t num_children = 1 + (node->children->next != NULL); + Array children_array = { + .items = xmalloc(num_children * sizeof(children_array.items[0])), + .capacity = num_children, + .size = num_children, + }; + for (size_t i = 0; i < num_children; i++) { + children_array.items[i] = NIL; + } + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("children"), + .value = ARRAY_OBJ(children_array), + }; + kvi_push(ast_conv_stack, ((ExprASTConvStackItem) { + .node_p = &node->children, + .ret_node_p = &children_array.items[0], + })); + } else if (node->next != NULL) { + kvi_push(ast_conv_stack, ((ExprASTConvStackItem) { + .node_p = &node->children, + .ret_node_p = cur_item.ret_node_p + 1, + })); + } else if (node != NULL) { + kv_drop(ast_conv_stack, 1); + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("type"), + .value = STRING_OBJ(cstr_to_string(east_node_type_tab[node->type])), + }; + Array start_array = { + .items = xmalloc(2 * sizeof(start_array.items[0])), + .capacity = 2, + .size = 2, + }; + start_array.items[0] = INTEGER_OBJ((Integer)node->start.line); + start_array.items[1] = INTEGER_OBJ((Integer)node->start.col); + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("start"), + .value = ARRAY_OBJ(start_array), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("len"), + .value = INTEGER_OBJ((Integer)node->len), + }; + switch (node->type) { + case kExprNodeDoubleQuotedString: + case kExprNodeSingleQuotedString: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("svalue"), + .value = STRING_OBJ(cstr_as_string(node->data.str.value)), + }; + break; + } + case kExprNodeOption: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("scope"), + .value = INTEGER_OBJ(node->data.opt.scope), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ident"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(node->data.opt.ident, + node->data.opt.ident_len), + .size = node->data.opt.ident_len, + })), + }; + break; + } + case kExprNodePlainIdentifier: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("scope"), + .value = INTEGER_OBJ(node->data.var.scope), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ident"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(node->data.var.ident, + node->data.var.ident_len), + .size = node->data.var.ident_len, + })), + }; + break; + } + case kExprNodePlainKey: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ident"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(node->data.var.ident, + node->data.var.ident_len), + .size = node->data.var.ident_len, + })), + }; + break; + } + case kExprNodeEnvironment: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ident"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(node->data.env.ident, + node->data.env.ident_len), + .size = node->data.env.ident_len, + })), + }; + break; + } + case kExprNodeRegister: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("name"), + .value = INTEGER_OBJ(node->data.reg.name), + }; + break; + } + case kExprNodeComparison: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("cmp_type"), + .value = STRING_OBJ(cstr_to_string( + eltkn_cmp_type_tab[node->data.cmp.type])), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ccs_strategy"), + .value = STRING_OBJ(cstr_to_string( + eltkn_cmp_type_tab[node->data.cmp.ccs])), + }; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("invert"), + .value = BOOLEAN_OBJ(node->data.cmp.inv), + }; + break; + } + case kExprNodeFloat: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("fvalue"), + .value = FLOAT_OBJ(node->data.flt.value), + }; + break; + } + case kExprNodeInteger: { + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("ivalue"), + .value = INTEGER_OBJ((Integer)( + node->data.num.value > API_INTEGER_MAX + ? API_INTEGER_MAX + : (Integer)node->data.num.value)), + }; + break; + } + case kExprNodeMissing: + case kExprNodeOpMissing: + case kExprNodeTernary: + case kExprNodeTernaryValue: + case kExprNodeSubscript: + case kExprNodeListLiteral: + case kExprNodeUnaryPlus: + case kExprNodeBinaryPlus: + case kExprNodeNested: + case kExprNodeCall: + case kExprNodeComplexIdentifier: + case kExprNodeUnknownFigure: + case kExprNodeLambda: + case kExprNodeDictLiteral: + case kExprNodeCurlyBracesIdentifier: + case kExprNodeComma: + case kExprNodeColon: + case kExprNodeArrow: + case kExprNodeConcat: + case kExprNodeConcatOrSubscript: + case kExprNodeOr: + case kExprNodeAnd: + case kExprNodeUnaryMinus: + case kExprNodeBinaryMinus: + case kExprNodeNot: + case kExprNodeMultiplication: + case kExprNodeDivision: + case kExprNodeMod: { + break; + } + } + assert(cur_item.ret_node_p->data.dictionary.size + == cur_item.ret_node_p->data.dictionary.capacity); + xfree(*cur_item.node_p); + *cur_item.node_p = NULL; + } + } + } + kvi_destroy(ast_conv_stack); assert(ret.size == ret.capacity); + // Should be a no-op actually, leaving it in case non-nodes will need to be + // freed later. viml_pexpr_free_ast(east); viml_parser_destroy(&pstate); - return (Dictionary)ARRAY_DICT_INIT; + return ret; } diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index fc184f56f5..b19aab22af 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -616,7 +616,7 @@ static const char *const eltkn_type_tab[] = { [kExprLexArrow] = "Arrow", }; -static const char *const eltkn_cmp_type_tab[] = { +const char *const eltkn_cmp_type_tab[] = { [kExprCmpEqual] = "Equal", [kExprCmpMatches] = "Matches", [kExprCmpGreater] = "Greater", @@ -624,7 +624,7 @@ static const char *const eltkn_cmp_type_tab[] = { [kExprCmpIdentical] = "Identical", }; -static const char *const ccs_tab[] = { +const char *const ccs_tab[] = { [kCCStrategyUseOption] = "UseOption", [kCCStrategyMatchCase] = "MatchCase", [kCCStrategyIgnoreCase] = "IgnoreCase", @@ -725,8 +725,7 @@ viml_pexpr_repr_token_end: return ret; } -#ifdef UNIT_TESTING -static const char *const east_node_type_tab[] = { +const char *const east_node_type_tab[] = { [kExprNodeMissing] = "Missing", [kExprNodeOpMissing] = "OpMissing", [kExprNodeTernary] = "Ternary", @@ -766,7 +765,6 @@ static const char *const east_node_type_tab[] = { [kExprNodeOption] = "Option", [kExprNodeEnvironment] = "Environment", }; -#endif /// Represent `int` character as a string /// @@ -2148,10 +2146,10 @@ viml_pexpr_parse_invalid_comma: } #define EXP_VAL_COLON "E15: Expected value, got colon: %.*s" case kExprLexColon: { + bool is_ternary = false; if (kv_size(ast_stack) < 2) { goto viml_pexpr_parse_invalid_colon; } - bool is_ternary = false; bool can_be_ternary = true; bool is_subscript = false; for (size_t i = 1; i < kv_size(ast_stack); i++) { diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 668c2a4c84..648f8cbc1f 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -341,6 +341,15 @@ typedef struct { /// Array mapping ExprASTNodeType to maximum amount of children node may have extern const uint8_t node_maxchildren[]; +/// Array mapping ExprASTNodeType values to their stringified versions +extern const char *const east_node_type_tab[]; + +/// Array mapping ExprComparisonType values to their stringified versions +extern const char *const eltkn_cmp_type_tab[]; + +/// Array mapping ExprCaseCompareStrategy values to their stringified versions +extern const char *const ccs_tab[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.h.generated.h" #endif From 7bc6de75263f58c6c4f999bc86a6454ae9f28b80 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 5 Nov 2017 02:41:44 +0300 Subject: [PATCH 072/109] api/vim,functests: Add tests for nvim_parse_expression, fix found bugs --- src/nvim/api/vim.c | 47 +- src/nvim/viml/parser/expressions.c | 2 +- test/functional/api/vim_spec.lua | 6868 ++++++++++++++++++++ test/helpers.lua | 6 + test/unit/viml/expressions/lexer_spec.lua | 2 +- test/unit/viml/expressions/parser_spec.lua | 5 +- test/unit/viml/helpers.lua | 5 - 7 files changed, 6908 insertions(+), 27 deletions(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 1aab87010e..5f725acaf7 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -995,21 +995,21 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, &pstate, parser_simple_get_line, &plines_p, colors_p); ExprAST east = viml_pexpr_parse(&pstate, pflags); - const size_t dict_size = ( + const size_t ret_size = ( 1 // "ast" - + (size_t)(east.err.arg != NULL) // "error" + + (size_t)(east.err.msg != NULL) // "error" + (size_t)highlight // "highlight" ); Dictionary ret = { - .items = xmalloc(dict_size * sizeof(ret.items[0])), + .items = xmalloc(ret_size * sizeof(ret.items[0])), .size = 0, - .capacity = dict_size, + .capacity = ret_size, }; ret.items[ret.size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("ast"), .value = NIL, }; - if (east.err.arg != NULL) { + if (east.err.msg != NULL) { Dictionary err_dict = { .items = xmalloc(2 * sizeof(err_dict.items[0])), .size = 2, @@ -1017,15 +1017,22 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, }; err_dict.items[0] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("message"), - .value = STRING_OBJ(cstr_to_string(east.err.arg)), - }; - err_dict.items[1] = (KeyValuePair) { - .key = STATIC_CSTR_TO_STRING("arg"), - .value = STRING_OBJ(((String) { - .data = xmemdupz(east.err.arg, (size_t)east.err.arg_len), - .size = (size_t)east.err.arg_len, - })), + .value = STRING_OBJ(cstr_to_string(east.err.msg)), }; + if (east.err.arg == NULL) { + err_dict.items[1] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("arg"), + .value = STRING_OBJ(STRING_INIT), + }; + } else { + err_dict.items[1] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("arg"), + .value = STRING_OBJ(((String) { + .data = xmemdupz(east.err.arg, (size_t)east.err.arg_len), + .size = (size_t)east.err.arg_len, + })), + }; + } ret.items[ret.size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("error"), .value = DICTIONARY_OBJ(err_dict), @@ -1055,6 +1062,7 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, .value = ARRAY_OBJ(hl), }; } + kvi_destroy(colors); // Walk over the AST, freeing nodes in process. ExprASTConvStack ast_conv_stack; @@ -1065,11 +1073,11 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, })); while (kv_size(ast_conv_stack)) { ExprASTConvStackItem cur_item = kv_last(ast_conv_stack); - if (*cur_item.node_p == NULL) { + ExprASTNode *const node = *cur_item.node_p; + if (node == NULL) { assert(kv_size(ast_conv_stack) == 1); kv_drop(ast_conv_stack, 1); } else { - ExprASTNode *const node = *cur_item.node_p; if (cur_item.ret_node_p->type == kObjectTypeNil) { const size_t ret_node_items_size = (size_t)( 3 // "type", "start" and "len" @@ -1116,7 +1124,7 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, })); } else if (node->next != NULL) { kvi_push(ast_conv_stack, ((ExprASTConvStackItem) { - .node_p = &node->children, + .node_p = &node->next, .ret_node_p = cur_item.ret_node_p + 1, })); } else if (node != NULL) { @@ -1145,7 +1153,10 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, case kExprNodeSingleQuotedString: { ret_node->items[ret_node->size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("svalue"), - .value = STRING_OBJ(cstr_as_string(node->data.str.value)), + .value = STRING_OBJ(((String) { + .data = node->data.str.value, + .size = node->data.str.size, + })), }; break; } @@ -1217,7 +1228,7 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, ret_node->items[ret_node->size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("ccs_strategy"), .value = STRING_OBJ(cstr_to_string( - eltkn_cmp_type_tab[node->data.cmp.ccs])), + ccs_tab[node->data.cmp.ccs])), }; ret_node->items[ret_node->size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("invert"), diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index b19aab22af..b10952a8ac 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1492,7 +1492,7 @@ static void parse_quoted_string(ParserState *const pstate, node->data.str.value = NULL; } else { char *v_p; - v_p = node->data.str.value = xmalloc(size); + v_p = node->data.str.value = xmallocz(size); p = s + 1; while (p < e) { const char *const chunk_e = memchr(p, '\'', (size_t)(e - p)); diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index 6b44698638..bb7785657d 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -1,5 +1,7 @@ local helpers = require('test.functional.helpers')(after_each) local Screen = require('test.functional.ui.screen') +local global_helpers = require('test.helpers') + local NIL = helpers.NIL local clear, nvim, eq, neq = helpers.clear, helpers.nvim, helpers.eq, helpers.neq local ok, nvim_async, feed = helpers.ok, helpers.nvim_async, helpers.feed @@ -10,6 +12,9 @@ local request = helpers.request local meth_pcall = helpers.meth_pcall local command = helpers.command +local intchar2lua = global_helpers.intchar2lua +local format_string = global_helpers.format_string + describe('api', function() before_each(clear) @@ -711,8 +716,6871 @@ describe('api', function() end) describe('nvim_parse_expression', function() + local function simplify_east_api_node(line, east_api_node) + if east_api_node.children then + for k, v in pairs(east_api_node.children) do + east_api_node.children[k] = simplify_east_api_node(line, v) + end + end + local typ = east_api_node.type + if typ == 'Register' then + typ = typ .. ('(name=%s)'):format( + tostring(intchar2lua(east_api_node.name))) + east_api_node.name = nil + elseif typ == 'PlainIdentifier' then + typ = typ .. ('(scope=%s,ident=%s)'):format( + tostring(intchar2lua(east_api_node.scope)), east_api_node.ident) + east_api_node.scope = nil + east_api_node.ident = nil + elseif typ == 'PlainKey' then + typ = typ .. ('(key=%s)'):format(east_api_node.ident) + east_api_node.ident = nil + elseif typ == 'Comparison' then + typ = typ .. ('(type=%s,inv=%u,ccs=%s)'):format( + east_api_node.cmp_type, east_api_node.invert and 1 or 0, + east_api_node.ccs_strategy) + east_api_node.ccs_strategy = nil + east_api_node.cmp_type = nil + east_api_node.invert = nil + elseif typ == 'Integer' then + typ = typ .. ('(val=%u)'):format(east_api_node.ivalue) + east_api_node.ivalue = nil + elseif typ == 'Float' then + typ = typ .. ('(val=%e)'):format(east_api_node.fvalue) + east_api_node.fvalue = nil + elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then + typ = format_string('%s(val=%q)', typ, east_api_node.svalue) + east_api_node.svalue = nil + elseif typ == 'Option' then + typ = ('%s(scope=%s,ident=%s)'):format( + typ, + tostring(intchar2lua(east_api_node.scope)), + east_api_node.ident) + east_api_node.ident = nil + east_api_node.scope = nil + elseif typ == 'Environment' then + typ = ('%s(ident=%s)'):format(typ, east_api_node.ident) + east_api_node.ident = nil + end + typ = ('%s:%u:%u:%s'):format( + typ, east_api_node.start[1], east_api_node.start[2], + line:sub(east_api_node.start[2] + 1, + east_api_node.start[2] + 1 + east_api_node.len - 1)) + assert(east_api_node.start[2] + east_api_node.len - 1 <= #line) + for k, _ in pairs(east_api_node.start) do + assert(({true, true})[k]) + end + east_api_node.start = nil + east_api_node.type = nil + east_api_node.len = nil + local can_simplify = true + for _, _ in pairs(east_api_node) do + if can_simplify then can_simplify = false end + end + if can_simplify then + return typ + else + east_api_node[1] = typ + return east_api_node + end + end + local function simplify_east_api(line, east_api) + if east_api.error then + east_api.err = east_api.error + east_api.error = nil + east_api.err.msg = east_api.err.message + east_api.err.message = nil + end + if east_api.ast then + east_api.ast = {simplify_east_api_node(line, east_api.ast)} + end + return east_api + end + local function simplify_east_hl(line, east_hl) + for i, v in ipairs(east_hl) do + east_hl[i] = ('%s:%u:%u:%s'):format( + v[4], + v[1], + v[2], + line:sub(v[2] + 1, v[3])) + end + return east_hl + end + local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) + if flags == 0 then + flags = "" + end + + local err, msg = pcall(function() + local east_api = meths.parse_expression(str, flags, true) + local east_hl = east_api.highlight + east_api.highlight = nil + local ast = simplify_east_api(str, east_api) + local hls = simplify_east_hl(str, east_hl) + eq(exp_ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exp_highlighting_fs) do + exp_highlighting[i], next_col = h(next_col) + end + eq(exp_highlighting, hls) + end + end) + if not err then + msg = format_string('Error while processing test (%r, %s):\n%s', + str, flags, msg) + error(msg) + end + end + local function hl(group, str, shift) + return function(next_col) + local col = next_col + (shift or 0) + return (('%s:%u:%u:%s'):format( + 'NVim' .. group, + 0, + col, + str)), (col + #str) + end + end + it('works with + and @a', function() + check_parsing('@a', 0, { + ast = { + 'Register(name=a):0:0:@a', + }, + }, { + hl('Register', '@a'), + }) + check_parsing('+@a', 0, { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + }) + check_parsing('@a+@b', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+@b+@c', 0, { + ast = { + { + 'BinaryPlus:0:5:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + 'Register(name=c):0:6:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing('+@a+@b', 0, { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:4:@b', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('+@a++@b', 0, { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'UnaryPlus:0:4:+', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a@b', 0, { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:2:@b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidRegister', '@b'), + }) + check_parsing(' @a \t @b', 0, { + ast = { + { + 'OpMissing:0:3:', + children = { + 'Register(name=a):0:0: @a', + 'Register(name=b):0:3: \t @b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a', 1), + hl('InvalidSpacing', ' \t '), + hl('Register', '@b'), + }) + check_parsing('+', 0, { + ast = { + 'UnaryPlus:0:0:+', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + }) + check_parsing(' +', 0, { + ast = { + 'UnaryPlus:0:0: +', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryPlus', '+', 1), + }) + check_parsing('@a+ ', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + }) + end) + it('works with @a, + and parenthesis', function() + check_parsing('(@a)', 0, { + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + }) + check_parsing('()', 0, { + ast = { + { + 'Nested:0:0:(', + children = { + 'Missing:0:1:', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing(')', 0, { + ast = { + { + 'Nested:0:0:', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+)', 0, { + ast = { + { + 'Nested:0:1:', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+@a(@b)', 0, { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + { + 'Call:0:3:(', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+@b(@c)', 0, { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a()', 0, { + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a ()', 0, { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:2: (', + children = { + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = '()', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidSpacing', ' '), + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing( + '@a + (@b)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing( + '@a + (+@b)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'UnaryPlus:0:6:+', + children = { + 'Register(name=b):0:7:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing( + '@a + (@b + @c)', 0, { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + }) + check_parsing('(@a)+@b', 0, { + ast = { + { + 'BinaryPlus:0:4:+', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:5:@b', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+(@b)(@c)', 0, { + -- 01234567890 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:7:(', + children = { + { + 'Nested:0:3:(', + children = { 'Register(name=b):0:4:@b' }, + }, + 'Register(name=c):0:8:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))(@c)', 0, { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:9:(', + children = { + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))+@c', 0, { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:9:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing( + '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', 0, {--[[ + | | | | | | | | || | | || | | ||| || || || || + 000000000011111111112222222222333333333344444444445555555 + 012345678901234567890123456789012345678901234567890123456 + ]] + ast = {{ + 'BinaryPlus:0:31: +', + children = { + { + 'BinaryPlus:0:23: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=d):0:16: @d', + 'Register(name=e):0:20:@e', + }, + }, + }, + }, + { + 'Nested:0:25: (', + children = { + { + 'UnaryPlus:0:27:+', + children = { + 'Register(name=f):0:28:@f', + }, + }, + }, + }, + }, + }, + { + 'Call:0:53:(', + children = { + { + 'Nested:0:33: (', + children = { + { + 'Call:0:48:(', + children = { + { + 'Call:0:44:(', + children = { + { + 'Nested:0:35:(', + children = { + { + 'UnaryPlus:0:36:+', + children = { + { + 'Call:0:39:(', + children = { + 'Register(name=g):0:37:@g', + 'Register(name=h):0:40:@h', + }, + }, + }, + }, + }, + }, + 'Register(name=j):0:45:@j', + }, + }, + 'Register(name=k):0:49:@k', + }, + }, + }, + }, + 'Register(name=l):0:54:@l', + }, + }, + }, + }}, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('NestingParenthesis', '('), + hl('UnaryPlus', '+'), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@k'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@l'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a)', 0, { + -- 012 + ast = { + { + 'Nested:0:2:', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Unexpected closing parenthesis: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('(@a', 0, { + -- 012 + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '(@a', + msg = 'E110: Missing closing parenthesis for nested expression: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + }) + check_parsing('@a(@b', 0, { + -- 01234 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + err = { + arg = '(@b', + msg = 'E116: Missing closing parenthesis for function call: %.*s', + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + }) + check_parsing('@a(@b, @c, @d, @e)', 0, { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Comma:0:5:,', + children = { + 'Register(name=b):0:3:@b', + { + 'Comma:0:9:,', + children = { + 'Register(name=c):0:6: @c', + { + 'Comma:0:13:,', + children = { + 'Register(name=d):0:10: @d', + 'Register(name=e):0:14: @e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c', 1), + hl('Comma', ','), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c))', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + { + 'Call:0:8:(', + children = { + 'Register(name=c):0:6:@c', + { + 'Comma:0:15:,', + children = { + { + 'Call:0:11:(', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=f):0:16: @f', + { + 'Comma:0:26:,', + children = { + { + 'Call:0:22:(', + children = { + 'Register(name=g):0:20:@g', + 'Register(name=h):0:23:@h', + }, + }, + { + 'Call:0:30:(', + children = { + 'Register(name=i):0:27: @i', + 'Register(name=j):0:31:@j', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', '('), + hl('Register', '@d'), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@f', 1), + hl('CallingParenthesis', '('), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@i', 1), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + end) + it('works with variable names, including curly braces ones', function() + check_parsing('var', 0, { + ast = { + 'PlainIdentifier(scope=0,ident=var):0:0:var', + }, + }, { + hl('IdentifierName', 'var'), + }) + check_parsing('g:var', 0, { + ast = { + 'PlainIdentifier(scope=g,ident=var):0:0:g:var', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'var'), + }) + check_parsing('g:', 0, { + ast = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + }) + check_parsing('{a}', 0, { + -- 012 + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + }) + check_parsing('{a:b}', 0, { + -- 012 + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + }) + check_parsing('{a:@b}', 0, { + -- 012345 + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + { + 'OpMissing:0:3:', + children={ + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, + }, + err = { + arg = '@b}', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidRegister', '@b'), + hl('Curly', '}'), + }) + check_parsing('{@a}', 0, { + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}{@b}', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'CurlyBracesIdentifier:0:4:{', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('Register', '@b'), + hl('Curly', '}'), + }) + check_parsing('g:{@a}', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'CurlyBracesIdentifier:0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}_test', 0, { + -- 012345678 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:4:_test', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + }) + check_parsing('g:{@a}_test', 0, { + -- 01234567890 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier:0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + }) + check_parsing('g:{@a}_test()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:11:(', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier:0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a} ()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:4: (', + children = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('g:{@a} ()', 0, { + -- 0123456789012 + ast = { + { + 'Call:0:6: (', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'CurlyBracesIdentifier:0:2:{', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a', 0, { + -- 012 + ast = { + { + 'UnknownFigure:0:0:{', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '{@a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('Register', '@a'), + }) + end) + it('works with lambdas and dictionaries', function() + check_parsing('{}', 0, { + ast = { + 'DictLiteral:0:0:{', + }, + }, { + hl('Dict', '{'), + hl('Dict', '}'), + }) + check_parsing('{->@a}', 0, { + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Arrow:0:1:->', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{->@a+@b}', 0, { + -- 012345678 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Arrow:0:1:->', + children = { + { + 'BinaryPlus:0:5:+', + children = { + 'Register(name=a):0:3:@a', + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('Lambda', '}'), + }) + check_parsing('{a->@a}', 0, { + -- 012345678 + ast = { + { + 'Lambda:0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2:->', + children = { + 'Register(name=a):0:4:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->@a}', 0, { + -- 012345678 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'Register(name=a):0:6:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c->@a}', 0, { + -- 01234567890 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + { + 'Arrow:0:6:->', + children = { + 'Register(name=a):0:8:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d->@a}', 0, { + -- 0123456789012 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:8:->', + children = { + 'Register(name=a):0:10:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d,->@a}', 0, { + -- 01234567890123 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:9:->', + children = { + 'Register(name=a):0:11:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Comma', ','), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->{c,d->{e,f->@a}}}', 0, { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + 'Lambda:0:6:{', + children = { + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + 'PlainIdentifier(scope=0,ident=d):0:9:d', + }, + }, + { + 'Arrow:0:10:->', + children = { + { + 'Lambda:0:12:{', + children = { + { + 'Comma:0:14:,', + children = { + 'PlainIdentifier(scope=0,ident=e):0:13:e', + 'PlainIdentifier(scope=0,ident=f):0:15:f', + }, + }, + { + 'Arrow:0:16:->', + children = { + 'Register(name=a):0:18:@a', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('IdentifierName', 'e'), + hl('Comma', ','), + hl('IdentifierName', 'f'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + hl('Lambda', '}'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->c,d}', 0, { + -- 0123456789 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',d}', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + hl('Lambda', '}'), + }) + check_parsing('a,b,c,d', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidComma', ','), + hl('IdentifierName', 'b'), + hl('InvalidComma', ','), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + }) + check_parsing('a,b,c,d,', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d,', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidComma', ','), + hl('IdentifierName', 'b'), + hl('InvalidComma', ','), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + hl('InvalidComma', ','), + }) + check_parsing(',', 0, { + -- 0123456789 + ast = { + { + 'Comma:0:0:,', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ',', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('InvalidComma', ','), + }) + check_parsing('{,a->@a}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + { + 'Arrow:0:3:->', + children = { + { + 'Comma:0:1:,', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:2:a', + }, + }, + 'Register(name=a):0:5:@a', + }, + }, + }, + }, + }, + err = { + arg = ',a->@a}', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('Curly', '{'), + hl('InvalidComma', ','), + hl('IdentifierName', 'a'), + hl('InvalidArrow', '->'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('}', 0, { + -- 0123456789 + ast = { + 'UnknownFigure:0:0:', + }, + err = { + arg = '}', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidFigureBrace', '}'), + }) + check_parsing('{->}', 0, { + -- 0123456789 + ast = { + { + 'Lambda:0:0:{', + children = { + 'Arrow:0:1:->', + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,b}', 0, { + -- 0123456789 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,}', 0, { + -- 0123456789 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('InvalidLambda', '}'), + }) + check_parsing('{@a:@b}', 0, { + -- 0123456789 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d}', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,}', 0, { + -- 01234567890123456789 + -- 0 1 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', 0, { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + { + 'Colon:0:21::', + children = { + 'Register(name=g):0:19:@g', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Register', '@g'), + hl('Colon', ':'), + hl('InvalidDict', '}'), + }) + check_parsing('{@a:@b,}', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{({f -> g})(@h)(@i)}', 0, { + -- 01234567890123456789 + -- 0 1 + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + { + 'Call:0:15:(', + children = { + { + 'Call:0:11:(', + children = { + { + 'Nested:0:1:(', + children = { + { + 'Lambda:0:2:{', + children = { + 'PlainIdentifier(scope=0,ident=f):0:3:f', + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:7: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:12:@h', + }, + }, + 'Register(name=i):0:16:@i', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('NestingParenthesis', '('), + hl('Lambda', '{'), + hl('IdentifierName', 'f'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + }) + check_parsing('a:{b()}c', 0, { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:7:', + children = { + { + 'CurlyBracesIdentifier:0:2:{', + children = { + { + 'Call:0:4:(', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + }) + check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', 0, { + -- 01234567890123456789012345678901234567890123456 + -- 0 1 2 3 4 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:42:', + children = { + { + 'CurlyBracesIdentifier:0:2:{', + children = { + { + 'Call:0:37:(', + children = { + { + 'Lambda:0:3:{', + children = { + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + { + 'Arrow:0:8: ->', + children = { + { + 'BinaryPlus:0:19: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + 'Register(name=d):0:11: @d', + 'Register(name=e):0:16: @e', + }, + }, + { + 'Call:0:32:(', + children = { + { + 'Nested:0:21: (', + children = { + { + 'Lambda:0:23:{', + children = { + 'PlainIdentifier(scope=0,ident=f):0:24:f', + { + 'Arrow:0:25: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:28: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:33:@h', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=i):0:38:@i', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=j):0:42:j', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Lambda', '{'), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('Arrow', '->', 1), + hl('Register', '@d', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Lambda', '{'), + hl('IdentifierName', 'f'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('IdentifierName', 'j'), + }) + check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', 0, { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:8: :', + children = { + { + 'BinaryPlus:0:3: +', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:5: @b', + }, + }, + { + 'BinaryPlus:0:13: +', + children = { + 'Register(name=c):0:10: @c', + 'Register(name=d):0:15: @d', + }, + }, + }, + }, + { + 'Colon:0:27: :', + children = { + { + 'BinaryPlus:0:22: +', + children = { + 'Register(name=e):0:19: @e', + 'Register(name=f):0:24: @f', + }, + }, + { + 'BinaryPlus:0:32: +', + children = { + 'Register(name=g):0:29: @g', + 'Register(name=i):0:34: @i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('Register', '@b', 1), + hl('Colon', ':', 1), + hl('Register', '@c', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@f', 1), + hl('Colon', ':', 1), + hl('Register', '@g', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@i', 1), + hl('Dict', '}'), + }) + check_parsing('-> -> ->', 0, { + -- 01234567 + ast = { + { + 'Arrow:0:0:->', + children = { + 'Missing:0:0:', + { + 'Arrow:0:2: ->', + children = { + 'Missing:0:2:', + { + 'Arrow:0:5: ->', + children = { + 'Missing:0:5:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> -> ->', + msg = 'E15: Unexpected arrow: %.*s', + }, + }, { + hl('InvalidArrow', '->'), + hl('InvalidArrow', '->', 1), + hl('InvalidArrow', '->', 1), + }) + check_parsing('a -> b -> c -> d', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Arrow:0:1: ->', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Arrow:0:6: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + { + 'Arrow:0:11: ->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:9: c', + 'PlainIdentifier(scope=0,ident=d):0:14: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> b -> c -> d', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'c', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'd', 1), + }) + check_parsing('{a -> b -> c}', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Lambda:0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2: ->', + children = { + { + 'Arrow:0:7: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:5: b', + 'PlainIdentifier(scope=0,ident=c):0:10: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> c}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'c', 1), + hl('Lambda', '}'), + }) + check_parsing('{a: -> b}', 0, { + -- 012345678 + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + { + 'Arrow:0:3: ->', + children = { + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'PlainIdentifier(scope=0,ident=b):0:6: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a:b -> b}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'b'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a#b -> b}', 0, { + -- 0123456789 + ast = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=a#b):0:1:a#b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a#b'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + check_parsing('{a : b : c}', 0, { + -- 01234567890 + -- 0 1 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Colon:0:6: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': c}', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('Dict', '}'), + }) + check_parsing('{', 0, { + -- 0 + ast = { + 'UnknownFigure:0:0:{', + }, + err = { + arg = '{', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + }) + check_parsing('{a', 0, { + -- 01 + ast = { + { + 'UnknownFigure:0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + err = { + arg = '{a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + }) + check_parsing('{a,b', 0, { + -- 0123 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '{a,b', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + }) + check_parsing('{a,b->', 0, { + -- 012345 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + 'Arrow:0:4:->', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + }) + check_parsing('{a,b->c', 0, { + -- 0123456 + ast = { + { + 'Lambda:0:0:{', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + }, + }, + }, + }, + }, + err = { + arg = '{a,b->c', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('IdentifierName', 'c'), + }) + check_parsing('{a : b', 0, { + -- 012345 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + err = { + arg = '{a : b', + msg = 'E723: Missing end of Dictionary \'}\': %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + }) + check_parsing('{a : b,', 0, { + -- 0123456 + ast = { + { + 'DictLiteral:0:0:{', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + }) + end) + it('works with ternary operator', function() + check_parsing('a ? b : c', 0, { + -- 012345678 + ast = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?', 1), + hl('IdentifierName', 'b', 1), + hl('TernaryColon', ':', 1), + hl('IdentifierName', 'c', 1), + }) + check_parsing('@a?@b?@c:@d:@e', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:11::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:8::', + children = { + 'Register(name=c):0:6:@c', + 'Register(name=d):0:9:@d', + }, + }, + }, + }, + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('TernaryColon', ':'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b:@c?@d:@e', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:5::', + children = { + 'Register(name=b):0:3:@b', + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', 0, { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:29::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:20::', + children = { + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + { + 'Ternary:0:14:?', + children = { + 'Register(name=e):0:12:@e', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=f):0:15:@f', + 'Register(name=g):0:18:@g', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Ternary:0:23:?', + children = { + 'Register(name=h):0:21:@h', + { + 'TernaryValue:0:26::', + children = { + 'Register(name=i):0:24:@i', + 'Register(name=j):0:27:@j', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=k):0:30:@k', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + hl('Ternary', '?'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('TernaryColon', ':'), + hl('Register', '@h'), + hl('Ternary', '?'), + hl('Register', '@i'), + hl('TernaryColon', ':'), + hl('Register', '@j'), + hl('TernaryColon', ':'), + hl('Register', '@k'), + }) + check_parsing('?', 0, { + -- 0 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + 'TernaryValue:0:0:?', + }, + }, + }, + err = { + arg = '?', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + }) + + check_parsing('?:', 0, { + -- 01 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = '?:', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + }) + + check_parsing('?::', 0, { + -- 012 + ast = { + { + 'Colon:0:2::', + children = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + 'Missing:0:2:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '?::', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + hl('InvalidColon', ':'), + }) + + check_parsing('a?b', 0, { + -- 012 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '?b', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + }) + check_parsing('a?b:', 0, { + -- 0123 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, + }, + err = { + arg = '?b:', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + }) + + check_parsing('a?b::c', 0, { + -- 012345 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:4::', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('TernaryColon', ':'), + hl('IdentifierName', 'c'), + }) + + check_parsing('a?b :', 0, { + -- 01234 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + hl('TernaryColon', ':', 1), + }) + + check_parsing('(@a?@b:@c)?@d:@e', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:13::', + children = { + 'Register(name=d):0:11:@d', + 'Register(name=e):0:14:@e', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + + check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', 0, { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:21::', + children = { + { + 'Nested:0:11:(', + children = { + { + 'Ternary:0:14:?', + children = { + 'Register(name=d):0:12:@d', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=e):0:15:@e', + 'Register(name=f):0:18:@f', + }, + }, + }, + }, + }, + }, + { + 'Nested:0:22:(', + children = { + { + 'Ternary:0:25:?', + children = { + 'Register(name=g):0:23:@g', + { + 'TernaryValue:0:28::', + children = { + 'Register(name=h):0:26:@h', + 'Register(name=i):0:29:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('NestingParenthesis', '('), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('TernaryColon', ':'), + hl('NestingParenthesis', '('), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', 0, { + -- 0123456789012345678901234567 + -- 0 1 2 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:19::', + children = { + { + 'Ternary:0:13:?', + children = { + 'Register(name=d):0:11:@d', + { + 'TernaryValue:0:16::', + children = { + 'Register(name=e):0:14:@e', + 'Register(name=f):0:17:@f', + }, + }, + }, + }, + { + 'Ternary:0:22:?', + children = { + 'Register(name=g):0:20:@g', + { + 'TernaryValue:0:25::', + children = { + 'Register(name=h):0:23:@h', + 'Register(name=i):0:26:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + }) + check_parsing('a?b{cdef}g:h', 0, { + -- 012345678901 + -- 0 1 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:10::', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'ComplexIdentifier:0:9:', + children = { + { + 'CurlyBracesIdentifier:0:3:{', + children = { + 'PlainIdentifier(scope=0,ident=cdef):0:4:cdef', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:9:g', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=h):0:11:h', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('IdentifierName', 'cdef'), + hl('Curly', '}'), + hl('IdentifierName', 'g'), + hl('TernaryColon', ':'), + hl('IdentifierName', 'h'), + }) + check_parsing('a ? b : c : d', 0, { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Colon:0:9: :', + children = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + err = { + arg = ': d', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?', 1), + hl('IdentifierName', 'b', 1), + hl('TernaryColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'd', 1), + }) + end) + it('works with comparison operators', function() + check_parsing('a == b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:1: ==', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ==? b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=IgnoreCase):0:1: ==?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ==# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=MatchCase):0:1: ==#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a !=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=1,ccs=MatchCase):0:1: !=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '!=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a <=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=Greater,inv=1,ccs=MatchCase):0:1: <=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a >=# b', 0, { + -- 0123456 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=0,ccs=MatchCase):0:1: >=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '>=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ># b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Greater,inv=0,ccs=MatchCase):0:1: >#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '>', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a <# b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:1: <#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a is#b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=MatchCase):0:1: is#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b'), + }) + + check_parsing('a is?b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=IgnoreCase):0:1: is?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b'), + }) + + check_parsing('a isnot b', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=Identical,inv=1,ccs=UseOption):0:1: isnot', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'isnot', 1), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a < b < c', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:5: <', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + err = { + arg = ' < c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('IdentifierName', 'c', 1), + }) + + check_parsing('a < b <# c', 0, { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:5: <#', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + err = { + arg = ' <# c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('InvalidComparisonModifier', '#'), + hl('IdentifierName', 'c', 1), + }) + + check_parsing('a += b', 0, { + -- 012345 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3:=', + children = { + { + 'BinaryPlus:0:1: +', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Missing:0:3:', + }, + }, + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + err = { + arg = '= b', + msg = 'E15: Expected == or =~: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('BinaryPlus', '+', 1), + hl('InvalidComparison', '='), + hl('IdentifierName', 'b', 1), + }) + check_parsing('a + b == c + d', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:5: ==', + children = { + { + 'BinaryPlus:0:1: +', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + { + 'BinaryPlus:0:10: +', + children = { + 'PlainIdentifier(scope=0,ident=c):0:8: c', + 'PlainIdentifier(scope=0,ident=d):0:12: d', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'b', 1), + hl('Comparison', '==', 1), + hl('IdentifierName', 'c', 1), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'd', 1), + }) + check_parsing('+ a == + b', 0, { + -- 0123456789 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3: ==', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1: a', + }, + }, + { + 'UnaryPlus:0:6: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:8: b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('IdentifierName', 'a', 1), + hl('Comparison', '==', 1), + hl('UnaryPlus', '+', 1), + hl('IdentifierName', 'b', 1), + }) + end) + it('works with concat/subscript', function() + check_parsing('.', 0, { + -- 0 + ast = { + { + 'ConcatOrSubscript:0:0:.', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = '.', + msg = 'E15: Unexpected dot: %.*s', + }, + }, { + hl('InvalidConcatOrSubscript', '.'), + }) + + check_parsing('a.', 0, { + -- 01 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + }) + + check_parsing('a.b', 0, { + -- 012 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=b):0:2:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', 'b'), + }) + + check_parsing('1.2', 0, { + -- 012 + ast = { + 'Float(val=1.200000e+00):0:0:1.2', + }, + }, { + hl('Float', '1.2'), + }) + + check_parsing('1.2 + 1.3e-5', 0, { + -- 012345678901 + -- 0 1 + ast = { + { + 'BinaryPlus:0:3: +', + children = { + 'Float(val=1.200000e+00):0:0:1.2', + 'Float(val=1.300000e-05):0:5: 1.3e-5', + }, + }, + }, + }, { + hl('Float', '1.2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('a . 1.2 + 1.3e-5', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'BinaryPlus:0:7: +', + children = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + 'Float(val=1.300000e-05):0:9: 1.3e-5', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('1.3e-5 + 1.2 . a', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:12: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'Float(val=1.200000e+00):0:8: 1.2', + }, + }, + 'PlainIdentifier(scope=0,ident=a):0:14: a', + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.2', 1), + hl('Concat', '.', 1), + hl('IdentifierName', 'a', 1), + }) + + check_parsing('1.3e-5 + a . 1.2', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:10: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'PlainIdentifier(scope=0,ident=a):0:8: a', + }, + }, + { + 'ConcatOrSubscript:0:14:.', + children = { + 'Integer(val=1):0:12: 1', + 'PlainKey(key=2):0:15:2', + }, + }, + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'a', 1), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('1.2.3', 0, { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'Integer(val=1):0:0:1', + 'PlainKey(key=2):0:2:2', + }, + }, + 'PlainKey(key=3):0:4:3', + }, + }, + }, + }, { + hl('Number', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '3'), + }) + + check_parsing('a.1.2', 0, { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=1):0:2:1', + }, + }, + 'PlainKey(key=2):0:4:2', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('a . 1.2', 0, { + -- 0123456 + ast = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('+a . +b', 0, { + -- 0123456 + ast = { + { + 'Concat:0:2: .', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'UnaryPlus:0:4: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:6:b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('UnaryPlus', '+', 1), + hl('IdentifierName', 'b'), + }) + + check_parsing('a. b', 0, { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a. 1', 0, { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2: 1', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('Number', '1', 1), + }) + end) + it('works with bracket subscripts', function() + check_parsing(':', 0, { + -- 0 + ast = { + { + 'Colon:0:0::', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ':', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('InvalidColon', ':'), + }) + check_parsing('a[]', 0, { + -- 012 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), + }) + check_parsing('a[b:]', 0, { + -- 01234 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[b:c]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=c):0:2:b:c', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + }) + check_parsing('a[b : c]', 0, { + -- 01234567 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + 'PlainIdentifier(scope=0,ident=c):0:5: c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[: b]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:2::', + children = { + 'Missing:0:2:', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('SubscriptColon', ':'), + hl('IdentifierName', 'b', 1), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[b :]', 0, { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptColon', ':', 1), + hl('SubscriptBracket', ']'), + }) + check_parsing('a[b][c][d](e)(f)(g)', 0, { + -- 0123456789012345678 + -- 0 1 + ast = { + { + 'Call:0:16:(', + children = { + { + 'Call:0:13:(', + children = { + { + 'Call:0:10:(', + children = { + { + 'Subscript:0:7:[', + children = { + { + 'Subscript:0:4:[', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:11:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:14:f', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:17:g', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'e'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'f'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'g'), + hl('CallingParenthesis', ')'), + }) + check_parsing('{a}{b}{c}[d][e][f]', 0, { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Subscript:0:15:[', + children = { + { + 'Subscript:0:12:[', + children = { + { + 'Subscript:0:9:[', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + { + 'CurlyBracesIdentifier:0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier:0:3:{', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + }, + }, + { + 'CurlyBracesIdentifier:0:6:{', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:10:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:13:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:16:f', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'c'), + hl('Curly', '}'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'e'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'f'), + hl('SubscriptBracket', ']'), + }) + end) + it('supports list literals', function() + check_parsing('[]', 0, { + -- 01 + ast = { + 'ListLiteral:0:0:[', + }, + }, { + hl('List', '['), + hl('List', ']'), + }) + + check_parsing('[a]', 0, { + -- 012 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + }) + + check_parsing('[a, b]', 0, { + -- 012345 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c]', 0, { + -- 012345678 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c, ]', 0, { + -- 01234567890 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('Comma', ','), + hl('List', ']', 1), + }) + + check_parsing('[a : b, c : d]', 0, { + -- 01234567890123 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + { + 'Colon:0:9: :', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7: c', + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': b, c : d]', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'd', 1), + hl('List', ']'), + }) + + check_parsing(']', 0, { + -- 0 + ast = { + 'ListLiteral:0:0:', + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidList', ']'), + }) + + check_parsing('a]', 0, { + -- 01 + ast = { + { + 'ListLiteral:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidList', ']'), + }) + + check_parsing('[] []', 0, { + -- 01234 + ast = { + { + 'OpMissing:0:2:', + children = { + 'ListLiteral:0:0:[', + 'ListLiteral:0:2: [', + }, + }, + }, + err = { + arg = '[]', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('InvalidSpacing', ' '), + hl('List', '['), + hl('List', ']'), + }) + + check_parsing('[][]', 0, { + -- 0123 + ast = { + { + 'Subscript:0:2:[', + children = { + 'ListLiteral:0:0:[', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), + }) + + check_parsing('[', 0, { + -- 0 + ast = { + 'ListLiteral:0:0:[', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('List', '['), + }) + + check_parsing('[1', 0, { + -- 01 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'Integer(val=1):0:1:1', + }, + }, + }, + err = { + arg = '[1', + msg = 'E697: Missing end of List \']\': %.*s', + }, + }, { + hl('List', '['), + hl('Number', '1'), + }) + end) + it('works with strings', function() + check_parsing('\'abc\'', 0, { + -- 01234 + ast = { + 'SingleQuotedString(val="abc"):0:0:\'abc\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuote', '\''), + }) + check_parsing('"abc"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="abc"):0:0:"abc"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuote', '"'), + }) + check_parsing('\'\'', 0, { + -- 01 + ast = { + 'SingleQuotedString(val=""):0:0:\'\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuote', '\''), + }) + check_parsing('""', 0, { + -- 01 + ast = { + 'DoubleQuotedString(val=""):0:0:""', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuote', '"'), + }) + check_parsing('"', 0, { + -- 0 + ast = { + 'DoubleQuotedString(val=""):0:0:"', + }, + err = { + arg = '"', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + }) + check_parsing('\'', 0, { + -- 0 + ast = { + 'SingleQuotedString(val=""):0:0:\'', + }, + err = { + arg = '\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + }) + check_parsing('"a', 0, { + -- 01 + ast = { + 'DoubleQuotedString(val="a"):0:0:"a', + }, + err = { + arg = '"a', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedBody', 'a'), + }) + check_parsing('\'a', 0, { + -- 01 + ast = { + 'SingleQuotedString(val="a"):0:0:\'a', + }, + err = { + arg = '\'a', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + hl('InvalidSingleQuotedBody', 'a'), + }) + check_parsing('\'abc\'\'def\'', 0, { + -- 0123456789 + ast = { + 'SingleQuotedString(val="abc\'def"):0:0:\'abc\'\'def\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'def'), + hl('SingleQuote', '\''), + }) + check_parsing('\'abc\'\'', 0, { + -- 012345 + ast = { + 'SingleQuotedString(val="abc\'"):0:0:\'abc\'\'', + }, + err = { + arg = '\'abc\'\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + hl('InvalidSingleQuotedBody', 'abc'), + hl('InvalidSingleQuotedQuote', '\'\''), + }) + check_parsing('\'\'\'\'\'\'\'\'', 0, { + -- 01234567 + ast = { + 'SingleQuotedString(val="\'\'\'"):0:0:\'\'\'\'\'\'\'\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuote', '\''), + }) + check_parsing('\'\'\'a\'\'\'\'bc\'', 0, { + -- 01234567890 + -- 0 1 + ast = { + 'SingleQuotedString(val="\'a\'\'bc"):0:0:\'\'\'a\'\'\'\'bc\'', + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'a'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'bc'), + hl('SingleQuote', '\''), + }) + check_parsing('"\\"\\"\\"\\""', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\"\\"\\"\\""):0:0:"\\"\\"\\"\\""', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuote', '"'), + }) + check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', 0, { + -- 0123456789012345678901234 + -- 0 1 2 + ast = { + 'DoubleQuotedString(val="abc\\"def\\"ghi\\"jkl\\"mno"):0:0:"abc\\"def\\"ghi\\"jkl\\"mno"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'def'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'ghi'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'jkl'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'mno'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\b\\e\\f\\r\\t\\\\"', 0, { + -- 0123456789012345 + -- 0 1 + ast = { + [[DoubleQuotedString(val="\8\27\12\13\9\\"):0:0:"\b\e\f\r\t\\"]], + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\b'), + hl('DoubleQuotedEscape', '\\e'), + hl('DoubleQuotedEscape', '\\f'), + hl('DoubleQuotedEscape', '\\r'), + hl('DoubleQuotedEscape', '\\t'), + hl('DoubleQuotedEscape', '\\\\'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\n\n"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\\n\\\n"):0:0:"\\n\n"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\n'), + hl('DoubleQuotedBody', '\n'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\x00"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\x00"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\xFF"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\255"):0:0:"\\xFF"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xFF'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\xF"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15"):0:0:"\\xF"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\u00AB"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="«"):0:0:"\\u00AB"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u00AB'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\U000000AB"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="«"):0:0:"\\U000000AB"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000000AB'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\x"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="x"):0:0:"\\x"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="x"):0:0:"\\x', + }, + err = { + arg = '"\\x', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\x'), + }) + + check_parsing('"\\xF', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="\\15"):0:0:"\\xF', + }, + err = { + arg = '"\\xF', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedEscape', '\\xF'), + }) + + check_parsing('"\\u"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="u"):0:0:"\\u"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="u"):0:0:"\\u', + }, + err = { + arg = '"\\u', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\u'), + }) + + check_parsing('"\\U', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="U"):0:0:"\\U', + }, + err = { + arg = '"\\U', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + }) + + check_parsing('"\\U"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="U"):0:0:"\\U"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\xFX"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15X"):0:0:"\\xFX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\XFX"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\15X"):0:0:"\\XFX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\XF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\xX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="xX"):0:0:"\\xX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\XX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="XX"):0:0:"\\XX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\X'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\uX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="uX"):0:0:"\\uX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\UX"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="UX"):0:0:"\\UX"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\x0X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\X0X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u0X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0X"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\x00X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\X00X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u00X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00X"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u0000X"', 0, { + -- 012345678 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\u0000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0000X"', 0, { + -- 012345678 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00000X"', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000000X"', 0, { + -- 01234567890 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U000000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0000000X"', 0, { + -- 012345678901 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00000000X"', 0, { + -- 0123456789012 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\x000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X000X"', 0, { + -- 01234567 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\X000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u00000X"', 0, { + -- 0123456789 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\u00000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000000000X"', 0, { + -- 01234567890123 + -- 0 1 + ast = { + 'DoubleQuotedString(val="\\0000X"):0:0:"\\U000000000X"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\0"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\00"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\00"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\000"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0"):0:0:"\\000"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0000"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0000"):0:0:"\\0000"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '0'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\8"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="8"):0:0:"\\8"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\08"', 0, { + -- 01234 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\08"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\008"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\008"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0008"', 0, { + -- 0123456 + ast = { + 'DoubleQuotedString(val="\\0008"):0:0:"\\0008"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\777"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\255"):0:0:"\\777"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\777'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\050"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\40"):0:0:"\\050"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\050'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\"', 0, { + -- 012345 + ast = { + 'DoubleQuotedString(val="\\21"):0:0:"\\"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\<', 0, { + -- 012 + ast = { + 'DoubleQuotedString(val="<"):0:0:"\\<', + }, + err = { + arg = '"\\<', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\<'), + }) + + check_parsing('"\\<"', 0, { + -- 0123 + ast = { + 'DoubleQuotedString(val="<"):0:0:"\\<"', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\<'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\ Date: Sun, 5 Nov 2017 21:06:12 +0300 Subject: [PATCH 073/109] tests: Add missing test cases --- src/nvim/api/vim.c | 6 + test/functional/api/vim_spec.lua | 1197 +++++++++++++------- test/helpers.lua | 42 + test/unit/viml/expressions/parser_spec.lua | 1107 ++++++++++-------- 4 files changed, 1455 insertions(+), 897 deletions(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 5f725acaf7..b12c595cb5 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -971,6 +971,11 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, switch (flags.data[i]) { case 'm': { pflags |= kExprFlagsMulti; break; } case 'E': { pflags |= kExprFlagsDisallowEOC; break; } + case NUL: { + api_set_error(err, kErrorTypeValidation, "Invalid flag: '\\0' (%u)", + (unsigned)flags.data[i]); + return (Dictionary)ARRAY_DICT_INIT; + } default: { api_set_error(err, kErrorTypeValidation, "Invalid flag: '%c' (%u)", flags.data[i], (unsigned)flags.data[i]); @@ -995,6 +1000,7 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, &pstate, parser_simple_get_line, &plines_p, colors_p); ExprAST east = viml_pexpr_parse(&pstate, pflags); + // FIXME add parse_length key const size_t ret_size = ( 1 // "ast" + (size_t)(east.err.msg != NULL) // "error" diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index bb7785657d..b904bd2a8f 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -12,8 +12,10 @@ local request = helpers.request local meth_pcall = helpers.meth_pcall local command = helpers.command +local REMOVE_THIS = global_helpers.REMOVE_THIS local intchar2lua = global_helpers.intchar2lua local format_string = global_helpers.format_string +local mergedicts_copy = global_helpers.mergedicts_copy describe('api', function() before_each(clear) @@ -717,6 +719,9 @@ describe('api', function() describe('nvim_parse_expression', function() local function simplify_east_api_node(line, east_api_node) + if east_api_node == NIL then + return nil + end if east_api_node.children then for k, v in pairs(east_api_node.children) do east_api_node.children[k] = simplify_east_api_node(line, v) @@ -806,31 +811,53 @@ describe('api', function() end return east_hl end - local function check_parsing(str, flags, exp_ast, exp_highlighting_fs) - if flags == 0 then - flags = "" - end - - local err, msg = pcall(function() - local east_api = meths.parse_expression(str, flags, true) - local east_hl = east_api.highlight - east_api.highlight = nil - local ast = simplify_east_api(str, east_api) - local hls = simplify_east_hl(str, east_hl) - eq(exp_ast, ast) - if exp_highlighting_fs then - local exp_highlighting = {} - local next_col = 0 - for i, h in ipairs(exp_highlighting_fs) do - exp_highlighting[i], next_col = h(next_col) + local FLAGS_TO_STR = { + [0] = "", + [1] = "m", + [2] = "E", + [3] = "mE", + } + local function check_parsing(str, exp_ast, exp_highlighting_fs, + nz_flags_exps) + nz_flags_exps = nz_flags_exps or {} + for _, flags in ipairs({0, 1, 2, 3}) do + local err, msg = pcall(function() + local east_api = meths.parse_expression(str, FLAGS_TO_STR[flags], true) + local east_hl = east_api.highlight + east_api.highlight = nil + local ast = simplify_east_api(str, east_api) + local hls = simplify_east_hl(str, east_hl) + local exps = { + ast = exp_ast, + hl_fs = exp_highlighting_fs, + } + local add_exps = nz_flags_exps[flags] + if not add_exps and flags == 3 then + add_exps = nz_flags_exps[1] or nz_flags_exps[2] end - eq(exp_highlighting, hls) + if add_exps then + if add_exps.ast then + exps.ast = mergedicts_copy(exps.ast, add_exps.ast) + end + if add_exps.hl_fs then + exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs) + end + end + eq(exps.ast, ast) + if exp_highlighting_fs then + local exp_highlighting = {} + local next_col = 0 + for i, h in ipairs(exps.hl_fs) do + exp_highlighting[i], next_col = h(next_col) + end + eq(exp_highlighting, hls) + end + end) + if not err then + msg = format_string('Error while processing test (%r, %s):\n%s', + str, FLAGS_TO_STR[flags], msg) + error(msg) end - end) - if not err then - msg = format_string('Error while processing test (%r, %s):\n%s', - str, flags, msg) - error(msg) end end local function hl(group, str, shift) @@ -844,14 +871,14 @@ describe('api', function() end end it('works with + and @a', function() - check_parsing('@a', 0, { + check_parsing('@a', { ast = { 'Register(name=a):0:0:@a', }, }, { hl('Register', '@a'), }) - check_parsing('+@a', 0, { + check_parsing('+@a', { ast = { { 'UnaryPlus:0:0:+', @@ -864,7 +891,7 @@ describe('api', function() hl('UnaryPlus', '+'), hl('Register', '@a'), }) - check_parsing('@a+@b', 0, { + check_parsing('@a+@b', { ast = { { 'BinaryPlus:0:2:+', @@ -879,7 +906,7 @@ describe('api', function() hl('BinaryPlus', '+'), hl('Register', '@b'), }) - check_parsing('@a+@b+@c', 0, { + check_parsing('@a+@b+@c', { ast = { { 'BinaryPlus:0:5:+', @@ -902,7 +929,7 @@ describe('api', function() hl('BinaryPlus', '+'), hl('Register', '@c'), }) - check_parsing('+@a+@b', 0, { + check_parsing('+@a+@b', { ast = { { 'BinaryPlus:0:3:+', @@ -923,7 +950,7 @@ describe('api', function() hl('BinaryPlus', '+'), hl('Register', '@b'), }) - check_parsing('+@a++@b', 0, { + check_parsing('+@a++@b', { ast = { { 'BinaryPlus:0:3:+', @@ -950,7 +977,7 @@ describe('api', function() hl('UnaryPlus', '+'), hl('Register', '@b'), }) - check_parsing('@a@b', 0, { + check_parsing('@a@b', { ast = { { 'OpMissing:0:2:', @@ -967,8 +994,20 @@ describe('api', function() }, { hl('Register', '@a'), hl('InvalidRegister', '@b'), + }, { + [1] = { + ast = { + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0:@a' + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + }, + }, }) - check_parsing(' @a \t @b', 0, { + check_parsing(' @a \t @b', { ast = { { 'OpMissing:0:3:', @@ -986,8 +1025,21 @@ describe('api', function() hl('Register', '@a', 1), hl('InvalidSpacing', ' \t '), hl('Register', '@b'), + }, { + [1] = { + ast = { + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0: @a' + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + }, + }, }) - check_parsing('+', 0, { + check_parsing('+', { ast = { 'UnaryPlus:0:0:+', }, @@ -998,7 +1050,7 @@ describe('api', function() }, { hl('UnaryPlus', '+'), }) - check_parsing(' +', 0, { + check_parsing(' +', { ast = { 'UnaryPlus:0:0: +', }, @@ -1009,7 +1061,7 @@ describe('api', function() }, { hl('UnaryPlus', '+', 1), }) - check_parsing('@a+ ', 0, { + check_parsing('@a+ ', { ast = { { 'BinaryPlus:0:2:+', @@ -1028,7 +1080,7 @@ describe('api', function() }) end) it('works with @a, + and parenthesis', function() - check_parsing('(@a)', 0, { + check_parsing('(@a)', { ast = { { 'Nested:0:0:(', @@ -1042,7 +1094,7 @@ describe('api', function() hl('Register', '@a'), hl('NestingParenthesis', ')'), }) - check_parsing('()', 0, { + check_parsing('()', { ast = { { 'Nested:0:0:(', @@ -1059,7 +1111,7 @@ describe('api', function() hl('NestingParenthesis', '('), hl('InvalidNestingParenthesis', ')'), }) - check_parsing(')', 0, { + check_parsing(')', { ast = { { 'Nested:0:0:', @@ -1075,7 +1127,7 @@ describe('api', function() }, { hl('InvalidNestingParenthesis', ')'), }) - check_parsing('+)', 0, { + check_parsing('+)', { ast = { { 'Nested:0:1:', @@ -1097,7 +1149,7 @@ describe('api', function() hl('UnaryPlus', '+'), hl('InvalidNestingParenthesis', ')'), }) - check_parsing('+@a(@b)', 0, { + check_parsing('+@a(@b)', { ast = { { 'UnaryPlus:0:0:+', @@ -1119,7 +1171,7 @@ describe('api', function() hl('Register', '@b'), hl('CallingParenthesis', ')'), }) - check_parsing('@a+@b(@c)', 0, { + check_parsing('@a+@b(@c)', { ast = { { 'BinaryPlus:0:2:+', @@ -1143,7 +1195,7 @@ describe('api', function() hl('Register', '@c'), hl('CallingParenthesis', ')'), }) - check_parsing('@a()', 0, { + check_parsing('@a()', { ast = { { 'Call:0:2:(', @@ -1157,7 +1209,7 @@ describe('api', function() hl('CallingParenthesis', '('), hl('CallingParenthesis', ')'), }) - check_parsing('@a ()', 0, { + check_parsing('@a ()', { ast = { { 'OpMissing:0:2:', @@ -1181,91 +1233,102 @@ describe('api', function() hl('InvalidSpacing', ' '), hl('NestingParenthesis', '('), hl('InvalidNestingParenthesis', ')'), + }, { + [1] = { + ast = { + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0:@a', + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + [4] = REMOVE_THIS, + }, + }, }) - check_parsing( - '@a + (@b)', 0, { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - 'Register(name=b):0:6:@b', - }, + check_parsing('@a + (@b)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', }, }, }, }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - }) - check_parsing( - '@a + (+@b)', 0, { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - { - 'UnaryPlus:0:6:+', - children = { - 'Register(name=b):0:7:@b', - }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing('@a + (+@b)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'UnaryPlus:0:6:+', + children = { + 'Register(name=b):0:7:@b', }, }, }, }, }, }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('UnaryPlus', '+'), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - }) - check_parsing( - '@a + (@b + @c)', 0, { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - { - 'BinaryPlus:0:8: +', - children = { - 'Register(name=b):0:6:@b', - 'Register(name=c):0:10: @c', - }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing('@a + (@b + @c)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', }, }, }, }, }, }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Register', '@b'), - hl('BinaryPlus', '+', 1), - hl('Register', '@c', 1), - hl('NestingParenthesis', ')'), - }) - check_parsing('(@a)+@b', 0, { + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + }) + check_parsing('(@a)+@b', { ast = { { 'BinaryPlus:0:4:+', @@ -1287,7 +1350,7 @@ describe('api', function() hl('BinaryPlus', '+'), hl('Register', '@b'), }) - check_parsing('@a+(@b)(@c)', 0, { + check_parsing('@a+(@b)(@c)', { -- 01234567890 ast = { { @@ -1317,7 +1380,7 @@ describe('api', function() hl('Register', '@c'), hl('CallingParenthesis', ')'), }) - check_parsing('@a+((@b))(@c)', 0, { + check_parsing('@a+((@b))(@c)', { -- 01234567890123456890123456789 -- 0 1 2 ast = { @@ -1355,7 +1418,7 @@ describe('api', function() hl('Register', '@c'), hl('CallingParenthesis', ')'), }) - check_parsing('@a+((@b))+@c', 0, { + check_parsing('@a+((@b))+@c', { -- 01234567890123456890123456789 -- 0 1 2 ast = { @@ -1393,7 +1456,7 @@ describe('api', function() hl('Register', '@c'), }) check_parsing( - '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', 0, {--[[ + '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', {--[[ | | | | | | | | || | | || | | ||| || || || || 000000000011111111112222222222333333333344444444445555555 012345678901234567890123456789012345678901234567890123456 @@ -1527,7 +1590,7 @@ describe('api', function() hl('Register', '@l'), hl('CallingParenthesis', ')'), }) - check_parsing('@a)', 0, { + check_parsing('@a)', { -- 012 ast = { { @@ -1545,7 +1608,7 @@ describe('api', function() hl('Register', '@a'), hl('InvalidNestingParenthesis', ')'), }) - check_parsing('(@a', 0, { + check_parsing('(@a', { -- 012 ast = { { @@ -1563,7 +1626,7 @@ describe('api', function() hl('NestingParenthesis', '('), hl('Register', '@a'), }) - check_parsing('@a(@b', 0, { + check_parsing('@a(@b', { -- 01234 ast = { { @@ -1583,7 +1646,7 @@ describe('api', function() hl('CallingParenthesis', '('), hl('Register', '@b'), }) - check_parsing('@a(@b, @c, @d, @e)', 0, { + check_parsing('@a(@b, @c, @d, @e)', { -- 012345678901234567 -- 0 1 ast = { @@ -1625,7 +1688,7 @@ describe('api', function() hl('Register', '@e', 1), hl('CallingParenthesis', ')'), }) - check_parsing('@a(@b(@c))', 0, { + check_parsing('@a(@b(@c))', { -- 01234567890123456789012345678901234567 -- 0 1 2 3 ast = { @@ -1652,7 +1715,7 @@ describe('api', function() hl('CallingParenthesis', ')'), hl('CallingParenthesis', ')'), }) - check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', 0, { + check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', { -- 01234567890123456789012345678901234567 -- 0 1 2 3 ast = { @@ -1742,14 +1805,14 @@ describe('api', function() }) end) it('works with variable names, including curly braces ones', function() - check_parsing('var', 0, { + check_parsing('var', { ast = { 'PlainIdentifier(scope=0,ident=var):0:0:var', }, }, { hl('IdentifierName', 'var'), }) - check_parsing('g:var', 0, { + check_parsing('g:var', { ast = { 'PlainIdentifier(scope=g,ident=var):0:0:g:var', }, @@ -1758,7 +1821,7 @@ describe('api', function() hl('IdentifierScopeDelimiter', ':'), hl('IdentifierName', 'var'), }) - check_parsing('g:', 0, { + check_parsing('g:', { ast = { 'PlainIdentifier(scope=g,ident=):0:0:g:', }, @@ -1766,7 +1829,7 @@ describe('api', function() hl('IdentifierScope', 'g'), hl('IdentifierScopeDelimiter', ':'), }) - check_parsing('{a}', 0, { + check_parsing('{a}', { -- 012 ast = { { @@ -1781,7 +1844,7 @@ describe('api', function() hl('IdentifierName', 'a'), hl('Curly', '}'), }) - check_parsing('{a:b}', 0, { + check_parsing('{a:b}', { -- 012 ast = { { @@ -1798,7 +1861,7 @@ describe('api', function() hl('IdentifierName', 'b'), hl('Curly', '}'), }) - check_parsing('{a:@b}', 0, { + check_parsing('{a:@b}', { -- 012345 ast = { { @@ -1825,7 +1888,7 @@ describe('api', function() hl('InvalidRegister', '@b'), hl('Curly', '}'), }) - check_parsing('{@a}', 0, { + check_parsing('{@a}', { ast = { { 'CurlyBracesIdentifier:0:0:{', @@ -1839,7 +1902,7 @@ describe('api', function() hl('Register', '@a'), hl('Curly', '}'), }) - check_parsing('{@a}{@b}', 0, { + check_parsing('{@a}{@b}', { -- 01234567 ast = { { @@ -1868,7 +1931,7 @@ describe('api', function() hl('Register', '@b'), hl('Curly', '}'), }) - check_parsing('g:{@a}', 0, { + check_parsing('g:{@a}', { -- 01234567 ast = { { @@ -1891,7 +1954,7 @@ describe('api', function() hl('Register', '@a'), hl('Curly', '}'), }) - check_parsing('{@a}_test', 0, { + check_parsing('{@a}_test', { -- 012345678 ast = { { @@ -1913,7 +1976,7 @@ describe('api', function() hl('Curly', '}'), hl('IdentifierName', '_test'), }) - check_parsing('g:{@a}_test', 0, { + check_parsing('g:{@a}_test', { -- 01234567890 ast = { { @@ -1943,7 +2006,7 @@ describe('api', function() hl('Curly', '}'), hl('IdentifierName', '_test'), }) - check_parsing('g:{@a}_test()', 0, { + check_parsing('g:{@a}_test()', { -- 0123456789012 ast = { { @@ -1980,7 +2043,7 @@ describe('api', function() hl('CallingParenthesis', '('), hl('CallingParenthesis', ')'), }) - check_parsing('{@a} ()', 0, { + check_parsing('{@a} ()', { -- 0123456789012 ast = { { @@ -2002,7 +2065,7 @@ describe('api', function() hl('CallingParenthesis', '(', 1), hl('CallingParenthesis', ')'), }) - check_parsing('g:{@a} ()', 0, { + check_parsing('g:{@a} ()', { -- 0123456789012 ast = { { @@ -2032,7 +2095,7 @@ describe('api', function() hl('CallingParenthesis', '(', 1), hl('CallingParenthesis', ')'), }) - check_parsing('{@a', 0, { + check_parsing('{@a', { -- 012 ast = { { @@ -2052,7 +2115,7 @@ describe('api', function() }) end) it('works with lambdas and dictionaries', function() - check_parsing('{}', 0, { + check_parsing('{}', { ast = { 'DictLiteral:0:0:{', }, @@ -2060,7 +2123,7 @@ describe('api', function() hl('Dict', '{'), hl('Dict', '}'), }) - check_parsing('{->@a}', 0, { + check_parsing('{->@a}', { ast = { { 'Lambda:0:0:{', @@ -2080,7 +2143,7 @@ describe('api', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{->@a+@b}', 0, { + check_parsing('{->@a+@b}', { -- 012345678 ast = { { @@ -2109,7 +2172,7 @@ describe('api', function() hl('Register', '@b'), hl('Lambda', '}'), }) - check_parsing('{a->@a}', 0, { + check_parsing('{a->@a}', { -- 012345678 ast = { { @@ -2132,7 +2195,7 @@ describe('api', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b->@a}', 0, { + check_parsing('{a,b->@a}', { -- 012345678 ast = { { @@ -2163,7 +2226,7 @@ describe('api', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b,c->@a}', 0, { + check_parsing('{a,b,c->@a}', { -- 01234567890 ast = { { @@ -2202,7 +2265,7 @@ describe('api', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b,c,d->@a}', 0, { + check_parsing('{a,b,c,d->@a}', { -- 0123456789012 ast = { { @@ -2249,7 +2312,7 @@ describe('api', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b,c,d,->@a}', 0, { + check_parsing('{a,b,c,d,->@a}', { -- 01234567890123 ast = { { @@ -2302,7 +2365,7 @@ describe('api', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b->{c,d->{e,f->@a}}}', 0, { + check_parsing('{a,b->{c,d->{e,f->@a}}}', { -- 01234567890123456789012 -- 0 1 2 ast = { @@ -2380,7 +2443,7 @@ describe('api', function() hl('Lambda', '}'), hl('Lambda', '}'), }) - check_parsing('{a,b->c,d}', 0, { + check_parsing('{a,b->c,d}', { -- 0123456789 ast = { { @@ -2423,7 +2486,7 @@ describe('api', function() hl('IdentifierName', 'd'), hl('Lambda', '}'), }) - check_parsing('a,b,c,d', 0, { + check_parsing('a,b,c,d', { -- 0123456789 ast = { { @@ -2459,7 +2522,7 @@ describe('api', function() hl('InvalidComma', ','), hl('IdentifierName', 'd'), }) - check_parsing('a,b,c,d,', 0, { + check_parsing('a,b,c,d,', { -- 0123456789 ast = { { @@ -2501,7 +2564,7 @@ describe('api', function() hl('IdentifierName', 'd'), hl('InvalidComma', ','), }) - check_parsing(',', 0, { + check_parsing(',', { -- 0123456789 ast = { { @@ -2518,7 +2581,7 @@ describe('api', function() }, { hl('InvalidComma', ','), }) - check_parsing('{,a->@a}', 0, { + check_parsing('{,a->@a}', { -- 0123456789 ast = { { @@ -2552,7 +2615,7 @@ describe('api', function() hl('Register', '@a'), hl('Curly', '}'), }) - check_parsing('}', 0, { + check_parsing('}', { -- 0123456789 ast = { 'UnknownFigure:0:0:', @@ -2564,7 +2627,7 @@ describe('api', function() }, { hl('InvalidFigureBrace', '}'), }) - check_parsing('{->}', 0, { + check_parsing('{->}', { -- 0123456789 ast = { { @@ -2583,7 +2646,7 @@ describe('api', function() hl('Arrow', '->'), hl('InvalidLambda', '}'), }) - check_parsing('{a,b}', 0, { + check_parsing('{a,b}', { -- 0123456789 ast = { { @@ -2610,7 +2673,7 @@ describe('api', function() hl('IdentifierName', 'b'), hl('InvalidLambda', '}'), }) - check_parsing('{a,}', 0, { + check_parsing('{a,}', { -- 0123456789 ast = { { @@ -2635,7 +2698,7 @@ describe('api', function() hl('Comma', ','), hl('InvalidLambda', '}'), }) - check_parsing('{@a:@b}', 0, { + check_parsing('{@a:@b}', { -- 0123456789 ast = { { @@ -2658,7 +2721,7 @@ describe('api', function() hl('Register', '@b'), hl('Dict', '}'), }) - check_parsing('{@a:@b,@c:@d}', 0, { + check_parsing('{@a:@b,@c:@d}', { -- 0123456789012 -- 0 1 ast = { @@ -2698,7 +2761,7 @@ describe('api', function() hl('Register', '@d'), hl('Dict', '}'), }) - check_parsing('{@a:@b,@c:@d,@e:@f,}', 0, { + check_parsing('{@a:@b,@c:@d,@e:@f,}', { -- 01234567890123456789 -- 0 1 ast = { @@ -2760,7 +2823,7 @@ describe('api', function() hl('Comma', ','), hl('Dict', '}'), }) - check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', 0, { + check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', { -- 01234567890123456789012 -- 0 1 2 ast = { @@ -2834,7 +2897,7 @@ describe('api', function() hl('Colon', ':'), hl('InvalidDict', '}'), }) - check_parsing('{@a:@b,}', 0, { + check_parsing('{@a:@b,}', { -- 01234567890123 -- 0 1 ast = { @@ -2864,7 +2927,7 @@ describe('api', function() hl('Comma', ','), hl('Dict', '}'), }) - check_parsing('{({f -> g})(@h)(@i)}', 0, { + check_parsing('{({f -> g})(@h)(@i)}', { -- 01234567890123456789 -- 0 1 ast = { @@ -2920,7 +2983,7 @@ describe('api', function() hl('CallingParenthesis', ')'), hl('Curly', '}'), }) - check_parsing('a:{b()}c', 0, { + check_parsing('a:{b()}c', { -- 01234567 ast = { { @@ -2957,7 +3020,7 @@ describe('api', function() hl('Curly', '}'), hl('IdentifierName', 'c'), }) - check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', 0, { + check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', { -- 01234567890123456789012345678901234567890123456 -- 0 1 2 3 4 ast = { @@ -3067,7 +3130,7 @@ describe('api', function() hl('Curly', '}'), hl('IdentifierName', 'j'), }) - check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', 0, { + check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', { -- 01234567890123456789012345678901234567 -- 0 1 2 3 ast = { @@ -3139,7 +3202,7 @@ describe('api', function() hl('Register', '@i', 1), hl('Dict', '}'), }) - check_parsing('-> -> ->', 0, { + check_parsing('-> -> ->', { -- 01234567 ast = { { @@ -3170,7 +3233,7 @@ describe('api', function() hl('InvalidArrow', '->', 1), hl('InvalidArrow', '->', 1), }) - check_parsing('a -> b -> c -> d', 0, { + check_parsing('a -> b -> c -> d', { -- 0123456789012345 -- 0 1 ast = { @@ -3207,7 +3270,7 @@ describe('api', function() hl('InvalidArrow', '->', 1), hl('IdentifierName', 'd', 1), }) - check_parsing('{a -> b -> c}', 0, { + check_parsing('{a -> b -> c}', { -- 0123456789012 -- 0 1 ast = { @@ -3243,7 +3306,7 @@ describe('api', function() hl('IdentifierName', 'c', 1), hl('Lambda', '}'), }) - check_parsing('{a: -> b}', 0, { + check_parsing('{a: -> b}', { -- 012345678 ast = { { @@ -3272,7 +3335,7 @@ describe('api', function() hl('Curly', '}'), }) - check_parsing('{a:b -> b}', 0, { + check_parsing('{a:b -> b}', { -- 0123456789 ast = { { @@ -3302,7 +3365,7 @@ describe('api', function() hl('Curly', '}'), }) - check_parsing('{a#b -> b}', 0, { + check_parsing('{a#b -> b}', { -- 0123456789 ast = { { @@ -3329,7 +3392,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), hl('Curly', '}'), }) - check_parsing('{a : b : c}', 0, { + check_parsing('{a : b : c}', { -- 01234567890 -- 0 1 ast = { @@ -3365,7 +3428,7 @@ describe('api', function() hl('IdentifierName', 'c', 1), hl('Dict', '}'), }) - check_parsing('{', 0, { + check_parsing('{', { -- 0 ast = { 'UnknownFigure:0:0:{', @@ -3377,7 +3440,7 @@ describe('api', function() }, { hl('FigureBrace', '{'), }) - check_parsing('{a', 0, { + check_parsing('{a', { -- 01 ast = { { @@ -3395,7 +3458,7 @@ describe('api', function() hl('FigureBrace', '{'), hl('IdentifierName', 'a'), }) - check_parsing('{a,b', 0, { + check_parsing('{a,b', { -- 0123 ast = { { @@ -3421,7 +3484,7 @@ describe('api', function() hl('Comma', ','), hl('IdentifierName', 'b'), }) - check_parsing('{a,b->', 0, { + check_parsing('{a,b->', { -- 012345 ast = { { @@ -3449,7 +3512,7 @@ describe('api', function() hl('IdentifierName', 'b'), hl('Arrow', '->'), }) - check_parsing('{a,b->c', 0, { + check_parsing('{a,b->c', { -- 0123456 ast = { { @@ -3483,7 +3546,7 @@ describe('api', function() hl('Arrow', '->'), hl('IdentifierName', 'c'), }) - check_parsing('{a : b', 0, { + check_parsing('{a : b', { -- 012345 ast = { { @@ -3509,7 +3572,7 @@ describe('api', function() hl('Colon', ':', 1), hl('IdentifierName', 'b', 1), }) - check_parsing('{a : b,', 0, { + check_parsing('{a : b,', { -- 0123456 ast = { { @@ -3543,7 +3606,7 @@ describe('api', function() }) end) it('works with ternary operator', function() - check_parsing('a ? b : c', 0, { + check_parsing('a ? b : c', { -- 012345678 ast = { { @@ -3567,7 +3630,7 @@ describe('api', function() hl('TernaryColon', ':', 1), hl('IdentifierName', 'c', 1), }) - check_parsing('@a?@b?@c:@d:@e', 0, { + check_parsing('@a?@b?@c:@d:@e', { -- 01234567890123 -- 0 1 ast = { @@ -3608,7 +3671,7 @@ describe('api', function() hl('TernaryColon', ':'), hl('Register', '@e'), }) - check_parsing('@a?@b:@c?@d:@e', 0, { + check_parsing('@a?@b:@c?@d:@e', { -- 01234567890123 -- 0 1 ast = { @@ -3649,7 +3712,7 @@ describe('api', function() hl('TernaryColon', ':'), hl('Register', '@e'), }) - check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', 0, { + check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', { -- 01234567890123456789012345678901 -- 0 1 2 3 ast = { @@ -3738,7 +3801,7 @@ describe('api', function() hl('TernaryColon', ':'), hl('Register', '@k'), }) - check_parsing('?', 0, { + check_parsing('?', { -- 0 ast = { { @@ -3757,7 +3820,7 @@ describe('api', function() hl('InvalidTernary', '?'), }) - check_parsing('?:', 0, { + check_parsing('?:', { -- 01 ast = { { @@ -3782,7 +3845,7 @@ describe('api', function() hl('InvalidTernaryColon', ':'), }) - check_parsing('?::', 0, { + check_parsing('?::', { -- 012 ast = { { @@ -3814,7 +3877,7 @@ describe('api', function() hl('InvalidColon', ':'), }) - check_parsing('a?b', 0, { + check_parsing('a?b', { -- 012 ast = { { @@ -3839,7 +3902,7 @@ describe('api', function() hl('Ternary', '?'), hl('IdentifierName', 'b'), }) - check_parsing('a?b:', 0, { + check_parsing('a?b:', { -- 0123 ast = { { @@ -3866,7 +3929,7 @@ describe('api', function() hl('IdentifierScopeDelimiter', ':'), }) - check_parsing('a?b::c', 0, { + check_parsing('a?b::c', { -- 012345 ast = { { @@ -3892,7 +3955,7 @@ describe('api', function() hl('IdentifierName', 'c'), }) - check_parsing('a?b :', 0, { + check_parsing('a?b :', { -- 01234 ast = { { @@ -3919,7 +3982,7 @@ describe('api', function() hl('TernaryColon', ':', 1), }) - check_parsing('(@a?@b:@c)?@d:@e', 0, { + check_parsing('(@a?@b:@c)?@d:@e', { -- 0123456789012345 -- 0 1 ast = { @@ -3968,7 +4031,7 @@ describe('api', function() hl('Register', '@e'), }) - check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', 0, { + check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', { -- 01234567890123456789012345678901 -- 0 1 2 3 ast = { @@ -4063,7 +4126,7 @@ describe('api', function() hl('NestingParenthesis', ')'), }) - check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', 0, { + check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', { -- 0123456789012345678901234567 -- 0 1 2 ast = { @@ -4143,7 +4206,7 @@ describe('api', function() hl('TernaryColon', ':'), hl('Register', '@i'), }) - check_parsing('a?b{cdef}g:h', 0, { + check_parsing('a?b{cdef}g:h', { -- 012345678901 -- 0 1 ast = { @@ -4189,7 +4252,7 @@ describe('api', function() hl('TernaryColon', ':'), hl('IdentifierName', 'h'), }) - check_parsing('a ? b : c : d', 0, { + check_parsing('a ? b : c : d', { -- 0123456789012 -- 0 1 ast = { @@ -4228,7 +4291,7 @@ describe('api', function() }) end) it('works with comparison operators', function() - check_parsing('a == b', 0, { + check_parsing('a == b', { -- 012345 ast = { { @@ -4245,7 +4308,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a ==? b', 0, { + check_parsing('a ==? b', { -- 0123456 ast = { { @@ -4263,7 +4326,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a ==# b', 0, { + check_parsing('a ==# b', { -- 0123456 ast = { { @@ -4281,7 +4344,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a !=# b', 0, { + check_parsing('a !=# b', { -- 0123456 ast = { { @@ -4299,7 +4362,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a <=# b', 0, { + check_parsing('a <=# b', { -- 0123456 ast = { { @@ -4317,7 +4380,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a >=# b', 0, { + check_parsing('a >=# b', { -- 0123456 ast = { { @@ -4335,7 +4398,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a ># b', 0, { + check_parsing('a ># b', { -- 012345 ast = { { @@ -4353,7 +4416,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a <# b', 0, { + check_parsing('a <# b', { -- 012345 ast = { { @@ -4371,7 +4434,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a is#b', 0, { + check_parsing('a is#b', { -- 012345 ast = { { @@ -4389,7 +4452,7 @@ describe('api', function() hl('IdentifierName', 'b'), }) - check_parsing('a is?b', 0, { + check_parsing('a is?b', { -- 012345 ast = { { @@ -4407,7 +4470,7 @@ describe('api', function() hl('IdentifierName', 'b'), }) - check_parsing('a isnot b', 0, { + check_parsing('a isnot b', { -- 012345678 ast = { { @@ -4424,7 +4487,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a < b < c', 0, { + check_parsing('a < b < c', { -- 012345678 ast = { { @@ -4453,7 +4516,7 @@ describe('api', function() hl('IdentifierName', 'c', 1), }) - check_parsing('a < b <# c', 0, { + check_parsing('a < b <# c', { -- 012345678 ast = { { @@ -4483,7 +4546,7 @@ describe('api', function() hl('IdentifierName', 'c', 1), }) - check_parsing('a += b', 0, { + check_parsing('a += b', { -- 012345 ast = { { @@ -4510,7 +4573,7 @@ describe('api', function() hl('InvalidComparison', '='), hl('IdentifierName', 'b', 1), }) - check_parsing('a + b == c + d', 0, { + check_parsing('a + b == c + d', { -- 01234567890123 -- 0 1 ast = { @@ -4543,7 +4606,7 @@ describe('api', function() hl('BinaryPlus', '+', 1), hl('IdentifierName', 'd', 1), }) - check_parsing('+ a == + b', 0, { + check_parsing('+ a == + b', { -- 0123456789 ast = { { @@ -4573,7 +4636,7 @@ describe('api', function() }) end) it('works with concat/subscript', function() - check_parsing('.', 0, { + check_parsing('.', { -- 0 ast = { { @@ -4591,7 +4654,7 @@ describe('api', function() hl('InvalidConcatOrSubscript', '.'), }) - check_parsing('a.', 0, { + check_parsing('a.', { -- 01 ast = { { @@ -4610,7 +4673,7 @@ describe('api', function() hl('ConcatOrSubscript', '.'), }) - check_parsing('a.b', 0, { + check_parsing('a.b', { -- 012 ast = { { @@ -4627,7 +4690,7 @@ describe('api', function() hl('IdentifierKey', 'b'), }) - check_parsing('1.2', 0, { + check_parsing('1.2', { -- 012 ast = { 'Float(val=1.200000e+00):0:0:1.2', @@ -4636,7 +4699,7 @@ describe('api', function() hl('Float', '1.2'), }) - check_parsing('1.2 + 1.3e-5', 0, { + check_parsing('1.2 + 1.3e-5', { -- 012345678901 -- 0 1 ast = { @@ -4654,7 +4717,7 @@ describe('api', function() hl('Float', '1.3e-5', 1), }) - check_parsing('a . 1.2 + 1.3e-5', 0, { + check_parsing('a . 1.2 + 1.3e-5', { -- 0123456789012345 -- 0 1 ast = { @@ -4688,7 +4751,7 @@ describe('api', function() hl('Float', '1.3e-5', 1), }) - check_parsing('1.3e-5 + 1.2 . a', 0, { + check_parsing('1.3e-5 + 1.2 . a', { -- 0123456789012345 -- 0 1 ast = { @@ -4714,7 +4777,7 @@ describe('api', function() hl('IdentifierName', 'a', 1), }) - check_parsing('1.3e-5 + a . 1.2', 0, { + check_parsing('1.3e-5 + a . 1.2', { -- 0123456789012345 -- 0 1 ast = { @@ -4748,7 +4811,7 @@ describe('api', function() hl('IdentifierKey', '2'), }) - check_parsing('1.2.3', 0, { + check_parsing('1.2.3', { -- 01234 ast = { { @@ -4773,7 +4836,7 @@ describe('api', function() hl('IdentifierKey', '3'), }) - check_parsing('a.1.2', 0, { + check_parsing('a.1.2', { -- 01234 ast = { { @@ -4798,7 +4861,7 @@ describe('api', function() hl('IdentifierKey', '2'), }) - check_parsing('a . 1.2', 0, { + check_parsing('a . 1.2', { -- 0123456 ast = { { @@ -4823,7 +4886,7 @@ describe('api', function() hl('IdentifierKey', '2'), }) - check_parsing('+a . +b', 0, { + check_parsing('+a . +b', { -- 0123456 ast = { { @@ -4852,7 +4915,7 @@ describe('api', function() hl('IdentifierName', 'b'), }) - check_parsing('a. b', 0, { + check_parsing('a. b', { -- 0123 ast = { { @@ -4869,7 +4932,7 @@ describe('api', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a. 1', 0, { + check_parsing('a. 1', { -- 0123 ast = { { @@ -4887,7 +4950,7 @@ describe('api', function() }) end) it('works with bracket subscripts', function() - check_parsing(':', 0, { + check_parsing(':', { -- 0 ast = { { @@ -4904,7 +4967,7 @@ describe('api', function() }, { hl('InvalidColon', ':'), }) - check_parsing('a[]', 0, { + check_parsing('a[]', { -- 012 ast = { { @@ -4923,7 +4986,7 @@ describe('api', function() hl('SubscriptBracket', '['), hl('InvalidSubscriptBracket', ']'), }) - check_parsing('a[b:]', 0, { + check_parsing('a[b:]', { -- 01234 ast = { { @@ -4942,7 +5005,7 @@ describe('api', function() hl('SubscriptBracket', ']'), }) - check_parsing('a[b:c]', 0, { + check_parsing('a[b:c]', { -- 012345 ast = { { @@ -4961,7 +5024,7 @@ describe('api', function() hl('IdentifierName', 'c'), hl('SubscriptBracket', ']'), }) - check_parsing('a[b : c]', 0, { + check_parsing('a[b : c]', { -- 01234567 ast = { { @@ -4987,7 +5050,7 @@ describe('api', function() hl('SubscriptBracket', ']'), }) - check_parsing('a[: b]', 0, { + check_parsing('a[: b]', { -- 012345 ast = { { @@ -5012,7 +5075,7 @@ describe('api', function() hl('SubscriptBracket', ']'), }) - check_parsing('a[b :]', 0, { + check_parsing('a[b :]', { -- 012345 ast = { { @@ -5035,7 +5098,7 @@ describe('api', function() hl('SubscriptColon', ':', 1), hl('SubscriptBracket', ']'), }) - check_parsing('a[b][c][d](e)(f)(g)', 0, { + check_parsing('a[b][c][d](e)(f)(g)', { -- 0123456789012345678 -- 0 1 ast = { @@ -5098,7 +5161,7 @@ describe('api', function() hl('IdentifierName', 'g'), hl('CallingParenthesis', ')'), }) - check_parsing('{a}{b}{c}[d][e][f]', 0, { + check_parsing('{a}{b}{c}[d][e][f]', { -- 012345678901234567 -- 0 1 ast = { @@ -5171,7 +5234,7 @@ describe('api', function() }) end) it('supports list literals', function() - check_parsing('[]', 0, { + check_parsing('[]', { -- 01 ast = { 'ListLiteral:0:0:[', @@ -5181,7 +5244,7 @@ describe('api', function() hl('List', ']'), }) - check_parsing('[a]', 0, { + check_parsing('[a]', { -- 012 ast = { { @@ -5197,7 +5260,7 @@ describe('api', function() hl('List', ']'), }) - check_parsing('[a, b]', 0, { + check_parsing('[a, b]', { -- 012345 ast = { { @@ -5221,7 +5284,7 @@ describe('api', function() hl('List', ']'), }) - check_parsing('[a, b, c]', 0, { + check_parsing('[a, b, c]', { -- 012345678 ast = { { @@ -5253,7 +5316,7 @@ describe('api', function() hl('List', ']'), }) - check_parsing('[a, b, c, ]', 0, { + check_parsing('[a, b, c, ]', { -- 01234567890 -- 0 1 ast = { @@ -5292,7 +5355,7 @@ describe('api', function() hl('List', ']', 1), }) - check_parsing('[a : b, c : d]', 0, { + check_parsing('[a : b, c : d]', { -- 01234567890123 -- 0 1 ast = { @@ -5337,7 +5400,7 @@ describe('api', function() hl('List', ']'), }) - check_parsing(']', 0, { + check_parsing(']', { -- 0 ast = { 'ListLiteral:0:0:', @@ -5350,7 +5413,7 @@ describe('api', function() hl('InvalidList', ']'), }) - check_parsing('a]', 0, { + check_parsing('a]', { -- 01 ast = { { @@ -5369,7 +5432,7 @@ describe('api', function() hl('InvalidList', ']'), }) - check_parsing('[] []', 0, { + check_parsing('[] []', { -- 01234 ast = { { @@ -5390,9 +5453,23 @@ describe('api', function() hl('InvalidSpacing', ' '), hl('List', '['), hl('List', ']'), + }, { + [1] = { + ast = { + err = REMOVE_THIS, + ast = { + 'ListLiteral:0:0:[', + }, + }, + hl_fs = { + [3] = REMOVE_THIS, + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + }, + }, }) - check_parsing('[][]', 0, { + check_parsing('[][]', { -- 0123 ast = { { @@ -5413,7 +5490,7 @@ describe('api', function() hl('InvalidSubscriptBracket', ']'), }) - check_parsing('[', 0, { + check_parsing('[', { -- 0 ast = { 'ListLiteral:0:0:[', @@ -5426,7 +5503,7 @@ describe('api', function() hl('List', '['), }) - check_parsing('[1', 0, { + check_parsing('[1', { -- 01 ast = { { @@ -5446,7 +5523,7 @@ describe('api', function() }) end) it('works with strings', function() - check_parsing('\'abc\'', 0, { + check_parsing('\'abc\'', { -- 01234 ast = { 'SingleQuotedString(val="abc"):0:0:\'abc\'', @@ -5456,7 +5533,7 @@ describe('api', function() hl('SingleQuotedBody', 'abc'), hl('SingleQuote', '\''), }) - check_parsing('"abc"', 0, { + check_parsing('"abc"', { -- 01234 ast = { 'DoubleQuotedString(val="abc"):0:0:"abc"', @@ -5466,7 +5543,7 @@ describe('api', function() hl('DoubleQuotedBody', 'abc'), hl('DoubleQuote', '"'), }) - check_parsing('\'\'', 0, { + check_parsing('\'\'', { -- 01 ast = { 'SingleQuotedString(val=""):0:0:\'\'', @@ -5475,7 +5552,7 @@ describe('api', function() hl('SingleQuote', '\''), hl('SingleQuote', '\''), }) - check_parsing('""', 0, { + check_parsing('""', { -- 01 ast = { 'DoubleQuotedString(val=""):0:0:""', @@ -5484,7 +5561,7 @@ describe('api', function() hl('DoubleQuote', '"'), hl('DoubleQuote', '"'), }) - check_parsing('"', 0, { + check_parsing('"', { -- 0 ast = { 'DoubleQuotedString(val=""):0:0:"', @@ -5496,7 +5573,7 @@ describe('api', function() }, { hl('InvalidDoubleQuote', '"'), }) - check_parsing('\'', 0, { + check_parsing('\'', { -- 0 ast = { 'SingleQuotedString(val=""):0:0:\'', @@ -5508,7 +5585,7 @@ describe('api', function() }, { hl('InvalidSingleQuote', '\''), }) - check_parsing('"a', 0, { + check_parsing('"a', { -- 01 ast = { 'DoubleQuotedString(val="a"):0:0:"a', @@ -5521,7 +5598,7 @@ describe('api', function() hl('InvalidDoubleQuote', '"'), hl('InvalidDoubleQuotedBody', 'a'), }) - check_parsing('\'a', 0, { + check_parsing('\'a', { -- 01 ast = { 'SingleQuotedString(val="a"):0:0:\'a', @@ -5534,7 +5611,7 @@ describe('api', function() hl('InvalidSingleQuote', '\''), hl('InvalidSingleQuotedBody', 'a'), }) - check_parsing('\'abc\'\'def\'', 0, { + check_parsing('\'abc\'\'def\'', { -- 0123456789 ast = { 'SingleQuotedString(val="abc\'def"):0:0:\'abc\'\'def\'', @@ -5546,7 +5623,7 @@ describe('api', function() hl('SingleQuotedBody', 'def'), hl('SingleQuote', '\''), }) - check_parsing('\'abc\'\'', 0, { + check_parsing('\'abc\'\'', { -- 012345 ast = { 'SingleQuotedString(val="abc\'"):0:0:\'abc\'\'', @@ -5560,7 +5637,7 @@ describe('api', function() hl('InvalidSingleQuotedBody', 'abc'), hl('InvalidSingleQuotedQuote', '\'\''), }) - check_parsing('\'\'\'\'\'\'\'\'', 0, { + check_parsing('\'\'\'\'\'\'\'\'', { -- 01234567 ast = { 'SingleQuotedString(val="\'\'\'"):0:0:\'\'\'\'\'\'\'\'', @@ -5572,7 +5649,7 @@ describe('api', function() hl('SingleQuotedQuote', '\'\''), hl('SingleQuote', '\''), }) - check_parsing('\'\'\'a\'\'\'\'bc\'', 0, { + check_parsing('\'\'\'a\'\'\'\'bc\'', { -- 01234567890 -- 0 1 ast = { @@ -5587,7 +5664,7 @@ describe('api', function() hl('SingleQuotedBody', 'bc'), hl('SingleQuote', '\''), }) - check_parsing('"\\"\\"\\"\\""', 0, { + check_parsing('"\\"\\"\\"\\""', { -- 0123456789 ast = { 'DoubleQuotedString(val="\\"\\"\\"\\""):0:0:"\\"\\"\\"\\""', @@ -5600,7 +5677,7 @@ describe('api', function() hl('DoubleQuotedEscape', '\\"'), hl('DoubleQuote', '"'), }) - check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', 0, { + check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', { -- 0123456789012345678901234 -- 0 1 2 ast = { @@ -5619,7 +5696,7 @@ describe('api', function() hl('DoubleQuotedBody', 'mno'), hl('DoubleQuote', '"'), }) - check_parsing('"\\b\\e\\f\\r\\t\\\\"', 0, { + check_parsing('"\\b\\e\\f\\r\\t\\\\"', { -- 0123456789012345 -- 0 1 ast = { @@ -5635,7 +5712,7 @@ describe('api', function() hl('DoubleQuotedEscape', '\\\\'), hl('DoubleQuote', '"'), }) - check_parsing('"\\n\n"', 0, { + check_parsing('"\\n\n"', { -- 01234 ast = { 'DoubleQuotedString(val="\\\n\\\n"):0:0:"\\n\n"', @@ -5646,7 +5723,7 @@ describe('api', function() hl('DoubleQuotedBody', '\n'), hl('DoubleQuote', '"'), }) - check_parsing('"\\x00"', 0, { + check_parsing('"\\x00"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0"):0:0:"\\x00"', @@ -5656,7 +5733,7 @@ describe('api', function() hl('DoubleQuotedEscape', '\\x00'), hl('DoubleQuote', '"'), }) - check_parsing('"\\xFF"', 0, { + check_parsing('"\\xFF"', { -- 012345 ast = { 'DoubleQuotedString(val="\255"):0:0:"\\xFF"', @@ -5666,7 +5743,7 @@ describe('api', function() hl('DoubleQuotedEscape', '\\xFF'), hl('DoubleQuote', '"'), }) - check_parsing('"\\xF"', 0, { + check_parsing('"\\xF"', { -- 012345 ast = { 'DoubleQuotedString(val="\\15"):0:0:"\\xF"', @@ -5676,7 +5753,7 @@ describe('api', function() hl('DoubleQuotedEscape', '\\xF'), hl('DoubleQuote', '"'), }) - check_parsing('"\\u00AB"', 0, { + check_parsing('"\\u00AB"', { -- 01234567 ast = { 'DoubleQuotedString(val="«"):0:0:"\\u00AB"', @@ -5686,7 +5763,7 @@ describe('api', function() hl('DoubleQuotedEscape', '\\u00AB'), hl('DoubleQuote', '"'), }) - check_parsing('"\\U000000AB"', 0, { + check_parsing('"\\U000000AB"', { -- 01234567 ast = { 'DoubleQuotedString(val="«"):0:0:"\\U000000AB"', @@ -5696,7 +5773,7 @@ describe('api', function() hl('DoubleQuotedEscape', '\\U000000AB'), hl('DoubleQuote', '"'), }) - check_parsing('"\\x"', 0, { + check_parsing('"\\x"', { -- 0123 ast = { 'DoubleQuotedString(val="x"):0:0:"\\x"', @@ -5707,7 +5784,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\x', 0, { + check_parsing('"\\x', { -- 012 ast = { 'DoubleQuotedString(val="x"):0:0:"\\x', @@ -5721,7 +5798,7 @@ describe('api', function() hl('InvalidDoubleQuotedUnknownEscape', '\\x'), }) - check_parsing('"\\xF', 0, { + check_parsing('"\\xF', { -- 0123 ast = { 'DoubleQuotedString(val="\\15"):0:0:"\\xF', @@ -5735,7 +5812,7 @@ describe('api', function() hl('InvalidDoubleQuotedEscape', '\\xF'), }) - check_parsing('"\\u"', 0, { + check_parsing('"\\u"', { -- 0123 ast = { 'DoubleQuotedString(val="u"):0:0:"\\u"', @@ -5746,7 +5823,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u', 0, { + check_parsing('"\\u', { -- 012 ast = { 'DoubleQuotedString(val="u"):0:0:"\\u', @@ -5760,7 +5837,7 @@ describe('api', function() hl('InvalidDoubleQuotedUnknownEscape', '\\u'), }) - check_parsing('"\\U', 0, { + check_parsing('"\\U', { -- 012 ast = { 'DoubleQuotedString(val="U"):0:0:"\\U', @@ -5774,7 +5851,7 @@ describe('api', function() hl('InvalidDoubleQuotedUnknownEscape', '\\U'), }) - check_parsing('"\\U"', 0, { + check_parsing('"\\U"', { -- 0123 ast = { 'DoubleQuotedString(val="U"):0:0:"\\U"', @@ -5785,7 +5862,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\xFX"', 0, { + check_parsing('"\\xFX"', { -- 012345 ast = { 'DoubleQuotedString(val="\\15X"):0:0:"\\xFX"', @@ -5797,7 +5874,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\XFX"', 0, { + check_parsing('"\\XFX"', { -- 012345 ast = { 'DoubleQuotedString(val="\\15X"):0:0:"\\XFX"', @@ -5809,7 +5886,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\xX"', 0, { + check_parsing('"\\xX"', { -- 01234 ast = { 'DoubleQuotedString(val="xX"):0:0:"\\xX"', @@ -5821,7 +5898,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\XX"', 0, { + check_parsing('"\\XX"', { -- 01234 ast = { 'DoubleQuotedString(val="XX"):0:0:"\\XX"', @@ -5833,7 +5910,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\uX"', 0, { + check_parsing('"\\uX"', { -- 01234 ast = { 'DoubleQuotedString(val="uX"):0:0:"\\uX"', @@ -5845,7 +5922,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\UX"', 0, { + check_parsing('"\\UX"', { -- 01234 ast = { 'DoubleQuotedString(val="UX"):0:0:"\\UX"', @@ -5857,7 +5934,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\x0X"', 0, { + check_parsing('"\\x0X"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\x0X"', @@ -5869,7 +5946,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\X0X"', 0, { + check_parsing('"\\X0X"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\X0X"', @@ -5881,7 +5958,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u0X"', 0, { + check_parsing('"\\u0X"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\u0X"', @@ -5893,7 +5970,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U0X"', 0, { + check_parsing('"\\U0X"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U0X"', @@ -5905,7 +5982,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\x00X"', 0, { + check_parsing('"\\x00X"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\x00X"', @@ -5917,7 +5994,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\X00X"', 0, { + check_parsing('"\\X00X"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\X00X"', @@ -5929,7 +6006,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u00X"', 0, { + check_parsing('"\\u00X"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\u00X"', @@ -5941,7 +6018,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U00X"', 0, { + check_parsing('"\\U00X"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U00X"', @@ -5953,7 +6030,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u000X"', 0, { + check_parsing('"\\u000X"', { -- 01234567 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\u000X"', @@ -5965,7 +6042,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U000X"', 0, { + check_parsing('"\\U000X"', { -- 01234567 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U000X"', @@ -5977,7 +6054,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u0000X"', 0, { + check_parsing('"\\u0000X"', { -- 012345678 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\u0000X"', @@ -5989,7 +6066,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U0000X"', 0, { + check_parsing('"\\U0000X"', { -- 012345678 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000X"', @@ -6001,7 +6078,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U00000X"', 0, { + check_parsing('"\\U00000X"', { -- 0123456789 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000X"', @@ -6013,7 +6090,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U000000X"', 0, { + check_parsing('"\\U000000X"', { -- 01234567890 -- 0 1 ast = { @@ -6026,7 +6103,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U0000000X"', 0, { + check_parsing('"\\U0000000X"', { -- 012345678901 -- 0 1 ast = { @@ -6039,7 +6116,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U00000000X"', 0, { + check_parsing('"\\U00000000X"', { -- 0123456789012 -- 0 1 ast = { @@ -6052,7 +6129,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\x000X"', 0, { + check_parsing('"\\x000X"', { -- 01234567 ast = { 'DoubleQuotedString(val="\\0000X"):0:0:"\\x000X"', @@ -6064,7 +6141,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\X000X"', 0, { + check_parsing('"\\X000X"', { -- 01234567 ast = { 'DoubleQuotedString(val="\\0000X"):0:0:"\\X000X"', @@ -6076,7 +6153,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u00000X"', 0, { + check_parsing('"\\u00000X"', { -- 0123456789 ast = { 'DoubleQuotedString(val="\\0000X"):0:0:"\\u00000X"', @@ -6088,7 +6165,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U000000000X"', 0, { + check_parsing('"\\U000000000X"', { -- 01234567890123 -- 0 1 ast = { @@ -6101,7 +6178,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\0"', 0, { + check_parsing('"\\0"', { -- 0123 ast = { 'DoubleQuotedString(val="\\0"):0:0:"\\0"', @@ -6112,7 +6189,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\00"', 0, { + check_parsing('"\\00"', { -- 01234 ast = { 'DoubleQuotedString(val="\\0"):0:0:"\\00"', @@ -6123,7 +6200,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\000"', 0, { + check_parsing('"\\000"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0"):0:0:"\\000"', @@ -6134,7 +6211,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\0000"', 0, { + check_parsing('"\\0000"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0000"):0:0:"\\0000"', @@ -6146,7 +6223,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\8"', 0, { + check_parsing('"\\8"', { -- 0123 ast = { 'DoubleQuotedString(val="8"):0:0:"\\8"', @@ -6157,7 +6234,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\08"', 0, { + check_parsing('"\\08"', { -- 01234 ast = { 'DoubleQuotedString(val="\\0008"):0:0:"\\08"', @@ -6169,7 +6246,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\008"', 0, { + check_parsing('"\\008"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0008"):0:0:"\\008"', @@ -6181,7 +6258,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\0008"', 0, { + check_parsing('"\\0008"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0008"):0:0:"\\0008"', @@ -6193,7 +6270,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\777"', 0, { + check_parsing('"\\777"', { -- 012345 ast = { 'DoubleQuotedString(val="\255"):0:0:"\\777"', @@ -6204,7 +6281,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\050"', 0, { + check_parsing('"\\050"', { -- 012345 ast = { 'DoubleQuotedString(val="\40"):0:0:"\\050"', @@ -6215,7 +6292,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\"', 0, { + check_parsing('"\\"', { -- 012345 ast = { 'DoubleQuotedString(val="\\21"):0:0:"\\"', @@ -6226,7 +6303,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\<', 0, { + check_parsing('"\\<', { -- 012 ast = { 'DoubleQuotedString(val="<"):0:0:"\\<', @@ -6240,7 +6317,7 @@ describe('api', function() hl('InvalidDoubleQuotedUnknownEscape', '\\<'), }) - check_parsing('"\\<"', 0, { + check_parsing('"\\<"', { -- 0123 ast = { 'DoubleQuotedString(val="<"):0:0:"\\<"', @@ -6251,7 +6328,7 @@ describe('api', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\@a}', 0, { + check_parsing('{->@a}', { ast = { { 'Lambda(\\di):0:0:{', @@ -1512,7 +1567,7 @@ describe('Expressions parser', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{->@a+@b}', 0, { + check_parsing('{->@a+@b}', { -- 012345678 ast = { { @@ -1541,7 +1596,7 @@ describe('Expressions parser', function() hl('Register', '@b'), hl('Lambda', '}'), }) - check_parsing('{a->@a}', 0, { + check_parsing('{a->@a}', { -- 012345678 ast = { { @@ -1564,7 +1619,7 @@ describe('Expressions parser', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b->@a}', 0, { + check_parsing('{a,b->@a}', { -- 012345678 ast = { { @@ -1595,7 +1650,7 @@ describe('Expressions parser', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b,c->@a}', 0, { + check_parsing('{a,b,c->@a}', { -- 01234567890 ast = { { @@ -1634,7 +1689,7 @@ describe('Expressions parser', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b,c,d->@a}', 0, { + check_parsing('{a,b,c,d->@a}', { -- 0123456789012 ast = { { @@ -1681,7 +1736,7 @@ describe('Expressions parser', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b,c,d,->@a}', 0, { + check_parsing('{a,b,c,d,->@a}', { -- 01234567890123 ast = { { @@ -1734,7 +1789,7 @@ describe('Expressions parser', function() hl('Register', '@a'), hl('Lambda', '}'), }) - check_parsing('{a,b->{c,d->{e,f->@a}}}', 0, { + check_parsing('{a,b->{c,d->{e,f->@a}}}', { -- 01234567890123456789012 -- 0 1 2 ast = { @@ -1812,7 +1867,7 @@ describe('Expressions parser', function() hl('Lambda', '}'), hl('Lambda', '}'), }) - check_parsing('{a,b->c,d}', 0, { + check_parsing('{a,b->c,d}', { -- 0123456789 ast = { { @@ -1855,7 +1910,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'd'), hl('Lambda', '}'), }) - check_parsing('a,b,c,d', 0, { + check_parsing('a,b,c,d', { -- 0123456789 ast = { { @@ -1891,7 +1946,7 @@ describe('Expressions parser', function() hl('InvalidComma', ','), hl('IdentifierName', 'd'), }) - check_parsing('a,b,c,d,', 0, { + check_parsing('a,b,c,d,', { -- 0123456789 ast = { { @@ -1933,7 +1988,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'd'), hl('InvalidComma', ','), }) - check_parsing(',', 0, { + check_parsing(',', { -- 0123456789 ast = { { @@ -1950,7 +2005,7 @@ describe('Expressions parser', function() }, { hl('InvalidComma', ','), }) - check_parsing('{,a->@a}', 0, { + check_parsing('{,a->@a}', { -- 0123456789 ast = { { @@ -1984,7 +2039,7 @@ describe('Expressions parser', function() hl('Register', '@a'), hl('Curly', '}'), }) - check_parsing('}', 0, { + check_parsing('}', { -- 0123456789 ast = { 'UnknownFigure(---):0:0:', @@ -1996,7 +2051,7 @@ describe('Expressions parser', function() }, { hl('InvalidFigureBrace', '}'), }) - check_parsing('{->}', 0, { + check_parsing('{->}', { -- 0123456789 ast = { { @@ -2015,7 +2070,7 @@ describe('Expressions parser', function() hl('Arrow', '->'), hl('InvalidLambda', '}'), }) - check_parsing('{a,b}', 0, { + check_parsing('{a,b}', { -- 0123456789 ast = { { @@ -2042,7 +2097,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b'), hl('InvalidLambda', '}'), }) - check_parsing('{a,}', 0, { + check_parsing('{a,}', { -- 0123456789 ast = { { @@ -2067,7 +2122,7 @@ describe('Expressions parser', function() hl('Comma', ','), hl('InvalidLambda', '}'), }) - check_parsing('{@a:@b}', 0, { + check_parsing('{@a:@b}', { -- 0123456789 ast = { { @@ -2090,7 +2145,7 @@ describe('Expressions parser', function() hl('Register', '@b'), hl('Dict', '}'), }) - check_parsing('{@a:@b,@c:@d}', 0, { + check_parsing('{@a:@b,@c:@d}', { -- 0123456789012 -- 0 1 ast = { @@ -2130,7 +2185,7 @@ describe('Expressions parser', function() hl('Register', '@d'), hl('Dict', '}'), }) - check_parsing('{@a:@b,@c:@d,@e:@f,}', 0, { + check_parsing('{@a:@b,@c:@d,@e:@f,}', { -- 01234567890123456789 -- 0 1 ast = { @@ -2192,7 +2247,7 @@ describe('Expressions parser', function() hl('Comma', ','), hl('Dict', '}'), }) - check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', 0, { + check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', { -- 01234567890123456789012 -- 0 1 2 ast = { @@ -2266,7 +2321,7 @@ describe('Expressions parser', function() hl('Colon', ':'), hl('InvalidDict', '}'), }) - check_parsing('{@a:@b,}', 0, { + check_parsing('{@a:@b,}', { -- 01234567890123 -- 0 1 ast = { @@ -2296,7 +2351,7 @@ describe('Expressions parser', function() hl('Comma', ','), hl('Dict', '}'), }) - check_parsing('{({f -> g})(@h)(@i)}', 0, { + check_parsing('{({f -> g})(@h)(@i)}', { -- 01234567890123456789 -- 0 1 ast = { @@ -2352,7 +2407,7 @@ describe('Expressions parser', function() hl('CallingParenthesis', ')'), hl('Curly', '}'), }) - check_parsing('a:{b()}c', 0, { + check_parsing('a:{b()}c', { -- 01234567 ast = { { @@ -2389,7 +2444,7 @@ describe('Expressions parser', function() hl('Curly', '}'), hl('IdentifierName', 'c'), }) - check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', 0, { + check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', { -- 01234567890123456789012345678901234567890123456 -- 0 1 2 3 4 ast = { @@ -2499,7 +2554,7 @@ describe('Expressions parser', function() hl('Curly', '}'), hl('IdentifierName', 'j'), }) - check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', 0, { + check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', { -- 01234567890123456789012345678901234567 -- 0 1 2 3 ast = { @@ -2571,7 +2626,7 @@ describe('Expressions parser', function() hl('Register', '@i', 1), hl('Dict', '}'), }) - check_parsing('-> -> ->', 0, { + check_parsing('-> -> ->', { -- 01234567 ast = { { @@ -2602,7 +2657,7 @@ describe('Expressions parser', function() hl('InvalidArrow', '->', 1), hl('InvalidArrow', '->', 1), }) - check_parsing('a -> b -> c -> d', 0, { + check_parsing('a -> b -> c -> d', { -- 0123456789012345 -- 0 1 ast = { @@ -2639,7 +2694,7 @@ describe('Expressions parser', function() hl('InvalidArrow', '->', 1), hl('IdentifierName', 'd', 1), }) - check_parsing('{a -> b -> c}', 0, { + check_parsing('{a -> b -> c}', { -- 0123456789012 -- 0 1 ast = { @@ -2675,7 +2730,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'c', 1), hl('Lambda', '}'), }) - check_parsing('{a: -> b}', 0, { + check_parsing('{a: -> b}', { -- 012345678 ast = { { @@ -2704,7 +2759,7 @@ describe('Expressions parser', function() hl('Curly', '}'), }) - check_parsing('{a:b -> b}', 0, { + check_parsing('{a:b -> b}', { -- 0123456789 ast = { { @@ -2734,7 +2789,7 @@ describe('Expressions parser', function() hl('Curly', '}'), }) - check_parsing('{a#b -> b}', 0, { + check_parsing('{a#b -> b}', { -- 0123456789 ast = { { @@ -2761,7 +2816,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), hl('Curly', '}'), }) - check_parsing('{a : b : c}', 0, { + check_parsing('{a : b : c}', { -- 01234567890 -- 0 1 ast = { @@ -2797,7 +2852,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'c', 1), hl('Dict', '}'), }) - check_parsing('{', 0, { + check_parsing('{', { -- 0 ast = { 'UnknownFigure(\\di):0:0:{', @@ -2809,7 +2864,7 @@ describe('Expressions parser', function() }, { hl('FigureBrace', '{'), }) - check_parsing('{a', 0, { + check_parsing('{a', { -- 01 ast = { { @@ -2827,7 +2882,7 @@ describe('Expressions parser', function() hl('FigureBrace', '{'), hl('IdentifierName', 'a'), }) - check_parsing('{a,b', 0, { + check_parsing('{a,b', { -- 0123 ast = { { @@ -2853,7 +2908,7 @@ describe('Expressions parser', function() hl('Comma', ','), hl('IdentifierName', 'b'), }) - check_parsing('{a,b->', 0, { + check_parsing('{a,b->', { -- 012345 ast = { { @@ -2881,7 +2936,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b'), hl('Arrow', '->'), }) - check_parsing('{a,b->c', 0, { + check_parsing('{a,b->c', { -- 0123456 ast = { { @@ -2915,7 +2970,7 @@ describe('Expressions parser', function() hl('Arrow', '->'), hl('IdentifierName', 'c'), }) - check_parsing('{a : b', 0, { + check_parsing('{a : b', { -- 012345 ast = { { @@ -2941,7 +2996,7 @@ describe('Expressions parser', function() hl('Colon', ':', 1), hl('IdentifierName', 'b', 1), }) - check_parsing('{a : b,', 0, { + check_parsing('{a : b,', { -- 0123456 ast = { { @@ -2975,7 +3030,7 @@ describe('Expressions parser', function() }) end) itp('works with ternary operator', function() - check_parsing('a ? b : c', 0, { + check_parsing('a ? b : c', { -- 012345678 ast = { { @@ -2999,7 +3054,7 @@ describe('Expressions parser', function() hl('TernaryColon', ':', 1), hl('IdentifierName', 'c', 1), }) - check_parsing('@a?@b?@c:@d:@e', 0, { + check_parsing('@a?@b?@c:@d:@e', { -- 01234567890123 -- 0 1 ast = { @@ -3040,7 +3095,7 @@ describe('Expressions parser', function() hl('TernaryColon', ':'), hl('Register', '@e'), }) - check_parsing('@a?@b:@c?@d:@e', 0, { + check_parsing('@a?@b:@c?@d:@e', { -- 01234567890123 -- 0 1 ast = { @@ -3081,7 +3136,7 @@ describe('Expressions parser', function() hl('TernaryColon', ':'), hl('Register', '@e'), }) - check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', 0, { + check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', { -- 01234567890123456789012345678901 -- 0 1 2 3 ast = { @@ -3170,7 +3225,7 @@ describe('Expressions parser', function() hl('TernaryColon', ':'), hl('Register', '@k'), }) - check_parsing('?', 0, { + check_parsing('?', { -- 0 ast = { { @@ -3189,7 +3244,7 @@ describe('Expressions parser', function() hl('InvalidTernary', '?'), }) - check_parsing('?:', 0, { + check_parsing('?:', { -- 01 ast = { { @@ -3214,7 +3269,7 @@ describe('Expressions parser', function() hl('InvalidTernaryColon', ':'), }) - check_parsing('?::', 0, { + check_parsing('?::', { -- 012 ast = { { @@ -3246,7 +3301,7 @@ describe('Expressions parser', function() hl('InvalidColon', ':'), }) - check_parsing('a?b', 0, { + check_parsing('a?b', { -- 012 ast = { { @@ -3271,7 +3326,7 @@ describe('Expressions parser', function() hl('Ternary', '?'), hl('IdentifierName', 'b'), }) - check_parsing('a?b:', 0, { + check_parsing('a?b:', { -- 0123 ast = { { @@ -3298,7 +3353,7 @@ describe('Expressions parser', function() hl('IdentifierScopeDelimiter', ':'), }) - check_parsing('a?b::c', 0, { + check_parsing('a?b::c', { -- 012345 ast = { { @@ -3324,7 +3379,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'c'), }) - check_parsing('a?b :', 0, { + check_parsing('a?b :', { -- 01234 ast = { { @@ -3351,7 +3406,7 @@ describe('Expressions parser', function() hl('TernaryColon', ':', 1), }) - check_parsing('(@a?@b:@c)?@d:@e', 0, { + check_parsing('(@a?@b:@c)?@d:@e', { -- 0123456789012345 -- 0 1 ast = { @@ -3400,7 +3455,7 @@ describe('Expressions parser', function() hl('Register', '@e'), }) - check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', 0, { + check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', { -- 01234567890123456789012345678901 -- 0 1 2 3 ast = { @@ -3495,7 +3550,7 @@ describe('Expressions parser', function() hl('NestingParenthesis', ')'), }) - check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', 0, { + check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', { -- 0123456789012345678901234567 -- 0 1 2 ast = { @@ -3575,7 +3630,7 @@ describe('Expressions parser', function() hl('TernaryColon', ':'), hl('Register', '@i'), }) - check_parsing('a?b{cdef}g:h', 0, { + check_parsing('a?b{cdef}g:h', { -- 012345678901 -- 0 1 ast = { @@ -3621,7 +3676,7 @@ describe('Expressions parser', function() hl('TernaryColon', ':'), hl('IdentifierName', 'h'), }) - check_parsing('a ? b : c : d', 0, { + check_parsing('a ? b : c : d', { -- 0123456789012 -- 0 1 ast = { @@ -3660,7 +3715,7 @@ describe('Expressions parser', function() }) end) itp('works with comparison operators', function() - check_parsing('a == b', 0, { + check_parsing('a == b', { -- 012345 ast = { { @@ -3677,7 +3732,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a ==? b', 0, { + check_parsing('a ==? b', { -- 0123456 ast = { { @@ -3695,7 +3750,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a ==# b', 0, { + check_parsing('a ==# b', { -- 0123456 ast = { { @@ -3713,7 +3768,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a !=# b', 0, { + check_parsing('a !=# b', { -- 0123456 ast = { { @@ -3731,7 +3786,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a <=# b', 0, { + check_parsing('a <=# b', { -- 0123456 ast = { { @@ -3749,7 +3804,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a >=# b', 0, { + check_parsing('a >=# b', { -- 0123456 ast = { { @@ -3767,7 +3822,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a ># b', 0, { + check_parsing('a ># b', { -- 012345 ast = { { @@ -3785,7 +3840,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a <# b', 0, { + check_parsing('a <# b', { -- 012345 ast = { { @@ -3803,7 +3858,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a is#b', 0, { + check_parsing('a is#b', { -- 012345 ast = { { @@ -3821,7 +3876,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b'), }) - check_parsing('a is?b', 0, { + check_parsing('a is?b', { -- 012345 ast = { { @@ -3839,7 +3894,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b'), }) - check_parsing('a isnot b', 0, { + check_parsing('a isnot b', { -- 012345678 ast = { { @@ -3856,7 +3911,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a < b < c', 0, { + check_parsing('a < b < c', { -- 012345678 ast = { { @@ -3885,7 +3940,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'c', 1), }) - check_parsing('a < b <# c', 0, { + check_parsing('a < b <# c', { -- 012345678 ast = { { @@ -3915,7 +3970,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'c', 1), }) - check_parsing('a += b', 0, { + check_parsing('a += b', { -- 012345 ast = { { @@ -3942,7 +3997,7 @@ describe('Expressions parser', function() hl('InvalidComparison', '='), hl('IdentifierName', 'b', 1), }) - check_parsing('a + b == c + d', 0, { + check_parsing('a + b == c + d', { -- 01234567890123 -- 0 1 ast = { @@ -3975,7 +4030,7 @@ describe('Expressions parser', function() hl('BinaryPlus', '+', 1), hl('IdentifierName', 'd', 1), }) - check_parsing('+ a == + b', 0, { + check_parsing('+ a == + b', { -- 0123456789 ast = { { @@ -4005,7 +4060,7 @@ describe('Expressions parser', function() }) end) itp('works with concat/subscript', function() - check_parsing('.', 0, { + check_parsing('.', { -- 0 ast = { { @@ -4023,7 +4078,7 @@ describe('Expressions parser', function() hl('InvalidConcatOrSubscript', '.'), }) - check_parsing('a.', 0, { + check_parsing('a.', { -- 01 ast = { { @@ -4042,7 +4097,7 @@ describe('Expressions parser', function() hl('ConcatOrSubscript', '.'), }) - check_parsing('a.b', 0, { + check_parsing('a.b', { -- 012 ast = { { @@ -4059,7 +4114,7 @@ describe('Expressions parser', function() hl('IdentifierKey', 'b'), }) - check_parsing('1.2', 0, { + check_parsing('1.2', { -- 012 ast = { 'Float(val=1.200000e+00):0:0:1.2', @@ -4068,7 +4123,7 @@ describe('Expressions parser', function() hl('Float', '1.2'), }) - check_parsing('1.2 + 1.3e-5', 0, { + check_parsing('1.2 + 1.3e-5', { -- 012345678901 -- 0 1 ast = { @@ -4086,7 +4141,7 @@ describe('Expressions parser', function() hl('Float', '1.3e-5', 1), }) - check_parsing('a . 1.2 + 1.3e-5', 0, { + check_parsing('a . 1.2 + 1.3e-5', { -- 0123456789012345 -- 0 1 ast = { @@ -4120,7 +4175,7 @@ describe('Expressions parser', function() hl('Float', '1.3e-5', 1), }) - check_parsing('1.3e-5 + 1.2 . a', 0, { + check_parsing('1.3e-5 + 1.2 . a', { -- 0123456789012345 -- 0 1 ast = { @@ -4146,7 +4201,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'a', 1), }) - check_parsing('1.3e-5 + a . 1.2', 0, { + check_parsing('1.3e-5 + a . 1.2', { -- 0123456789012345 -- 0 1 ast = { @@ -4180,7 +4235,7 @@ describe('Expressions parser', function() hl('IdentifierKey', '2'), }) - check_parsing('1.2.3', 0, { + check_parsing('1.2.3', { -- 01234 ast = { { @@ -4205,7 +4260,7 @@ describe('Expressions parser', function() hl('IdentifierKey', '3'), }) - check_parsing('a.1.2', 0, { + check_parsing('a.1.2', { -- 01234 ast = { { @@ -4230,7 +4285,7 @@ describe('Expressions parser', function() hl('IdentifierKey', '2'), }) - check_parsing('a . 1.2', 0, { + check_parsing('a . 1.2', { -- 0123456 ast = { { @@ -4255,7 +4310,7 @@ describe('Expressions parser', function() hl('IdentifierKey', '2'), }) - check_parsing('+a . +b', 0, { + check_parsing('+a . +b', { -- 0123456 ast = { { @@ -4284,7 +4339,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b'), }) - check_parsing('a. b', 0, { + check_parsing('a. b', { -- 0123 ast = { { @@ -4301,7 +4356,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'b', 1), }) - check_parsing('a. 1', 0, { + check_parsing('a. 1', { -- 0123 ast = { { @@ -4319,7 +4374,7 @@ describe('Expressions parser', function() }) end) itp('works with bracket subscripts', function() - check_parsing(':', 0, { + check_parsing(':', { -- 0 ast = { { @@ -4336,7 +4391,7 @@ describe('Expressions parser', function() }, { hl('InvalidColon', ':'), }) - check_parsing('a[]', 0, { + check_parsing('a[]', { -- 012 ast = { { @@ -4355,7 +4410,7 @@ describe('Expressions parser', function() hl('SubscriptBracket', '['), hl('InvalidSubscriptBracket', ']'), }) - check_parsing('a[b:]', 0, { + check_parsing('a[b:]', { -- 01234 ast = { { @@ -4374,7 +4429,7 @@ describe('Expressions parser', function() hl('SubscriptBracket', ']'), }) - check_parsing('a[b:c]', 0, { + check_parsing('a[b:c]', { -- 012345 ast = { { @@ -4393,7 +4448,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'c'), hl('SubscriptBracket', ']'), }) - check_parsing('a[b : c]', 0, { + check_parsing('a[b : c]', { -- 01234567 ast = { { @@ -4419,7 +4474,7 @@ describe('Expressions parser', function() hl('SubscriptBracket', ']'), }) - check_parsing('a[: b]', 0, { + check_parsing('a[: b]', { -- 012345 ast = { { @@ -4444,7 +4499,7 @@ describe('Expressions parser', function() hl('SubscriptBracket', ']'), }) - check_parsing('a[b :]', 0, { + check_parsing('a[b :]', { -- 012345 ast = { { @@ -4467,7 +4522,7 @@ describe('Expressions parser', function() hl('SubscriptColon', ':', 1), hl('SubscriptBracket', ']'), }) - check_parsing('a[b][c][d](e)(f)(g)', 0, { + check_parsing('a[b][c][d](e)(f)(g)', { -- 0123456789012345678 -- 0 1 ast = { @@ -4530,7 +4585,7 @@ describe('Expressions parser', function() hl('IdentifierName', 'g'), hl('CallingParenthesis', ')'), }) - check_parsing('{a}{b}{c}[d][e][f]', 0, { + check_parsing('{a}{b}{c}[d][e][f]', { -- 012345678901234567 -- 0 1 ast = { @@ -4603,7 +4658,7 @@ describe('Expressions parser', function() }) end) itp('supports list literals', function() - check_parsing('[]', 0, { + check_parsing('[]', { -- 01 ast = { 'ListLiteral:0:0:[', @@ -4613,7 +4668,7 @@ describe('Expressions parser', function() hl('List', ']'), }) - check_parsing('[a]', 0, { + check_parsing('[a]', { -- 012 ast = { { @@ -4629,7 +4684,7 @@ describe('Expressions parser', function() hl('List', ']'), }) - check_parsing('[a, b]', 0, { + check_parsing('[a, b]', { -- 012345 ast = { { @@ -4653,7 +4708,7 @@ describe('Expressions parser', function() hl('List', ']'), }) - check_parsing('[a, b, c]', 0, { + check_parsing('[a, b, c]', { -- 012345678 ast = { { @@ -4685,7 +4740,7 @@ describe('Expressions parser', function() hl('List', ']'), }) - check_parsing('[a, b, c, ]', 0, { + check_parsing('[a, b, c, ]', { -- 01234567890 -- 0 1 ast = { @@ -4724,7 +4779,7 @@ describe('Expressions parser', function() hl('List', ']', 1), }) - check_parsing('[a : b, c : d]', 0, { + check_parsing('[a : b, c : d]', { -- 01234567890123 -- 0 1 ast = { @@ -4769,7 +4824,7 @@ describe('Expressions parser', function() hl('List', ']'), }) - check_parsing(']', 0, { + check_parsing(']', { -- 0 ast = { 'ListLiteral:0:0:', @@ -4782,7 +4837,7 @@ describe('Expressions parser', function() hl('InvalidList', ']'), }) - check_parsing('a]', 0, { + check_parsing('a]', { -- 01 ast = { { @@ -4801,7 +4856,7 @@ describe('Expressions parser', function() hl('InvalidList', ']'), }) - check_parsing('[] []', 0, { + check_parsing('[] []', { -- 01234 ast = { { @@ -4822,9 +4877,23 @@ describe('Expressions parser', function() hl('InvalidSpacing', ' '), hl('List', '['), hl('List', ']'), + }, { + [1] = { + ast = { + err = REMOVE_THIS, + ast = { + 'ListLiteral:0:0:[', + }, + }, + hl_fs = { + [3] = REMOVE_THIS, + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + }, + }, }) - check_parsing('[][]', 0, { + check_parsing('[][]', { -- 0123 ast = { { @@ -4845,7 +4914,7 @@ describe('Expressions parser', function() hl('InvalidSubscriptBracket', ']'), }) - check_parsing('[', 0, { + check_parsing('[', { -- 0 ast = { 'ListLiteral:0:0:[', @@ -4858,7 +4927,7 @@ describe('Expressions parser', function() hl('List', '['), }) - check_parsing('[1', 0, { + check_parsing('[1', { -- 01 ast = { { @@ -4878,7 +4947,7 @@ describe('Expressions parser', function() }) end) itp('works with strings', function() - check_parsing('\'abc\'', 0, { + check_parsing('\'abc\'', { -- 01234 ast = { 'SingleQuotedString(val="abc"):0:0:\'abc\'', @@ -4888,7 +4957,7 @@ describe('Expressions parser', function() hl('SingleQuotedBody', 'abc'), hl('SingleQuote', '\''), }) - check_parsing('"abc"', 0, { + check_parsing('"abc"', { -- 01234 ast = { 'DoubleQuotedString(val="abc"):0:0:"abc"', @@ -4898,7 +4967,7 @@ describe('Expressions parser', function() hl('DoubleQuotedBody', 'abc'), hl('DoubleQuote', '"'), }) - check_parsing('\'\'', 0, { + check_parsing('\'\'', { -- 01 ast = { 'SingleQuotedString(val=NULL):0:0:\'\'', @@ -4907,7 +4976,7 @@ describe('Expressions parser', function() hl('SingleQuote', '\''), hl('SingleQuote', '\''), }) - check_parsing('""', 0, { + check_parsing('""', { -- 01 ast = { 'DoubleQuotedString(val=NULL):0:0:""', @@ -4916,7 +4985,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), hl('DoubleQuote', '"'), }) - check_parsing('"', 0, { + check_parsing('"', { -- 0 ast = { 'DoubleQuotedString(val=NULL):0:0:"', @@ -4928,7 +4997,7 @@ describe('Expressions parser', function() }, { hl('InvalidDoubleQuote', '"'), }) - check_parsing('\'', 0, { + check_parsing('\'', { -- 0 ast = { 'SingleQuotedString(val=NULL):0:0:\'', @@ -4940,7 +5009,7 @@ describe('Expressions parser', function() }, { hl('InvalidSingleQuote', '\''), }) - check_parsing('"a', 0, { + check_parsing('"a', { -- 01 ast = { 'DoubleQuotedString(val="a"):0:0:"a', @@ -4953,7 +5022,7 @@ describe('Expressions parser', function() hl('InvalidDoubleQuote', '"'), hl('InvalidDoubleQuotedBody', 'a'), }) - check_parsing('\'a', 0, { + check_parsing('\'a', { -- 01 ast = { 'SingleQuotedString(val="a"):0:0:\'a', @@ -4966,7 +5035,7 @@ describe('Expressions parser', function() hl('InvalidSingleQuote', '\''), hl('InvalidSingleQuotedBody', 'a'), }) - check_parsing('\'abc\'\'def\'', 0, { + check_parsing('\'abc\'\'def\'', { -- 0123456789 ast = { 'SingleQuotedString(val="abc\'def"):0:0:\'abc\'\'def\'', @@ -4978,7 +5047,7 @@ describe('Expressions parser', function() hl('SingleQuotedBody', 'def'), hl('SingleQuote', '\''), }) - check_parsing('\'abc\'\'', 0, { + check_parsing('\'abc\'\'', { -- 012345 ast = { 'SingleQuotedString(val="abc\'"):0:0:\'abc\'\'', @@ -4992,7 +5061,7 @@ describe('Expressions parser', function() hl('InvalidSingleQuotedBody', 'abc'), hl('InvalidSingleQuotedQuote', '\'\''), }) - check_parsing('\'\'\'\'\'\'\'\'', 0, { + check_parsing('\'\'\'\'\'\'\'\'', { -- 01234567 ast = { 'SingleQuotedString(val="\'\'\'"):0:0:\'\'\'\'\'\'\'\'', @@ -5004,7 +5073,7 @@ describe('Expressions parser', function() hl('SingleQuotedQuote', '\'\''), hl('SingleQuote', '\''), }) - check_parsing('\'\'\'a\'\'\'\'bc\'', 0, { + check_parsing('\'\'\'a\'\'\'\'bc\'', { -- 01234567890 -- 0 1 ast = { @@ -5019,7 +5088,7 @@ describe('Expressions parser', function() hl('SingleQuotedBody', 'bc'), hl('SingleQuote', '\''), }) - check_parsing('"\\"\\"\\"\\""', 0, { + check_parsing('"\\"\\"\\"\\""', { -- 0123456789 ast = { 'DoubleQuotedString(val="\\"\\"\\"\\""):0:0:"\\"\\"\\"\\""', @@ -5032,7 +5101,7 @@ describe('Expressions parser', function() hl('DoubleQuotedEscape', '\\"'), hl('DoubleQuote', '"'), }) - check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', 0, { + check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', { -- 0123456789012345678901234 -- 0 1 2 ast = { @@ -5051,7 +5120,7 @@ describe('Expressions parser', function() hl('DoubleQuotedBody', 'mno'), hl('DoubleQuote', '"'), }) - check_parsing('"\\b\\e\\f\\r\\t\\\\"', 0, { + check_parsing('"\\b\\e\\f\\r\\t\\\\"', { -- 0123456789012345 -- 0 1 ast = { @@ -5067,7 +5136,7 @@ describe('Expressions parser', function() hl('DoubleQuotedEscape', '\\\\'), hl('DoubleQuote', '"'), }) - check_parsing('"\\n\n"', 0, { + check_parsing('"\\n\n"', { -- 01234 ast = { 'DoubleQuotedString(val="\\\n\\\n"):0:0:"\\n\n"', @@ -5078,7 +5147,7 @@ describe('Expressions parser', function() hl('DoubleQuotedBody', '\n'), hl('DoubleQuote', '"'), }) - check_parsing('"\\x00"', 0, { + check_parsing('"\\x00"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0"):0:0:"\\x00"', @@ -5088,7 +5157,7 @@ describe('Expressions parser', function() hl('DoubleQuotedEscape', '\\x00'), hl('DoubleQuote', '"'), }) - check_parsing('"\\xFF"', 0, { + check_parsing('"\\xFF"', { -- 012345 ast = { 'DoubleQuotedString(val="\255"):0:0:"\\xFF"', @@ -5098,7 +5167,7 @@ describe('Expressions parser', function() hl('DoubleQuotedEscape', '\\xFF'), hl('DoubleQuote', '"'), }) - check_parsing('"\\xF"', 0, { + check_parsing('"\\xF"', { -- 012345 ast = { 'DoubleQuotedString(val="\\15"):0:0:"\\xF"', @@ -5108,7 +5177,7 @@ describe('Expressions parser', function() hl('DoubleQuotedEscape', '\\xF'), hl('DoubleQuote', '"'), }) - check_parsing('"\\u00AB"', 0, { + check_parsing('"\\u00AB"', { -- 01234567 ast = { 'DoubleQuotedString(val="«"):0:0:"\\u00AB"', @@ -5118,7 +5187,7 @@ describe('Expressions parser', function() hl('DoubleQuotedEscape', '\\u00AB'), hl('DoubleQuote', '"'), }) - check_parsing('"\\U000000AB"', 0, { + check_parsing('"\\U000000AB"', { -- 01234567 ast = { 'DoubleQuotedString(val="«"):0:0:"\\U000000AB"', @@ -5128,7 +5197,7 @@ describe('Expressions parser', function() hl('DoubleQuotedEscape', '\\U000000AB'), hl('DoubleQuote', '"'), }) - check_parsing('"\\x"', 0, { + check_parsing('"\\x"', { -- 0123 ast = { 'DoubleQuotedString(val="x"):0:0:"\\x"', @@ -5139,7 +5208,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\x', 0, { + check_parsing('"\\x', { -- 012 ast = { 'DoubleQuotedString(val="x"):0:0:"\\x', @@ -5153,7 +5222,7 @@ describe('Expressions parser', function() hl('InvalidDoubleQuotedUnknownEscape', '\\x'), }) - check_parsing('"\\xF', 0, { + check_parsing('"\\xF', { -- 0123 ast = { 'DoubleQuotedString(val="\\15"):0:0:"\\xF', @@ -5167,7 +5236,7 @@ describe('Expressions parser', function() hl('InvalidDoubleQuotedEscape', '\\xF'), }) - check_parsing('"\\u"', 0, { + check_parsing('"\\u"', { -- 0123 ast = { 'DoubleQuotedString(val="u"):0:0:"\\u"', @@ -5178,7 +5247,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u', 0, { + check_parsing('"\\u', { -- 012 ast = { 'DoubleQuotedString(val="u"):0:0:"\\u', @@ -5192,7 +5261,7 @@ describe('Expressions parser', function() hl('InvalidDoubleQuotedUnknownEscape', '\\u'), }) - check_parsing('"\\U', 0, { + check_parsing('"\\U', { -- 012 ast = { 'DoubleQuotedString(val="U"):0:0:"\\U', @@ -5206,7 +5275,7 @@ describe('Expressions parser', function() hl('InvalidDoubleQuotedUnknownEscape', '\\U'), }) - check_parsing('"\\U"', 0, { + check_parsing('"\\U"', { -- 0123 ast = { 'DoubleQuotedString(val="U"):0:0:"\\U"', @@ -5217,7 +5286,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\xFX"', 0, { + check_parsing('"\\xFX"', { -- 012345 ast = { 'DoubleQuotedString(val="\\15X"):0:0:"\\xFX"', @@ -5229,7 +5298,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\XFX"', 0, { + check_parsing('"\\XFX"', { -- 012345 ast = { 'DoubleQuotedString(val="\\15X"):0:0:"\\XFX"', @@ -5241,7 +5310,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\xX"', 0, { + check_parsing('"\\xX"', { -- 01234 ast = { 'DoubleQuotedString(val="xX"):0:0:"\\xX"', @@ -5253,7 +5322,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\XX"', 0, { + check_parsing('"\\XX"', { -- 01234 ast = { 'DoubleQuotedString(val="XX"):0:0:"\\XX"', @@ -5265,7 +5334,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\uX"', 0, { + check_parsing('"\\uX"', { -- 01234 ast = { 'DoubleQuotedString(val="uX"):0:0:"\\uX"', @@ -5277,7 +5346,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\UX"', 0, { + check_parsing('"\\UX"', { -- 01234 ast = { 'DoubleQuotedString(val="UX"):0:0:"\\UX"', @@ -5289,7 +5358,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\x0X"', 0, { + check_parsing('"\\x0X"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\x0X"', @@ -5301,7 +5370,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\X0X"', 0, { + check_parsing('"\\X0X"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\X0X"', @@ -5313,7 +5382,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u0X"', 0, { + check_parsing('"\\u0X"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\u0X"', @@ -5325,7 +5394,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U0X"', 0, { + check_parsing('"\\U0X"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U0X"', @@ -5337,7 +5406,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\x00X"', 0, { + check_parsing('"\\x00X"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\x00X"', @@ -5349,7 +5418,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\X00X"', 0, { + check_parsing('"\\X00X"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\X00X"', @@ -5361,7 +5430,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u00X"', 0, { + check_parsing('"\\u00X"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\u00X"', @@ -5373,7 +5442,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U00X"', 0, { + check_parsing('"\\U00X"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U00X"', @@ -5385,7 +5454,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u000X"', 0, { + check_parsing('"\\u000X"', { -- 01234567 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\u000X"', @@ -5397,7 +5466,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U000X"', 0, { + check_parsing('"\\U000X"', { -- 01234567 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U000X"', @@ -5409,7 +5478,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u0000X"', 0, { + check_parsing('"\\u0000X"', { -- 012345678 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\u0000X"', @@ -5421,7 +5490,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U0000X"', 0, { + check_parsing('"\\U0000X"', { -- 012345678 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000X"', @@ -5433,7 +5502,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U00000X"', 0, { + check_parsing('"\\U00000X"', { -- 0123456789 ast = { 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000X"', @@ -5445,7 +5514,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U000000X"', 0, { + check_parsing('"\\U000000X"', { -- 01234567890 -- 0 1 ast = { @@ -5458,7 +5527,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U0000000X"', 0, { + check_parsing('"\\U0000000X"', { -- 012345678901 -- 0 1 ast = { @@ -5471,7 +5540,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U00000000X"', 0, { + check_parsing('"\\U00000000X"', { -- 0123456789012 -- 0 1 ast = { @@ -5484,7 +5553,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\x000X"', 0, { + check_parsing('"\\x000X"', { -- 01234567 ast = { 'DoubleQuotedString(val="\\0000X"):0:0:"\\x000X"', @@ -5496,7 +5565,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\X000X"', 0, { + check_parsing('"\\X000X"', { -- 01234567 ast = { 'DoubleQuotedString(val="\\0000X"):0:0:"\\X000X"', @@ -5508,7 +5577,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\u00000X"', 0, { + check_parsing('"\\u00000X"', { -- 0123456789 ast = { 'DoubleQuotedString(val="\\0000X"):0:0:"\\u00000X"', @@ -5520,7 +5589,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\U000000000X"', 0, { + check_parsing('"\\U000000000X"', { -- 01234567890123 -- 0 1 ast = { @@ -5533,7 +5602,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\0"', 0, { + check_parsing('"\\0"', { -- 0123 ast = { 'DoubleQuotedString(val="\\0"):0:0:"\\0"', @@ -5544,7 +5613,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\00"', 0, { + check_parsing('"\\00"', { -- 01234 ast = { 'DoubleQuotedString(val="\\0"):0:0:"\\00"', @@ -5555,7 +5624,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\000"', 0, { + check_parsing('"\\000"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0"):0:0:"\\000"', @@ -5566,7 +5635,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\0000"', 0, { + check_parsing('"\\0000"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0000"):0:0:"\\0000"', @@ -5578,7 +5647,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\8"', 0, { + check_parsing('"\\8"', { -- 0123 ast = { 'DoubleQuotedString(val="8"):0:0:"\\8"', @@ -5589,7 +5658,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\08"', 0, { + check_parsing('"\\08"', { -- 01234 ast = { 'DoubleQuotedString(val="\\0008"):0:0:"\\08"', @@ -5601,7 +5670,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\008"', 0, { + check_parsing('"\\008"', { -- 012345 ast = { 'DoubleQuotedString(val="\\0008"):0:0:"\\008"', @@ -5613,7 +5682,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\0008"', 0, { + check_parsing('"\\0008"', { -- 0123456 ast = { 'DoubleQuotedString(val="\\0008"):0:0:"\\0008"', @@ -5625,7 +5694,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\777"', 0, { + check_parsing('"\\777"', { -- 012345 ast = { 'DoubleQuotedString(val="\255"):0:0:"\\777"', @@ -5636,7 +5705,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\050"', 0, { + check_parsing('"\\050"', { -- 012345 ast = { 'DoubleQuotedString(val="\40"):0:0:"\\050"', @@ -5647,7 +5716,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\"', 0, { + check_parsing('"\\"', { -- 012345 ast = { 'DoubleQuotedString(val="\\21"):0:0:"\\"', @@ -5658,7 +5727,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\<', 0, { + check_parsing('"\\<', { -- 012 ast = { 'DoubleQuotedString(val="<"):0:0:"\\<', @@ -5672,7 +5741,7 @@ describe('Expressions parser', function() hl('InvalidDoubleQuotedUnknownEscape', '\\<'), }) - check_parsing('"\\<"', 0, { + check_parsing('"\\<"', { -- 0123 ast = { 'DoubleQuotedString(val="<"):0:0:"\\<"', @@ -5683,7 +5752,7 @@ describe('Expressions parser', function() hl('DoubleQuote', '"'), }) - check_parsing('"\\ Date: Mon, 6 Nov 2017 01:15:18 +0300 Subject: [PATCH 074/109] =?UTF-8?q?api/vim:=20Add=20=E2=80=9Clen=E2=80=9D?= =?UTF-8?q?=20dictionary=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows determining where parsing ended which may be needed for e.g. parsing `:echo` with that API function. --- src/nvim/api/vim.c | 15 +++++++++++++-- test/functional/api/vim_spec.lua | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index b12c595cb5..8de37e2cf3 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -922,6 +922,12 @@ typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack; /// Must contain exactly one "%.*s". /// "arg": String, error message argument. /// +/// "len": Amount of bytes successfully parsed. With flags equal to "" +/// that should be equal to the length of expr string. +/// +/// @note: “Sucessfully parsed” here means “participated in AST +/// creation”, not “till the first error”. +/// /// "ast": actual AST, either nil or a dictionary with the following /// keys: /// @@ -1000,9 +1006,8 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, &pstate, parser_simple_get_line, &plines_p, colors_p); ExprAST east = viml_pexpr_parse(&pstate, pflags); - // FIXME add parse_length key const size_t ret_size = ( - 1 // "ast" + 2 // "ast", "len" + (size_t)(east.err.msg != NULL) // "error" + (size_t)highlight // "highlight" ); @@ -1015,6 +1020,12 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, .key = STATIC_CSTR_TO_STRING("ast"), .value = NIL, }; + ret.items[ret.size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("len"), + .value = INTEGER_OBJ((Integer)(pstate.pos.line == 1 + ? plines[0].size + : pstate.pos.col)), + }; if (east.err.msg != NULL) { Dictionary err_dict = { .items = xmalloc(2 * sizeof(err_dict.items[0])), diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index b904bd2a8f..714b1988fb 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -799,6 +799,9 @@ describe('api', function() if east_api.ast then east_api.ast = {simplify_east_api_node(line, east_api.ast)} end + if east_api.len == #line then + east_api.len = nil + end return east_api end local function simplify_east_hl(line, east_hl) @@ -997,6 +1000,7 @@ describe('api', function() }, { [1] = { ast = { + len = 2, err = REMOVE_THIS, ast = { 'Register(name=a):0:0:@a' @@ -1028,6 +1032,7 @@ describe('api', function() }, { [1] = { ast = { + len = 6, err = REMOVE_THIS, ast = { 'Register(name=a):0:0: @a' @@ -1236,6 +1241,7 @@ describe('api', function() }, { [1] = { ast = { + len = 3, err = REMOVE_THIS, ast = { 'Register(name=a):0:0:@a', @@ -5456,6 +5462,7 @@ describe('api', function() }, { [1] = { ast = { + len = 3, err = REMOVE_THIS, ast = { 'ListLiteral:0:0:[', @@ -7135,6 +7142,7 @@ describe('api', function() }, { [1] = { ast = { + len = 4, err = REMOVE_THIS, ast = { 'Option(scope=0,ident=xxx):0:0:&xxx', @@ -7520,6 +7528,7 @@ describe('api', function() }, { [1] = { ast = { + len = 1, err = REMOVE_THIS, ast = { 'Integer(val=1):0:0:1', @@ -7652,6 +7661,7 @@ describe('api', function() end) it('respects highlight argument', function() eq({ + len = 1, ast = { ivalue = 1, len = 1, @@ -7660,6 +7670,7 @@ describe('api', function() }, }, meths.parse_expression('1', '', false)) eq({ + len = 1, ast = { ivalue = 1, len = 1, @@ -7674,6 +7685,7 @@ describe('api', function() it('works (KLEE tests)', function() check_parsing('\0002&A:\000', { ast = {}, + len = 0, err = { arg = '\0002&A:\0', msg = 'E15: Expected value, got EOC: %.*s', @@ -7682,6 +7694,7 @@ describe('api', function() }, { [2] = { ast = { + len = REMOVE_THIS, ast = { { 'Colon:0:4::', @@ -7711,6 +7724,7 @@ describe('api', function() }, [3] = { ast = { + len = 2, ast = { 'Integer(val=2):0:1:2', }, @@ -7754,6 +7768,7 @@ describe('api', function() check_parsing('|"\\U\\', { -- 01234 ast = {}, + len = 0, err = { arg = '|"\\U\\', msg = 'E15: Expected value, got EOC: %.*s', @@ -7762,6 +7777,7 @@ describe('api', function() }, { [2] = { ast = { + len = REMOVE_THIS, ast = { { 'Or:0:0:|', @@ -7786,6 +7802,7 @@ describe('api', function() check_parsing('|"\\e"', { -- 01234 ast = {}, + len = 0, err = { arg = '|"\\e"', msg = 'E15: Expected value, got EOC: %.*s', @@ -7794,6 +7811,7 @@ describe('api', function() }, { [2] = { ast = { + len = REMOVE_THIS, ast = { { 'Or:0:0:|', @@ -7818,6 +7836,7 @@ describe('api', function() check_parsing('|\029', { -- 01 ast = {}, + len = 0, err = { arg = '|\029', msg = 'E15: Expected value, got EOC: %.*s', @@ -7826,6 +7845,7 @@ describe('api', function() }, { [2] = { ast = { + len = REMOVE_THIS, ast = { { 'Or:0:0:|', @@ -7892,6 +7912,7 @@ describe('api', function() }, { [1] = { ast = { + len = 1, ast = { 'UnknownFigure:0:0:', }, @@ -7917,6 +7938,7 @@ describe('api', function() }, }, }, + len = 2, err = { arg = ':?\000\000\000\000\000\000\000', msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', @@ -7926,6 +7948,9 @@ describe('api', function() hl('InvalidTernary', '?'), }, { [2] = { + ast = { + len = REMOVE_THIS, + }, hl_fs = { [3] = hl('InvalidSpacing', '\0'), [4] = hl('InvalidSpacing', '\0'), From 05f775b5f248d922c9539432235738cc53e7edd7 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 6 Nov 2017 01:57:22 +0300 Subject: [PATCH 075/109] viml/parser/expressions: Briefly document some differences --- src/nvim/viml/parser/expressions.c | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index b10952a8ac..998edb1ed4 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -3,6 +3,44 @@ /// VimL expression parser +// Planned incompatibilities (to be included into vim_diff.txt when this parser +// will be an actual part of VimL evaluation process): +// +// 1. Expressions are first fully parsed and only then executed. This means +// that while ":echo [system('touch abc')" will create file "abc" in Vim and +// only then raise syntax error regarding missing comma in list in Neovim +// trying to execute that will immediately raise syntax error regarding +// missing list end without actually executing anything. +// 2. Expressions are first fully parsed, without considering any runtime +// information. This means things like that "d.a" does not change its +// meaning depending on type of "d" (or whether Vim is currently executing or +// skipping). For compatibility reasons the dot thus may either be “concat +// or subscript” operator or just “concat” operator. +// 3. Expressions parser is aware whether it is called for :echo or =. +// This means that while "=1 | 2" is equivalent to "=1" +// because "| 2" part is left to be treated as a command separator and then +// ignored in Neovim it is an error. +// 4. Expressions parser has generally better error reporting. But for +// compatibility reasons most errors have error code E15 while error messages +// are significantly different from Vim’s E15. Also some error codes were +// retired because of being harder to emulate or because of them being +// a result of differences in parsing process: e.g. with ":echo {a, b}" Vim +// will attempt to parse expression as lambda, fail, check whether it is +// a curly-braces-name, fail again, and evaluate that as a dictionary, giving +// error regarding undefined variable "a" (or about missing colon). Neovim +// will not try to evaluate anything here: comma right after an argument name +// means that expression may not be anything, but lambda, so the resulting +// error message will never be about missing variable or colon: it will be +// about missing arrow (or a continuation of argument list). +// 5. Failing to parse expression always gives exactly one error message: no +// more stack of error messages like > +// +// :echo [1, +// E697: Missing end of List ']': +// E15: Invalid expression: [1, +// +// < , just exactly one E697 message. + #include #include #include From c85f485aa78352f31421d209176b2ed57b91b86e Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 6 Nov 2017 19:06:24 +0300 Subject: [PATCH 076/109] charset: Move vim_str2nr flags from vim.h to charset.h --- src/nvim/charset.c | 2 +- src/nvim/charset.h | 15 +++++++++++++++ src/nvim/vim.h | 7 ------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/nvim/charset.c b/src/nvim/charset.c index 5aebf90194..980b4ed426 100644 --- a/src/nvim/charset.c +++ b/src/nvim/charset.c @@ -1615,7 +1615,7 @@ bool vim_isblankline(char_u *lbuf) /// hexadecimal, '0' = octal, 'b' or 'B' is binary. When using /// STR2NR_FORCE is always zero. /// @param len Returns the detected length of number. -/// @param what Recognizes what number passed. +/// @param what Recognizes what number passed, @see ChStr2NrFlags. /// @param nptr Returns the signed result. /// @param unptr Returns the unsigned result. /// @param maxlen Max length of string to check. diff --git a/src/nvim/charset.h b/src/nvim/charset.h index c69582c4c6..7198c8d405 100644 --- a/src/nvim/charset.h +++ b/src/nvim/charset.h @@ -15,6 +15,21 @@ ?((int)(uint8_t)(c)) \ :((int)(c))) +/// Flags for vim_str2nr() +typedef enum { + STR2NR_DEC = 0, + STR2NR_BIN = (1 << 0), ///< Allow binary numbers. + STR2NR_OCT = (1 << 1), ///< Allow octal numbers. + STR2NR_HEX = (1 << 2), ///< Allow hexadecimal numbers. + /// Force one of the above variants. + /// + /// STR2NR_FORCE|STR2NR_DEC is actually not different from supplying zero + /// as flags, but still present for completeness. + STR2NR_FORCE = (1 << 3), + /// Recognize all formats vim_str2nr() can recognize. + STR2NR_ALL = STR2NR_BIN | STR2NR_OCT | STR2NR_HEX, +} ChStr2NrFlags; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "charset.h.generated.h" #endif diff --git a/src/nvim/vim.h b/src/nvim/vim.h index 62ffc7433e..096c57450f 100644 --- a/src/nvim/vim.h +++ b/src/nvim/vim.h @@ -28,13 +28,6 @@ /// length of a buffer to store a number in ASCII (64 bits binary + NUL) enum { NUMBUFLEN = 65 }; -// flags for vim_str2nr() -#define STR2NR_BIN 1 -#define STR2NR_OCT 2 -#define STR2NR_HEX 4 -#define STR2NR_ALL (STR2NR_BIN + STR2NR_OCT + STR2NR_HEX) -#define STR2NR_FORCE 8 // only when ONE of the above is used - #define MAX_TYPENR 65535 #define ROOT_UID 0 From 42959d0e8f9e779ba4983016b24967bf02abb59f Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 6 Nov 2017 20:15:05 +0300 Subject: [PATCH 077/109] unittests: Add tests for vim_str2nr --- test/helpers.lua | 8 + test/unit/charset/vim_str2nr_spec.lua | 294 +++++++++++++++++++++++++- 2 files changed, 301 insertions(+), 1 deletion(-) diff --git a/test/helpers.lua b/test/helpers.lua index 20fe23821f..16b9818f12 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -310,6 +310,13 @@ local function mergedicts_copy(d1, d2) return ret end +local function updated(d, d2) + for k, v in pairs(d2) do + d[k] = v + end + return d +end + local function concat_tables(...) local ret = {} for i = 1, select('#', ...) do @@ -460,4 +467,5 @@ return { format_luav = format_luav, format_string = format_string, intchar2lua = intchar2lua, + updated = updated, } diff --git a/test/unit/charset/vim_str2nr_spec.lua b/test/unit/charset/vim_str2nr_spec.lua index cfbc77bc2a..9309dc380c 100644 --- a/test/unit/charset/vim_str2nr_spec.lua +++ b/test/unit/charset/vim_str2nr_spec.lua @@ -1 +1,293 @@ --- FIXME +local helpers = require("test.unit.helpers")(after_each) +local global_helpers = require('test.helpers') + +local itp = helpers.gen_itp(it) + +local cimport = helpers.cimport +local ffi = helpers.ffi +local eq = helpers.eq + +local shallowcopy = global_helpers.shallowcopy +local updated = global_helpers.updated + +local lib = cimport('./src/nvim/charset.h') + +local ARGTYPES = { + num = ffi.typeof('varnumber_T[1]'), + unum = ffi.typeof('uvarnumber_T[1]'), + pre = ffi.typeof('int[1]'), + len = ffi.typeof('int[1]'), +} + +local icnt = -42 +local ucnt = 4242 + +local function arginit(arg) + if arg == 'unum' then + ucnt = ucnt + 1 + return ARGTYPES[arg]({ucnt}) + else + icnt = icnt - 1 + return ARGTYPES[arg]({icnt}) + end +end + +local function test_vim_str2nr(s, what, exp, maxlen) + local comb = {[''] = {}} + for k, _ in pairs(exp) do + for ck, cv in pairs(comb) do + comb[ck .. ',' .. k] = updated(shallowcopy(cv), { [k] = arginit(k) }) + end + end + maxlen = maxlen or #s + for _, cv in pairs(comb) do + lib.vim_str2nr(s, cv.pre, cv.len, what, cv.num, cv.unum, maxlen) + for cck, ccv in pairs(cv) do + if exp[cck] ~= tonumber(ccv[0]) then + error(('Failed check (%s = %d) in test (s=%s, w=%u, m=%d): %d'):format( + cck, exp[cck], s, tonumber(what), maxlen, tonumber(ccv[0]) + )) + end + end + end +end + +describe('vim_str2nr()', function() + itp('works fine when it has nothing to do', function() + test_vim_str2nr('', 0, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_ALL, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_BIN, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_OCT, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_HEX, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_FORCE + lib.STR2NR_DEC, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_FORCE + lib.STR2NR_BIN, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_FORCE + lib.STR2NR_OCT, {len = 0, num = 0, unum = 0, pre = 0}, 0) + test_vim_str2nr('', lib.STR2NR_FORCE + lib.STR2NR_HEX, {len = 0, num = 0, unum = 0, pre = 0}, 0) + end) + itp('works with decimal numbers', function() + for _, flags in ipairs({ + 0, + lib.STR2NR_BIN, + lib.STR2NR_OCT, + lib.STR2NR_HEX, + lib.STR2NR_BIN + lib.STR2NR_OCT, + lib.STR2NR_BIN + lib.STR2NR_HEX, + lib.STR2NR_OCT + lib.STR2NR_HEX, + lib.STR2NR_ALL, + lib.STR2NR_FORCE + lib.STR2NR_DEC, + }) do + -- Check that all digits are recognized + test_vim_str2nr( '12345', flags, {len = 5, num = 12345, unum = 12345, pre = 0}, 0) + test_vim_str2nr( '67890', flags, {len = 5, num = 67890, unum = 67890, pre = 0}, 0) + test_vim_str2nr( '12345A', flags, {len = 5, num = 12345, unum = 12345, pre = 0}, 0) + test_vim_str2nr( '67890A', flags, {len = 5, num = 67890, unum = 67890, pre = 0}, 0) + + test_vim_str2nr( '42', flags, {len = 2, num = 42, unum = 42, pre = 0}, 0) + test_vim_str2nr( '42', flags, {len = 1, num = 4, unum = 4, pre = 0}, 1) + test_vim_str2nr( '42', flags, {len = 2, num = 42, unum = 42, pre = 0}, 2) + test_vim_str2nr( '42', flags, {len = 2, num = 42, unum = 42, pre = 0}, 3) -- includes NUL byte in maxlen + + test_vim_str2nr( '42x', flags, {len = 2, num = 42, unum = 42, pre = 0}, 0) + test_vim_str2nr( '42x', flags, {len = 2, num = 42, unum = 42, pre = 0}, 3) + + test_vim_str2nr('-42', flags, {len = 3, num = -42, unum = 42, pre = 0}, 3) + test_vim_str2nr('-42', flags, {len = 1, num = 0, unum = 0, pre = 0}, 1) + + test_vim_str2nr('-42x', flags, {len = 3, num = -42, unum = 42, pre = 0}, 0) + test_vim_str2nr('-42x', flags, {len = 3, num = -42, unum = 42, pre = 0}, 4) + end + end) + itp('works with binary numbers', function() + for _, flags in ipairs({ + lib.STR2NR_BIN, + lib.STR2NR_BIN + lib.STR2NR_OCT, + lib.STR2NR_BIN + lib.STR2NR_HEX, + lib.STR2NR_ALL, + lib.STR2NR_FORCE + lib.STR2NR_BIN, + }) do + local bin + local BIN + if flags > lib.STR2NR_FORCE then + bin = 0 + BIN = 0 + else + bin = ('b'):byte() + BIN = ('B'):byte() + end + + test_vim_str2nr( '0b101', flags, {len = 5, num = 5, unum = 5, pre = bin}, 0) + test_vim_str2nr( '0b101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '0b101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr( '0b101', flags, {len = 3, num = 1, unum = 1, pre = bin}, 3) + test_vim_str2nr( '0b101', flags, {len = 4, num = 2, unum = 2, pre = bin}, 4) + test_vim_str2nr( '0b101', flags, {len = 5, num = 5, unum = 5, pre = bin}, 5) + test_vim_str2nr( '0b101', flags, {len = 5, num = 5, unum = 5, pre = bin}, 6) + + test_vim_str2nr( '0b1012', flags, {len = 5, num = 5, unum = 5, pre = bin}, 0) + test_vim_str2nr( '0b1012', flags, {len = 5, num = 5, unum = 5, pre = bin}, 6) + + test_vim_str2nr('-0b101', flags, {len = 6, num = -5, unum = 5, pre = bin}, 0) + test_vim_str2nr('-0b101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-0b101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-0b101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 3) + test_vim_str2nr('-0b101', flags, {len = 4, num = -1, unum = 1, pre = bin}, 4) + test_vim_str2nr('-0b101', flags, {len = 5, num = -2, unum = 2, pre = bin}, 5) + test_vim_str2nr('-0b101', flags, {len = 6, num = -5, unum = 5, pre = bin}, 6) + test_vim_str2nr('-0b101', flags, {len = 6, num = -5, unum = 5, pre = bin}, 7) + + test_vim_str2nr('-0b1012', flags, {len = 6, num = -5, unum = 5, pre = bin}, 0) + test_vim_str2nr('-0b1012', flags, {len = 6, num = -5, unum = 5, pre = bin}, 7) + + test_vim_str2nr( '0B101', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 0) + test_vim_str2nr( '0B101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '0B101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr( '0B101', flags, {len = 3, num = 1, unum = 1, pre = BIN}, 3) + test_vim_str2nr( '0B101', flags, {len = 4, num = 2, unum = 2, pre = BIN}, 4) + test_vim_str2nr( '0B101', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 5) + test_vim_str2nr( '0B101', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 6) + + test_vim_str2nr( '0B1012', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 0) + test_vim_str2nr( '0B1012', flags, {len = 5, num = 5, unum = 5, pre = BIN}, 6) + + test_vim_str2nr('-0B101', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 0) + test_vim_str2nr('-0B101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-0B101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-0B101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 3) + test_vim_str2nr('-0B101', flags, {len = 4, num = -1, unum = 1, pre = BIN}, 4) + test_vim_str2nr('-0B101', flags, {len = 5, num = -2, unum = 2, pre = BIN}, 5) + test_vim_str2nr('-0B101', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 6) + test_vim_str2nr('-0B101', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 7) + + test_vim_str2nr('-0B1012', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 0) + test_vim_str2nr('-0B1012', flags, {len = 6, num = -5, unum = 5, pre = BIN}, 7) + + if flags > lib.STR2NR_FORCE then + test_vim_str2nr('-101', flags, {len = 4, num = -5, unum = 5, pre = 0}, 0) + end + end + end) + itp('works with octal numbers', function() + for _, flags in ipairs({ + lib.STR2NR_OCT, + lib.STR2NR_OCT + lib.STR2NR_BIN, + lib.STR2NR_OCT + lib.STR2NR_HEX, + lib.STR2NR_ALL, + lib.STR2NR_FORCE + lib.STR2NR_OCT, + }) do + local oct + if flags > lib.STR2NR_FORCE then + oct = 0 + else + oct = ('0'):byte() + end + + -- Check that all digits are recognized + test_vim_str2nr( '012345670', flags, {len = 9, num = 2739128, unum = 2739128, pre = oct}, 0) + + test_vim_str2nr( '054', flags, {len = 3, num = 44, unum = 44, pre = oct}, 0) + test_vim_str2nr( '054', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '054', flags, {len = 2, num = 5, unum = 5, pre = oct}, 2) + test_vim_str2nr( '054', flags, {len = 3, num = 44, unum = 44, pre = oct}, 3) + test_vim_str2nr( '0548', flags, {len = 3, num = 44, unum = 44, pre = oct}, 3) + test_vim_str2nr( '054', flags, {len = 3, num = 44, unum = 44, pre = oct}, 4) + + test_vim_str2nr( '054x', flags, {len = 3, num = 44, unum = 44, pre = oct}, 4) + test_vim_str2nr( '054x', flags, {len = 3, num = 44, unum = 44, pre = oct}, 0) + + test_vim_str2nr('-054', flags, {len = 4, num = -44, unum = 44, pre = oct}, 0) + test_vim_str2nr('-054', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-054', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-054', flags, {len = 3, num = -5, unum = 5, pre = oct}, 3) + test_vim_str2nr('-054', flags, {len = 4, num = -44, unum = 44, pre = oct}, 4) + test_vim_str2nr('-0548', flags, {len = 4, num = -44, unum = 44, pre = oct}, 4) + test_vim_str2nr('-054', flags, {len = 4, num = -44, unum = 44, pre = oct}, 5) + + test_vim_str2nr('-054x', flags, {len = 4, num = -44, unum = 44, pre = oct}, 5) + test_vim_str2nr('-054x', flags, {len = 4, num = -44, unum = 44, pre = oct}, 0) + + if flags > lib.STR2NR_FORCE then + test_vim_str2nr('-54', flags, {len = 3, num = -44, unum = 44, pre = 0}, 0) + test_vim_str2nr('-0548', flags, {len = 4, num = -44, unum = 44, pre = 0}, 5) + test_vim_str2nr('-0548', flags, {len = 4, num = -44, unum = 44, pre = 0}, 0) + else + test_vim_str2nr('-0548', flags, {len = 5, num = -548, unum = 548, pre = 0}, 5) + test_vim_str2nr('-0548', flags, {len = 5, num = -548, unum = 548, pre = 0}, 0) + end + end + end) + itp('works with hexadecimal numbers', function() + for _, flags in ipairs({ + lib.STR2NR_HEX, + lib.STR2NR_HEX + lib.STR2NR_BIN, + lib.STR2NR_HEX + lib.STR2NR_OCT, + lib.STR2NR_ALL, + lib.STR2NR_FORCE + lib.STR2NR_HEX, + }) do + local hex + local HEX + if flags > lib.STR2NR_FORCE then + hex = 0 + HEX = 0 + else + hex = ('x'):byte() + HEX = ('X'):byte() + end + + -- Check that all digits are recognized + test_vim_str2nr('0x12345', flags, {len = 7, num = 74565, unum = 74565, pre = hex}, 0) + test_vim_str2nr('0x67890', flags, {len = 7, num = 424080, unum = 424080, pre = hex}, 0) + test_vim_str2nr('0xABCDEF', flags, {len = 8, num = 11259375, unum = 11259375, pre = hex}, 0) + test_vim_str2nr('0xabcdef', flags, {len = 8, num = 11259375, unum = 11259375, pre = hex}, 0) + + test_vim_str2nr( '0x101', flags, {len = 5, num = 257, unum =257, pre = hex}, 0) + test_vim_str2nr( '0x101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '0x101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr( '0x101', flags, {len = 3, num = 1, unum = 1, pre = hex}, 3) + test_vim_str2nr( '0x101', flags, {len = 4, num = 16, unum = 16, pre = hex}, 4) + test_vim_str2nr( '0x101', flags, {len = 5, num = 257, unum =257, pre = hex}, 5) + test_vim_str2nr( '0x101', flags, {len = 5, num = 257, unum =257, pre = hex}, 6) + + test_vim_str2nr( '0x101G', flags, {len = 5, num = 257, unum =257, pre = hex}, 0) + test_vim_str2nr( '0x101G', flags, {len = 5, num = 257, unum =257, pre = hex}, 6) + + test_vim_str2nr('-0x101', flags, {len = 6, num =-257, unum =257, pre = hex}, 0) + test_vim_str2nr('-0x101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-0x101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-0x101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 3) + test_vim_str2nr('-0x101', flags, {len = 4, num = -1, unum = 1, pre = hex}, 4) + test_vim_str2nr('-0x101', flags, {len = 5, num = -16, unum = 16, pre = hex}, 5) + test_vim_str2nr('-0x101', flags, {len = 6, num =-257, unum =257, pre = hex}, 6) + test_vim_str2nr('-0x101', flags, {len = 6, num =-257, unum =257, pre = hex}, 7) + + test_vim_str2nr('-0x101G', flags, {len = 6, num =-257, unum =257, pre = hex}, 0) + test_vim_str2nr('-0x101G', flags, {len = 6, num =-257, unum =257, pre = hex}, 7) + + test_vim_str2nr( '0X101', flags, {len = 5, num = 257, unum =257, pre = HEX}, 0) + test_vim_str2nr( '0X101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr( '0X101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr( '0X101', flags, {len = 3, num = 1, unum = 1, pre = HEX}, 3) + test_vim_str2nr( '0X101', flags, {len = 4, num = 16, unum = 16, pre = HEX}, 4) + test_vim_str2nr( '0X101', flags, {len = 5, num = 257, unum =257, pre = HEX}, 5) + test_vim_str2nr( '0X101', flags, {len = 5, num = 257, unum =257, pre = HEX}, 6) + + test_vim_str2nr( '0X101G', flags, {len = 5, num = 257, unum =257, pre = HEX}, 0) + test_vim_str2nr( '0X101G', flags, {len = 5, num = 257, unum =257, pre = HEX}, 6) + + test_vim_str2nr('-0X101', flags, {len = 6, num =-257, unum =257, pre = HEX}, 0) + test_vim_str2nr('-0X101', flags, {len = 1, num = 0, unum = 0, pre = 0 }, 1) + test_vim_str2nr('-0X101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 2) + test_vim_str2nr('-0X101', flags, {len = 2, num = 0, unum = 0, pre = 0 }, 3) + test_vim_str2nr('-0X101', flags, {len = 4, num = -1, unum = 1, pre = HEX}, 4) + test_vim_str2nr('-0X101', flags, {len = 5, num = -16, unum = 16, pre = HEX}, 5) + test_vim_str2nr('-0X101', flags, {len = 6, num =-257, unum =257, pre = HEX}, 6) + test_vim_str2nr('-0X101', flags, {len = 6, num =-257, unum =257, pre = HEX}, 7) + + test_vim_str2nr('-0X101G', flags, {len = 6, num =-257, unum =257, pre = HEX}, 0) + test_vim_str2nr('-0X101G', flags, {len = 6, num =-257, unum =257, pre = HEX}, 7) + + if flags > lib.STR2NR_FORCE then + test_vim_str2nr('-101', flags, {len = 4, num = -257, unum = 257, pre = 0}, 0) + end + end + end) +end) From f2660bee6aca35be3d0ddb1d225784476c13cd27 Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 6 Nov 2017 20:20:31 +0300 Subject: [PATCH 078/109] *: Fix some typos found by oni-link --- src/nvim/ex_getln.c | 1 - src/nvim/generators/gen_declarations.lua | 6 +++--- src/nvim/keymap.c | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c index f64efe08a1..3b751530e8 100644 --- a/src/nvim/ex_getln.c +++ b/src/nvim/ex_getln.c @@ -2501,7 +2501,6 @@ static bool color_cmdline(CmdlineInfo *colored_ccline) can_free_cb = true; } else if (colored_ccline->cmdfirstc == '=') { color_expr_cmdline(colored_ccline, ccline_colors); - can_free_cb = false; } if (!tl_ret || !dgc_ret) { goto color_cmdline_error; diff --git a/src/nvim/generators/gen_declarations.lua b/src/nvim/generators/gen_declarations.lua index 0c73376ba0..065c043557 100755 --- a/src/nvim/generators/gen_declarations.lua +++ b/src/nvim/generators/gen_declarations.lua @@ -169,10 +169,10 @@ Usage: gendeclarations.lua definitions.c static.h non-static.h definitions.i -Generates declarations for a C file defitions.c, putting declarations for +Generates declarations for a C file definitions.c, putting declarations for static functions into static.h and declarations for non-static functions into non-static.h. File `definitions.i' should contain an already preprocessed -version of defintions.c and it is the only one which is actually parsed, +version of definitions.c and it is the only one which is actually parsed, definitions.c is needed only to determine functions from which file out of all functions found in definitions.i are needed. @@ -181,7 +181,7 @@ Additionally uses the following environment variables: NVIM_GEN_DECLARATIONS_LINE_NUMBERS: If set to 1 then all generated declarations receive a comment with file name and line number after the declaration. This may be useful for - debugging gen_declarations script, but not much beyound that with + debugging gen_declarations script, but not much beyond that with configured development environment (i.e. with ctags/cscope/finding definitions with clang/etc). diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index 0c8e47b02e..aca21c20a5 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -714,7 +714,7 @@ int find_special_key_in_table(int c) /// with "t_" the next two characters are interpreted as /// a termcap name. /// -/// @return Key code or 0 if ton found. +/// @return Key code or 0 if not found. int get_special_key_code(const char_u *name) FUNC_ATTR_NONNULL_ALL FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT { From 4aebd00a9eeeb2f56ff53dd4e383825e997ee7be Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 6 Nov 2017 20:28:37 +0300 Subject: [PATCH 079/109] *: Fix linter errors --- src/nvim/api/vim.c | 20 ++++++++++---------- src/nvim/viml/parser/expressions.c | 2 +- test/helpers.lua | 6 +++--- test/unit/charset/vim_str2nr_spec.lua | 1 - test/unit/viml/helpers.lua | 1 - 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 0596c3ebd8..bac19ef363 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -930,7 +930,7 @@ typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack; /// @note: “Sucessfully parsed” here means “participated in AST /// creation”, not “till the first error”. /// -/// "ast": actual AST, either nil or a dictionary with the following +/// "ast": actual AST, either nil or a dictionary with the following /// keys: /// /// "type": node type, one of the value names from ExprASTNodeType @@ -1009,10 +1009,10 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, ExprAST east = viml_pexpr_parse(&pstate, pflags); const size_t ret_size = ( - 2 // "ast", "len" - + (size_t)(east.err.msg != NULL) // "error" - + (size_t)highlight // "highlight" - ); + 2 // "ast", "len" + + (size_t)(east.err.msg != NULL) // "error" + + (size_t)highlight // "highlight" + + 0); Dictionary ret = { .items = xmalloc(ret_size * sizeof(ret.items[0])), .size = 0, @@ -1242,12 +1242,12 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, ret_node->items[ret_node->size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("cmp_type"), .value = STRING_OBJ(cstr_to_string( - eltkn_cmp_type_tab[node->data.cmp.type])), + eltkn_cmp_type_tab[node->data.cmp.type])), }; ret_node->items[ret_node->size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("ccs_strategy"), .value = STRING_OBJ(cstr_to_string( - ccs_tab[node->data.cmp.ccs])), + ccs_tab[node->data.cmp.ccs])), }; ret_node->items[ret_node->size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("invert"), @@ -1266,9 +1266,9 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, ret_node->items[ret_node->size++] = (KeyValuePair) { .key = STATIC_CSTR_TO_STRING("ivalue"), .value = INTEGER_OBJ((Integer)( - node->data.num.value > API_INTEGER_MAX - ? API_INTEGER_MAX - : (Integer)node->data.num.value)), + node->data.num.value > API_INTEGER_MAX + ? API_INTEGER_MAX + : (Integer)node->data.num.value)), }; break; } diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 998edb1ed4..71eda2cdc1 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -22,7 +22,7 @@ // ignored in Neovim it is an error. // 4. Expressions parser has generally better error reporting. But for // compatibility reasons most errors have error code E15 while error messages -// are significantly different from Vim’s E15. Also some error codes were +// are significantly different from Vim’s E15. Also some error codes were // retired because of being harder to emulate or because of them being // a result of differences in parsing process: e.g. with ":echo {a, b}" Vim // will attempt to parse expression as lambda, fail, check whether it is diff --git a/test/helpers.lua b/test/helpers.lua index 16b9818f12..d24fae745b 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -301,10 +301,10 @@ local function mergedicts_copy(d1, d2) for k, v in pairs(d2) do if d2[k] == REMOVE_THIS then ret[k] = nil - elseif type(d1[k]) == 'table' and type(d2[k]) == 'table' then - ret[k] = mergedicts_copy(d1[k], d2[k]) + elseif type(d1[k]) == 'table' and type(v) == 'table' then + ret[k] = mergedicts_copy(d1[k], v) else - ret[k] = d2[k] + ret[k] = v end end return ret diff --git a/test/unit/charset/vim_str2nr_spec.lua b/test/unit/charset/vim_str2nr_spec.lua index 9309dc380c..22504649f6 100644 --- a/test/unit/charset/vim_str2nr_spec.lua +++ b/test/unit/charset/vim_str2nr_spec.lua @@ -5,7 +5,6 @@ local itp = helpers.gen_itp(it) local cimport = helpers.cimport local ffi = helpers.ffi -local eq = helpers.eq local shallowcopy = global_helpers.shallowcopy local updated = global_helpers.updated diff --git a/test/unit/viml/helpers.lua b/test/unit/viml/helpers.lua index aeb886a66f..9d2d7b61c7 100644 --- a/test/unit/viml/helpers.lua +++ b/test/unit/viml/helpers.lua @@ -112,7 +112,6 @@ return { pline2lua = pline2lua, pstate_str = pstate_str, new_pstate = new_pstate, - intchar2lua = intchar2lua, conv_cmp_type = conv_cmp_type, pstate_set_str = pstate_set_str, } From bbb21e5dd3891727c272ffd3aa4ce2a4841a1f0b Mon Sep 17 00:00:00 2001 From: ZyX Date: Sat, 11 Nov 2017 23:50:37 +0300 Subject: [PATCH 080/109] unittests: Add a way to show some custom messages only when crashed --- test/unit/helpers.lua | 17 ++++++++++++++++- test/unit/viml/expressions/parser_spec.lua | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index 96aa505739..2c148630dd 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -529,9 +529,13 @@ local hook_numlen = 5 local hook_msglen = 1 + 1 + 1 + (1 + hook_fnamelen) + (1 + hook_sfnamelen) + (1 + hook_numlen) + 1 local tracehelp = dedent([[ + Trace: either in the format described below or custom debug output starting + with `>`. Latter lines still have the same width in byte. + ┌ Trace type: _r_eturn from function , function _c_all, _l_ine executed, │ _t_ail return, _C_ount (should not actually appear), - │ _s_aved from previous run for reference. + │ _s_aved from previous run for reference, _>_ for custom debug + │ output. │┏ Function type: _L_ua function, _C_ function, _m_ain part of chunk, │┃ function that did _t_ail call. │┃┌ Function name type: _g_lobal, _l_ocal, _m_ethod, _f_ield, _u_pvalue, @@ -629,7 +633,17 @@ end local trace_end_msg = ('E%s\n'):format((' '):rep(hook_msglen - 2)) +local _debug_log + +local debug_log = only_separate(function(...) + return _debug_log(...) +end) + local function itp_child(wr, func) + _debug_log = function(s) + s = s:sub(1, hook_msglen - 2) + sc.write(wr, '>' .. s .. (' '):rep(hook_msglen - 2 - #s) .. '\n') + end init() collectgarbage('stop') child_sethook(wr) @@ -845,6 +859,7 @@ local module = { make_enum_conv_tab = make_enum_conv_tab, ptr2addr = ptr2addr, ptr2key = ptr2key, + debug_log = debug_log, } return function() return module diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index e5d0f2b84c..cfc9fe95ac 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -8,6 +8,7 @@ local child_call_once = helpers.child_call_once local alloc_log_new = helpers.alloc_log_new local kvi_destroy = helpers.kvi_destroy local conv_enum = helpers.conv_enum +local debug_log = helpers.debug_log local ptr2key = helpers.ptr2key local cimport = helpers.cimport local ffi = helpers.ffi @@ -233,6 +234,7 @@ describe('Expressions parser', function() local function check_parsing(str, exp_ast, exp_highlighting_fs, nz_flags_exps) nz_flags_exps = nz_flags_exps or {} for _, flags in ipairs({0, 1, 2, 3}) do + debug_log(('Running test case (%s, %u)'):format(str, flags)) local err, msg = pcall(function() if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then print(str, flags) From 1aa6276c29d562a6287519e6755a613eabca5c31 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 12 Nov 2017 00:03:45 +0300 Subject: [PATCH 081/109] viml/parser/expressions: Replace lambda-specific WantedNode entries This way code will be easier to adapt to handling (partially) non-expressions like :let lvalue part or :function definitions, and that would be needed in the future both for proper completion support and for the Ex commands parser. --- src/nvim/viml/parser/expressions.c | 101 +++++++++++++++-------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 71eda2cdc1..e23c58bfd1 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -73,12 +73,21 @@ typedef enum { /// For unrestricted expressions as well, implies that top item in AST stack /// points to NULL. kENodeValue, - /// Argument: only allows simple argument names. - kENodeArgument, - /// Argument separator: only allows commas. - kENodeArgumentSeparator, } ExprASTWantedNode; +/// Parse type: what is being parsed currently +typedef enum { + /// Parsing regular VimL expression + kEPTExpr = 0, + /// Parsing lambda arguments + /// + /// Just like parsing function arguments, but it is valid to be ended with an + /// arrow only. + kEPTLambdaArguments, +} ExprASTParseType; + +typedef kvec_withinit_t(ExprASTParseType, 4) ExprASTParseTypeStack; + /// Operator priority level typedef enum { kEOpLvlInvalid = 0, @@ -1238,9 +1247,7 @@ static bool viml_pexpr_handle_bop(const ParserState *const pstate, ret = false; } } - *want_node_p = (*want_node_p == kENodeArgumentSeparator - ? kENodeArgument - : kENodeValue); + *want_node_p = kENodeValue; return ret; } @@ -1790,8 +1797,6 @@ static void parse_quoted_string(ParserState *const pstate, static const int want_node_to_lexer_flags[] = { [kENodeValue] = kELFlagIsNotCmp, [kENodeOperator] = kELFlagForbidScope, - [kENodeArgument] = kELFlagIsNotCmp, - [kENodeArgumentSeparator] = kELFlagForbidScope, }; /// Number of characters to highlight as NumberPrefix depending on the base @@ -1827,12 +1832,13 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) // (list start) last stack item contains NULL. Otherwise last stack item is // supposed to contain last “finished” value: e.g. "1" or "+(1, 1)" (node // representing "1+1"). - // - // Both kENodeValue and kENodeArgument stand for “value” nodes. ExprASTStack ast_stack; kvi_init(ast_stack); kvi_push(ast_stack, &ast.root); ExprASTWantedNode want_node = kENodeValue; + ExprASTParseTypeStack pt_stack; + kvi_init(pt_stack); + kvi_push(pt_stack, kEPTExpr); LexExprToken prev_token = { .type = kExprLexMissing }; bool highlighted_prev_spacing = false; // Lambda node, valid when parsing lambda arguments only. @@ -1884,8 +1890,7 @@ viml_pexpr_parse_process_token: assert(kv_size(ast_stack) >= 1); ExprASTNode *cur_node = NULL; #ifndef NDEBUG - const bool want_value = ( - want_node == kENodeValue || want_node == kENodeArgument); + const bool want_value = (want_node == kENodeValue); assert(want_value == (*top_node_p == NULL)); assert(kv_A(ast_stack, 0) == &ast.root); // Check that stack item i + 1 points to stack items’ i *last* child. @@ -1933,14 +1938,15 @@ viml_pexpr_parse_process_token: // circumstances, and in any case runtime and not parse time errors. (*kv_Z(ast_stack, 1))->type = kExprNodeConcat; } - if ((want_node == kENodeArgumentSeparator - && tok_type != kExprLexComma - && tok_type != kExprLexArrow) - || (want_node == kENodeArgument - && !(cur_token.type == kExprLexPlainIdentifier - && cur_token.data.var.scope == kExprVarScopeMissing - && !cur_token.data.var.autoload) - && tok_type != kExprLexArrow)) { + if (kv_last(pt_stack) == kEPTLambdaArguments + && ((want_node == kENodeOperator + && tok_type != kExprLexComma + && tok_type != kExprLexArrow) + || (want_node == kENodeValue + && !(cur_token.type == kExprLexPlainIdentifier + && cur_token.data.var.scope == kExprVarScopeMissing + && !cur_token.data.var.autoload) + && tok_type != kExprLexArrow))) { lambda_node->data.fig.type_guesses.allow_lambda = false; if (lambda_node->children != NULL && lambda_node->children->type == kExprNodeComma) { @@ -1956,16 +1962,11 @@ viml_pexpr_parse_process_token: // Else it may appear that possibly-lambda node is actually a dictionary // or curly-braces-name identifier. lambda_node = NULL; - if (want_node == kENodeArgumentSeparator) { - want_node = kENodeOperator; - } else { - want_node = kENodeValue; - } + kv_drop(pt_stack, 1); } } - assert(lambda_node == NULL - || want_node == kENodeArgumentSeparator - || want_node == kENodeArgument); + const ExprASTParseType cur_pt = kv_last(pt_stack); + assert(lambda_node == NULL || cur_pt == kEPTLambdaArguments); switch (tok_type) { case kExprLexMissing: case kExprLexSpacing: @@ -2129,7 +2130,7 @@ viml_pexpr_parse_process_token: break; } case kExprLexComma: { - assert(want_node != kENodeArgument); + assert(!(want_node == kENodeValue && cur_pt == kEPTLambdaArguments)); if (want_node == kENodeValue) { // Value level: comma appearing here is not valid. // Note: in Vim string(,x) will give E116, this is not the case here. @@ -2138,13 +2139,11 @@ viml_pexpr_parse_process_token: NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeMissing); cur_node->len = 0; *top_node_p = cur_node; - want_node = (want_node == kENodeArgument - ? kENodeArgumentSeparator - : kENodeOperator); + want_node = kENodeOperator; } - if (want_node == kENodeArgumentSeparator) { - assert(lambda_node->data.fig.type_guesses.allow_lambda); + if (cur_pt == kEPTLambdaArguments) { assert(lambda_node != NULL); + assert(lambda_node->data.fig.type_guesses.allow_lambda); SELECT_FIGURE_BRACE_TYPE(lambda_node, Lambda, Lambda); } if (kv_size(ast_stack) < 2) { @@ -2156,7 +2155,8 @@ viml_pexpr_parse_process_token: const ExprASTNodeType eastnode_type = (*eastnode_p)->type; const ExprOpLvl eastnode_lvl = node_lvl(**eastnode_p); if (eastnode_type == kExprNodeLambda) { - assert(want_node == kENodeArgumentSeparator); + assert(cur_pt == kEPTLambdaArguments + && want_node == kENodeOperator); break; } else if (eastnode_type == kExprNodeDictLiteral || eastnode_type == kExprNodeListLiteral @@ -2410,9 +2410,6 @@ viml_pexpr_parse_bracket_closing_error: case kExprNodeUnknownFigure: { if (new_top_node->children == NULL) { // No children of curly braces node indicates empty dictionary. - - // Should actually be kENodeArgument, but that was changed - // earlier. assert(want_node == kENodeValue); assert(new_top_node->data.fig.type_guesses.allow_dict); SELECT_FIGURE_BRACE_TYPE(new_top_node, DictLiteral, Dict); @@ -2475,7 +2472,7 @@ viml_pexpr_parse_figure_brace_closing_error: } *top_node_p = cur_node; kvi_push(ast_stack, &cur_node->children); - want_node = kENodeArgument; + kvi_push(pt_stack, kEPTLambdaArguments); lambda_node = cur_node; } else { ADD_IDENT( @@ -2495,9 +2492,12 @@ viml_pexpr_parse_figure_brace_closing_error: break; } case kExprLexArrow: { - if (want_node == kENodeArgumentSeparator - || want_node == kENodeArgument) { - if (want_node == kENodeArgument) { + if (cur_pt == kEPTLambdaArguments) { + kv_drop(pt_stack, 1); + assert(kv_size(pt_stack)); + if (want_node == kENodeValue) { + // Wanting value means trailing comma and NULL at the top of the + // stack. kv_drop(ast_stack, 1); } assert(kv_size(ast_stack) >= 1); @@ -2509,7 +2509,7 @@ viml_pexpr_parse_figure_brace_closing_error: SELECT_FIGURE_BRACE_TYPE(lambda_node, Lambda, Lambda); NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeArrow); if (lambda_node->children == NULL) { - assert(want_node == kENodeArgument); + assert(want_node == kENodeValue); lambda_node->children = cur_node; kvi_push(ast_stack, &lambda_node->children); } else { @@ -2535,10 +2535,8 @@ viml_pexpr_parse_figure_brace_closing_error: const ExprVarScope scope = (cur_token.type == kExprLexInvalid ? kExprVarScopeMissing : cur_token.data.var.scope); - if (want_node == kENodeValue || want_node == kENodeArgument) { - want_node = (want_node == kENodeArgument - ? kENodeArgumentSeparator - : kENodeOperator); + if (want_node == kENodeValue) { + want_node = kENodeOperator; NEW_NODE_WITH_CUR_POS(cur_node, (node_is_key ? kExprNodePlainKey @@ -2755,7 +2753,12 @@ viml_pexpr_parse_cycle_end: viml_parser_advance(pstate, cur_token.len); } while (true); viml_pexpr_parse_end: - if (want_node == kENodeValue) { + assert(kv_size(pt_stack)); + assert(kv_size(ast_stack)); + if (want_node == kENodeValue + // Blacklist some parse type entries as their presence means better error + // message in the other branch. + && kv_last(pt_stack) != kEPTLambdaArguments) { east_set_error(pstate, &ast.err, _("E15: Expected value, got EOC: %.*s"), pstate->pos); } else if (kv_size(ast_stack) != 1) { From c7495ebcc0918ffd682083408895451318e41d1f Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 12 Nov 2017 02:18:43 +0300 Subject: [PATCH 082/109] viml/parser/expressions: Add support for parsing assignments --- src/nvim/api/vim.c | 37 ++- src/nvim/syntax.c | 20 ++ src/nvim/viml/parser/expressions.c | 237 ++++++++++++++++--- src/nvim/viml/parser/expressions.h | 40 +++- test/symbolic/klee/viml_expressions_parser.c | 3 +- test/unit/viml/expressions/lexer_spec.lua | 10 +- test/unit/viml/expressions/parser_spec.lua | 23 +- test/unit/viml/helpers.lua | 13 + 8 files changed, 325 insertions(+), 58 deletions(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index bac19ef363..ce554d351c 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -900,14 +900,21 @@ typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack; /// Parse a VimL expression /// /// @param[in] expr Expression to parse. Is always treated as a single line. -/// @param[in] flags Flags: "m" if multiple expressions in a row are allowed -/// (only the first one will be parsed), "E" if EOC tokens -/// are not allowed (determines whether they will stop -/// parsing process or be recognized as an operator/space, -/// though also yielding an error). +/// @param[in] flags Flags: /// -/// Use only "m" to parse like for "=", only "E" to -/// parse like for ":echo", empty string for ":let". +/// - "m" if multiple expressions in a row are allowed (only +/// the first one will be parsed), +/// - "E" if EOC tokens are not allowed (determines whether +/// they will stop parsing process or be recognized as an +/// operator/space, though also yielding an error). +/// - "l" when needing to start parsing with lvalues for +/// ":let" or ":for". +/// +/// Common flag sets: +/// - "m" to parse like for ":echo". +/// - "E" to parse like for "=". +/// - empty string for ":call". +/// - "lm" to parse for ":let". /// @param[in] highlight If true, return value will also include "highlight" /// key containing array of 4-tuples (arrays) (Integer, /// Integer, Integer, String), where first three numbers @@ -964,6 +971,9 @@ typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack; /// value names from ExprCaseCompareStrategy, /// stringified without "kCCStrategy" prefix. Only /// present for "Comparison" nodes. +/// "augmentation": String, augmentation type for "Assignment" nodes. +/// Is either an empty string, "Add", "Subtract" or +/// "Concat" for "=", "+=", "-=" or ".=" respectively. /// "invert": Boolean, true if result of comparison needs to be /// inverted. Only present for "Comparison" nodes. /// "ivalue": Integer, integer value for "Integer" nodes. @@ -979,6 +989,7 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, switch (flags.data[i]) { case 'm': { pflags |= kExprFlagsMulti; break; } case 'E': { pflags |= kExprFlagsDisallowEOC; break; } + case 'l': { pflags |= kExprFlagsParseLet; break; } case NUL: { api_set_error(err, kErrorTypeValidation, "Invalid flag: '\\0' (%u)", (unsigned)flags.data[i]); @@ -1114,6 +1125,7 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, + (node->type == kExprNodeFloat) // "fvalue" + (node->type == kExprNodeDoubleQuotedString || node->type == kExprNodeSingleQuotedString) // "svalue" + + (node->type == kExprNodeAssignment) // "augmentation" + 0); Dictionary ret_node = { .items = xmalloc(ret_node_items_size * sizeof(ret_node.items[0])), @@ -1272,6 +1284,17 @@ Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, }; break; } + case kExprNodeAssignment: { + const ExprAssignmentType asgn_type = node->data.ass.type; + ret_node->items[ret_node->size++] = (KeyValuePair) { + .key = STATIC_CSTR_TO_STRING("augmentation"), + .value = STRING_OBJ( + asgn_type == kExprAsgnPlain + ? (String)STRING_INIT + : cstr_to_string(expr_asgn_type_tab[asgn_type])), + }; + break; + } case kExprNodeMissing: case kExprNodeOpMissing: case kExprNodeTernary: diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index e0bf74567d..9f98b26905 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -6021,11 +6021,21 @@ static const char *highlight_init_dark[] = { }; static const char *highlight_init_cmdline[] = { + // XXX When modifying a list modify it in both valid and invalid halfs. + // TODO(ZyX-I): merge valid and invalid groups via a macros. + // NVimInternalError should appear only when highlighter has a bug. "NVimInternalError ctermfg=Red ctermbg=Red guifg=Red guibg=Red", // Highlight groups (links) used by parser: + "default link NVimAssignment Operator", + "default link NVimPlainAssignment NVimAssignment", + "default link NVimAugmentedAssignment NVimAssignment", + "default link NVimAssignmentWithAddition NVimAugmentedAssignment", + "default link NVimAssignmentWithSubtraction NVimAugmentedAssignment", + "default link NVimAssignmentWithConcatenation NVimAugmentedAssignment", + "default link NVimOperator Operator", "default link NVimUnaryOperator NVimOperator", @@ -6113,6 +6123,16 @@ static const char *highlight_init_cmdline[] = { "default link NVimInvalid Error", + "default link NVimInvalidAssignment NVimInvalid", + "default link NVimInvalidPlainAssignment NVimInvalidAssignment", + "default link NVimInvalidAugmentedAssignment NVimInvalidAssignment", + "default link NVimInvalidAssignmentWithAddition " + "NVimInvalidAugmentedAssignment", + "default link NVimInvalidAssignmentWithSubtraction " + "NVimInvalidAugmentedAssignment", + "default link NVimInvalidAssignmentWithConcatenation " + "NVimInvalidAugmentedAssignment", + "default link NVimInvalidOperator NVimInvalid", "default link NVimInvalidUnaryOperator NVimInvalidOperator", diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index e23c58bfd1..13f7131744 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -84,6 +84,10 @@ typedef enum { /// Just like parsing function arguments, but it is valid to be ended with an /// arrow only. kEPTLambdaArguments, + /// Assignment: parsing for :let + kEPTAssignment, + /// Single assignment: used when lists are not allowed (i.e. when nesting) + kEPTSingleAssignment, } ExprASTParseType; typedef kvec_withinit_t(ExprASTParseType, 4) ExprASTParseTypeStack; @@ -93,6 +97,7 @@ typedef enum { kEOpLvlInvalid = 0, kEOpLvlComplexIdentifier, kEOpLvlParens, + kEOpLvlAssignment, kEOpLvlArrow, kEOpLvlComma, kEOpLvlColon, @@ -217,8 +222,6 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) } CHAR(kExprLexQuestion, '?') CHAR(kExprLexColon, ':') - CHAR(kExprLexDot, '.') - CHAR(kExprLexPlus, '+') CHAR(kExprLexComma, ',') #undef CHAR @@ -532,12 +535,8 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) case '!': case '=': { if (pline.size == 1) { -viml_pexpr_next_token_invalid_comparison: - ret.type = (schar == '!' ? kExprLexNot : kExprLexInvalid); - if (ret.type == kExprLexInvalid) { - ret.data.err.msg = _("E15: Expected == or =~: %.*s"); - ret.data.err.type = kExprLexComparison; - } + ret.type = (schar == '!' ? kExprLexNot : kExprLexAssignment); + ret.data.ass.type = kExprAsgnPlain; break; } ret.type = kExprLexComparison; @@ -548,8 +547,11 @@ viml_pexpr_next_token_invalid_comparison: } else if (pline.data[1] == '~') { ret.data.cmp.type = kExprCmpMatches; ret.len++; + } else if (schar == '!') { + ret.type = kExprLexNot; } else { - goto viml_pexpr_next_token_invalid_comparison; + ret.type = kExprLexAssignment; + ret.data.ass.type = kExprAsgnPlain; } GET_CCS(ret, pline); break; @@ -571,17 +573,37 @@ viml_pexpr_next_token_invalid_comparison: break; } - // Minus sign or arrow from lambdas. + // Minus sign, arrow from lambdas or augmented assignment. case '-': { if (pline.size > 1 && pline.data[1] == '>') { ret.len++; ret.type = kExprLexArrow; + } else if (pline.size > 1 && pline.data[1] == '=') { + ret.len++; + ret.type = kExprLexAssignment; + ret.data.ass.type = kExprAsgnSubtract; } else { ret.type = kExprLexMinus; } break; } + // Sign or augmented assignment. +#define CHAR_OR_ASSIGN(ch, ch_type, ass_type) \ + case ch: { \ + if (pline.size > 1 && pline.data[1] == '=') { \ + ret.len++; \ + ret.type = kExprLexAssignment; \ + ret.data.ass.type = ass_type; \ + } else { \ + ret.type = ch_type; \ + } \ + break; \ + } + CHAR_OR_ASSIGN('+', kExprLexPlus, kExprAsgnAdd) + CHAR_OR_ASSIGN('.', kExprLexDot, kExprAsgnConcat) +#undef CHAR_OR_ASSIGN + // Expression end because Ex command ended. case NUL: case NL: { @@ -661,6 +683,7 @@ static const char *const eltkn_type_tab[] = { [kExprLexParenthesis] = "Parenthesis", [kExprLexComma] = "Comma", [kExprLexArrow] = "Arrow", + [kExprLexAssignment] = "Assignment", }; const char *const eltkn_cmp_type_tab[] = { @@ -671,6 +694,13 @@ const char *const eltkn_cmp_type_tab[] = { [kExprCmpIdentical] = "Identical", }; +const char *const expr_asgn_type_tab[] = { + [kExprAsgnPlain] = "Plain", + [kExprAsgnAdd] = "Add", + [kExprAsgnSubtract] = "Subtract", + [kExprAsgnConcat] = "Concat", +}; + const char *const ccs_tab[] = { [kCCStrategyUseOption] = "UseOption", [kCCStrategyMatchCase] = "MatchCase", @@ -732,6 +762,8 @@ const char *viml_pexpr_repr_token(const ParserState *const pstate, (int)token.data.cmp.inv) TKNARGS(kExprLexMultiplication, "(type=%s)", eltkn_mul_type_tab[token.data.mul.type]) + TKNARGS(kExprLexAssignment, "(type=%s)", + expr_asgn_type_tab[token.data.ass.type]) TKNARGS(kExprLexRegister, "(name=%s)", intchar2str(token.data.reg.name)) case kExprLexDoubleQuotedString: TKNARGS(kExprLexSingleQuotedString, "(closed=%i)", @@ -811,6 +843,7 @@ const char *const east_node_type_tab[] = { [kExprNodeMod] = "Mod", [kExprNodeOption] = "Option", [kExprNodeEnvironment] = "Environment", + [kExprNodeAssignment] = "Assignment", }; /// Represent `int` character as a string @@ -933,6 +966,7 @@ const uint8_t node_maxchildren[] = { [kExprNodeMod] = 2, [kExprNodeOption] = 0, [kExprNodeEnvironment] = 0, + [kExprNodeAssignment] = 2, }; /// Free memory occupied by AST @@ -993,6 +1027,7 @@ void viml_pexpr_free_ast(ExprAST ast) case kExprNodeLambda: case kExprNodeDictLiteral: case kExprNodeCurlyBracesIdentifier: + case kExprNodeAssignment: case kExprNodeComma: case kExprNodeColon: case kExprNodeArrow: @@ -1111,6 +1146,8 @@ static struct { [kExprNodeCurlyBracesIdentifier] = { kEOpLvlComplexIdentifier, kEOpAssLeft }, + [kExprNodeAssignment] = { kEOpLvlAssignment, kEOpAssLeft }, + [kExprNodeComplexIdentifier] = { kEOpLvlValue, kEOpAssLeft }, [kExprNodePlainIdentifier] = { kEOpLvlValue, kEOpAssNo }, @@ -1478,6 +1515,17 @@ static inline void east_set_error(const ParserState *const pstate, } \ } while (0) +/// Determine whether given parse type is an assignment +/// +/// @param[in] pt Checked parse type. +/// +/// @return true if parsing an assignment, false otherwise. +static inline bool pt_is_assignment(const ExprASTParseType pt) + FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT +{ + return (pt == kEPTAssignment || pt == kEPTSingleAssignment); +} + /// Structure used to define “string shifts” necessary to map string /// highlighting to actual strings. typedef struct { @@ -1839,6 +1887,9 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) ExprASTParseTypeStack pt_stack; kvi_init(pt_stack); kvi_push(pt_stack, kEPTExpr); + if (flags & kExprFlagsParseLet) { + kvi_push(pt_stack, kEPTAssignment); + } LexExprToken prev_token = { .type = kExprLexMissing }; bool highlighted_prev_spacing = false; // Lambda node, valid when parsing lambda arguments only. @@ -1938,33 +1989,83 @@ viml_pexpr_parse_process_token: // circumstances, and in any case runtime and not parse time errors. (*kv_Z(ast_stack, 1))->type = kExprNodeConcat; } - if (kv_last(pt_stack) == kEPTLambdaArguments - && ((want_node == kENodeOperator + // Pop some stack pt_stack items in case of misplaced nodes. + const bool is_single_assignment = kv_last(pt_stack) == kEPTSingleAssignment; + switch (kv_last(pt_stack)) { + case kEPTExpr: { + break; + } + case kEPTLambdaArguments: { + if ((want_node == kENodeOperator && tok_type != kExprLexComma && tok_type != kExprLexArrow) || (want_node == kENodeValue && !(cur_token.type == kExprLexPlainIdentifier && cur_token.data.var.scope == kExprVarScopeMissing && !cur_token.data.var.autoload) - && tok_type != kExprLexArrow))) { - lambda_node->data.fig.type_guesses.allow_lambda = false; - if (lambda_node->children != NULL - && lambda_node->children->type == kExprNodeComma) { - // If lambda has comma child this means that parser has already seen at - // least "{arg1,", so node cannot possibly be anything, but lambda. + && tok_type != kExprLexArrow)) { + lambda_node->data.fig.type_guesses.allow_lambda = false; + if (lambda_node->children != NULL + && lambda_node->children->type == kExprNodeComma) { + // If lambda has comma child this means that parser has already seen at + // least "{arg1,", so node cannot possibly be anything, but lambda. - // Vim may give E121 or E720 in this case, but it does not look right to - // have either because both are results of reevaluation possibly-lambda - // node as a dictionary and here this is not going to happen. - ERROR_FROM_TOKEN_AND_MSG( - cur_token, _("E15: Expected lambda arguments list or arrow: %.*s")); - } else { - // Else it may appear that possibly-lambda node is actually a dictionary - // or curly-braces-name identifier. - lambda_node = NULL; - kv_drop(pt_stack, 1); + // Vim may give E121 or E720 in this case, but it does not look right to + // have either because both are results of reevaluation possibly-lambda + // node as a dictionary and here this is not going to happen. + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Expected lambda arguments list or arrow: %.*s")); + } else { + // Else it may appear that possibly-lambda node is actually a dictionary + // or curly-braces-name identifier. + lambda_node = NULL; + kv_drop(pt_stack, 1); + } + } + break; + } + case kEPTSingleAssignment: { + if (tok_type == kExprLexBracket && !cur_token.data.brc.closing) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E475: Nested lists not allowed when assigning: %.*s")); + kv_drop(pt_stack, 2); + assert(kv_size(pt_stack)); + assert(kv_last(pt_stack) == kEPTExpr); + break; + } + FALLTHROUGH; + } + case kEPTAssignment: { + if (want_node == kENodeValue + && tok_type != kExprLexBracket + && tok_type != kExprLexPlainIdentifier + && (tok_type != kExprLexFigureBrace || cur_token.data.brc.closing) + && !(node_is_key && tok_type == kExprLexNumber) + && tok_type != kExprLexEnv + && tok_type != kExprLexOption + && tok_type != kExprLexRegister) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected value part of assignment lvalue: %.*s")); + kv_drop(pt_stack, 1); + } else if (want_node == kENodeOperator + && tok_type != kExprLexBracket + && (tok_type != kExprLexFigureBrace + || cur_token.data.brc.closing) + && tok_type != kExprLexDot + && (tok_type != kExprLexComma || !is_single_assignment) + && tok_type != kExprLexAssignment) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E15: Expected assignment operator or subscript: %.*s")); + kv_drop(pt_stack, 1); + } + assert(kv_size(pt_stack)); + break; } } + assert(kv_size(pt_stack)); const ExprASTParseType cur_pt = kv_last(pt_stack); assert(lambda_node == NULL || cur_pt == kEPTLambdaArguments); switch (tok_type) { @@ -2339,21 +2440,41 @@ viml_pexpr_parse_bracket_closing_error: } kvi_push(ast_stack, new_top_node_p); want_node = kENodeOperator; + if (cur_pt == kEPTSingleAssignment) { + kv_drop(pt_stack, 1); + } else if (cur_pt == kEPTAssignment) { + assert(ast.err.msg); + } else if (cur_pt == kEPTExpr + && kv_size(pt_stack) > 1 + && pt_is_assignment(kv_Z(pt_stack, 1))) { + kv_drop(pt_stack, 1); + } } else { if (want_node == kENodeValue) { - // Value means list literal. + // Value means list literal or list assignment. HL_CUR_TOKEN(List); NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeListLiteral); *top_node_p = cur_node; kvi_push(ast_stack, &cur_node->children); want_node = kENodeValue; + if (cur_pt == kEPTAssignment) { + // Additional assignment parse type allows to easily forbid nested + // lists. + kvi_push(pt_stack, kEPTSingleAssignment); + } } else { + // Operator means subscript, also in assignment. But in assignment + // subscript may be pretty much any expression, so need to push + // kEPTExpr. if (prev_token.type == kExprLexSpacing) { OP_MISSING; } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeSubscript); ADD_OP_NODE(cur_node); HL_CUR_TOKEN(SubscriptBracket); + if (pt_is_assignment(cur_pt)) { + kvi_push(pt_stack, kEPTExpr); + } } } break; @@ -2458,15 +2579,31 @@ viml_pexpr_parse_figure_brace_closing_error: } kvi_push(ast_stack, new_top_node_p); want_node = kENodeOperator; + if (cur_pt == kEPTExpr + && kv_size(pt_stack) > 1 + && pt_is_assignment(kv_Z(pt_stack, 1))) { + kv_drop(pt_stack, 1); + } } else { if (want_node == kENodeValue) { HL_CUR_TOKEN(FigureBrace); // Value: may be any of lambda, dictionary literal and curly braces // name. - NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnknownFigure); - cur_node->data.fig.type_guesses.allow_lambda = true; - cur_node->data.fig.type_guesses.allow_dict = true; - cur_node->data.fig.type_guesses.allow_ident = true; + + // Though if we are in an assignment this may only be a curly braces + // name. + if (pt_is_assignment(cur_pt)) { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeCurlyBracesIdentifier); + cur_node->data.fig.type_guesses.allow_lambda = false; + cur_node->data.fig.type_guesses.allow_dict = false; + cur_node->data.fig.type_guesses.allow_ident = true; + kvi_push(pt_stack, kEPTExpr); + } else { + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeUnknownFigure); + cur_node->data.fig.type_guesses.allow_lambda = true; + cur_node->data.fig.type_guesses.allow_dict = true; + cur_node->data.fig.type_guesses.allow_ident = true; + } if (pstate->colors) { cur_node->data.fig.opening_hl_idx = kv_size(*pstate->colors) - 1; } @@ -2484,6 +2621,9 @@ viml_pexpr_parse_figure_brace_closing_error: cur_node->data.fig.type_guesses.allow_dict = false; cur_node->data.fig.type_guesses.allow_ident = true; kvi_push(ast_stack, &cur_node->children); + if (pt_is_assignment(cur_pt)) { + kvi_push(pt_stack, kEPTExpr); + } want_node = kENodeValue; } while (0), Curly); @@ -2746,6 +2886,36 @@ viml_pexpr_parse_no_paren_closing_error: {} want_node = kENodeOperator; break; } + case kExprLexAssignment: { + if (cur_pt == kEPTAssignment) { + kv_drop(pt_stack, 1); + } else if (cur_pt == kEPTSingleAssignment) { + kv_drop(pt_stack, 2); + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E475: Expected closing bracket to end list assignment " + "lvalue: %.*s")); + } else { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Misplaced assignment: %.*s")); + } + assert(kv_size(pt_stack)); + assert(kv_last(pt_stack) == kEPTExpr); + ADD_VALUE_IF_MISSING(_("E15: Unexpected assignment: %.*s")); + NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeAssignment); + cur_node->data.ass.type = cur_token.data.ass.type; + switch (cur_token.data.ass.type) { +#define HL_ASGN(asgn, hl) \ + case kExprAsgn##asgn: { HL_CUR_TOKEN(hl); break; } + HL_ASGN(Plain, PlainAssignment) + HL_ASGN(Add, AssignmentWithAddition) + HL_ASGN(Subtract, AssignmentWithSubtraction) + HL_ASGN(Concat, AssignmentWithConcatenation) +#undef HL_ASGN + } + ADD_OP_NODE(cur_node); + break; + } } viml_pexpr_parse_cycle_end: prev_token = cur_token; @@ -2862,6 +3032,7 @@ viml_pexpr_parse_end: // FIXME: Investigate whether above are OK to be present in the stack. break; } + case kExprNodeAssignment: case kExprNodeMod: case kExprNodeDivision: case kExprNodeMultiplication: diff --git a/src/nvim/viml/parser/expressions.h b/src/nvim/viml/parser/expressions.h index 648f8cbc1f..23e172da75 100644 --- a/src/nvim/viml/parser/expressions.h +++ b/src/nvim/viml/parser/expressions.h @@ -51,6 +51,9 @@ typedef enum { kExprLexParenthesis, ///< Parenthesis, either opening or closing. kExprLexComma, ///< Comma. kExprLexArrow, ///< Arrow, like from lambda expressions. + kExprLexAssignment, ///< Assignment: `=` or `{op}=`. + // XXX When modifying this enum you need to also modify eltkn_type_tab in + // expressions.c and tests and, possibly, viml_pexpr_repr_token. } LexExprTokenType; typedef enum { @@ -68,6 +71,14 @@ typedef enum { kExprOptScopeLocal = 'l', } ExprOptScope; +/// All possible assignment types: `=` and `{op}=`. +typedef enum { + kExprAsgnPlain = 0, ///< Plain assignment: `=`. + kExprAsgnAdd, ///< Assignment augmented with addition: `+=`. + kExprAsgnSubtract, ///< Assignment augmented with subtraction: `-=`. + kExprAsgnConcat, ///< Assignment augmented with concatenation: `.=`. +} ExprAssignmentType; + #define EXPR_OPT_SCOPE_LIST \ ((char[]){ kExprOptScopeGlobal, kExprOptScopeLocal }) @@ -147,6 +158,10 @@ typedef struct { uint8_t base; ///< Base: 2, 8, 10 or 16. bool is_float; ///< True if number is a floating-point. } num; ///< For kExprLexNumber + + struct { + ExprAssignmentType type; + } ass; ///< For kExprLexAssignment } data; ///< Additional data, if needed. } LexExprToken; @@ -170,8 +185,8 @@ typedef enum { /// “EOC” is something like "|". It is fine with emitting EOC at the end of /// string still, with or without this flag set. kELFlagForbidEOC = (1 << 4), - // WARNING: whenever you add a new flag, alter klee_assume() statement in - // viml_expressions_lexer.c. + // XXX Whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_lexer.c. } LexExprFlags; /// Expression AST node type @@ -233,6 +248,10 @@ typedef enum { kExprNodeMod, kExprNodeOption, kExprNodeEnvironment, + kExprNodeAssignment, + // XXX When modifying this list also modify east_node_type_tab both in parser + // and in tests, and you most likely will also have to alter list of + // highlight groups stored in highlight_init_cmdline variable. } ExprASTNodeType; typedef struct expr_ast_node ExprASTNode; @@ -301,6 +320,9 @@ struct expr_ast_node { const char *ident; ///< Environment variable name start. size_t ident_len; ///< Environment variable name length. } env; ///< For kExprNodeEnvironment. + struct { + ExprAssignmentType type; + } ass; ///< For kExprNodeAssignment } data; }; @@ -314,8 +336,15 @@ enum { /// When parsing expressions input by user bar is assumed to be a binary /// operator and other two are spacings. kExprFlagsDisallowEOC = (1 << 1), - // WARNING: whenever you add a new flag, alter klee_assume() statement in - // viml_expressions_parser.c. + /// Parse :let argument + /// + /// That mean that top level node must be an assignment and first nodes + /// belong to lvalues. + kExprFlagsParseLet = (1 << 2), + // XXX whenever you add a new flag, alter klee_assume() statement in + // viml_expressions_parser.c, nvim_parse_expression() flags parsing + // alongside with its documentation and flag sets in check_parsing() + // function in expressions parser functional and unit tests. } ExprParserFlags; /// AST error definition @@ -350,6 +379,9 @@ extern const char *const eltkn_cmp_type_tab[]; /// Array mapping ExprCaseCompareStrategy values to their stringified versions extern const char *const ccs_tab[]; +/// Array mapping ExprAssignmentType values to their stringified versions +extern const char *const expr_asgn_type_tab[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "viml/parser/expressions.h.generated.h" #endif diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index 9448937430..fd75e8d355 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -48,7 +48,8 @@ int main(const int argc, const char *const *const argv, klee_make_symbolic(&shift, sizeof(shift), "shift"); klee_make_symbolic(&flags, sizeof(flags), "flags"); klee_assume(shift < INPUT_SIZE); - klee_assume(flags <= (kExprFlagsMulti|kExprFlagsDisallowEOC)); + klee_assume( + flags <= (kExprFlagsMulti|kExprFlagsDisallowEOC|kExprFlagsParseLet)); #endif ParserLine plines[] = { diff --git a/test/unit/viml/expressions/lexer_spec.lua b/test/unit/viml/expressions/lexer_spec.lua index 75a641c48a..1b57a24ad5 100644 --- a/test/unit/viml/expressions/lexer_spec.lua +++ b/test/unit/viml/expressions/lexer_spec.lua @@ -13,6 +13,7 @@ local conv_ccs = viml_helpers.conv_ccs local new_pstate = viml_helpers.new_pstate local conv_cmp_type = viml_helpers.conv_cmp_type local pstate_set_str = viml_helpers.pstate_set_str +local conv_expr_asgn_type = viml_helpers.conv_expr_asgn_type local shallowcopy = global_helpers.shallowcopy local intchar2lua = global_helpers.intchar2lua @@ -52,6 +53,8 @@ child_call_once(function() [tonumber(lib.kExprLexParenthesis)] = 'Parenthesis', [tonumber(lib.kExprLexComma)] = 'Comma', [tonumber(lib.kExprLexArrow)] = 'Arrow', + + [tonumber(lib.kExprLexAssignment)] = 'Assignment', } eltkn_mul_type_tab = { @@ -118,6 +121,8 @@ local function eltkn2lua(pstate, tkn) ret.data.val = tonumber(tkn.data.num.is_float and tkn.data.num.val.floating or tkn.data.num.val.integer) + elseif ret.type == 'Assignment' then + ret.data = { type = conv_expr_asgn_type(tkn.data.ass.type) } elseif ret.type == 'Invalid' then ret.data = { error = ffi.string(tkn.data.err.msg) } end @@ -198,7 +203,9 @@ describe('Expressions lexer', function() singl_eltkn_test('Question', '?') singl_eltkn_test('Colon', ':') singl_eltkn_test('Dot', '.') + singl_eltkn_test('Assignment', '.=', {type='Concat'}) singl_eltkn_test('Plus', '+') + singl_eltkn_test('Assignment', '+=', {type='Add'}) singl_eltkn_test('Comma', ',') singl_eltkn_test('Multiplication', '*', {type='Mul'}) singl_eltkn_test('Multiplication', '/', {type='Div'}) @@ -266,12 +273,13 @@ describe('Expressions lexer', function() singl_eltkn_test('DoubleQuotedString', '"x\\"', {closed=false}) singl_eltkn_test('DoubleQuotedString', '"\\"x', {closed=false}) singl_eltkn_test('Not', '!') - singl_eltkn_test('Invalid', '=', {error='E15: Expected == or =~: %.*s'}) + singl_eltkn_test('Assignment', '=', {type='Plain'}) comparison_test('==', '!=', 'Equal') comparison_test('=~', '!~', 'Matches') comparison_test('>', '<=', 'Greater') comparison_test('>=', '<', 'GreaterOrEqual') singl_eltkn_test('Minus', '-') + singl_eltkn_test('Assignment', '-=', {type='Subtract'}) singl_eltkn_test('Arrow', '->') singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'}) simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'}) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index cfc9fe95ac..810d7bfbc6 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -18,6 +18,7 @@ local conv_ccs = viml_helpers.conv_ccs local new_pstate = viml_helpers.new_pstate local conv_cmp_type = viml_helpers.conv_cmp_type local pstate_set_str = viml_helpers.pstate_set_str +local conv_expr_asgn_type = viml_helpers.conv_expr_asgn_type local mergedicts_copy = global_helpers.mergedicts_copy local format_string = global_helpers.format_string @@ -109,6 +110,7 @@ make_enum_conv_tab(lib, { 'kExprNodeMod', 'kExprNodeOption', 'kExprNodeEnvironment', + 'kExprNodeAssignment', }, 'kExprNode', function(ret) east_node_type_tab = ret end) local function conv_east_node_type(typ) @@ -174,6 +176,8 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) typ = ('%s(ident=%s)'):format( typ, ffi.string(eastnode.data.env.ident, eastnode.data.env.ident_len)) + elseif typ == 'Assignment' then + typ = ('%s(%s)'):format(typ, conv_expr_asgn_type(eastnode.data.ass.type)) end ret_str = typ .. ':' .. ret_str local can_simplify = not ret.children @@ -3976,27 +3980,20 @@ describe('Expressions parser', function() -- 012345 ast = { { - 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3:=', + 'Assignment(Add):0:1: +=', children = { - { - 'BinaryPlus:0:1: +', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'Missing:0:3:', - }, - }, + 'PlainIdentifier(scope=0,ident=a):0:0:a', 'PlainIdentifier(scope=0,ident=b):0:4: b', }, }, }, err = { - arg = '= b', - msg = 'E15: Expected == or =~: %.*s', + arg = '+= b', + msg = 'E15: Misplaced assignment: %.*s', }, }, { hl('IdentifierName', 'a'), - hl('BinaryPlus', '+', 1), - hl('InvalidComparison', '='), + hl('InvalidAssignmentWithAddition', '+=', 1), hl('IdentifierName', 'b', 1), }) check_parsing('a + b == c + d', { @@ -7347,4 +7344,6 @@ describe('Expressions parser', function() }, }) end) + -- FIXME: Test assignments thoroughly + -- FIXME: Test that parsing assignments can be used for `:for` pre-`in` part. end) diff --git a/test/unit/viml/helpers.lua b/test/unit/viml/helpers.lua index 9d2d7b61c7..9d8102e023 100644 --- a/test/unit/viml/helpers.lua +++ b/test/unit/viml/helpers.lua @@ -107,6 +107,18 @@ local function conv_ccs(ccs) return conv_enum(ccs_tab, ccs) end +local expr_asgn_type_tab +make_enum_conv_tab(lib, { + 'kExprAsgnPlain', + 'kExprAsgnAdd', + 'kExprAsgnSubtract', + 'kExprAsgnConcat', +}, 'kExprAsgn', function(ret) expr_asgn_type_tab = ret end) + +local function conv_expr_asgn_type(expr_asgn_type) + return conv_enum(expr_asgn_type_tab, expr_asgn_type) +end + return { conv_ccs = conv_ccs, pline2lua = pline2lua, @@ -114,4 +126,5 @@ return { new_pstate = new_pstate, conv_cmp_type = conv_cmp_type, pstate_set_str = pstate_set_str, + conv_expr_asgn_type = conv_expr_asgn_type, } From 45445e2e03f1cbfa25dde76ccf3e24d0d297cabe Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 12 Nov 2017 03:52:26 +0300 Subject: [PATCH 083/109] unittests: Add some more edge test cases --- src/nvim/viml/parser/expressions.c | 7 + test/unit/viml/expressions/parser_spec.lua | 143 +++++++++++++++++++++ 2 files changed, 150 insertions(+) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 13f7131744..07bac89997 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -40,6 +40,13 @@ // E15: Invalid expression: [1, // // < , just exactly one E697 message. +// 6. Some expressions involving calling parenthesis which are treated +// separately by Vim even when not separated by spaces are treated as one +// expression by Neovim: e.g. ":echo (1)(1)" will yield runtime error after +// failing to call "1", while Vim will echo "1 1". Reasoning is the same: +// type of what is in the first expression is generally not known when +// parsing, so to have separate expressions like this separate them with +// spaces. #include #include diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 810d7bfbc6..df45cae304 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -1233,6 +1233,132 @@ describe('Expressions parser', function() hl('CallingParenthesis', ')'), hl('CallingParenthesis', ')'), }) + check_parsing('()()', { + -- 0123 + ast = { + { + 'Call:0:2:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = ')()', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a)()', { + -- 012345 + ast = { + { + 'Call:0:4:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a)(@b)', { + -- 01234567 + ast = { + { + 'Call:0:4:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:5:@b', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a) (@b)', { + -- 012345678 + ast = { + { + 'OpMissing:0:4:', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + err = { + arg = '(@b)', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('InvalidSpacing', ' '), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }, { + [1] = { + ast = { + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + REMOVE_THIS, + }, + }, + }, + err = REMOVE_THIS, + }, + hl_fs = { + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + [6] = REMOVE_THIS, + [7] = REMOVE_THIS, + }, + }, + }) end) itp('works with variable names, including curly braces ones', function() check_parsing('var', { @@ -1543,6 +1669,21 @@ describe('Expressions parser', function() hl('FigureBrace', '{'), hl('Register', '@a'), }) + check_parsing('a ()', { + -- 0123 + ast = { + { + 'Call:0:1: (', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) end) itp('works with lambdas and dictionaries', function() check_parsing('{}', { @@ -7346,4 +7487,6 @@ describe('Expressions parser', function() end) -- FIXME: Test assignments thoroughly -- FIXME: Test that parsing assignments can be used for `:for` pre-`in` part. + -- FIXME: Somehow make functional tests use the same code. Or, at least, + -- create an automated script which will do the import. end) From 556451a7f2fd513db33b9d7ac1b653d356b7b915 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 12 Nov 2017 16:59:36 +0300 Subject: [PATCH 084/109] unittests,syntax: Check for sanity of highlight_init_cmdline Also fixes some errors found. --- src/nvim/syntax.c | 14 +- src/nvim/syntax.h | 3 + test/functional/ui/cmdline_highlight_spec.lua | 1 - test/unit/viml/expressions/parser_spec.lua | 159 +++++++++++++++++- 4 files changed, 169 insertions(+), 8 deletions(-) diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index 9f98b26905..c9e99d82f8 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -6020,7 +6020,7 @@ static const char *highlight_init_dark[] = { NULL }; -static const char *highlight_init_cmdline[] = { +const char *const highlight_init_cmdline[] = { // XXX When modifying a list modify it in both valid and invalid halfs. // TODO(ZyX-I): merge valid and invalid groups via a macros. @@ -6047,6 +6047,7 @@ static const char *highlight_init_cmdline[] = { "default link NVimComparison NVimBinaryOperator", "default link NVimComparisonModifier NVimComparison", "default link NVimBinaryPlus NVimBinaryOperator", + "default link NVimBinaryMinus NVimBinaryOperator", "default link NVimConcat NVimBinaryOperator", "default link NVimConcatOrSubscript NVimConcat", "default link NVimOr NVimBinaryOperator", @@ -6108,9 +6109,6 @@ static const char *highlight_init_cmdline[] = { "default link NVimDoubleQuote NVimStringQuote", "default link NVimDoubleQuotedBody NVimStringBody", "default link NVimDoubleQuotedEscape NVimStringSpecial", - // Not actually invalid, but we highlight user that he is doing something - // wrong. - "default link NVimDoubleQuotedUnknownEscape NVimInvalidValue", "default link NVimFigureBrace NVimInternalError", "default link NVimSingleQuotedUnknownEscape NVimInternalError", @@ -6144,6 +6142,7 @@ static const char *highlight_init_cmdline[] = { "default link NVimInvalidComparison NVimInvalidBinaryOperator", "default link NVimInvalidComparisonModifier NVimInvalidComparison", "default link NVimInvalidBinaryPlus NVimInvalidBinaryOperator", + "default link NVimInvalidBinaryMinus NVimInvalidBinaryOperator", "default link NVimInvalidConcat NVimInvalidBinaryOperator", "default link NVimInvalidConcatOrSubscript NVimInvalidConcat", "default link NVimInvalidOr NVimInvalidBinaryOperator", @@ -6217,12 +6216,17 @@ static const char *highlight_init_cmdline[] = { "default link NVimInvalidFigureBrace NVimInvalidDelimiter", "default link NVimInvalidSpacing ErrorMsg", + + // Not actually invalid, but we highlight user that he is doing something + // wrong. + "default link NVimDoubleQuotedUnknownEscape NVimInvalidValue", + NULL, }; /// Create default links for NVim* highlight groups used for cmdline coloring void syn_init_cmdline_highlight(bool reset, bool init) { - for (size_t i = 0 ; i < ARRAY_SIZE(highlight_init_cmdline) ; i++) { + for (size_t i = 0 ; highlight_init_cmdline[i] != NULL ; i++) { do_highlight(highlight_init_cmdline[i], reset, init); } } diff --git a/src/nvim/syntax.h b/src/nvim/syntax.h index bb733ead30..c9b0665ec8 100644 --- a/src/nvim/syntax.h +++ b/src/nvim/syntax.h @@ -45,6 +45,9 @@ typedef struct { } color_name_table_T; extern color_name_table_T color_name_table[]; +/// Array of highlight definitions, used for unit testing +extern const char *const highlight_init_cmdline[]; + #ifdef INCLUDE_GENERATED_DECLARATIONS # include "syntax.h.generated.h" #endif diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua index b16b4a5602..ab195f9f1e 100644 --- a/test/functional/ui/cmdline_highlight_spec.lua +++ b/test/functional/ui/cmdline_highlight_spec.lua @@ -899,7 +899,6 @@ describe('Expressions coloring support', function() ]]) end) -- FIXME: Test expr coloring when using -u NORC and -u NONE. - -- FIXME: Test all highlight groups, using long expression. -- FIXME: Test different ways of triggering expression highlighting (:=, -- i=, :e, "=). end) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index df45cae304..cfcd63f005 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -7,11 +7,13 @@ local make_enum_conv_tab = helpers.make_enum_conv_tab local child_call_once = helpers.child_call_once local alloc_log_new = helpers.alloc_log_new local kvi_destroy = helpers.kvi_destroy +local array_size = helpers.array_size local conv_enum = helpers.conv_enum local debug_log = helpers.debug_log local ptr2key = helpers.ptr2key local cimport = helpers.cimport local ffi = helpers.ffi +local neq = helpers.neq local eq = helpers.eq local conv_ccs = viml_helpers.conv_ccs @@ -26,10 +28,158 @@ local format_luav = global_helpers.format_luav local intchar2lua = global_helpers.intchar2lua local REMOVE_THIS = global_helpers.REMOVE_THIS -local lib = cimport('./src/nvim/viml/parser/expressions.h') +local lib = cimport('./src/nvim/viml/parser/expressions.h', + './src/nvim/syntax.h') local alloc_log = alloc_log_new() +local predefined_hl_defs = { + -- From highlight_init_both + Conceal=true, + Cursor=true, + lCursor=true, + DiffText=true, + ErrorMsg=true, + IncSearch=true, + ModeMsg=true, + NonText=true, + PmenuSbar=true, + StatusLine=true, + StatusLineNC=true, + TabLineFill=true, + TabLineSel=true, + TermCursor=true, + VertSplit=true, + WildMenu=true, + EndOfBuffer=true, + QuickFixLine=true, + Substitute=true, + Whitespace=true, + + -- From highlight_init_(dark|light) + ColorColumn=true, + CursorColumn=true, + CursorLine=true, + CursorLineNr=true, + DiffAdd=true, + DiffChange=true, + DiffDelete=true, + Directory=true, + FoldColumn=true, + Folded=true, + LineNr=true, + MatchParen=true, + MoreMsg=true, + Pmenu=true, + PmenuSel=true, + PmenuThumb=true, + Question=true, + Search=true, + SignColumn=true, + SpecialKey=true, + SpellBad=true, + SpellCap=true, + SpellLocal=true, + SpellRare=true, + TabLine=true, + Title=true, + Visual=true, + WarningMsg=true, + Normal=true, + + -- From syncolor.vim, if &background + Comment=true, + Constant=true, + Special=true, + Identifier=true, + Statement=true, + PreProc=true, + Type=true, + Underlined=true, + Ignore=true, + + -- From syncolor.vim, below if &background + Error=true, + Todo=true, + + -- From syncolor.vim, links at the bottom + String=true, + Character=true, + Number=true, + Boolean=true, + Float=true, + Function=true, + Conditional=true, + Repeat=true, + Label=true, + Operator=true, + Keyword=true, + Exception=true, + Include=true, + Define=true, + Macro=true, + PreCondit=true, + StorageClass=true, + Structure=true, + Typedef=true, + Tag=true, + SpecialChar=true, + Delimiter=true, + SpecialComment=true, + Debug=true, +} + +local nvim_hl_defs = {} + +child_call_once(function() + local i = 0 + while lib.highlight_init_cmdline[i] ~= nil do + local hl_args = lib.highlight_init_cmdline[i] + local s = ffi.string(hl_args) + local err, msg = pcall(function() + if s:sub(1, 13) == 'default link ' then + local new_grp, grp_link = s:match('^default link (%w+) (%w+)$') + neq(nil, new_grp) + -- Note: group to link to must be already defined at the time of + -- linking, otherwise it will be created as cleared. So existence + -- of the group is checked here and not in the next pass over + -- nvim_hl_defs. + eq(true, not not (nvim_hl_defs[grp_link] + or predefined_hl_defs[grp_link])) + nvim_hl_defs[new_grp] = {'link', grp_link} + else + local new_grp, grp_args = s:match('^(%w+) (.*)') + neq(nil, new_grp) + eq(false, not not (nvim_hl_defs[grp_link] + or predefined_hl_defs[grp_link])) + nvim_hl_defs[new_grp] = {'definition', grp_args} + end + end) + if not err then + msg = format_string( + 'Error while processing string %s at position %u:\n%s', s, i, msg) + error(msg) + end + i = i + 1 + end + for k, _ in ipairs(nvim_hl_defs) do + eq('NVim', k:sub(1, 4)) + -- NVimInvalid + -- 12345678901 + local err, msg = pcall(function() + if k:sub(5, 11) == 'Invalid' then + neq(nil, nvim_hl_defs['NVim' .. k:sub(12)]) + else + neq(nil, nvim_hl_defs['NVimInvalid' .. k:sub(5)]) + end + end) + if not err then + msg = format_string('Error while processing group %s:\n%s', k, msg) + error(msg) + end + end +end) + local function format_check(expr, flags, ast, hls) -- That forces specific order. print( format_string('\ncheck_parsing(%r, %u, {', expr, flags)) @@ -237,6 +387,8 @@ end) describe('Expressions parser', function() local function check_parsing(str, exp_ast, exp_highlighting_fs, nz_flags_exps) nz_flags_exps = nz_flags_exps or {} + local format_check_data = function() + end for _, flags in ipairs({0, 1, 2, 3}) do debug_log(('Running test case (%s, %u)'):format(str, flags)) local err, msg = pcall(function() @@ -266,7 +418,7 @@ describe('Expressions parser', function() end end if exp_ast == nil then - format_check(str, flags, ast, hls) + format_check(str, ast, hls) else eq(exps.ast, ast) if exp_highlighting_fs then @@ -292,6 +444,9 @@ describe('Expressions parser', function() end local function hl(group, str, shift) return function(next_col) + if nvim_hl_defs['NVim' .. group] == nil then + error(('Unknown group: NVim%s'):format(group)) + end local col = next_col + (shift or 0) return (('%s:%u:%u:%s'):format( 'NVim' .. group, From 39c75d31be98e4cbb63a01c398d89e231f71f380 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 12 Nov 2017 18:52:49 +0300 Subject: [PATCH 085/109] unittests: Fix automatic test case generation --- test/helpers.lua | 95 ++++++++++++++++----- test/unit/viml/expressions/parser_spec.lua | 98 +++++++++++++++++----- 2 files changed, 149 insertions(+), 44 deletions(-) diff --git a/test/helpers.lua b/test/helpers.lua index d24fae745b..6c6611d061 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -310,6 +310,43 @@ local function mergedicts_copy(d1, d2) return ret end +-- dictdiff: find a diff so that mergedicts_copy(d1, diff) is equal to d2 +-- +-- Note: does not copy values from d2 +local function dictdiff(d1, d2) + local ret = {} + local hasdiff = false + for k, v in pairs(d1) do + if d2[k] == nil then + hasdiff = true + ret[k] = REMOVE_THIS + elseif type(v) == type(d2[k]) then + if type(v) == 'table' and type(d2[k]) == 'table' then + local subdiff = dictdiff(v, d2[k]) + if subdiff ~= nil then + hasdiff = true + ret[k] = subdiff + end + elseif v ~= d2[k] then + ret[k] = d2[k] + end + else + ret[k] = d2[k] + end + end + for k, v in pairs(d2) do + if d1[k] == nil then + ret[k] = v + hasdiff = true + end + end + if hasdiff then + return ret + else + return nil + end +end + local function updated(d, d2) for k, v in pairs(d2) do d[k] = v @@ -365,39 +402,52 @@ local SUBTBL = { local format_luav -format_luav = function(v, indent) +format_luav = function(v, indent, opts) + opts = opts or {} local linesep = '\n' - local next_indent = nil + local next_indent_arg = nil if indent == nil then indent = '' linesep = '' else - next_indent = indent .. ' ' + next_indent_arg = indent .. ' ' end + local next_indent = indent .. ' ' local ret = '' if type(v) == 'string' then - ret = tostring(v):gsub('[\'\\]', '\\%0'):gsub('[%z\1-\31]', function(match) - return SUBTBL[match:byte() + 1] - end) - ret = '\'' .. ret .. '\'' + if opts.literal_strings then + ret = v + else + ret = tostring(v):gsub('[\'\\]', '\\%0'):gsub( + '[%z\1-\31]', function(match) + return SUBTBL[match:byte() + 1] + end) + ret = '\'' .. ret .. '\'' + end elseif type(v) == 'table' then - local processed_keys = {} - ret = '{' .. linesep - for i, subv in ipairs(v) do - ret = ret .. (next_indent or '') .. format_luav(subv, next_indent) .. ',\n' - processed_keys[i] = true - end - for k, subv in pairs(v) do - if not processed_keys[k] then - if type(k) == 'string' and k:match('^[a-zA-Z_][a-zA-Z0-9_]*$') then - ret = ret .. next_indent .. k .. ' = ' - else - ret = ret .. next_indent .. '[' .. format_luav(k) .. '] = ' - end - ret = ret .. format_luav(subv, next_indent) .. ',\n' + if v == REMOVE_THIS then + ret = 'REMOVE_THIS' + else + local processed_keys = {} + ret = '{' .. linesep + for i, subv in ipairs(v) do + ret = ('%s%s%s,\n'):format(ret, next_indent, + format_luav(subv, next_indent_arg, opts)) + processed_keys[i] = true end + for k, subv in pairs(v) do + if not processed_keys[k] then + if type(k) == 'string' and k:match('^[a-zA-Z_][a-zA-Z0-9_]*$') then + ret = ret .. next_indent .. k .. ' = ' + else + ret = ('%s%s[%s] = '):format(ret, next_indent, + format_luav(k, nil, opts)) + end + ret = ret .. format_luav(subv, next_indent_arg, opts) .. ',\n' + end + end + ret = ret .. indent .. '}' end - ret = ret .. indent .. '}' elseif type(v) == 'number' then if v % 1 == 0 then ret = ('%d'):format(v) @@ -461,6 +511,7 @@ return { shallowcopy = shallowcopy, deepcopy = deepcopy, mergedicts_copy = mergedicts_copy, + dictdiff = dictdiff, REMOVE_THIS = REMOVE_THIS, concat_tables = concat_tables, dedent = dedent, diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index cfcd63f005..25a41518eb 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -27,6 +27,7 @@ local format_string = global_helpers.format_string local format_luav = global_helpers.format_luav local intchar2lua = global_helpers.intchar2lua local REMOVE_THIS = global_helpers.REMOVE_THIS +local dictdiff = global_helpers.dictdiff local lib = cimport('./src/nvim/viml/parser/expressions.h', './src/nvim/syntax.h') @@ -180,9 +181,31 @@ child_call_once(function() end end) -local function format_check(expr, flags, ast, hls) +local function hls_to_hl_fs(hls) + local ret = {} + local next_col = 0 + for i, v in ipairs(hls) do + local group, line, col, str = v:match('^NVim([a-zA-Z]+):(%d+):(%d+):(.*)$') + col = tonumber(col) + line = tonumber(line) + assert(line == 0) + local col_shift = col - next_col + assert(col_shift >= 0) + next_col = col + #str + ret[i] = format_string('hl(%r, %r%s)', + group, + str, + (col_shift == 0 + and '' + or (', %u'):format(col_shift))) + end + return ret +end + +local function format_check(expr, format_check_data) -- That forces specific order. - print( format_string('\ncheck_parsing(%r, %u, {', expr, flags)) + local zdata = format_check_data[0] + print(format_string('\ncheck_parsing(%r, {', expr, flags)) local digits = ' -- ' local digits2 = ' -- ' for i = 0, #expr - 1 do @@ -195,27 +218,56 @@ local function format_check(expr, flags, ast, hls) if #expr > 10 then print(digits2) end - print(' ast = ' .. format_luav(ast.ast, ' ') .. ',') - if ast.err then + print(' ast = ' .. format_luav(zdata.ast.ast, ' ') .. ',') + if zdata.ast.err then print(' err = {') - print(' arg = ' .. format_luav(ast.err.arg) .. ',') - print(' msg = ' .. format_luav(ast.err.msg) .. ',') + print(' arg = ' .. format_luav(zdata.ast.err.arg) .. ',') + print(' msg = ' .. format_luav(zdata.ast.err.msg) .. ',') print(' },') end print('}, {') - local next_col = 0 - for _, v in ipairs(hls) do - local group, line, col, str = v:match('NVim([a-zA-Z]+):(%d+):(%d+):(.*)') - col = tonumber(col) - line = tonumber(line) - assert(line == 0) - local col_shift = col - next_col - assert(col_shift >= 0) - next_col = col + #str - print(format_string(' hl(%r, %r%s),', - group, - str, - (col_shift == 0 and '' or (', %u'):format(col_shift)))) + for _, v in ipairs(zdata.hl_fs) do + print(' ' .. v .. ',') + end + local diffs = {} + local diffs_num = 0 + for flags, v in pairs(format_check_data) do + if flags ~= 0 then + diffs[flags] = dictdiff(zdata, v) + if diffs[flags] then + if flags == 3 then + if (dictdiff(format_check_data[1], format_check_data[3]) == nil + or dictdiff(format_check_data[2], format_check_data[3]) == nil) then + diffs[flags] = nil + else + diffs_num = diffs_num + 1 + end + else + diffs_num = diffs_num + 1 + end + end + end + end + if diffs_num ~= 0 then + print('}, {') + local flags = 1 + while diffs_num ~= 0 do + if diffs[flags] then + diffs_num = diffs_num - 1 + local diff = diffs[flags] + print((' [%u] = {'):format(flags)) + if diff.ast then + print(' ast = ' .. format_luav(diff.ast, ' ')) + end + if diff.hl_fs then + print(' hl_fs = ' .. format_luav(diff.hl_fs, ' ', { + literal_strings=true + })) + end + print(' },') + end + flags = flags + 1 + end end print('})') end @@ -387,8 +439,7 @@ end) describe('Expressions parser', function() local function check_parsing(str, exp_ast, exp_highlighting_fs, nz_flags_exps) nz_flags_exps = nz_flags_exps or {} - local format_check_data = function() - end + local format_check_data = {} for _, flags in ipairs({0, 1, 2, 3}) do debug_log(('Running test case (%s, %u)'):format(str, flags)) local err, msg = pcall(function() @@ -418,7 +469,7 @@ describe('Expressions parser', function() end end if exp_ast == nil then - format_check(str, ast, hls) + format_check_data[flags] = {ast=ast, hl_fs=hls_to_hl_fs(hls)} else eq(exps.ast, ast) if exp_highlighting_fs then @@ -441,6 +492,9 @@ describe('Expressions parser', function() error(msg) end end + if exp_ast == nil then + format_check(str, format_check_data) + end end local function hl(group, str, shift) return function(next_col) From 342239a9c53cf4857d18c0583d4cab1fdca534fa Mon Sep 17 00:00:00 2001 From: ZyX Date: Mon, 13 Nov 2017 01:10:39 +0300 Subject: [PATCH 086/109] unittests,viml/parser/expressions: Start adding asgn parsing tests --- src/nvim/viml/parser/expressions.c | 50 +++- test/helpers.lua | 53 +++- test/symbolic/klee/viml_expressions_parser.c | 14 + test/unit/viml/expressions/parser_spec.lua | 280 +++++++++++++++++-- 4 files changed, 348 insertions(+), 49 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 07bac89997..4bd3652292 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1363,7 +1363,6 @@ static inline ParserPosition recol_pos(const ParserPosition pos, } \ } while (0) -// TODO(ZyX-I): actual condition /// Check whether it is possible to have next expression after current /// /// For :echo: `:echo @a @a` is a valid expression. `:echo (@a @a)` is not. @@ -1901,6 +1900,7 @@ ExprAST viml_pexpr_parse(ParserState *const pstate, const int flags) bool highlighted_prev_spacing = false; // Lambda node, valid when parsing lambda arguments only. ExprASTNode *lambda_node = NULL; + size_t asgn_level = 0; do { const bool is_concat_or_subscript = ( want_node == kENodeValue @@ -2063,6 +2063,9 @@ viml_pexpr_parse_process_token: && tok_type != kExprLexDot && (tok_type != kExprLexComma || !is_single_assignment) && tok_type != kExprLexAssignment) { + if (flags & kExprFlagsMulti && MAY_HAVE_NEXT_EXPR) { + goto viml_pexpr_parse_end; + } ERROR_FROM_TOKEN_AND_MSG( cur_token, _("E15: Expected assignment operator or subscript: %.*s")); @@ -2429,6 +2432,10 @@ viml_pexpr_parse_valid_colon: ExprASTNode *new_top_node = *new_top_node_p; switch (new_top_node->type) { case kExprNodeListLiteral: { + if (pt_is_assignment(cur_pt) && new_top_node->children == NULL) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E475: Unable to assign to empty list: %.*s")); + } HL_CUR_TOKEN(List); break; } @@ -2447,14 +2454,18 @@ viml_pexpr_parse_bracket_closing_error: } kvi_push(ast_stack, new_top_node_p); want_node = kENodeOperator; - if (cur_pt == kEPTSingleAssignment) { - kv_drop(pt_stack, 1); - } else if (cur_pt == kEPTAssignment) { - assert(ast.err.msg); - } else if (cur_pt == kEPTExpr - && kv_size(pt_stack) > 1 - && pt_is_assignment(kv_Z(pt_stack, 1))) { - kv_drop(pt_stack, 1); + if (kv_size(ast_stack) <= asgn_level) { + assert(kv_size(ast_stack) == asgn_level); + asgn_level = 0; + if (cur_pt == kEPTSingleAssignment) { + kv_drop(pt_stack, 1); + } else if (cur_pt == kEPTAssignment) { + assert(ast.err.msg); + } else if (cur_pt == kEPTExpr + && kv_size(pt_stack) > 1 + && pt_is_assignment(kv_Z(pt_stack, 1))) { + kv_drop(pt_stack, 1); + } } } else { if (want_node == kENodeValue) { @@ -2480,6 +2491,7 @@ viml_pexpr_parse_bracket_closing_error: ADD_OP_NODE(cur_node); HL_CUR_TOKEN(SubscriptBracket); if (pt_is_assignment(cur_pt)) { + asgn_level = kv_size(ast_stack); kvi_push(pt_stack, kEPTExpr); } } @@ -2586,10 +2598,14 @@ viml_pexpr_parse_figure_brace_closing_error: } kvi_push(ast_stack, new_top_node_p); want_node = kENodeOperator; - if (cur_pt == kEPTExpr - && kv_size(pt_stack) > 1 - && pt_is_assignment(kv_Z(pt_stack, 1))) { - kv_drop(pt_stack, 1); + if (kv_size(ast_stack) <= asgn_level) { + assert(kv_size(ast_stack) == asgn_level); + if (cur_pt == kEPTExpr + && kv_size(pt_stack) > 1 + && pt_is_assignment(kv_Z(pt_stack, 1))) { + kv_drop(pt_stack, 1); + asgn_level = 0; + } } } else { if (want_node == kENodeValue) { @@ -2635,6 +2651,10 @@ viml_pexpr_parse_figure_brace_closing_error: } while (0), Curly); } + if (pt_is_assignment(cur_pt) + && !pt_is_assignment(kv_last(pt_stack))) { + asgn_level = kv_size(ast_stack); + } } break; } @@ -2755,6 +2775,10 @@ viml_pexpr_parse_figure_brace_closing_error: case kExprLexDot: { ADD_VALUE_IF_MISSING(_("E15: Unexpected dot: %.*s")); if (prev_token.type == kExprLexSpacing) { + if (cur_pt == kEPTAssignment) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, _("E15: Cannot concatenate in assignments: %.*s")); + } NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeConcat); HL_CUR_TOKEN(Concat); } else { diff --git a/test/helpers.lua b/test/helpers.lua index 6c6611d061..ada690a4d2 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -264,6 +264,9 @@ local function which(exe) end local function shallowcopy(orig) + if type(orig) ~= 'table' then + return orig + end local copy = {} for orig_key, orig_value in pairs(orig) do copy[orig_key] = orig_value @@ -312,7 +315,7 @@ end -- dictdiff: find a diff so that mergedicts_copy(d1, diff) is equal to d2 -- --- Note: does not copy values from d2 +-- Note: does not do copies of d2 values used. local function dictdiff(d1, d2) local ret = {} local hasdiff = false @@ -321,7 +324,7 @@ local function dictdiff(d1, d2) hasdiff = true ret[k] = REMOVE_THIS elseif type(v) == type(d2[k]) then - if type(v) == 'table' and type(d2[k]) == 'table' then + if type(v) == 'table' then local subdiff = dictdiff(v, d2[k]) if subdiff ~= nil then hasdiff = true @@ -329,14 +332,16 @@ local function dictdiff(d1, d2) end elseif v ~= d2[k] then ret[k] = d2[k] + hasdiff = true end else ret[k] = d2[k] + hasdiff = true end end for k, v in pairs(d2) do if d1[k] == nil then - ret[k] = v + ret[k] = shallowcopy(v) hasdiff = true end end @@ -406,13 +411,18 @@ format_luav = function(v, indent, opts) opts = opts or {} local linesep = '\n' local next_indent_arg = nil + local indent_shift = opts.indent_shift or ' ' + local next_indent + local nl = '\n' if indent == nil then indent = '' linesep = '' + next_indent = '' + nl = ' ' else - next_indent_arg = indent .. ' ' + next_indent_arg = indent .. indent_shift + next_indent = indent .. indent_shift end - local next_indent = indent .. ' ' local ret = '' if type(v) == 'string' then if opts.literal_strings then @@ -430,10 +440,12 @@ format_luav = function(v, indent, opts) else local processed_keys = {} ret = '{' .. linesep + local non_empty = false for i, subv in ipairs(v) do - ret = ('%s%s%s,\n'):format(ret, next_indent, - format_luav(subv, next_indent_arg, opts)) + ret = ('%s%s%s,%s'):format(ret, next_indent, + format_luav(subv, next_indent_arg, opts), nl) processed_keys[i] = true + non_empty = true end for k, subv in pairs(v) do if not processed_keys[k] then @@ -443,9 +455,13 @@ format_luav = function(v, indent, opts) ret = ('%s%s[%s] = '):format(ret, next_indent, format_luav(k, nil, opts)) end - ret = ret .. format_luav(subv, next_indent_arg, opts) .. ',\n' + ret = ret .. format_luav(subv, next_indent_arg, opts) .. ',' .. nl + non_empty = true end end + if nl == ' ' and non_empty then + ret = ret:sub(1, -3) + end ret = ret .. indent .. '}' end elseif type(v) == 'number' then @@ -495,6 +511,25 @@ local function intchar2lua(ch) return (20 <= ch and ch < 127) and ('%c'):format(ch) or ch end +local fixtbl_metatable = { + __newindex = function() + assert(false) + end, +} + +local function fixtbl(tbl) + return setmetatable(tbl, fixtbl_metatable) +end + +local function fixtbl_rec(tbl) + for k, v in pairs(tbl) do + if type(v) == 'table' then + fixtbl_rec(v) + end + end + return fixtbl(tbl) +end + return { eq = eq, neq = neq, @@ -519,4 +554,6 @@ return { format_string = format_string, intchar2lua = intchar2lua, updated = updated, + fixtbl = fixtbl, + fixtbl_rec = fixtbl_rec, } diff --git a/test/symbolic/klee/viml_expressions_parser.c b/test/symbolic/klee/viml_expressions_parser.c index fd75e8d355..9a876ed3fa 100644 --- a/test/symbolic/klee/viml_expressions_parser.c +++ b/test/symbolic/klee/viml_expressions_parser.c @@ -93,6 +93,20 @@ int main(const int argc, const char *const *const argv, const ExprAST ast = viml_pexpr_parse(&pstate, (int)flags); assert(ast.root != NULL || ast.err.msg); + if (flags & kExprFlagsParseLet) { + assert(ast.err.msg != NULL + || ast.root->type == kExprNodeAssignment + || (ast.root->type == kExprNodeListLiteral + && ast.root->children != NULL) + || ast.root->type == kExprNodeComplexIdentifier + || ast.root->type == kExprNodeCurlyBracesIdentifier + || ast.root->type == kExprNodePlainIdentifier + || ast.root->type == kExprNodeRegister + || ast.root->type == kExprNodeEnvironment + || ast.root->type == kExprNodeOption + || ast.root->type == kExprNodeSubscript + || ast.root->type == kExprNodeConcatOrSubscript); + } // Can’t possibly have more highlight tokens then there are bytes in string. assert(kv_size(colors) <= INPUT_SIZE - shift); kvi_destroy(colors); diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 25a41518eb..d6d8dd0807 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -202,12 +202,20 @@ local function hls_to_hl_fs(hls) return ret end -local function format_check(expr, format_check_data) +local function format_check(expr, format_check_data, opts) -- That forces specific order. - local zdata = format_check_data[0] - print(format_string('\ncheck_parsing(%r, {', expr, flags)) - local digits = ' -- ' - local digits2 = ' -- ' + local zflags = opts.flags[1] + local zdata = format_check_data[zflags] + local dig_len = 0 + if opts.funcname then + print(format_string('\n%s(%r, {', opts.funcname, expr)) + dig_len = #opts.funcname + 2 + else + print(format_string('\n_check_parsing(%r, %r, {', opts, expr)) + dig_len = #('_check_parsing(, \'') + #(format_string('%r', opts)) + end + local digits = ' --' .. (' '):rep(dig_len - #(' --')) + local digits2 = digits:sub(1, -10) for i = 0, #expr - 1 do if i % 10 == 0 then digits2 = ('%s%10u'):format(digits2, i / 10) @@ -232,12 +240,15 @@ local function format_check(expr, format_check_data) local diffs = {} local diffs_num = 0 for flags, v in pairs(format_check_data) do - if flags ~= 0 then + if flags ~= zflags then diffs[flags] = dictdiff(zdata, v) if diffs[flags] then - if flags == 3 then - if (dictdiff(format_check_data[1], format_check_data[3]) == nil - or dictdiff(format_check_data[2], format_check_data[3]) == nil) then + if flags == 3 + zflags then + if (dictdiff(format_check_data[1 + zflags], + format_check_data[3 + zflags]) == nil + or dictdiff(format_check_data[2 + zflags], + format_check_data[3 + zflags]) == nil) + then diffs[flags] = nil else diffs_num = diffs_num + 1 @@ -437,10 +448,12 @@ child_call_once(function() end) describe('Expressions parser', function() - local function check_parsing(str, exp_ast, exp_highlighting_fs, nz_flags_exps) + local function _check_parsing(opts, str, exp_ast, exp_highlighting_fs, + nz_flags_exps) + local zflags = opts.flags[1] nz_flags_exps = nz_flags_exps or {} local format_check_data = {} - for _, flags in ipairs({0, 1, 2, 3}) do + for _, flags in ipairs(opts.flags) do debug_log(('Running test case (%s, %u)'):format(str, flags)) local err, msg = pcall(function() if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then @@ -452,25 +465,25 @@ describe('Expressions parser', function() local east = lib.viml_pexpr_parse(pstate, flags) local ast = east2lua(pstate, east) local hls = phl2lua(pstate) - local exps = { - ast = exp_ast, - hl_fs = exp_highlighting_fs, - } - local add_exps = nz_flags_exps[flags] - if not add_exps and flags == 3 then - add_exps = nz_flags_exps[1] or nz_flags_exps[2] - end - if add_exps then - if add_exps.ast then - exps.ast = mergedicts_copy(exps.ast, add_exps.ast) - end - if add_exps.hl_fs then - exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs) - end - end if exp_ast == nil then format_check_data[flags] = {ast=ast, hl_fs=hls_to_hl_fs(hls)} else + local exps = { + ast = exp_ast, + hl_fs = exp_highlighting_fs, + } + local add_exps = nz_flags_exps[flags] + if not add_exps and flags == 3 + zflags then + add_exps = nz_flags_exps[1 + zflags] or nz_flags_exps[2 + zflags] + end + if add_exps then + if add_exps.ast then + exps.ast = mergedicts_copy(exps.ast, add_exps.ast) + end + if add_exps.hl_fs then + exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs) + end + end eq(exps.ast, ast) if exp_highlighting_fs then local exp_highlighting = {} @@ -493,9 +506,18 @@ describe('Expressions parser', function() end end if exp_ast == nil then - format_check(str, format_check_data) + format_check(str, format_check_data, opts) end end + local function check_parsing(...) + return _check_parsing({flags={0, 1, 2, 3}, funcname='check_parsing'}, ...) + end + local function check_asgn_parsing(...) + return _check_parsing({ + flags={4, 5, 6, 7}, + funcname='check_asgn_parsing', + }, ...) + end local function hl(group, str, shift) return function(next_col) if nvim_hl_defs['NVim' .. group] == nil then @@ -7694,6 +7716,208 @@ describe('Expressions parser', function() }, }) end) + itp('works with assignments', function() + check_asgn_parsing('a=b', { + -- 012 + ast = { + { + 'Assignment(Plain):0:1:=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('PlainAssignment', '='), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a+=b', { + -- 0123 + ast = { + { + 'Assignment(Add):0:1:+=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('AssignmentWithAddition', '+='), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a-=b', { + -- 0123 + ast = { + { + 'Assignment(Subtract):0:1:-=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('AssignmentWithSubtraction', '-='), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a.=b', { + -- 0123 + ast = { + { + 'Assignment(Concat):0:1:.=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('AssignmentWithConcatenation', '.='), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a', { + -- 0 + ast = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, { + hl('IdentifierName', 'a'), + }) + + check_asgn_parsing('a b', { + -- 012 + ast = { + { + 'OpMissing:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:1: b', + }, + }, + }, + err = { + arg = 'b', + msg = 'E15: Expected assignment operator or subscript: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidSpacing', ' '), + hl('IdentifierName', 'b'), + }, { + [5] = { + ast = { + ast = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + err = REMOVE_THIS, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + } + }, + }) + + check_asgn_parsing('[a, b, c]', { + -- 012345678 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('List', ']'), + }) + + check_asgn_parsing('[a, b]', { + -- 012345 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('List', ']'), + }) + + check_asgn_parsing('[a]', { + -- 012 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + }) + + check_asgn_parsing('[]', { + -- 01 + ast = { + 'ListLiteral:0:0:[', + }, + err = { + arg = ']', + msg = 'E475: Unable to assign to empty list: %.*s', + }, + }, { + hl('List', '['), + hl('InvalidList', ']'), + }) + + -- check_asgn_parsing('a[1 + 2] += 3') + -- check_asgn_parsing('a[{-> {b{3}: 4}[5]}()] += 6') + -- check_asgn_parsing('a{1}.2[{-> {b{3}: 4}[5]}()]') + end) -- FIXME: Test assignments thoroughly -- FIXME: Test that parsing assignments can be used for `:for` pre-`in` part. -- FIXME: Somehow make functional tests use the same code. Or, at least, From c287893225bad586af486b37546f5982e5b1cd03 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 19:22:54 +0300 Subject: [PATCH 087/109] viml/parser/expressions,unittests: Do better testing, fix found issues --- src/nvim/viml/parser/expressions.c | 40 +- test/functional/ui/cmdline_highlight_spec.lua | 1 + test/unit/viml/expressions/parser_spec.lua | 771 +++++++++++++++++- 3 files changed, 786 insertions(+), 26 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 4bd3652292..0824b3ca7d 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -2031,18 +2031,7 @@ viml_pexpr_parse_process_token: } break; } - case kEPTSingleAssignment: { - if (tok_type == kExprLexBracket && !cur_token.data.brc.closing) { - ERROR_FROM_TOKEN_AND_MSG( - cur_token, - _("E475: Nested lists not allowed when assigning: %.*s")); - kv_drop(pt_stack, 2); - assert(kv_size(pt_stack)); - assert(kv_last(pt_stack) == kEPTExpr); - break; - } - FALLTHROUGH; - } + case kEPTSingleAssignment: case kEPTAssignment: { if (want_node == kENodeValue && tok_type != kExprLexBracket @@ -2062,7 +2051,13 @@ viml_pexpr_parse_process_token: || cur_token.data.brc.closing) && tok_type != kExprLexDot && (tok_type != kExprLexComma || !is_single_assignment) - && tok_type != kExprLexAssignment) { + && tok_type != kExprLexAssignment + // Curly brace identifiers: will contain plain identifier or + // another curly brace in position where operator is wanted. + && !((tok_type == kExprLexPlainIdentifier + || (tok_type == kExprLexFigureBrace + && !cur_token.data.brc.closing)) + && prev_token.type != kExprLexSpacing)) { if (flags & kExprFlagsMulti && MAY_HAVE_NEXT_EXPR) { goto viml_pexpr_parse_end; } @@ -2457,9 +2452,7 @@ viml_pexpr_parse_bracket_closing_error: if (kv_size(ast_stack) <= asgn_level) { assert(kv_size(ast_stack) == asgn_level); asgn_level = 0; - if (cur_pt == kEPTSingleAssignment) { - kv_drop(pt_stack, 1); - } else if (cur_pt == kEPTAssignment) { + if (cur_pt == kEPTAssignment) { assert(ast.err.msg); } else if (cur_pt == kEPTExpr && kv_size(pt_stack) > 1 @@ -2467,10 +2460,12 @@ viml_pexpr_parse_bracket_closing_error: kv_drop(pt_stack, 1); } } + if (cur_pt == kEPTSingleAssignment && kv_size(ast_stack) == 1) { + kv_drop(pt_stack, 1); + } } else { if (want_node == kENodeValue) { // Value means list literal or list assignment. - HL_CUR_TOKEN(List); NEW_NODE_WITH_CUR_POS(cur_node, kExprNodeListLiteral); *top_node_p = cur_node; kvi_push(ast_stack, &cur_node->children); @@ -2479,7 +2474,12 @@ viml_pexpr_parse_bracket_closing_error: // Additional assignment parse type allows to easily forbid nested // lists. kvi_push(pt_stack, kEPTSingleAssignment); + } else if (cur_pt == kEPTSingleAssignment) { + ERROR_FROM_TOKEN_AND_MSG( + cur_token, + _("E475: Nested lists not allowed when assigning: %.*s")); } + HL_CUR_TOKEN(List); } else { // Operator means subscript, also in assignment. But in assignment // subscript may be pretty much any expression, so need to push @@ -2491,7 +2491,8 @@ viml_pexpr_parse_bracket_closing_error: ADD_OP_NODE(cur_node); HL_CUR_TOKEN(SubscriptBracket); if (pt_is_assignment(cur_pt)) { - asgn_level = kv_size(ast_stack); + assert(want_node == kENodeValue); // Subtract 1 for NULL at top. + asgn_level = kv_size(ast_stack) - 1; kvi_push(pt_stack, kEPTExpr); } } @@ -2653,7 +2654,8 @@ viml_pexpr_parse_figure_brace_closing_error: } if (pt_is_assignment(cur_pt) && !pt_is_assignment(kv_last(pt_stack))) { - asgn_level = kv_size(ast_stack); + assert(want_node == kENodeValue); // Subtract 1 for NULL at top. + asgn_level = kv_size(ast_stack) - 1; } } break; diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua index ab195f9f1e..023673738d 100644 --- a/test/functional/ui/cmdline_highlight_spec.lua +++ b/test/functional/ui/cmdline_highlight_spec.lua @@ -901,4 +901,5 @@ describe('Expressions coloring support', function() -- FIXME: Test expr coloring when using -u NORC and -u NONE. -- FIXME: Test different ways of triggering expression highlighting (:=, -- i=, :e, "=). + -- FIXME: Test with various invalid unicode and multibyte characters. end) diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index d6d8dd0807..eca76aa9cd 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -268,12 +268,12 @@ local function format_check(expr, format_check_data, opts) local diff = diffs[flags] print((' [%u] = {'):format(flags)) if diff.ast then - print(' ast = ' .. format_luav(diff.ast, ' ')) + print(' ast = ' .. format_luav(diff.ast, ' ') .. ',') end if diff.hl_fs then print(' hl_fs = ' .. format_luav(diff.hl_fs, ' ', { literal_strings=true - })) + }) .. ',') end print(' },') end @@ -7914,12 +7914,769 @@ describe('Expressions parser', function() hl('InvalidList', ']'), }) - -- check_asgn_parsing('a[1 + 2] += 3') - -- check_asgn_parsing('a[{-> {b{3}: 4}[5]}()] += 6') - -- check_asgn_parsing('a{1}.2[{-> {b{3}: 4}[5]}()]') + check_asgn_parsing('a[1] += 3', { + -- 012345678 + ast = { + { + 'Assignment(Add):0:4: +=', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2:1', + }, + }, + 'Integer(val=3):0:7: 3', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Number', '1'), + hl('SubscriptBracket', ']'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '3', 1), + }) + + check_asgn_parsing('a[1 + 2] += 3', { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Assignment(Add):0:8: +=', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'BinaryPlus:0:3: +', + children = { + 'Integer(val=1):0:2:1', + 'Integer(val=2):0:5: 2', + }, + }, + }, + }, + 'Integer(val=3):0:11: 3', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Number', '1'), + hl('BinaryPlus', '+', 1), + hl('Number', '2', 1), + hl('SubscriptBracket', ']'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '3', 1), + }) + + check_asgn_parsing('a[{-> {b{3}: 4}[5]}()] += 6', { + -- 012345678901234567890123456 + -- 0 1 2 + ast = { + { + 'Assignment(Add):0:22: +=', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Call:0:19:(', + children = { + { + 'Lambda(\\di):0:2:{', + children = { + { + 'Arrow:0:3:->', + children = { + { + 'Subscript:0:15:[', + children = { + { + 'DictLiteral(-di):0:5: {', + children = { + { + 'Colon:0:11::', + children = { + { + 'ComplexIdentifier:0:8:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:7:b', + { + 'CurlyBracesIdentifier(--i):0:8:{', + children = { + 'Integer(val=3):0:9:3', + }, + }, + }, + }, + 'Integer(val=4):0:12: 4', + }, + }, + }, + }, + 'Integer(val=5):0:16:5', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + 'Integer(val=6):0:25: 6', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Dict', '{', 1), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('Number', '3'), + hl('Curly', '}'), + hl('Colon', ':'), + hl('Number', '4', 1), + hl('Dict', '}'), + hl('SubscriptBracket', '['), + hl('Number', '5'), + hl('SubscriptBracket', ']'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('SubscriptBracket', ']'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '6', 1), + }) + + check_asgn_parsing('a{1}.2[{-> {b{3}: 4}[5]}()]', { + -- 012345678901234567890123456 + -- 0 1 2 + ast = { + { + 'Subscript:0:6:[', + children = { + { + 'ConcatOrSubscript:0:4:.', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'CurlyBracesIdentifier(--i):0:1:{', + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + 'PlainKey(key=2):0:5:2', + }, + }, + { + 'Call:0:24:(', + children = { + { + 'Lambda(\\di):0:7:{', + children = { + { + 'Arrow:0:8:->', + children = { + { + 'Subscript:0:20:[', + children = { + { + 'DictLiteral(-di):0:10: {', + children = { + { + 'Colon:0:16::', + children = { + { + 'ComplexIdentifier:0:13:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:12:b', + { + 'CurlyBracesIdentifier(--i):0:13:{', + children = { + 'Integer(val=3):0:14:3', + }, + }, + }, + }, + 'Integer(val=4):0:17: 4', + }, + }, + }, + }, + 'Integer(val=5):0:21:5', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('Number', '1'), + hl('Curly', '}'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('SubscriptBracket', '['), + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Dict', '{', 1), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('Number', '3'), + hl('Curly', '}'), + hl('Colon', ':'), + hl('Number', '4', 1), + hl('Dict', '}'), + hl('SubscriptBracket', '['), + hl('Number', '5'), + hl('SubscriptBracket', ']'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('SubscriptBracket', ']'), + }) + + check_asgn_parsing('a', { + -- 0 + ast = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, { + hl('IdentifierName', 'a'), + }) + + check_asgn_parsing('{a}', { + -- 012 + ast = { + { + 'CurlyBracesIdentifier(--i):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + }) + + check_asgn_parsing('{a}b', { + -- 0123 + ast = { + { + 'ComplexIdentifier:0:3:', + children = { + { + 'CurlyBracesIdentifier(--i):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a{b}c', { + -- 01234 + ast = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier(--i):0:1:{', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + }) + + check_asgn_parsing('a{b}c[0]', { + -- 01234567 + ast = { + { + 'Subscript:0:5:[', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier(--i):0:1:{', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + 'Integer(val=0):0:6:0', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', '['), + hl('Number', '0'), + hl('SubscriptBracket', ']'), + }) + + check_asgn_parsing('a{b}c.0', { + -- 0123456 + ast = { + { + 'ConcatOrSubscript:0:5:.', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + 'CurlyBracesIdentifier(--i):0:1:{', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + 'PlainKey(key=0):0:6:0', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '0'), + }) + + check_asgn_parsing('[a{b}c[0].0]', { + -- 012345678901 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'ConcatOrSubscript:0:9:.', + children = { + { + 'Subscript:0:6:[', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'ComplexIdentifier:0:5:', + children = { + { + 'CurlyBracesIdentifier(--i):0:2:{', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + 'Integer(val=0):0:7:0', + }, + }, + 'PlainKey(key=0):0:10:0', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', '['), + hl('Number', '0'), + hl('SubscriptBracket', ']'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '0'), + hl('List', ']'), + }) + + check_asgn_parsing('{a}{b}', { + -- 012345 + ast = { + { + 'ComplexIdentifier:0:3:', + children = { + { + 'CurlyBracesIdentifier(--i):0:0:{', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'CurlyBracesIdentifier(--i):0:3:{', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + }, + }, + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + }) + + check_asgn_parsing('a.b{c}{d}', { + -- 012345678 + ast = { + { + 'OpMissing:0:3:', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=b):0:2:b', + }, + }, + { + 'ComplexIdentifier:0:6:', + children = { + { + 'CurlyBracesIdentifier(--i):0:3:{', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + { + 'CurlyBracesIdentifier(--i):0:6:{', + children = { + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '{c}{d}', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', 'b'), + hl('InvalidFigureBrace', '{'), + hl('IdentifierName', 'c'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'd'), + hl('Curly', '}'), + }) + + check_asgn_parsing('[a] = 1', { + -- 0123456 + ast = { + { + 'Assignment(Plain):0:3: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('[a[b], [c, [d, [e]]]] = 1', { + -- 0123456789012345678901234 + -- 0 1 2 + ast = { + { + 'Assignment(Plain):0:21: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:5:,', + children = { + { + 'Subscript:0:2:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'ListLiteral:0:6: [', + children = { + { + 'Comma:0:9:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:8:c', + { + 'ListLiteral:0:10: [', + children = { + { + 'Comma:0:13:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:12:d', + { + 'ListLiteral:0:14: [', + children = { + 'PlainIdentifier(scope=0,ident=e):0:16:e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + 'Integer(val=1):0:23: 1', + }, + }, + }, + err = { + arg = '[c, [d, [e]]]] = 1', + msg = 'E475: Nested lists not allowed when assigning: %.*s', + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptBracket', ']'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'd'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'e'), + hl('List', ']'), + hl('List', ']'), + hl('List', ']'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('$X += 1', { + -- 0123456 + ast = { + { + 'Assignment(Add):0:2: +=', + children = { + 'Environment(ident=X):0:0:$X', + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'X'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('@a .= 1', { + -- 0123456 + ast = { + { + 'Assignment(Concat):0:2: .=', + children = { + 'Register(name=a):0:0:@a', + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('AssignmentWithConcatenation', '.=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('&option -= 1', { + -- 012345678901 + -- 0 1 + ast = { + { + 'Assignment(Subtract):0:7: -=', + children = { + 'Option(scope=0,ident=option):0:0:&option', + 'Integer(val=1):0:10: 1', + }, + }, + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'option'), + hl('AssignmentWithSubtraction', '-=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('[$X, @a, &l:option] = [1, 2, 3]', { + -- 0123456789012345678901234567890 + -- 0 1 2 3 + ast = { + { + 'Assignment(Plain):0:19: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:3:,', + children = { + 'Environment(ident=X):0:1:$X', + { + 'Comma:0:7:,', + children = { + 'Register(name=a):0:4: @a', + 'Option(scope=l,ident=option):0:8: &l:option', + }, + }, + }, + }, + }, + }, + { + 'ListLiteral:0:21: [', + children = { + { + 'Comma:0:24:,', + children = { + 'Integer(val=1):0:23:1', + { + 'Comma:0:27:,', + children = { + 'Integer(val=2):0:25: 2', + 'Integer(val=3):0:28: 3', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'X'), + hl('Comma', ','), + hl('Register', '@a', 1), + hl('Comma', ','), + hl('OptionSigil', '&', 1), + hl('OptionScope', 'l'), + hl('OptionScopeDelimiter', ':'), + hl('OptionName', 'option'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('List', '[', 1), + hl('Number', '1'), + hl('Comma', ','), + hl('Number', '2', 1), + hl('Comma', ','), + hl('Number', '3', 1), + hl('List', ']'), + }) end) - -- FIXME: Test assignments thoroughly - -- FIXME: Test that parsing assignments can be used for `:for` pre-`in` part. -- FIXME: Somehow make functional tests use the same code. Or, at least, -- create an automated script which will do the import. end) From 764cf3251de4f141fa451633dbdb48440ddf218a Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 19:27:21 +0300 Subject: [PATCH 088/109] charset: Add missing include needed for vim_str2nr --- src/nvim/charset.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nvim/charset.h b/src/nvim/charset.h index 7198c8d405..eb64b6128a 100644 --- a/src/nvim/charset.h +++ b/src/nvim/charset.h @@ -4,6 +4,7 @@ #include "nvim/types.h" #include "nvim/pos.h" #include "nvim/buffer_defs.h" +#include "nvim/eval/typval.h" /// Return the folded-case equivalent of the given character /// From 53fa435a1f081e1d7e1890e3ecd978b85c39e0eb Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 19:34:15 +0300 Subject: [PATCH 089/109] functests: Fix ui/cmdline test --- test/functional/ui/cmdline_spec.lua | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/functional/ui/cmdline_spec.lua b/test/functional/ui/cmdline_spec.lua index 0f8302b036..136ab09d4f 100644 --- a/test/functional/ui/cmdline_spec.lua +++ b/test/functional/ui/cmdline_spec.lua @@ -226,7 +226,11 @@ describe('external cmdline', function() prompt = "", special = {'"', true}, },{ - content = { { {}, "1+2" } }, + content = { + { {}, "1" }, + { {}, "+" }, + { {}, "2" }, + }, firstc = "=", indent = 0, pos = 3, @@ -303,7 +307,7 @@ describe('external cmdline', function() pos = 0, prompt = "", }}, cmdline) - eq({{{{}, 'function Foo()'}}}, block) + eq({ { { {}, 'function Foo()'} } }, block) end) feed('line1') @@ -314,8 +318,8 @@ describe('external cmdline', function() ~ | | ]], nil, nil, function() - eq({{{{}, 'function Foo()'}}, - {{{}, ' line1'}}}, block) + eq({ { { {}, 'function Foo()'} }, + { { {}, ' line1'} } }, block) end) block = {} @@ -327,8 +331,8 @@ describe('external cmdline', function() ~ | ^ | ]], nil, nil, function() - eq({{{{}, 'function Foo()'}}, - {{{}, ' line1'}}}, block) + eq({ { { {}, 'function Foo()'} }, + { { {}, ' line1'} } }, block) end) From a94255a7ac22649311f858e39b217543d0d7e5e8 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 20:20:06 +0300 Subject: [PATCH 090/109] tests: Use single test file for unit and functional parser tests --- test/functional/api/vim_spec.lua | 7132 +--------------- test/unit/viml/expressions/parser_spec.lua | 8180 +----------------- test/unit/viml/expressions/parser_tests.lua | 8185 +++++++++++++++++++ 3 files changed, 8241 insertions(+), 15256 deletions(-) create mode 100644 test/unit/viml/expressions/parser_tests.lua diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index 714b1988fb..3939bc9b52 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -766,6 +766,11 @@ describe('api', function() elseif typ == 'Environment' then typ = ('%s(ident=%s)'):format(typ, east_api_node.ident) east_api_node.ident = nil + elseif typ == 'Assignment' then + local aug = east_api_node.augmentation + if aug == '' then aug = 'Plain' end + typ = ('%s(%s)'):format(typ, aug) + east_api_node.augmentation = nil end typ = ('%s:%u:%u:%s'):format( typ, east_api_node.start[1], east_api_node.start[2], @@ -798,6 +803,9 @@ describe('api', function() end if east_api.ast then east_api.ast = {simplify_east_api_node(line, east_api.ast)} + if #east_api.ast == 0 then + east_api.ast = nil + end end if east_api.len == #line then east_api.len = nil @@ -819,11 +827,19 @@ describe('api', function() [1] = "m", [2] = "E", [3] = "mE", + [4] = "l", + [5] = "lm", + [6] = "lE", + [7] = "lmE", } - local function check_parsing(str, exp_ast, exp_highlighting_fs, - nz_flags_exps) + local function _check_parsing(opts, str, exp_ast, exp_highlighting_fs, + nz_flags_exps) + if type(str) ~= 'string' then + return + end + local zflags = opts.flags[1] nz_flags_exps = nz_flags_exps or {} - for _, flags in ipairs({0, 1, 2, 3}) do + for _, flags in ipairs(opts.flags) do local err, msg = pcall(function() local east_api = meths.parse_expression(str, FLAGS_TO_STR[flags], true) local east_hl = east_api.highlight @@ -835,8 +851,8 @@ describe('api', function() hl_fs = exp_highlighting_fs, } local add_exps = nz_flags_exps[flags] - if not add_exps and flags == 3 then - add_exps = nz_flags_exps[1] or nz_flags_exps[2] + if not add_exps and flags == 3 + zflags then + add_exps = nz_flags_exps[1 + zflags] or nz_flags_exps[2 + zflags] end if add_exps then if add_exps.ast then @@ -873,7096 +889,22 @@ describe('api', function() str)), (col + #str) end end - it('works with + and @a', function() - check_parsing('@a', { - ast = { - 'Register(name=a):0:0:@a', - }, - }, { - hl('Register', '@a'), - }) - check_parsing('+@a', { - ast = { - { - 'UnaryPlus:0:0:+', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('Register', '@a'), - }) - check_parsing('@a+@b', { - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - 'Register(name=b):0:3:@b', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - }) - check_parsing('@a+@b+@c', { - ast = { - { - 'BinaryPlus:0:5:+', - children = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - 'Register(name=b):0:3:@b', - }, - }, - 'Register(name=c):0:6:@c', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - hl('BinaryPlus', '+'), - hl('Register', '@c'), - }) - check_parsing('+@a+@b', { - ast = { - { - 'BinaryPlus:0:3:+', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'Register(name=a):0:1:@a', - }, - }, - 'Register(name=b):0:4:@b', - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - }) - check_parsing('+@a++@b', { - ast = { - { - 'BinaryPlus:0:3:+', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'Register(name=a):0:1:@a', - }, - }, - { - 'UnaryPlus:0:4:+', - children = { - 'Register(name=b):0:5:@b', - }, - }, - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('UnaryPlus', '+'), - hl('Register', '@b'), - }) - check_parsing('@a@b', { - ast = { - { - 'OpMissing:0:2:', - children = { - 'Register(name=a):0:0:@a', - 'Register(name=b):0:2:@b', - }, - }, - }, - err = { - arg = '@b', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('Register', '@a'), - hl('InvalidRegister', '@b'), - }, { - [1] = { - ast = { - len = 2, - err = REMOVE_THIS, - ast = { - 'Register(name=a):0:0:@a' - }, - }, - hl_fs = { - [2] = REMOVE_THIS, - }, - }, - }) - check_parsing(' @a \t @b', { - ast = { - { - 'OpMissing:0:3:', - children = { - 'Register(name=a):0:0: @a', - 'Register(name=b):0:3: \t @b', - }, - }, - }, - err = { - arg = '@b', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('Register', '@a', 1), - hl('InvalidSpacing', ' \t '), - hl('Register', '@b'), - }, { - [1] = { - ast = { - len = 6, - err = REMOVE_THIS, - ast = { - 'Register(name=a):0:0: @a' - }, - }, - hl_fs = { - [2] = REMOVE_THIS, - [3] = REMOVE_THIS, - }, - }, - }) - check_parsing('+', { - ast = { - 'UnaryPlus:0:0:+', - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('UnaryPlus', '+'), - }) - check_parsing(' +', { - ast = { - 'UnaryPlus:0:0: +', - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('UnaryPlus', '+', 1), - }) - check_parsing('@a+ ', { - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - }) - end) - it('works with @a, + and parenthesis', function() - check_parsing('(@a)', { - ast = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('NestingParenthesis', ')'), - }) - check_parsing('()', { - ast = { - { - 'Nested:0:0:(', - children = { - 'Missing:0:1:', - }, - }, - }, - err = { - arg = ')', - msg = 'E15: Expected value, got parenthesis: %.*s', - }, - }, { - hl('NestingParenthesis', '('), - hl('InvalidNestingParenthesis', ')'), - }) - check_parsing(')', { - ast = { - { - 'Nested:0:0:', - children = { - 'Missing:0:0:', - }, - }, - }, - err = { - arg = ')', - msg = 'E15: Expected value, got parenthesis: %.*s', - }, - }, { - hl('InvalidNestingParenthesis', ')'), - }) - check_parsing('+)', { - ast = { - { - 'Nested:0:1:', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'Missing:0:1:', - }, - }, - }, - }, - }, - err = { - arg = ')', - msg = 'E15: Expected value, got parenthesis: %.*s', - }, - }, { - hl('UnaryPlus', '+'), - hl('InvalidNestingParenthesis', ')'), - }) - check_parsing('+@a(@b)', { - ast = { - { - 'UnaryPlus:0:0:+', - children = { - { - 'Call:0:3:(', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a+@b(@c)', { - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:5:(', - children = { - 'Register(name=b):0:3:@b', - 'Register(name=c):0:6:@c', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a()', { - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a ()', { - ast = { - { - 'OpMissing:0:2:', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:2: (', - children = { - 'Missing:0:4:', - }, - }, - }, - }, - }, - err = { - arg = '()', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('Register', '@a'), - hl('InvalidSpacing', ' '), - hl('NestingParenthesis', '('), - hl('InvalidNestingParenthesis', ')'), - }, { - [1] = { - ast = { - len = 3, - err = REMOVE_THIS, - ast = { - 'Register(name=a):0:0:@a', - }, - }, - hl_fs = { - [2] = REMOVE_THIS, - [3] = REMOVE_THIS, - [4] = REMOVE_THIS, - }, - }, - }) - check_parsing('@a + (@b)', { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - 'Register(name=b):0:6:@b', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - }) - check_parsing('@a + (+@b)', { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - { - 'UnaryPlus:0:6:+', - children = { - 'Register(name=b):0:7:@b', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('UnaryPlus', '+'), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - }) - check_parsing('@a + (@b + @c)', { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - { - 'BinaryPlus:0:8: +', - children = { - 'Register(name=b):0:6:@b', - 'Register(name=c):0:10: @c', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Register', '@b'), - hl('BinaryPlus', '+', 1), - hl('Register', '@c', 1), - hl('NestingParenthesis', ')'), - }) - check_parsing('(@a)+@b', { - ast = { - { - 'BinaryPlus:0:4:+', - children = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - 'Register(name=b):0:5:@b', - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('NestingParenthesis', ')'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - }) - check_parsing('@a+(@b)(@c)', { - -- 01234567890 - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:7:(', - children = { - { - 'Nested:0:3:(', - children = { 'Register(name=b):0:4:@b' }, - }, - 'Register(name=c):0:8:@c', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('NestingParenthesis', '('), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a+((@b))(@c)', { - -- 01234567890123456890123456789 - -- 0 1 2 - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:9:(', - children = { - { - 'Nested:0:3:(', - children = { - { - 'Nested:0:4:(', - children = { 'Register(name=b):0:5:@b' } - }, - }, - }, - 'Register(name=c):0:10:@c', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('NestingParenthesis', '('), - hl('NestingParenthesis', '('), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a+((@b))+@c', { - -- 01234567890123456890123456789 - -- 0 1 2 - ast = { - { - 'BinaryPlus:0:9:+', - children = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:3:(', - children = { - { - 'Nested:0:4:(', - children = { 'Register(name=b):0:5:@b' } - }, - }, - }, - }, - }, - 'Register(name=c):0:10:@c', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('NestingParenthesis', '('), - hl('NestingParenthesis', '('), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - hl('NestingParenthesis', ')'), - hl('BinaryPlus', '+'), - hl('Register', '@c'), - }) - check_parsing( - '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', {--[[ - | | | | | | | | || | | || | | ||| || || || || - 000000000011111111112222222222333333333344444444445555555 - 012345678901234567890123456789012345678901234567890123456 - ]] - ast = {{ - 'BinaryPlus:0:31: +', - children = { - { - 'BinaryPlus:0:23: +', - children = { - { - 'BinaryPlus:0:14: +', - children = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - { - 'BinaryPlus:0:8: +', - children = { - 'Register(name=b):0:6:@b', - 'Register(name=c):0:10: @c', - }, - }, - }, - }, - }, - }, - { - 'Call:0:19:(', - children = { - 'Register(name=d):0:16: @d', - 'Register(name=e):0:20:@e', - }, - }, - }, - }, - { - 'Nested:0:25: (', - children = { - { - 'UnaryPlus:0:27:+', - children = { - 'Register(name=f):0:28:@f', - }, - }, - }, - }, - }, - }, - { - 'Call:0:53:(', - children = { - { - 'Nested:0:33: (', - children = { - { - 'Call:0:48:(', - children = { - { - 'Call:0:44:(', - children = { - { - 'Nested:0:35:(', - children = { - { - 'UnaryPlus:0:36:+', - children = { - { - 'Call:0:39:(', - children = { - 'Register(name=g):0:37:@g', - 'Register(name=h):0:40:@h', - }, - }, - }, - }, - }, - }, - 'Register(name=j):0:45:@j', - }, - }, - 'Register(name=k):0:49:@k', - }, - }, - }, - }, - 'Register(name=l):0:54:@l', - }, - }, - }, - }}, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Register', '@b'), - hl('BinaryPlus', '+', 1), - hl('Register', '@c', 1), - hl('NestingParenthesis', ')'), - hl('BinaryPlus', '+', 1), - hl('Register', '@d', 1), - hl('CallingParenthesis', '('), - hl('Register', '@e'), - hl('CallingParenthesis', ')'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('UnaryPlus', '+'), - hl('Register', '@f'), - hl('NestingParenthesis', ')'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('NestingParenthesis', '('), - hl('UnaryPlus', '+'), - hl('Register', '@g'), - hl('CallingParenthesis', '('), - hl('Register', '@h'), - hl('CallingParenthesis', ')'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@j'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@k'), - hl('CallingParenthesis', ')'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@l'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a)', { - -- 012 - ast = { - { - 'Nested:0:2:', - children = { - 'Register(name=a):0:0:@a', - }, - }, - }, - err = { - arg = ')', - msg = 'E15: Unexpected closing parenthesis: %.*s', - }, - }, { - hl('Register', '@a'), - hl('InvalidNestingParenthesis', ')'), - }) - check_parsing('(@a', { - -- 012 - ast = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - err = { - arg = '(@a', - msg = 'E110: Missing closing parenthesis for nested expression: %.*s', - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - }) - check_parsing('@a(@b', { - -- 01234 - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - 'Register(name=b):0:3:@b', - }, - }, - }, - err = { - arg = '(@b', - msg = 'E116: Missing closing parenthesis for function call: %.*s', - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - }) - check_parsing('@a(@b, @c, @d, @e)', { - -- 012345678901234567 - -- 0 1 - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - { - 'Comma:0:5:,', - children = { - 'Register(name=b):0:3:@b', - { - 'Comma:0:9:,', - children = { - 'Register(name=c):0:6: @c', - { - 'Comma:0:13:,', - children = { - 'Register(name=d):0:10: @d', - 'Register(name=e):0:14: @e', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('Comma', ','), - hl('Register', '@c', 1), - hl('Comma', ','), - hl('Register', '@d', 1), - hl('Comma', ','), - hl('Register', '@e', 1), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a(@b(@c))', { - -- 01234567890123456789012345678901234567 - -- 0 1 2 3 - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:5:(', - children = { - 'Register(name=b):0:3:@b', - 'Register(name=c):0:6:@c', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', { - -- 01234567890123456789012345678901234567 - -- 0 1 2 3 - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:5:(', - children = { - 'Register(name=b):0:3:@b', - { - 'Call:0:8:(', - children = { - 'Register(name=c):0:6:@c', - { - 'Comma:0:15:,', - children = { - { - 'Call:0:11:(', - children = { - 'Register(name=d):0:9:@d', - 'Register(name=e):0:12:@e', - }, - }, - { - 'Call:0:19:(', - children = { - 'Register(name=f):0:16: @f', - { - 'Comma:0:26:,', - children = { - { - 'Call:0:22:(', - children = { - 'Register(name=g):0:20:@g', - 'Register(name=h):0:23:@h', - }, - }, - { - 'Call:0:30:(', - children = { - 'Register(name=i):0:27: @i', - 'Register(name=j):0:31:@j', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', '('), - hl('Register', '@d'), - hl('CallingParenthesis', '('), - hl('Register', '@e'), - hl('CallingParenthesis', ')'), - hl('Comma', ','), - hl('Register', '@f', 1), - hl('CallingParenthesis', '('), - hl('Register', '@g'), - hl('CallingParenthesis', '('), - hl('Register', '@h'), - hl('CallingParenthesis', ')'), - hl('Comma', ','), - hl('Register', '@i', 1), - hl('CallingParenthesis', '('), - hl('Register', '@j'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - }) - end) - it('works with variable names, including curly braces ones', function() - check_parsing('var', { - ast = { - 'PlainIdentifier(scope=0,ident=var):0:0:var', - }, - }, { - hl('IdentifierName', 'var'), - }) - check_parsing('g:var', { - ast = { - 'PlainIdentifier(scope=g,ident=var):0:0:g:var', - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('IdentifierName', 'var'), - }) - check_parsing('g:', { - ast = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - }) - check_parsing('{a}', { - -- 012 - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('IdentifierName', 'a'), - hl('Curly', '}'), - }) - check_parsing('{a:b}', { - -- 012 - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - 'PlainIdentifier(scope=a,ident=b):0:1:a:b', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - }) - check_parsing('{a:@b}', { - -- 012345 - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - { - 'OpMissing:0:3:', - children={ - 'PlainIdentifier(scope=a,ident=):0:1:a:', - 'Register(name=b):0:3:@b', - }, - }, - }, - }, - }, - err = { - arg = '@b}', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('Curly', '{'), - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('InvalidRegister', '@b'), - hl('Curly', '}'), - }) - check_parsing('{@a}', { - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - }) - check_parsing('{@a}{@b}', { - -- 01234567 - ast = { - { - 'ComplexIdentifier:0:4:', - children = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - { - 'CurlyBracesIdentifier:0:4:{', - children = { - 'Register(name=b):0:5:@b', - }, - }, - }, - }, - }, - }, { - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('Curly', '{'), - hl('Register', '@b'), - hl('Curly', '}'), - }) - check_parsing('g:{@a}', { - -- 01234567 - ast = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - { - 'CurlyBracesIdentifier:0:2:{', - children = { - 'Register(name=a):0:3:@a', - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - }) - check_parsing('{@a}_test', { - -- 012345678 - ast = { - { - 'ComplexIdentifier:0:4:', - children = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - 'PlainIdentifier(scope=0,ident=_test):0:4:_test', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('IdentifierName', '_test'), - }) - check_parsing('g:{@a}_test', { - -- 01234567890 - ast = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - { - 'ComplexIdentifier:0:6:', - children = { - { - 'CurlyBracesIdentifier:0:2:{', - children = { - 'Register(name=a):0:3:@a', - }, - }, - 'PlainIdentifier(scope=0,ident=_test):0:6:_test', - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('IdentifierName', '_test'), - }) - check_parsing('g:{@a}_test()', { - -- 0123456789012 - ast = { - { - 'Call:0:11:(', - children = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - { - 'ComplexIdentifier:0:6:', - children = { - { - 'CurlyBracesIdentifier:0:2:{', - children = { - 'Register(name=a):0:3:@a', - }, - }, - 'PlainIdentifier(scope=0,ident=_test):0:6:_test', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('IdentifierName', '_test'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - }) - check_parsing('{@a} ()', { - -- 0123456789012 - ast = { - { - 'Call:0:4: (', - children = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, - }, - }, { - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('CallingParenthesis', '(', 1), - hl('CallingParenthesis', ')'), - }) - check_parsing('g:{@a} ()', { - -- 0123456789012 - ast = { - { - 'Call:0:6: (', - children = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - { - 'CurlyBracesIdentifier:0:2:{', - children = { - 'Register(name=a):0:3:@a', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('CallingParenthesis', '(', 1), - hl('CallingParenthesis', ')'), - }) - check_parsing('{@a', { - -- 012 - ast = { - { - 'UnknownFigure:0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - err = { - arg = '{@a', - msg = 'E15: Missing closing figure brace: %.*s', - }, - }, { - hl('FigureBrace', '{'), - hl('Register', '@a'), - }) - end) - it('works with lambdas and dictionaries', function() - check_parsing('{}', { - ast = { - 'DictLiteral:0:0:{', - }, - }, { - hl('Dict', '{'), - hl('Dict', '}'), - }) - check_parsing('{->@a}', { - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Arrow:0:1:->', - children = { - 'Register(name=a):0:3:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{->@a+@b}', { - -- 012345678 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Arrow:0:1:->', - children = { - { - 'BinaryPlus:0:5:+', - children = { - 'Register(name=a):0:3:@a', - 'Register(name=b):0:6:@b', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - hl('Lambda', '}'), - }) - check_parsing('{a->@a}', { - -- 012345678 - ast = { - { - 'Lambda:0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Arrow:0:2:->', - children = { - 'Register(name=a):0:4:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b->@a}', { - -- 012345678 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'Arrow:0:4:->', - children = { - 'Register(name=a):0:6:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b,c->@a}', { - -- 01234567890 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:4:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - 'PlainIdentifier(scope=0,ident=c):0:5:c', - }, - }, - }, - }, - { - 'Arrow:0:6:->', - children = { - 'Register(name=a):0:8:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Comma', ','), - hl('IdentifierName', 'c'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b,c,d->@a}', { - -- 0123456789012 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:4:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - { - 'Comma:0:6:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:5:c', - 'PlainIdentifier(scope=0,ident=d):0:7:d', - }, - }, - }, - }, - }, - }, - { - 'Arrow:0:8:->', - children = { - 'Register(name=a):0:10:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Comma', ','), - hl('IdentifierName', 'c'), - hl('Comma', ','), - hl('IdentifierName', 'd'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b,c,d,->@a}', { - -- 01234567890123 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:4:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - { - 'Comma:0:6:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:5:c', - { - 'Comma:0:8:,', - children = { - 'PlainIdentifier(scope=0,ident=d):0:7:d', - }, - }, - }, - }, - }, - }, - }, - }, - { - 'Arrow:0:9:->', - children = { - 'Register(name=a):0:11:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Comma', ','), - hl('IdentifierName', 'c'), - hl('Comma', ','), - hl('IdentifierName', 'd'), - hl('Comma', ','), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b->{c,d->{e,f->@a}}}', { - -- 01234567890123456789012 - -- 0 1 2 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'Arrow:0:4:->', - children = { - { - 'Lambda:0:6:{', - children = { - { - 'Comma:0:8:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:7:c', - 'PlainIdentifier(scope=0,ident=d):0:9:d', - }, - }, - { - 'Arrow:0:10:->', - children = { - { - 'Lambda:0:12:{', - children = { - { - 'Comma:0:14:,', - children = { - 'PlainIdentifier(scope=0,ident=e):0:13:e', - 'PlainIdentifier(scope=0,ident=f):0:15:f', - }, - }, - { - 'Arrow:0:16:->', - children = { - 'Register(name=a):0:18:@a', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - hl('Lambda', '{'), - hl('IdentifierName', 'c'), - hl('Comma', ','), - hl('IdentifierName', 'd'), - hl('Arrow', '->'), - hl('Lambda', '{'), - hl('IdentifierName', 'e'), - hl('Comma', ','), - hl('IdentifierName', 'f'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - hl('Lambda', '}'), - hl('Lambda', '}'), - }) - check_parsing('{a,b->c,d}', { - -- 0123456789 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'Arrow:0:4:->', - children = { - { - 'Comma:0:7:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:6:c', - 'PlainIdentifier(scope=0,ident=d):0:8:d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ',d}', - msg = 'E15: Comma outside of call, lambda or literal: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - hl('IdentifierName', 'c'), - hl('InvalidComma', ','), - hl('IdentifierName', 'd'), - hl('Lambda', '}'), - }) - check_parsing('a,b,c,d', { - -- 0123456789 - ast = { - { - 'Comma:0:1:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Comma:0:3:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:4:c', - 'PlainIdentifier(scope=0,ident=d):0:6:d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ',b,c,d', - msg = 'E15: Comma outside of call, lambda or literal: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidComma', ','), - hl('IdentifierName', 'b'), - hl('InvalidComma', ','), - hl('IdentifierName', 'c'), - hl('InvalidComma', ','), - hl('IdentifierName', 'd'), - }) - check_parsing('a,b,c,d,', { - -- 0123456789 - ast = { - { - 'Comma:0:1:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Comma:0:3:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:4:c', - { - 'Comma:0:7:,', - children = { - 'PlainIdentifier(scope=0,ident=d):0:6:d', - }, - }, - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ',b,c,d,', - msg = 'E15: Comma outside of call, lambda or literal: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidComma', ','), - hl('IdentifierName', 'b'), - hl('InvalidComma', ','), - hl('IdentifierName', 'c'), - hl('InvalidComma', ','), - hl('IdentifierName', 'd'), - hl('InvalidComma', ','), - }) - check_parsing(',', { - -- 0123456789 - ast = { - { - 'Comma:0:0:,', - children = { - 'Missing:0:0:', - }, - }, - }, - err = { - arg = ',', - msg = 'E15: Expected value, got comma: %.*s', - }, - }, { - hl('InvalidComma', ','), - }) - check_parsing('{,a->@a}', { - -- 0123456789 - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - { - 'Arrow:0:3:->', - children = { - { - 'Comma:0:1:,', - children = { - 'Missing:0:1:', - 'PlainIdentifier(scope=0,ident=a):0:2:a', - }, - }, - 'Register(name=a):0:5:@a', - }, - }, - }, - }, - }, - err = { - arg = ',a->@a}', - msg = 'E15: Expected value, got comma: %.*s', - }, - }, { - hl('Curly', '{'), - hl('InvalidComma', ','), - hl('IdentifierName', 'a'), - hl('InvalidArrow', '->'), - hl('Register', '@a'), - hl('Curly', '}'), - }) - check_parsing('}', { - -- 0123456789 - ast = { - 'UnknownFigure:0:0:', - }, - err = { - arg = '}', - msg = 'E15: Unexpected closing figure brace: %.*s', - }, - }, { - hl('InvalidFigureBrace', '}'), - }) - check_parsing('{->}', { - -- 0123456789 - ast = { - { - 'Lambda:0:0:{', - children = { - 'Arrow:0:1:->', - }, - }, - }, - err = { - arg = '}', - msg = 'E15: Expected value, got closing figure brace: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('Arrow', '->'), - hl('InvalidLambda', '}'), - }) - check_parsing('{a,b}', { - -- 0123456789 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - }, - }, - }, - err = { - arg = '}', - msg = 'E15: Expected lambda arguments list or arrow: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('InvalidLambda', '}'), - }) - check_parsing('{a,}', { - -- 0123456789 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - }, - }, - err = { - arg = '}', - msg = 'E15: Expected lambda arguments list or arrow: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('InvalidLambda', '}'), - }) - check_parsing('{@a:@b}', { - -- 0123456789 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Dict', '}'), - }) - check_parsing('{@a:@b,@c:@d}', { - -- 0123456789012 - -- 0 1 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - { - 'Colon:0:9::', - children = { - 'Register(name=c):0:7:@c', - 'Register(name=d):0:10:@d', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Comma', ','), - hl('Register', '@c'), - hl('Colon', ':'), - hl('Register', '@d'), - hl('Dict', '}'), - }) - check_parsing('{@a:@b,@c:@d,@e:@f,}', { - -- 01234567890123456789 - -- 0 1 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - { - 'Comma:0:12:,', - children = { - { - 'Colon:0:9::', - children = { - 'Register(name=c):0:7:@c', - 'Register(name=d):0:10:@d', - }, - }, - { - 'Comma:0:18:,', - children = { - { - 'Colon:0:15::', - children = { - 'Register(name=e):0:13:@e', - 'Register(name=f):0:16:@f', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Comma', ','), - hl('Register', '@c'), - hl('Colon', ':'), - hl('Register', '@d'), - hl('Comma', ','), - hl('Register', '@e'), - hl('Colon', ':'), - hl('Register', '@f'), - hl('Comma', ','), - hl('Dict', '}'), - }) - check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', { - -- 01234567890123456789012 - -- 0 1 2 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - { - 'Comma:0:12:,', - children = { - { - 'Colon:0:9::', - children = { - 'Register(name=c):0:7:@c', - 'Register(name=d):0:10:@d', - }, - }, - { - 'Comma:0:18:,', - children = { - { - 'Colon:0:15::', - children = { - 'Register(name=e):0:13:@e', - 'Register(name=f):0:16:@f', - }, - }, - { - 'Colon:0:21::', - children = { - 'Register(name=g):0:19:@g', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '}', - msg = 'E15: Expected value, got closing figure brace: %.*s', - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Comma', ','), - hl('Register', '@c'), - hl('Colon', ':'), - hl('Register', '@d'), - hl('Comma', ','), - hl('Register', '@e'), - hl('Colon', ':'), - hl('Register', '@f'), - hl('Comma', ','), - hl('Register', '@g'), - hl('Colon', ':'), - hl('InvalidDict', '}'), - }) - check_parsing('{@a:@b,}', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Comma', ','), - hl('Dict', '}'), - }) - check_parsing('{({f -> g})(@h)(@i)}', { - -- 01234567890123456789 - -- 0 1 - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - { - 'Call:0:15:(', - children = { - { - 'Call:0:11:(', - children = { - { - 'Nested:0:1:(', - children = { - { - 'Lambda:0:2:{', - children = { - 'PlainIdentifier(scope=0,ident=f):0:3:f', - { - 'Arrow:0:4: ->', - children = { - 'PlainIdentifier(scope=0,ident=g):0:7: g', - }, - }, - }, - }, - }, - }, - 'Register(name=h):0:12:@h', - }, - }, - 'Register(name=i):0:16:@i', - }, - }, - }, - }, - }, - }, { - hl('Curly', '{'), - hl('NestingParenthesis', '('), - hl('Lambda', '{'), - hl('IdentifierName', 'f'), - hl('Arrow', '->', 1), - hl('IdentifierName', 'g', 1), - hl('Lambda', '}'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@h'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@i'), - hl('CallingParenthesis', ')'), - hl('Curly', '}'), - }) - check_parsing('a:{b()}c', { - -- 01234567 - ast = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=a,ident=):0:0:a:', - { - 'ComplexIdentifier:0:7:', - children = { - { - 'CurlyBracesIdentifier:0:2:{', - children = { - { - 'Call:0:4:(', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=c):0:7:c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - hl('Curly', '}'), - hl('IdentifierName', 'c'), - }) - check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', { - -- 01234567890123456789012345678901234567890123456 - -- 0 1 2 3 4 - ast = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=a,ident=):0:0:a:', - { - 'ComplexIdentifier:0:42:', - children = { - { - 'CurlyBracesIdentifier:0:2:{', - children = { - { - 'Call:0:37:(', - children = { - { - 'Lambda:0:3:{', - children = { - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4:b', - 'PlainIdentifier(scope=0,ident=c):0:6: c', - }, - }, - { - 'Arrow:0:8: ->', - children = { - { - 'BinaryPlus:0:19: +', - children = { - { - 'BinaryPlus:0:14: +', - children = { - 'Register(name=d):0:11: @d', - 'Register(name=e):0:16: @e', - }, - }, - { - 'Call:0:32:(', - children = { - { - 'Nested:0:21: (', - children = { - { - 'Lambda:0:23:{', - children = { - 'PlainIdentifier(scope=0,ident=f):0:24:f', - { - 'Arrow:0:25: ->', - children = { - 'PlainIdentifier(scope=0,ident=g):0:28: g', - }, - }, - }, - }, - }, - }, - 'Register(name=h):0:33:@h', - }, - }, - }, - }, - }, - }, - }, - }, - 'Register(name=i):0:38:@i', - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=j):0:42:j', - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Lambda', '{'), - hl('IdentifierName', 'b'), - hl('Comma', ','), - hl('IdentifierName', 'c', 1), - hl('Arrow', '->', 1), - hl('Register', '@d', 1), - hl('BinaryPlus', '+', 1), - hl('Register', '@e', 1), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Lambda', '{'), - hl('IdentifierName', 'f'), - hl('Arrow', '->', 1), - hl('IdentifierName', 'g', 1), - hl('Lambda', '}'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@h'), - hl('CallingParenthesis', ')'), - hl('Lambda', '}'), - hl('CallingParenthesis', '('), - hl('Register', '@i'), - hl('CallingParenthesis', ')'), - hl('Curly', '}'), - hl('IdentifierName', 'j'), - }) - check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', { - -- 01234567890123456789012345678901234567 - -- 0 1 2 3 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Comma:0:18:,', - children = { - { - 'Colon:0:8: :', - children = { - { - 'BinaryPlus:0:3: +', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:5: @b', - }, - }, - { - 'BinaryPlus:0:13: +', - children = { - 'Register(name=c):0:10: @c', - 'Register(name=d):0:15: @d', - }, - }, - }, - }, - { - 'Colon:0:27: :', - children = { - { - 'BinaryPlus:0:22: +', - children = { - 'Register(name=e):0:19: @e', - 'Register(name=f):0:24: @f', - }, - }, - { - 'BinaryPlus:0:32: +', - children = { - 'Register(name=g):0:29: @g', - 'Register(name=i):0:34: @i', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('Register', '@b', 1), - hl('Colon', ':', 1), - hl('Register', '@c', 1), - hl('BinaryPlus', '+', 1), - hl('Register', '@d', 1), - hl('Comma', ','), - hl('Register', '@e', 1), - hl('BinaryPlus', '+', 1), - hl('Register', '@f', 1), - hl('Colon', ':', 1), - hl('Register', '@g', 1), - hl('BinaryPlus', '+', 1), - hl('Register', '@i', 1), - hl('Dict', '}'), - }) - check_parsing('-> -> ->', { - -- 01234567 - ast = { - { - 'Arrow:0:0:->', - children = { - 'Missing:0:0:', - { - 'Arrow:0:2: ->', - children = { - 'Missing:0:2:', - { - 'Arrow:0:5: ->', - children = { - 'Missing:0:5:', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '-> -> ->', - msg = 'E15: Unexpected arrow: %.*s', - }, - }, { - hl('InvalidArrow', '->'), - hl('InvalidArrow', '->', 1), - hl('InvalidArrow', '->', 1), - }) - check_parsing('a -> b -> c -> d', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'Arrow:0:1: ->', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Arrow:0:6: ->', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4: b', - { - 'Arrow:0:11: ->', - children = { - 'PlainIdentifier(scope=0,ident=c):0:9: c', - 'PlainIdentifier(scope=0,ident=d):0:14: d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '-> b -> c -> d', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'c', 1), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'd', 1), - }) - check_parsing('{a -> b -> c}', { - -- 0123456789012 - -- 0 1 - ast = { - { - 'Lambda:0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Arrow:0:2: ->', - children = { - { - 'Arrow:0:7: ->', - children = { - 'PlainIdentifier(scope=0,ident=b):0:5: b', - 'PlainIdentifier(scope=0,ident=c):0:10: c', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '-> c}', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Arrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'c', 1), - hl('Lambda', '}'), - }) - check_parsing('{a: -> b}', { - -- 012345678 - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - { - 'Arrow:0:3: ->', - children = { - 'PlainIdentifier(scope=a,ident=):0:1:a:', - 'PlainIdentifier(scope=0,ident=b):0:6: b', - }, - }, - }, - }, - }, - err = { - arg = '-> b}', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('Curly', '{'), - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('Curly', '}'), - }) - - check_parsing('{a:b -> b}', { - -- 0123456789 - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - { - 'Arrow:0:4: ->', - children = { - 'PlainIdentifier(scope=a,ident=b):0:1:a:b', - 'PlainIdentifier(scope=0,ident=b):0:7: b', - }, - }, - }, - }, - }, - err = { - arg = '-> b}', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('Curly', '{'), - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('IdentifierName', 'b'), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('Curly', '}'), - }) - - check_parsing('{a#b -> b}', { - -- 0123456789 - ast = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - { - 'Arrow:0:4: ->', - children = { - 'PlainIdentifier(scope=0,ident=a#b):0:1:a#b', - 'PlainIdentifier(scope=0,ident=b):0:7: b', - }, - }, - }, - }, - }, - err = { - arg = '-> b}', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('Curly', '{'), - hl('IdentifierName', 'a#b'), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('Curly', '}'), - }) - check_parsing('{a : b : c}', { - -- 01234567890 - -- 0 1 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Colon:0:2: :', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Colon:0:6: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4: b', - 'PlainIdentifier(scope=0,ident=c):0:8: c', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ': c}', - msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', - }, - }, { - hl('Dict', '{'), - hl('IdentifierName', 'a'), - hl('Colon', ':', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidColon', ':', 1), - hl('IdentifierName', 'c', 1), - hl('Dict', '}'), - }) - check_parsing('{', { - -- 0 - ast = { - 'UnknownFigure:0:0:{', - }, - err = { - arg = '{', - msg = 'E15: Missing closing figure brace: %.*s', - }, - }, { - hl('FigureBrace', '{'), - }) - check_parsing('{a', { - -- 01 - ast = { - { - 'UnknownFigure:0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - err = { - arg = '{a', - msg = 'E15: Missing closing figure brace: %.*s', - }, - }, { - hl('FigureBrace', '{'), - hl('IdentifierName', 'a'), - }) - check_parsing('{a,b', { - -- 0123 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - }, - }, - }, - err = { - arg = '{a,b', - msg = 'E15: Missing closing figure brace for lambda: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - }) - check_parsing('{a,b->', { - -- 012345 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - 'Arrow:0:4:->', - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - }) - check_parsing('{a,b->c', { - -- 0123456 - ast = { - { - 'Lambda:0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'Arrow:0:4:->', - children = { - 'PlainIdentifier(scope=0,ident=c):0:6:c', - }, - }, - }, - }, - }, - err = { - arg = '{a,b->c', - msg = 'E15: Missing closing figure brace for lambda: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - hl('IdentifierName', 'c'), - }) - check_parsing('{a : b', { - -- 012345 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Colon:0:2: :', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, - }, - err = { - arg = '{a : b', - msg = 'E723: Missing end of Dictionary \'}\': %.*s', - }, - }, { - hl('Dict', '{'), - hl('IdentifierName', 'a'), - hl('Colon', ':', 1), - hl('IdentifierName', 'b', 1), - }) - check_parsing('{a : b,', { - -- 0123456 - ast = { - { - 'DictLiteral:0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:2: :', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('Dict', '{'), - hl('IdentifierName', 'a'), - hl('Colon', ':', 1), - hl('IdentifierName', 'b', 1), - hl('Comma', ','), - }) - end) - it('works with ternary operator', function() - check_parsing('a ? b : c', { - -- 012345678 - ast = { - { - 'Ternary:0:1: ?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:5: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:7: c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?', 1), - hl('IdentifierName', 'b', 1), - hl('TernaryColon', ':', 1), - hl('IdentifierName', 'c', 1), - }) - check_parsing('@a?@b?@c:@d:@e', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'Ternary:0:2:?', - children = { - 'Register(name=a):0:0:@a', - { - 'TernaryValue:0:11::', - children = { - { - 'Ternary:0:5:?', - children = { - 'Register(name=b):0:3:@b', - { - 'TernaryValue:0:8::', - children = { - 'Register(name=c):0:6:@c', - 'Register(name=d):0:9:@d', - }, - }, - }, - }, - 'Register(name=e):0:12:@e', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('Ternary', '?'), - hl('Register', '@c'), - hl('TernaryColon', ':'), - hl('Register', '@d'), - hl('TernaryColon', ':'), - hl('Register', '@e'), - }) - check_parsing('@a?@b:@c?@d:@e', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'Ternary:0:2:?', - children = { - 'Register(name=a):0:0:@a', - { - 'TernaryValue:0:5::', - children = { - 'Register(name=b):0:3:@b', - { - 'Ternary:0:8:?', - children = { - 'Register(name=c):0:6:@c', - { - 'TernaryValue:0:11::', - children = { - 'Register(name=d):0:9:@d', - 'Register(name=e):0:12:@e', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('TernaryColon', ':'), - hl('Register', '@c'), - hl('Ternary', '?'), - hl('Register', '@d'), - hl('TernaryColon', ':'), - hl('Register', '@e'), - }) - check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', { - -- 01234567890123456789012345678901 - -- 0 1 2 3 - ast = { - { - 'Ternary:0:2:?', - children = { - 'Register(name=a):0:0:@a', - { - 'TernaryValue:0:29::', - children = { - { - 'Ternary:0:5:?', - children = { - 'Register(name=b):0:3:@b', - { - 'TernaryValue:0:20::', - children = { - { - 'Ternary:0:8:?', - children = { - 'Register(name=c):0:6:@c', - { - 'TernaryValue:0:11::', - children = { - 'Register(name=d):0:9:@d', - { - 'Ternary:0:14:?', - children = { - 'Register(name=e):0:12:@e', - { - 'TernaryValue:0:17::', - children = { - 'Register(name=f):0:15:@f', - 'Register(name=g):0:18:@g', - }, - }, - }, - }, - }, - }, - }, - }, - { - 'Ternary:0:23:?', - children = { - 'Register(name=h):0:21:@h', - { - 'TernaryValue:0:26::', - children = { - 'Register(name=i):0:24:@i', - 'Register(name=j):0:27:@j', - }, - }, - }, - }, - }, - }, - }, - }, - 'Register(name=k):0:30:@k', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('Ternary', '?'), - hl('Register', '@c'), - hl('Ternary', '?'), - hl('Register', '@d'), - hl('TernaryColon', ':'), - hl('Register', '@e'), - hl('Ternary', '?'), - hl('Register', '@f'), - hl('TernaryColon', ':'), - hl('Register', '@g'), - hl('TernaryColon', ':'), - hl('Register', '@h'), - hl('Ternary', '?'), - hl('Register', '@i'), - hl('TernaryColon', ':'), - hl('Register', '@j'), - hl('TernaryColon', ':'), - hl('Register', '@k'), - }) - check_parsing('?', { - -- 0 - ast = { - { - 'Ternary:0:0:?', - children = { - 'Missing:0:0:', - 'TernaryValue:0:0:?', - }, - }, - }, - err = { - arg = '?', - msg = 'E15: Expected value, got question mark: %.*s', - }, - }, { - hl('InvalidTernary', '?'), - }) - - check_parsing('?:', { - -- 01 - ast = { - { - 'Ternary:0:0:?', - children = { - 'Missing:0:0:', - { - 'TernaryValue:0:1::', - children = { - 'Missing:0:1:', - }, - }, - }, - }, - }, - err = { - arg = '?:', - msg = 'E15: Expected value, got question mark: %.*s', - }, - }, { - hl('InvalidTernary', '?'), - hl('InvalidTernaryColon', ':'), - }) - - check_parsing('?::', { - -- 012 - ast = { - { - 'Colon:0:2::', - children = { - { - 'Ternary:0:0:?', - children = { - 'Missing:0:0:', - { - 'TernaryValue:0:1::', - children = { - 'Missing:0:1:', - 'Missing:0:2:', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '?::', - msg = 'E15: Expected value, got question mark: %.*s', - }, - }, { - hl('InvalidTernary', '?'), - hl('InvalidTernaryColon', ':'), - hl('InvalidColon', ':'), - }) - - check_parsing('a?b', { - -- 012 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - }, - }, - }, - err = { - arg = '?b', - msg = 'E109: Missing \':\' after \'?\': %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierName', 'b'), - }) - check_parsing('a?b:', { - -- 0123 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:1:?', - children = { - 'PlainIdentifier(scope=b,ident=):0:2:b:', - }, - }, - }, - }, - }, - err = { - arg = '?b:', - msg = 'E109: Missing \':\' after \'?\': %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierScope', 'b'), - hl('IdentifierScopeDelimiter', ':'), - }) - - check_parsing('a?b::c', { - -- 012345 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:4::', - children = { - 'PlainIdentifier(scope=b,ident=):0:2:b:', - 'PlainIdentifier(scope=0,ident=c):0:5:c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierScope', 'b'), - hl('IdentifierScopeDelimiter', ':'), - hl('TernaryColon', ':'), - hl('IdentifierName', 'c'), - }) - - check_parsing('a?b :', { - -- 01234 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:3: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierName', 'b'), - hl('TernaryColon', ':', 1), - }) - - check_parsing('(@a?@b:@c)?@d:@e', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'Ternary:0:10:?', - children = { - { - 'Nested:0:0:(', - children = { - { - 'Ternary:0:3:?', - children = { - 'Register(name=a):0:1:@a', - { - 'TernaryValue:0:6::', - children = { - 'Register(name=b):0:4:@b', - 'Register(name=c):0:7:@c', - }, - }, - }, - }, - }, - }, - { - 'TernaryValue:0:13::', - children = { - 'Register(name=d):0:11:@d', - 'Register(name=e):0:14:@e', - }, - }, - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('TernaryColon', ':'), - hl('Register', '@c'), - hl('NestingParenthesis', ')'), - hl('Ternary', '?'), - hl('Register', '@d'), - hl('TernaryColon', ':'), - hl('Register', '@e'), - }) - - check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', { - -- 01234567890123456789012345678901 - -- 0 1 2 3 - ast = { - { - 'Ternary:0:10:?', - children = { - { - 'Nested:0:0:(', - children = { - { - 'Ternary:0:3:?', - children = { - 'Register(name=a):0:1:@a', - { - 'TernaryValue:0:6::', - children = { - 'Register(name=b):0:4:@b', - 'Register(name=c):0:7:@c', - }, - }, - }, - }, - }, - }, - { - 'TernaryValue:0:21::', - children = { - { - 'Nested:0:11:(', - children = { - { - 'Ternary:0:14:?', - children = { - 'Register(name=d):0:12:@d', - { - 'TernaryValue:0:17::', - children = { - 'Register(name=e):0:15:@e', - 'Register(name=f):0:18:@f', - }, - }, - }, - }, - }, - }, - { - 'Nested:0:22:(', - children = { - { - 'Ternary:0:25:?', - children = { - 'Register(name=g):0:23:@g', - { - 'TernaryValue:0:28::', - children = { - 'Register(name=h):0:26:@h', - 'Register(name=i):0:29:@i', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('TernaryColon', ':'), - hl('Register', '@c'), - hl('NestingParenthesis', ')'), - hl('Ternary', '?'), - hl('NestingParenthesis', '('), - hl('Register', '@d'), - hl('Ternary', '?'), - hl('Register', '@e'), - hl('TernaryColon', ':'), - hl('Register', '@f'), - hl('NestingParenthesis', ')'), - hl('TernaryColon', ':'), - hl('NestingParenthesis', '('), - hl('Register', '@g'), - hl('Ternary', '?'), - hl('Register', '@h'), - hl('TernaryColon', ':'), - hl('Register', '@i'), - hl('NestingParenthesis', ')'), - }) - - check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', { - -- 0123456789012345678901234567 - -- 0 1 2 - ast = { - { - 'Ternary:0:10:?', - children = { - { - 'Nested:0:0:(', - children = { - { - 'Ternary:0:3:?', - children = { - 'Register(name=a):0:1:@a', - { - 'TernaryValue:0:6::', - children = { - 'Register(name=b):0:4:@b', - 'Register(name=c):0:7:@c', - }, - }, - }, - }, - }, - }, - { - 'TernaryValue:0:19::', - children = { - { - 'Ternary:0:13:?', - children = { - 'Register(name=d):0:11:@d', - { - 'TernaryValue:0:16::', - children = { - 'Register(name=e):0:14:@e', - 'Register(name=f):0:17:@f', - }, - }, - }, - }, - { - 'Ternary:0:22:?', - children = { - 'Register(name=g):0:20:@g', - { - 'TernaryValue:0:25::', - children = { - 'Register(name=h):0:23:@h', - 'Register(name=i):0:26:@i', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('TernaryColon', ':'), - hl('Register', '@c'), - hl('NestingParenthesis', ')'), - hl('Ternary', '?'), - hl('Register', '@d'), - hl('Ternary', '?'), - hl('Register', '@e'), - hl('TernaryColon', ':'), - hl('Register', '@f'), - hl('TernaryColon', ':'), - hl('Register', '@g'), - hl('Ternary', '?'), - hl('Register', '@h'), - hl('TernaryColon', ':'), - hl('Register', '@i'), - }) - check_parsing('a?b{cdef}g:h', { - -- 012345678901 - -- 0 1 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:10::', - children = { - { - 'ComplexIdentifier:0:3:', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - { - 'ComplexIdentifier:0:9:', - children = { - { - 'CurlyBracesIdentifier:0:3:{', - children = { - 'PlainIdentifier(scope=0,ident=cdef):0:4:cdef', - }, - }, - 'PlainIdentifier(scope=0,ident=g):0:9:g', - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=h):0:11:h', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierName', 'b'), - hl('Curly', '{'), - hl('IdentifierName', 'cdef'), - hl('Curly', '}'), - hl('IdentifierName', 'g'), - hl('TernaryColon', ':'), - hl('IdentifierName', 'h'), - }) - check_parsing('a ? b : c : d', { - -- 0123456789012 - -- 0 1 - ast = { - { - 'Colon:0:9: :', - children = { - { - 'Ternary:0:1: ?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:5: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:7: c', - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=d):0:11: d', - }, - }, - }, - err = { - arg = ': d', - msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?', 1), - hl('IdentifierName', 'b', 1), - hl('TernaryColon', ':', 1), - hl('IdentifierName', 'c', 1), - hl('InvalidColon', ':', 1), - hl('IdentifierName', 'd', 1), - }) - end) - it('works with comparison operators', function() - check_parsing('a == b', { - -- 012345 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=UseOption):0:1: ==', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '==', 1), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a ==? b', { - -- 0123456 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=IgnoreCase):0:1: ==?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '==', 1), - hl('ComparisonModifier', '?'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a ==# b', { - -- 0123456 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=MatchCase):0:1: ==#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '==', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a !=# b', { - -- 0123456 - ast = { - { - 'Comparison(type=Equal,inv=1,ccs=MatchCase):0:1: !=#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '!=', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a <=# b', { - -- 0123456 - ast = { - { - 'Comparison(type=Greater,inv=1,ccs=MatchCase):0:1: <=#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '<=', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a >=# b', { - -- 0123456 - ast = { - { - 'Comparison(type=GreaterOrEqual,inv=0,ccs=MatchCase):0:1: >=#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '>=', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a ># b', { - -- 012345 - ast = { - { - 'Comparison(type=Greater,inv=0,ccs=MatchCase):0:1: >#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '>', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a <# b', { - -- 012345 - ast = { - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:1: <#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '<', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a is#b', { - -- 012345 - ast = { - { - 'Comparison(type=Identical,inv=0,ccs=MatchCase):0:1: is#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5:b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', 'is', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b'), - }) - - check_parsing('a is?b', { - -- 012345 - ast = { - { - 'Comparison(type=Identical,inv=0,ccs=IgnoreCase):0:1: is?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5:b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', 'is', 1), - hl('ComparisonModifier', '?'), - hl('IdentifierName', 'b'), - }) - - check_parsing('a isnot b', { - -- 012345678 - ast = { - { - 'Comparison(type=Identical,inv=1,ccs=UseOption):0:1: isnot', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:7: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', 'isnot', 1), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a < b < c', { - -- 012345678 - ast = { - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:5: <', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:7: c', - }, - }, - }, - }, - }, - err = { - arg = ' < c', - msg = 'E15: Operator is not associative: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '<', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidComparison', '<', 1), - hl('IdentifierName', 'c', 1), - }) - - check_parsing('a < b <# c', { - -- 012345678 - ast = { - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:5: <#', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:8: c', - }, - }, - }, - }, - }, - err = { - arg = ' <# c', - msg = 'E15: Operator is not associative: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '<', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidComparison', '<', 1), - hl('InvalidComparisonModifier', '#'), - hl('IdentifierName', 'c', 1), - }) - - check_parsing('a += b', { - -- 012345 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3:=', - children = { - { - 'BinaryPlus:0:1: +', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'Missing:0:3:', - }, - }, - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - err = { - arg = '= b', - msg = 'E15: Expected == or =~: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('BinaryPlus', '+', 1), - hl('InvalidComparison', '='), - hl('IdentifierName', 'b', 1), - }) - check_parsing('a + b == c + d', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=UseOption):0:5: ==', - children = { - { - 'BinaryPlus:0:1: +', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:3: b', - }, - }, - { - 'BinaryPlus:0:10: +', - children = { - 'PlainIdentifier(scope=0,ident=c):0:8: c', - 'PlainIdentifier(scope=0,ident=d):0:12: d', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('BinaryPlus', '+', 1), - hl('IdentifierName', 'b', 1), - hl('Comparison', '==', 1), - hl('IdentifierName', 'c', 1), - hl('BinaryPlus', '+', 1), - hl('IdentifierName', 'd', 1), - }) - check_parsing('+ a == + b', { - -- 0123456789 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3: ==', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1: a', - }, - }, - { - 'UnaryPlus:0:6: +', - children = { - 'PlainIdentifier(scope=0,ident=b):0:8: b', - }, - }, - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('IdentifierName', 'a', 1), - hl('Comparison', '==', 1), - hl('UnaryPlus', '+', 1), - hl('IdentifierName', 'b', 1), - }) - end) - it('works with concat/subscript', function() - check_parsing('.', { - -- 0 - ast = { - { - 'ConcatOrSubscript:0:0:.', - children = { - 'Missing:0:0:', - }, - }, - }, - err = { - arg = '.', - msg = 'E15: Unexpected dot: %.*s', - }, - }, { - hl('InvalidConcatOrSubscript', '.'), - }) - - check_parsing('a.', { - -- 01 - ast = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - }) - - check_parsing('a.b', { - -- 012 - ast = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainKey(key=b):0:2:b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', 'b'), - }) - - check_parsing('1.2', { - -- 012 - ast = { - 'Float(val=1.200000e+00):0:0:1.2', - }, - }, { - hl('Float', '1.2'), - }) - - check_parsing('1.2 + 1.3e-5', { - -- 012345678901 - -- 0 1 - ast = { - { - 'BinaryPlus:0:3: +', - children = { - 'Float(val=1.200000e+00):0:0:1.2', - 'Float(val=1.300000e-05):0:5: 1.3e-5', - }, - }, - }, - }, { - hl('Float', '1.2'), - hl('BinaryPlus', '+', 1), - hl('Float', '1.3e-5', 1), - }) - - check_parsing('a . 1.2 + 1.3e-5', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'BinaryPlus:0:7: +', - children = { - { - 'Concat:0:1: .', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'ConcatOrSubscript:0:5:.', - children = { - 'Integer(val=1):0:3: 1', - 'PlainKey(key=2):0:6:2', - }, - }, - }, - }, - 'Float(val=1.300000e-05):0:9: 1.3e-5', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Concat', '.', 1), - hl('Number', '1', 1), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - hl('BinaryPlus', '+', 1), - hl('Float', '1.3e-5', 1), - }) - - check_parsing('1.3e-5 + 1.2 . a', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'Concat:0:12: .', - children = { - { - 'BinaryPlus:0:6: +', - children = { - 'Float(val=1.300000e-05):0:0:1.3e-5', - 'Float(val=1.200000e+00):0:8: 1.2', - }, - }, - 'PlainIdentifier(scope=0,ident=a):0:14: a', - }, - }, - }, - }, { - hl('Float', '1.3e-5'), - hl('BinaryPlus', '+', 1), - hl('Float', '1.2', 1), - hl('Concat', '.', 1), - hl('IdentifierName', 'a', 1), - }) - - check_parsing('1.3e-5 + a . 1.2', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'Concat:0:10: .', - children = { - { - 'BinaryPlus:0:6: +', - children = { - 'Float(val=1.300000e-05):0:0:1.3e-5', - 'PlainIdentifier(scope=0,ident=a):0:8: a', - }, - }, - { - 'ConcatOrSubscript:0:14:.', - children = { - 'Integer(val=1):0:12: 1', - 'PlainKey(key=2):0:15:2', - }, - }, - }, - }, - }, - }, { - hl('Float', '1.3e-5'), - hl('BinaryPlus', '+', 1), - hl('IdentifierName', 'a', 1), - hl('Concat', '.', 1), - hl('Number', '1', 1), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - }) - - check_parsing('1.2.3', { - -- 01234 - ast = { - { - 'ConcatOrSubscript:0:3:.', - children = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'Integer(val=1):0:0:1', - 'PlainKey(key=2):0:2:2', - }, - }, - 'PlainKey(key=3):0:4:3', - }, - }, - }, - }, { - hl('Number', '1'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '3'), - }) - - check_parsing('a.1.2', { - -- 01234 - ast = { - { - 'ConcatOrSubscript:0:3:.', - children = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainKey(key=1):0:2:1', - }, - }, - 'PlainKey(key=2):0:4:2', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '1'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - }) - - check_parsing('a . 1.2', { - -- 0123456 - ast = { - { - 'Concat:0:1: .', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'ConcatOrSubscript:0:5:.', - children = { - 'Integer(val=1):0:3: 1', - 'PlainKey(key=2):0:6:2', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Concat', '.', 1), - hl('Number', '1', 1), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - }) - - check_parsing('+a . +b', { - -- 0123456 - ast = { - { - 'Concat:0:2: .', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - { - 'UnaryPlus:0:4: +', - children = { - 'PlainIdentifier(scope=0,ident=b):0:6:b', - }, - }, - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('IdentifierName', 'a'), - hl('Concat', '.', 1), - hl('UnaryPlus', '+', 1), - hl('IdentifierName', 'b'), - }) - - check_parsing('a. b', { - -- 0123 - ast = { - { - 'Concat:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:2: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a. 1', { - -- 0123 - ast = { - { - 'Concat:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'Integer(val=1):0:2: 1', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('Number', '1', 1), - }) - end) - it('works with bracket subscripts', function() - check_parsing(':', { - -- 0 - ast = { - { - 'Colon:0:0::', - children = { - 'Missing:0:0:', - }, - }, - }, - err = { - arg = ':', - msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', - }, - }, { - hl('InvalidColon', ':'), - }) - check_parsing('a[]', { - -- 012 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - }, - }, - }, - err = { - arg = ']', - msg = 'E15: Expected value, got closing bracket: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('InvalidSubscriptBracket', ']'), - }) - check_parsing('a[b:]', { - -- 01234 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=b,ident=):0:2:b:', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierScope', 'b'), - hl('IdentifierScopeDelimiter', ':'), - hl('SubscriptBracket', ']'), - }) - - check_parsing('a[b:c]', { - -- 012345 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=b,ident=c):0:2:b:c', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierScope', 'b'), - hl('IdentifierScopeDelimiter', ':'), - hl('IdentifierName', 'c'), - hl('SubscriptBracket', ']'), - }) - check_parsing('a[b : c]', { - -- 01234567 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Colon:0:3: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - 'PlainIdentifier(scope=0,ident=c):0:5: c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'b'), - hl('SubscriptColon', ':', 1), - hl('IdentifierName', 'c', 1), - hl('SubscriptBracket', ']'), - }) - - check_parsing('a[: b]', { - -- 012345 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Colon:0:2::', - children = { - 'Missing:0:2:', - 'PlainIdentifier(scope=0,ident=b):0:3: b', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('SubscriptColon', ':'), - hl('IdentifierName', 'b', 1), - hl('SubscriptBracket', ']'), - }) - - check_parsing('a[b :]', { - -- 012345 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Colon:0:3: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'b'), - hl('SubscriptColon', ':', 1), - hl('SubscriptBracket', ']'), - }) - check_parsing('a[b][c][d](e)(f)(g)', { - -- 0123456789012345678 - -- 0 1 - ast = { - { - 'Call:0:16:(', - children = { - { - 'Call:0:13:(', - children = { - { - 'Call:0:10:(', - children = { - { - 'Subscript:0:7:[', - children = { - { - 'Subscript:0:4:[', - children = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - 'PlainIdentifier(scope=0,ident=c):0:5:c', - }, - }, - 'PlainIdentifier(scope=0,ident=d):0:8:d', - }, - }, - 'PlainIdentifier(scope=0,ident=e):0:11:e', - }, - }, - 'PlainIdentifier(scope=0,ident=f):0:14:f', - }, - }, - 'PlainIdentifier(scope=0,ident=g):0:17:g', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'b'), - hl('SubscriptBracket', ']'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'c'), - hl('SubscriptBracket', ']'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'd'), - hl('SubscriptBracket', ']'), - hl('CallingParenthesis', '('), - hl('IdentifierName', 'e'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('IdentifierName', 'f'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('IdentifierName', 'g'), - hl('CallingParenthesis', ')'), - }) - check_parsing('{a}{b}{c}[d][e][f]', { - -- 012345678901234567 - -- 0 1 - ast = { - { - 'Subscript:0:15:[', - children = { - { - 'Subscript:0:12:[', - children = { - { - 'Subscript:0:9:[', - children = { - { - 'ComplexIdentifier:0:3:', - children = { - { - 'CurlyBracesIdentifier:0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - { - 'ComplexIdentifier:0:6:', - children = { - { - 'CurlyBracesIdentifier:0:3:{', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4:b', - }, - }, - { - 'CurlyBracesIdentifier:0:6:{', - children = { - 'PlainIdentifier(scope=0,ident=c):0:7:c', - }, - }, - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=d):0:10:d', - }, - }, - 'PlainIdentifier(scope=0,ident=e):0:13:e', - }, - }, - 'PlainIdentifier(scope=0,ident=f):0:16:f', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('IdentifierName', 'a'), - hl('Curly', '}'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - hl('Curly', '{'), - hl('IdentifierName', 'c'), - hl('Curly', '}'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'd'), - hl('SubscriptBracket', ']'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'e'), - hl('SubscriptBracket', ']'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'f'), - hl('SubscriptBracket', ']'), - }) - end) - it('supports list literals', function() - check_parsing('[]', { - -- 01 - ast = { - 'ListLiteral:0:0:[', - }, - }, { - hl('List', '['), - hl('List', ']'), - }) - - check_parsing('[a]', { - -- 012 - ast = { - { - 'ListLiteral:0:0:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('List', ']'), - }) - - check_parsing('[a, b]', { - -- 012345 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3: b', - }, - }, - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b', 1), - hl('List', ']'), - }) - - check_parsing('[a, b, c]', { - -- 012345678 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:6: c', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b', 1), - hl('Comma', ','), - hl('IdentifierName', 'c', 1), - hl('List', ']'), - }) - - check_parsing('[a, b, c, ]', { - -- 01234567890 - -- 0 1 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - { - 'Comma:0:8:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:6: c', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b', 1), - hl('Comma', ','), - hl('IdentifierName', 'c', 1), - hl('Comma', ','), - hl('List', ']', 1), - }) - - check_parsing('[a : b, c : d]', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:2: :', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - { - 'Colon:0:9: :', - children = { - 'PlainIdentifier(scope=0,ident=c):0:7: c', - 'PlainIdentifier(scope=0,ident=d):0:11: d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ': b, c : d]', - msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('InvalidColon', ':', 1), - hl('IdentifierName', 'b', 1), - hl('Comma', ','), - hl('IdentifierName', 'c', 1), - hl('InvalidColon', ':', 1), - hl('IdentifierName', 'd', 1), - hl('List', ']'), - }) - - check_parsing(']', { - -- 0 - ast = { - 'ListLiteral:0:0:', - }, - err = { - arg = ']', - msg = 'E15: Unexpected closing figure brace: %.*s', - }, - }, { - hl('InvalidList', ']'), - }) - - check_parsing('a]', { - -- 01 - ast = { - { - 'ListLiteral:0:1:', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - }, - }, - }, - err = { - arg = ']', - msg = 'E15: Unexpected closing figure brace: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidList', ']'), - }) - - check_parsing('[] []', { - -- 01234 - ast = { - { - 'OpMissing:0:2:', - children = { - 'ListLiteral:0:0:[', - 'ListLiteral:0:2: [', - }, - }, - }, - err = { - arg = '[]', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('List', '['), - hl('List', ']'), - hl('InvalidSpacing', ' '), - hl('List', '['), - hl('List', ']'), - }, { - [1] = { - ast = { - len = 3, - err = REMOVE_THIS, - ast = { - 'ListLiteral:0:0:[', - }, - }, - hl_fs = { - [3] = REMOVE_THIS, - [4] = REMOVE_THIS, - [5] = REMOVE_THIS, - }, - }, - }) - - check_parsing('[][]', { - -- 0123 - ast = { - { - 'Subscript:0:2:[', - children = { - 'ListLiteral:0:0:[', - }, - }, - }, - err = { - arg = ']', - msg = 'E15: Expected value, got closing bracket: %.*s', - }, - }, { - hl('List', '['), - hl('List', ']'), - hl('SubscriptBracket', '['), - hl('InvalidSubscriptBracket', ']'), - }) - - check_parsing('[', { - -- 0 - ast = { - 'ListLiteral:0:0:[', - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('List', '['), - }) - - check_parsing('[1', { - -- 01 - ast = { - { - 'ListLiteral:0:0:[', - children = { - 'Integer(val=1):0:1:1', - }, - }, - }, - err = { - arg = '[1', - msg = 'E697: Missing end of List \']\': %.*s', - }, - }, { - hl('List', '['), - hl('Number', '1'), - }) - end) - it('works with strings', function() - check_parsing('\'abc\'', { - -- 01234 - ast = { - 'SingleQuotedString(val="abc"):0:0:\'abc\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuotedBody', 'abc'), - hl('SingleQuote', '\''), - }) - check_parsing('"abc"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="abc"):0:0:"abc"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedBody', 'abc'), - hl('DoubleQuote', '"'), - }) - check_parsing('\'\'', { - -- 01 - ast = { - 'SingleQuotedString(val=""):0:0:\'\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuote', '\''), - }) - check_parsing('""', { - -- 01 - ast = { - 'DoubleQuotedString(val=""):0:0:""', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuote', '"'), - }) - check_parsing('"', { - -- 0 - ast = { - 'DoubleQuotedString(val=""):0:0:"', - }, - err = { - arg = '"', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - }) - check_parsing('\'', { - -- 0 - ast = { - 'SingleQuotedString(val=""):0:0:\'', - }, - err = { - arg = '\'', - msg = 'E115: Missing single quote: %.*s', - }, - }, { - hl('InvalidSingleQuote', '\''), - }) - check_parsing('"a', { - -- 01 - ast = { - 'DoubleQuotedString(val="a"):0:0:"a', - }, - err = { - arg = '"a', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedBody', 'a'), - }) - check_parsing('\'a', { - -- 01 - ast = { - 'SingleQuotedString(val="a"):0:0:\'a', - }, - err = { - arg = '\'a', - msg = 'E115: Missing single quote: %.*s', - }, - }, { - hl('InvalidSingleQuote', '\''), - hl('InvalidSingleQuotedBody', 'a'), - }) - check_parsing('\'abc\'\'def\'', { - -- 0123456789 - ast = { - 'SingleQuotedString(val="abc\'def"):0:0:\'abc\'\'def\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuotedBody', 'abc'), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedBody', 'def'), - hl('SingleQuote', '\''), - }) - check_parsing('\'abc\'\'', { - -- 012345 - ast = { - 'SingleQuotedString(val="abc\'"):0:0:\'abc\'\'', - }, - err = { - arg = '\'abc\'\'', - msg = 'E115: Missing single quote: %.*s', - }, - }, { - hl('InvalidSingleQuote', '\''), - hl('InvalidSingleQuotedBody', 'abc'), - hl('InvalidSingleQuotedQuote', '\'\''), - }) - check_parsing('\'\'\'\'\'\'\'\'', { - -- 01234567 - ast = { - 'SingleQuotedString(val="\'\'\'"):0:0:\'\'\'\'\'\'\'\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuote', '\''), - }) - check_parsing('\'\'\'a\'\'\'\'bc\'', { - -- 01234567890 - -- 0 1 - ast = { - 'SingleQuotedString(val="\'a\'\'bc"):0:0:\'\'\'a\'\'\'\'bc\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedBody', 'a'), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedBody', 'bc'), - hl('SingleQuote', '\''), - }) - check_parsing('"\\"\\"\\"\\""', { - -- 0123456789 - ast = { - 'DoubleQuotedString(val="\\"\\"\\"\\""):0:0:"\\"\\"\\"\\""', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuote', '"'), - }) - check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', { - -- 0123456789012345678901234 - -- 0 1 2 - ast = { - 'DoubleQuotedString(val="abc\\"def\\"ghi\\"jkl\\"mno"):0:0:"abc\\"def\\"ghi\\"jkl\\"mno"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedBody', 'abc'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedBody', 'def'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedBody', 'ghi'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedBody', 'jkl'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedBody', 'mno'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\b\\e\\f\\r\\t\\\\"', { - -- 0123456789012345 - -- 0 1 - ast = { - [[DoubleQuotedString(val="\8\27\12\13\9\\"):0:0:"\b\e\f\r\t\\"]], - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\b'), - hl('DoubleQuotedEscape', '\\e'), - hl('DoubleQuotedEscape', '\\f'), - hl('DoubleQuotedEscape', '\\r'), - hl('DoubleQuotedEscape', '\\t'), - hl('DoubleQuotedEscape', '\\\\'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\n\n"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="\\\n\\\n"):0:0:"\\n\n"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\n'), - hl('DoubleQuotedBody', '\n'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\x00"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0"):0:0:"\\x00"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\x00'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\xFF"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\255"):0:0:"\\xFF"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\xFF'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\xF"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\15"):0:0:"\\xF"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\xF'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\u00AB"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="«"):0:0:"\\u00AB"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u00AB'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\U000000AB"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="«"):0:0:"\\U000000AB"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U000000AB'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\x"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="x"):0:0:"\\x"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\x'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\x', { - -- 012 - ast = { - 'DoubleQuotedString(val="x"):0:0:"\\x', - }, - err = { - arg = '"\\x', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedUnknownEscape', '\\x'), - }) - - check_parsing('"\\xF', { - -- 0123 - ast = { - 'DoubleQuotedString(val="\\15"):0:0:"\\xF', - }, - err = { - arg = '"\\xF', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedEscape', '\\xF'), - }) - - check_parsing('"\\u"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="u"):0:0:"\\u"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\u'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u', { - -- 012 - ast = { - 'DoubleQuotedString(val="u"):0:0:"\\u', - }, - err = { - arg = '"\\u', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedUnknownEscape', '\\u'), - }) - - check_parsing('"\\U', { - -- 012 - ast = { - 'DoubleQuotedString(val="U"):0:0:"\\U', - }, - err = { - arg = '"\\U', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedUnknownEscape', '\\U'), - }) - - check_parsing('"\\U"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="U"):0:0:"\\U"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\U'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\xFX"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\15X"):0:0:"\\xFX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\xF'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\XFX"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\15X"):0:0:"\\XFX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\XF'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\xX"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="xX"):0:0:"\\xX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\x'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\XX"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="XX"):0:0:"\\XX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\X'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\uX"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="uX"):0:0:"\\uX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\u'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\UX"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="UX"):0:0:"\\UX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\U'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\x0X"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\x0X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\x0'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\X0X"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\X0X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\X0'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u0X"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\u0X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u0'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U0X"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U0X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U0'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\x00X"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\x00X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\x00'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\X00X"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\X00X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\X00'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u00X"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\u00X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u00'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U00X"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U00X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U00'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u000X"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\u000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U000X"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u0000X"', { - -- 012345678 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\u0000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u0000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U0000X"', { - -- 012345678 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U0000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U00000X"', { - -- 0123456789 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U00000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U000000X"', { - -- 01234567890 - -- 0 1 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U000000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U000000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U0000000X"', { - -- 012345678901 - -- 0 1 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U0000000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U00000000X"', { - -- 0123456789012 - -- 0 1 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U00000000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\x000X"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="\\0000X"):0:0:"\\x000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\x00'), - hl('DoubleQuotedBody', '0X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\X000X"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="\\0000X"):0:0:"\\X000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\X00'), - hl('DoubleQuotedBody', '0X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u00000X"', { - -- 0123456789 - ast = { - 'DoubleQuotedString(val="\\0000X"):0:0:"\\u00000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u0000'), - hl('DoubleQuotedBody', '0X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U000000000X"', { - -- 01234567890123 - -- 0 1 - ast = { - 'DoubleQuotedString(val="\\0000X"):0:0:"\\U000000000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U00000000'), - hl('DoubleQuotedBody', '0X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\0"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="\\0"):0:0:"\\0"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\0'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\00"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="\\0"):0:0:"\\00"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\00'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\000"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0"):0:0:"\\000"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\000'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\0000"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0000"):0:0:"\\0000"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\000'), - hl('DoubleQuotedBody', '0'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\8"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="8"):0:0:"\\8"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\8'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\08"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="\\0008"):0:0:"\\08"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\0'), - hl('DoubleQuotedBody', '8'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\008"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0008"):0:0:"\\008"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\00'), - hl('DoubleQuotedBody', '8'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\0008"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0008"):0:0:"\\0008"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\000'), - hl('DoubleQuotedBody', '8'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\777"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\255"):0:0:"\\777"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\777'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\050"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\40"):0:0:"\\050"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\050'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\21"):0:0:"\\"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\<', { - -- 012 - ast = { - 'DoubleQuotedString(val="<"):0:0:"\\<', - }, - err = { - arg = '"\\<', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedUnknownEscape', '\\<'), - }) - - check_parsing('"\\<"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="<"):0:0:"\\<"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\<'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\ 10 then print(digits2) end + if zdata.ast.len then + print((' len = %u,'):format(zdata.ast.len)) + end print(' ast = ' .. format_luav(zdata.ast.ast, ' ') .. ',') if zdata.ast.err then print(' err = {') @@ -414,14 +417,21 @@ eastnodelist2lua = function(pstate, eastnode, checked_nodes) return ret end -local function east2lua(pstate, east) +local function east2lua(str, pstate, east) local checked_nodes = {} + local len = tonumber(pstate.pos.col) + if pstate.pos.line == 1 then + len = tonumber(pstate.reader.lines.items[0].size) + end + if type(str) == 'string' and len == #str then + len = nil + end return { err = east.err.msg ~= nil and { msg = ffi.string(east.err.msg), - arg = ('%s'):format( - ffi.string(east.err.arg, east.err.arg_len)), + arg = ffi.string(east.err.arg, east.err.arg_len), } or nil, + len = len, ast = eastnodelist2lua(pstate, east.root, checked_nodes), } end @@ -463,7 +473,7 @@ describe('Expressions parser', function() local pstate = new_pstate({str}) local east = lib.viml_pexpr_parse(pstate, flags) - local ast = east2lua(pstate, east) + local ast = east2lua(str, pstate, east) local hls = phl2lua(pstate) if exp_ast == nil then format_check_data[flags] = {ast=ast, hl_fs=hls_to_hl_fs(hls)} @@ -509,15 +519,6 @@ describe('Expressions parser', function() format_check(str, format_check_data, opts) end end - local function check_parsing(...) - return _check_parsing({flags={0, 1, 2, 3}, funcname='check_parsing'}, ...) - end - local function check_asgn_parsing(...) - return _check_parsing({ - flags={4, 5, 6, 7}, - funcname='check_asgn_parsing', - }, ...) - end local function hl(group, str, shift) return function(next_col) if nvim_hl_defs['NVim' .. group] == nil then @@ -531,8152 +532,9 @@ describe('Expressions parser', function() str)), (col + #str) end end - itp('works with + and @a', function() - check_parsing('@a', { - ast = { - 'Register(name=a):0:0:@a', - }, - }, { - hl('Register', '@a'), - }) - check_parsing('+@a', { - ast = { - { - 'UnaryPlus:0:0:+', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('Register', '@a'), - }) - check_parsing('@a+@b', { - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - 'Register(name=b):0:3:@b', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - }) - check_parsing('@a+@b+@c', { - ast = { - { - 'BinaryPlus:0:5:+', - children = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - 'Register(name=b):0:3:@b', - }, - }, - 'Register(name=c):0:6:@c', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - hl('BinaryPlus', '+'), - hl('Register', '@c'), - }) - check_parsing('+@a+@b', { - ast = { - { - 'BinaryPlus:0:3:+', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'Register(name=a):0:1:@a', - }, - }, - 'Register(name=b):0:4:@b', - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - }) - check_parsing('+@a++@b', { - ast = { - { - 'BinaryPlus:0:3:+', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'Register(name=a):0:1:@a', - }, - }, - { - 'UnaryPlus:0:4:+', - children = { - 'Register(name=b):0:5:@b', - }, - }, - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('UnaryPlus', '+'), - hl('Register', '@b'), - }) - check_parsing('@a@b', { - ast = { - { - 'OpMissing:0:2:', - children = { - 'Register(name=a):0:0:@a', - 'Register(name=b):0:2:@b', - }, - }, - }, - err = { - arg = '@b', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('Register', '@a'), - hl('InvalidRegister', '@b'), - }, { - [1] = { - ast = { - err = REMOVE_THIS, - ast = { - 'Register(name=a):0:0:@a' - }, - }, - hl_fs = { - [2] = REMOVE_THIS, - }, - }, - }) - check_parsing(' @a \t @b', { - ast = { - { - 'OpMissing:0:3:', - children = { - 'Register(name=a):0:0: @a', - 'Register(name=b):0:3: \t @b', - }, - }, - }, - err = { - arg = '@b', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('Register', '@a', 1), - hl('InvalidSpacing', ' \t '), - hl('Register', '@b'), - }, { - [1] = { - ast = { - err = REMOVE_THIS, - ast = { - 'Register(name=a):0:0: @a' - }, - }, - hl_fs = { - [2] = REMOVE_THIS, - [3] = REMOVE_THIS, - }, - }, - }) - check_parsing('+', { - ast = { - 'UnaryPlus:0:0:+', - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('UnaryPlus', '+'), - }) - check_parsing(' +', { - ast = { - 'UnaryPlus:0:0: +', - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('UnaryPlus', '+', 1), - }) - check_parsing('@a+ ', { - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - }) - end) - itp('works with @a, + and parenthesis', function() - check_parsing('(@a)', { - ast = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('NestingParenthesis', ')'), - }) - check_parsing('()', { - ast = { - { - 'Nested:0:0:(', - children = { - 'Missing:0:1:', - }, - }, - }, - err = { - arg = ')', - msg = 'E15: Expected value, got parenthesis: %.*s', - }, - }, { - hl('NestingParenthesis', '('), - hl('InvalidNestingParenthesis', ')'), - }) - check_parsing(')', { - ast = { - { - 'Nested:0:0:', - children = { - 'Missing:0:0:', - }, - }, - }, - err = { - arg = ')', - msg = 'E15: Expected value, got parenthesis: %.*s', - }, - }, { - hl('InvalidNestingParenthesis', ')'), - }) - check_parsing('+)', { - ast = { - { - 'Nested:0:1:', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'Missing:0:1:', - }, - }, - }, - }, - }, - err = { - arg = ')', - msg = 'E15: Expected value, got parenthesis: %.*s', - }, - }, { - hl('UnaryPlus', '+'), - hl('InvalidNestingParenthesis', ')'), - }) - check_parsing('+@a(@b)', { - ast = { - { - 'UnaryPlus:0:0:+', - children = { - { - 'Call:0:3:(', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a+@b(@c)', { - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:5:(', - children = { - 'Register(name=b):0:3:@b', - 'Register(name=c):0:6:@c', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a()', { - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a ()', { - ast = { - { - 'OpMissing:0:2:', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:2: (', - children = { - 'Missing:0:4:', - }, - }, - }, - }, - }, - err = { - arg = '()', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('Register', '@a'), - hl('InvalidSpacing', ' '), - hl('NestingParenthesis', '('), - hl('InvalidNestingParenthesis', ')'), - }, { - [1] = { - ast = { - err = REMOVE_THIS, - ast = { - 'Register(name=a):0:0:@a', - }, - }, - hl_fs = { - [2] = REMOVE_THIS, - [3] = REMOVE_THIS, - [4] = REMOVE_THIS, - }, - }, - }) - check_parsing('@a + (@b)', { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - 'Register(name=b):0:6:@b', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - }) - check_parsing('@a + (+@b)', { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - { - 'UnaryPlus:0:6:+', - children = { - 'Register(name=b):0:7:@b', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('UnaryPlus', '+'), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - }) - check_parsing('@a + (@b + @c)', { - ast = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - { - 'BinaryPlus:0:8: +', - children = { - 'Register(name=b):0:6:@b', - 'Register(name=c):0:10: @c', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Register', '@b'), - hl('BinaryPlus', '+', 1), - hl('Register', '@c', 1), - hl('NestingParenthesis', ')'), - }) - check_parsing('(@a)+@b', { - ast = { - { - 'BinaryPlus:0:4:+', - children = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - 'Register(name=b):0:5:@b', - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('NestingParenthesis', ')'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - }) - check_parsing('@a+(@b)(@c)', { - -- 01234567890 - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:7:(', - children = { - { - 'Nested:0:3:(', - children = { 'Register(name=b):0:4:@b' }, - }, - 'Register(name=c):0:8:@c', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('NestingParenthesis', '('), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a+((@b))(@c)', { - -- 01234567890123456890123456789 - -- 0 1 2 - ast = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:9:(', - children = { - { - 'Nested:0:3:(', - children = { - { - 'Nested:0:4:(', - children = { 'Register(name=b):0:5:@b' } - }, - }, - }, - 'Register(name=c):0:10:@c', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('NestingParenthesis', '('), - hl('NestingParenthesis', '('), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a+((@b))+@c', { - -- 01234567890123456890123456789 - -- 0 1 2 - ast = { - { - 'BinaryPlus:0:9:+', - children = { - { - 'BinaryPlus:0:2:+', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:3:(', - children = { - { - 'Nested:0:4:(', - children = { 'Register(name=b):0:5:@b' } - }, - }, - }, - }, - }, - 'Register(name=c):0:10:@c', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('NestingParenthesis', '('), - hl('NestingParenthesis', '('), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - hl('NestingParenthesis', ')'), - hl('BinaryPlus', '+'), - hl('Register', '@c'), - }) - check_parsing( - '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', {--[[ - | | | | | | | | || | | || | | ||| || || || || - 000000000011111111112222222222333333333344444444445555555 - 012345678901234567890123456789012345678901234567890123456 - ]] - ast = {{ - 'BinaryPlus:0:31: +', - children = { - { - 'BinaryPlus:0:23: +', - children = { - { - 'BinaryPlus:0:14: +', - children = { - { - 'BinaryPlus:0:2: +', - children = { - 'Register(name=a):0:0:@a', - { - 'Nested:0:4: (', - children = { - { - 'BinaryPlus:0:8: +', - children = { - 'Register(name=b):0:6:@b', - 'Register(name=c):0:10: @c', - }, - }, - }, - }, - }, - }, - { - 'Call:0:19:(', - children = { - 'Register(name=d):0:16: @d', - 'Register(name=e):0:20:@e', - }, - }, - }, - }, - { - 'Nested:0:25: (', - children = { - { - 'UnaryPlus:0:27:+', - children = { - 'Register(name=f):0:28:@f', - }, - }, - }, - }, - }, - }, - { - 'Call:0:53:(', - children = { - { - 'Nested:0:33: (', - children = { - { - 'Call:0:48:(', - children = { - { - 'Call:0:44:(', - children = { - { - 'Nested:0:35:(', - children = { - { - 'UnaryPlus:0:36:+', - children = { - { - 'Call:0:39:(', - children = { - 'Register(name=g):0:37:@g', - 'Register(name=h):0:40:@h', - }, - }, - }, - }, - }, - }, - 'Register(name=j):0:45:@j', - }, - }, - 'Register(name=k):0:49:@k', - }, - }, - }, - }, - 'Register(name=l):0:54:@l', - }, - }, - }, - }}, - }, { - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Register', '@b'), - hl('BinaryPlus', '+', 1), - hl('Register', '@c', 1), - hl('NestingParenthesis', ')'), - hl('BinaryPlus', '+', 1), - hl('Register', '@d', 1), - hl('CallingParenthesis', '('), - hl('Register', '@e'), - hl('CallingParenthesis', ')'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('UnaryPlus', '+'), - hl('Register', '@f'), - hl('NestingParenthesis', ')'), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('NestingParenthesis', '('), - hl('UnaryPlus', '+'), - hl('Register', '@g'), - hl('CallingParenthesis', '('), - hl('Register', '@h'), - hl('CallingParenthesis', ')'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@j'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@k'), - hl('CallingParenthesis', ')'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@l'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a)', { - -- 012 - ast = { - { - 'Nested:0:2:', - children = { - 'Register(name=a):0:0:@a', - }, - }, - }, - err = { - arg = ')', - msg = 'E15: Unexpected closing parenthesis: %.*s', - }, - }, { - hl('Register', '@a'), - hl('InvalidNestingParenthesis', ')'), - }) - check_parsing('(@a', { - -- 012 - ast = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - err = { - arg = '(@a', - msg = 'E110: Missing closing parenthesis for nested expression: %.*s', - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - }) - check_parsing('@a(@b', { - -- 01234 - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - 'Register(name=b):0:3:@b', - }, - }, - }, - err = { - arg = '(@b', - msg = 'E116: Missing closing parenthesis for function call: %.*s', - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - }) - check_parsing('@a(@b, @c, @d, @e)', { - -- 012345678901234567 - -- 0 1 - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - { - 'Comma:0:5:,', - children = { - 'Register(name=b):0:3:@b', - { - 'Comma:0:9:,', - children = { - 'Register(name=c):0:6: @c', - { - 'Comma:0:13:,', - children = { - 'Register(name=d):0:10: @d', - 'Register(name=e):0:14: @e', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('Comma', ','), - hl('Register', '@c', 1), - hl('Comma', ','), - hl('Register', '@d', 1), - hl('Comma', ','), - hl('Register', '@e', 1), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a(@b(@c))', { - -- 01234567890123456789012345678901234567 - -- 0 1 2 3 - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:5:(', - children = { - 'Register(name=b):0:3:@b', - 'Register(name=c):0:6:@c', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - }) - check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', { - -- 01234567890123456789012345678901234567 - -- 0 1 2 3 - ast = { - { - 'Call:0:2:(', - children = { - 'Register(name=a):0:0:@a', - { - 'Call:0:5:(', - children = { - 'Register(name=b):0:3:@b', - { - 'Call:0:8:(', - children = { - 'Register(name=c):0:6:@c', - { - 'Comma:0:15:,', - children = { - { - 'Call:0:11:(', - children = { - 'Register(name=d):0:9:@d', - 'Register(name=e):0:12:@e', - }, - }, - { - 'Call:0:19:(', - children = { - 'Register(name=f):0:16: @f', - { - 'Comma:0:26:,', - children = { - { - 'Call:0:22:(', - children = { - 'Register(name=g):0:20:@g', - 'Register(name=h):0:23:@h', - }, - }, - { - 'Call:0:30:(', - children = { - 'Register(name=i):0:27: @i', - 'Register(name=j):0:31:@j', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('CallingParenthesis', '('), - hl('Register', '@c'), - hl('CallingParenthesis', '('), - hl('Register', '@d'), - hl('CallingParenthesis', '('), - hl('Register', '@e'), - hl('CallingParenthesis', ')'), - hl('Comma', ','), - hl('Register', '@f', 1), - hl('CallingParenthesis', '('), - hl('Register', '@g'), - hl('CallingParenthesis', '('), - hl('Register', '@h'), - hl('CallingParenthesis', ')'), - hl('Comma', ','), - hl('Register', '@i', 1), - hl('CallingParenthesis', '('), - hl('Register', '@j'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', ')'), - }) - check_parsing('()()', { - -- 0123 - ast = { - { - 'Call:0:2:(', - children = { - { - 'Nested:0:0:(', - children = { - 'Missing:0:1:', - }, - }, - }, - }, - }, - err = { - arg = ')()', - msg = 'E15: Expected value, got parenthesis: %.*s', - }, - }, { - hl('NestingParenthesis', '('), - hl('InvalidNestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - }) - check_parsing('(@a)()', { - -- 012345 - ast = { - { - 'Call:0:4:(', - children = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - }) - check_parsing('(@a)(@b)', { - -- 01234567 - ast = { - { - 'Call:0:4:(', - children = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - 'Register(name=b):0:5:@b', - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@b'), - hl('CallingParenthesis', ')'), - }) - check_parsing('(@a) (@b)', { - -- 012345678 - ast = { - { - 'OpMissing:0:4:', - children = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - }, - }, - { - 'Nested:0:4: (', - children = { - 'Register(name=b):0:6:@b', - }, - }, - }, - }, - }, - err = { - arg = '(@b)', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('NestingParenthesis', ')'), - hl('InvalidSpacing', ' '), - hl('NestingParenthesis', '('), - hl('Register', '@b'), - hl('NestingParenthesis', ')'), - }, { - [1] = { - ast = { - ast = { - { - 'Nested:0:0:(', - children = { - 'Register(name=a):0:1:@a', - REMOVE_THIS, - }, - }, - }, - err = REMOVE_THIS, - }, - hl_fs = { - [4] = REMOVE_THIS, - [5] = REMOVE_THIS, - [6] = REMOVE_THIS, - [7] = REMOVE_THIS, - }, - }, - }) - end) - itp('works with variable names, including curly braces ones', function() - check_parsing('var', { - ast = { - 'PlainIdentifier(scope=0,ident=var):0:0:var', - }, - }, { - hl('IdentifierName', 'var'), - }) - check_parsing('g:var', { - ast = { - 'PlainIdentifier(scope=g,ident=var):0:0:g:var', - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('IdentifierName', 'var'), - }) - check_parsing('g:', { - ast = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - }) - check_parsing('{a}', { - -- 012 - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('IdentifierName', 'a'), - hl('Curly', '}'), - }) - check_parsing('{a:b}', { - -- 012 - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - 'PlainIdentifier(scope=a,ident=b):0:1:a:b', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - }) - check_parsing('{a:@b}', { - -- 012345 - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - { - 'OpMissing:0:3:', - children={ - 'PlainIdentifier(scope=a,ident=):0:1:a:', - 'Register(name=b):0:3:@b', - }, - }, - }, - }, - }, - err = { - arg = '@b}', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('Curly', '{'), - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('InvalidRegister', '@b'), - hl('Curly', '}'), - }) - check_parsing('{@a}', { - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - }) - check_parsing('{@a}{@b}', { - -- 01234567 - ast = { - { - 'ComplexIdentifier:0:4:', - children = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - { - 'CurlyBracesIdentifier(--i):0:4:{', - children = { - 'Register(name=b):0:5:@b', - }, - }, - }, - }, - }, - }, { - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('Curly', '{'), - hl('Register', '@b'), - hl('Curly', '}'), - }) - check_parsing('g:{@a}', { - -- 01234567 - ast = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - { - 'CurlyBracesIdentifier(--i):0:2:{', - children = { - 'Register(name=a):0:3:@a', - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - }) - check_parsing('{@a}_test', { - -- 012345678 - ast = { - { - 'ComplexIdentifier:0:4:', - children = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - 'PlainIdentifier(scope=0,ident=_test):0:4:_test', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('IdentifierName', '_test'), - }) - check_parsing('g:{@a}_test', { - -- 01234567890 - ast = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - { - 'ComplexIdentifier:0:6:', - children = { - { - 'CurlyBracesIdentifier(--i):0:2:{', - children = { - 'Register(name=a):0:3:@a', - }, - }, - 'PlainIdentifier(scope=0,ident=_test):0:6:_test', - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('IdentifierName', '_test'), - }) - check_parsing('g:{@a}_test()', { - -- 0123456789012 - ast = { - { - 'Call:0:11:(', - children = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - { - 'ComplexIdentifier:0:6:', - children = { - { - 'CurlyBracesIdentifier(--i):0:2:{', - children = { - 'Register(name=a):0:3:@a', - }, - }, - 'PlainIdentifier(scope=0,ident=_test):0:6:_test', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('IdentifierName', '_test'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - }) - check_parsing('{@a} ()', { - -- 0123456789012 - ast = { - { - 'Call:0:4: (', - children = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - }, - }, - }, { - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('CallingParenthesis', '(', 1), - hl('CallingParenthesis', ')'), - }) - check_parsing('g:{@a} ()', { - -- 0123456789012 - ast = { - { - 'Call:0:6: (', - children = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=g,ident=):0:0:g:', - { - 'CurlyBracesIdentifier(--i):0:2:{', - children = { - 'Register(name=a):0:3:@a', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'g'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Register', '@a'), - hl('Curly', '}'), - hl('CallingParenthesis', '(', 1), - hl('CallingParenthesis', ')'), - }) - check_parsing('{@a', { - -- 012 - ast = { - { - 'UnknownFigure(-di):0:0:{', - children = { - 'Register(name=a):0:1:@a', - }, - }, - }, - err = { - arg = '{@a', - msg = 'E15: Missing closing figure brace: %.*s', - }, - }, { - hl('FigureBrace', '{'), - hl('Register', '@a'), - }) - check_parsing('a ()', { - -- 0123 - ast = { - { - 'Call:0:1: (', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('CallingParenthesis', '(', 1), - hl('CallingParenthesis', ')'), - }) - end) - itp('works with lambdas and dictionaries', function() - check_parsing('{}', { - ast = { - 'DictLiteral(-di):0:0:{', - }, - }, { - hl('Dict', '{'), - hl('Dict', '}'), - }) - check_parsing('{->@a}', { - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Arrow:0:1:->', - children = { - 'Register(name=a):0:3:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{->@a+@b}', { - -- 012345678 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Arrow:0:1:->', - children = { - { - 'BinaryPlus:0:5:+', - children = { - 'Register(name=a):0:3:@a', - 'Register(name=b):0:6:@b', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('BinaryPlus', '+'), - hl('Register', '@b'), - hl('Lambda', '}'), - }) - check_parsing('{a->@a}', { - -- 012345678 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Arrow:0:2:->', - children = { - 'Register(name=a):0:4:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b->@a}', { - -- 012345678 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'Arrow:0:4:->', - children = { - 'Register(name=a):0:6:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b,c->@a}', { - -- 01234567890 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:4:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - 'PlainIdentifier(scope=0,ident=c):0:5:c', - }, - }, - }, - }, - { - 'Arrow:0:6:->', - children = { - 'Register(name=a):0:8:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Comma', ','), - hl('IdentifierName', 'c'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b,c,d->@a}', { - -- 0123456789012 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:4:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - { - 'Comma:0:6:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:5:c', - 'PlainIdentifier(scope=0,ident=d):0:7:d', - }, - }, - }, - }, - }, - }, - { - 'Arrow:0:8:->', - children = { - 'Register(name=a):0:10:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Comma', ','), - hl('IdentifierName', 'c'), - hl('Comma', ','), - hl('IdentifierName', 'd'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b,c,d,->@a}', { - -- 01234567890123 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:4:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - { - 'Comma:0:6:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:5:c', - { - 'Comma:0:8:,', - children = { - 'PlainIdentifier(scope=0,ident=d):0:7:d', - }, - }, - }, - }, - }, - }, - }, - }, - { - 'Arrow:0:9:->', - children = { - 'Register(name=a):0:11:@a', - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Comma', ','), - hl('IdentifierName', 'c'), - hl('Comma', ','), - hl('IdentifierName', 'd'), - hl('Comma', ','), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - }) - check_parsing('{a,b->{c,d->{e,f->@a}}}', { - -- 01234567890123456789012 - -- 0 1 2 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'Arrow:0:4:->', - children = { - { - 'Lambda(\\di):0:6:{', - children = { - { - 'Comma:0:8:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:7:c', - 'PlainIdentifier(scope=0,ident=d):0:9:d', - }, - }, - { - 'Arrow:0:10:->', - children = { - { - 'Lambda(\\di):0:12:{', - children = { - { - 'Comma:0:14:,', - children = { - 'PlainIdentifier(scope=0,ident=e):0:13:e', - 'PlainIdentifier(scope=0,ident=f):0:15:f', - }, - }, - { - 'Arrow:0:16:->', - children = { - 'Register(name=a):0:18:@a', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - hl('Lambda', '{'), - hl('IdentifierName', 'c'), - hl('Comma', ','), - hl('IdentifierName', 'd'), - hl('Arrow', '->'), - hl('Lambda', '{'), - hl('IdentifierName', 'e'), - hl('Comma', ','), - hl('IdentifierName', 'f'), - hl('Arrow', '->'), - hl('Register', '@a'), - hl('Lambda', '}'), - hl('Lambda', '}'), - hl('Lambda', '}'), - }) - check_parsing('{a,b->c,d}', { - -- 0123456789 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'Arrow:0:4:->', - children = { - { - 'Comma:0:7:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:6:c', - 'PlainIdentifier(scope=0,ident=d):0:8:d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ',d}', - msg = 'E15: Comma outside of call, lambda or literal: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - hl('IdentifierName', 'c'), - hl('InvalidComma', ','), - hl('IdentifierName', 'd'), - hl('Lambda', '}'), - }) - check_parsing('a,b,c,d', { - -- 0123456789 - ast = { - { - 'Comma:0:1:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Comma:0:3:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:4:c', - 'PlainIdentifier(scope=0,ident=d):0:6:d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ',b,c,d', - msg = 'E15: Comma outside of call, lambda or literal: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidComma', ','), - hl('IdentifierName', 'b'), - hl('InvalidComma', ','), - hl('IdentifierName', 'c'), - hl('InvalidComma', ','), - hl('IdentifierName', 'd'), - }) - check_parsing('a,b,c,d,', { - -- 0123456789 - ast = { - { - 'Comma:0:1:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Comma:0:3:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:4:c', - { - 'Comma:0:7:,', - children = { - 'PlainIdentifier(scope=0,ident=d):0:6:d', - }, - }, - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ',b,c,d,', - msg = 'E15: Comma outside of call, lambda or literal: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidComma', ','), - hl('IdentifierName', 'b'), - hl('InvalidComma', ','), - hl('IdentifierName', 'c'), - hl('InvalidComma', ','), - hl('IdentifierName', 'd'), - hl('InvalidComma', ','), - }) - check_parsing(',', { - -- 0123456789 - ast = { - { - 'Comma:0:0:,', - children = { - 'Missing:0:0:', - }, - }, - }, - err = { - arg = ',', - msg = 'E15: Expected value, got comma: %.*s', - }, - }, { - hl('InvalidComma', ','), - }) - check_parsing('{,a->@a}', { - -- 0123456789 - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - { - 'Arrow:0:3:->', - children = { - { - 'Comma:0:1:,', - children = { - 'Missing:0:1:', - 'PlainIdentifier(scope=0,ident=a):0:2:a', - }, - }, - 'Register(name=a):0:5:@a', - }, - }, - }, - }, - }, - err = { - arg = ',a->@a}', - msg = 'E15: Expected value, got comma: %.*s', - }, - }, { - hl('Curly', '{'), - hl('InvalidComma', ','), - hl('IdentifierName', 'a'), - hl('InvalidArrow', '->'), - hl('Register', '@a'), - hl('Curly', '}'), - }) - check_parsing('}', { - -- 0123456789 - ast = { - 'UnknownFigure(---):0:0:', - }, - err = { - arg = '}', - msg = 'E15: Unexpected closing figure brace: %.*s', - }, - }, { - hl('InvalidFigureBrace', '}'), - }) - check_parsing('{->}', { - -- 0123456789 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - 'Arrow:0:1:->', - }, - }, - }, - err = { - arg = '}', - msg = 'E15: Expected value, got closing figure brace: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('Arrow', '->'), - hl('InvalidLambda', '}'), - }) - check_parsing('{a,b}', { - -- 0123456789 - ast = { - { - 'Lambda(-di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - }, - }, - }, - err = { - arg = '}', - msg = 'E15: Expected lambda arguments list or arrow: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('InvalidLambda', '}'), - }) - check_parsing('{a,}', { - -- 0123456789 - ast = { - { - 'Lambda(-di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - }, - }, - err = { - arg = '}', - msg = 'E15: Expected lambda arguments list or arrow: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('InvalidLambda', '}'), - }) - check_parsing('{@a:@b}', { - -- 0123456789 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Dict', '}'), - }) - check_parsing('{@a:@b,@c:@d}', { - -- 0123456789012 - -- 0 1 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - { - 'Colon:0:9::', - children = { - 'Register(name=c):0:7:@c', - 'Register(name=d):0:10:@d', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Comma', ','), - hl('Register', '@c'), - hl('Colon', ':'), - hl('Register', '@d'), - hl('Dict', '}'), - }) - check_parsing('{@a:@b,@c:@d,@e:@f,}', { - -- 01234567890123456789 - -- 0 1 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - { - 'Comma:0:12:,', - children = { - { - 'Colon:0:9::', - children = { - 'Register(name=c):0:7:@c', - 'Register(name=d):0:10:@d', - }, - }, - { - 'Comma:0:18:,', - children = { - { - 'Colon:0:15::', - children = { - 'Register(name=e):0:13:@e', - 'Register(name=f):0:16:@f', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Comma', ','), - hl('Register', '@c'), - hl('Colon', ':'), - hl('Register', '@d'), - hl('Comma', ','), - hl('Register', '@e'), - hl('Colon', ':'), - hl('Register', '@f'), - hl('Comma', ','), - hl('Dict', '}'), - }) - check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', { - -- 01234567890123456789012 - -- 0 1 2 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - { - 'Comma:0:12:,', - children = { - { - 'Colon:0:9::', - children = { - 'Register(name=c):0:7:@c', - 'Register(name=d):0:10:@d', - }, - }, - { - 'Comma:0:18:,', - children = { - { - 'Colon:0:15::', - children = { - 'Register(name=e):0:13:@e', - 'Register(name=f):0:16:@f', - }, - }, - { - 'Colon:0:21::', - children = { - 'Register(name=g):0:19:@g', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '}', - msg = 'E15: Expected value, got closing figure brace: %.*s', - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Comma', ','), - hl('Register', '@c'), - hl('Colon', ':'), - hl('Register', '@d'), - hl('Comma', ','), - hl('Register', '@e'), - hl('Colon', ':'), - hl('Register', '@f'), - hl('Comma', ','), - hl('Register', '@g'), - hl('Colon', ':'), - hl('InvalidDict', '}'), - }) - check_parsing('{@a:@b,}', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:3::', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:4:@b', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('Colon', ':'), - hl('Register', '@b'), - hl('Comma', ','), - hl('Dict', '}'), - }) - check_parsing('{({f -> g})(@h)(@i)}', { - -- 01234567890123456789 - -- 0 1 - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - { - 'Call:0:15:(', - children = { - { - 'Call:0:11:(', - children = { - { - 'Nested:0:1:(', - children = { - { - 'Lambda(\\di):0:2:{', - children = { - 'PlainIdentifier(scope=0,ident=f):0:3:f', - { - 'Arrow:0:4: ->', - children = { - 'PlainIdentifier(scope=0,ident=g):0:7: g', - }, - }, - }, - }, - }, - }, - 'Register(name=h):0:12:@h', - }, - }, - 'Register(name=i):0:16:@i', - }, - }, - }, - }, - }, - }, { - hl('Curly', '{'), - hl('NestingParenthesis', '('), - hl('Lambda', '{'), - hl('IdentifierName', 'f'), - hl('Arrow', '->', 1), - hl('IdentifierName', 'g', 1), - hl('Lambda', '}'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@h'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@i'), - hl('CallingParenthesis', ')'), - hl('Curly', '}'), - }) - check_parsing('a:{b()}c', { - -- 01234567 - ast = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=a,ident=):0:0:a:', - { - 'ComplexIdentifier:0:7:', - children = { - { - 'CurlyBracesIdentifier(--i):0:2:{', - children = { - { - 'Call:0:4:(', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=c):0:7:c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - hl('Curly', '}'), - hl('IdentifierName', 'c'), - }) - check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', { - -- 01234567890123456789012345678901234567890123456 - -- 0 1 2 3 4 - ast = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=a,ident=):0:0:a:', - { - 'ComplexIdentifier:0:42:', - children = { - { - 'CurlyBracesIdentifier(--i):0:2:{', - children = { - { - 'Call:0:37:(', - children = { - { - 'Lambda(\\di):0:3:{', - children = { - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4:b', - 'PlainIdentifier(scope=0,ident=c):0:6: c', - }, - }, - { - 'Arrow:0:8: ->', - children = { - { - 'BinaryPlus:0:19: +', - children = { - { - 'BinaryPlus:0:14: +', - children = { - 'Register(name=d):0:11: @d', - 'Register(name=e):0:16: @e', - }, - }, - { - 'Call:0:32:(', - children = { - { - 'Nested:0:21: (', - children = { - { - 'Lambda(\\di):0:23:{', - children = { - 'PlainIdentifier(scope=0,ident=f):0:24:f', - { - 'Arrow:0:25: ->', - children = { - 'PlainIdentifier(scope=0,ident=g):0:28: g', - }, - }, - }, - }, - }, - }, - 'Register(name=h):0:33:@h', - }, - }, - }, - }, - }, - }, - }, - }, - 'Register(name=i):0:38:@i', - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=j):0:42:j', - }, - }, - }, - }, - }, - }, { - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('Curly', '{'), - hl('Lambda', '{'), - hl('IdentifierName', 'b'), - hl('Comma', ','), - hl('IdentifierName', 'c', 1), - hl('Arrow', '->', 1), - hl('Register', '@d', 1), - hl('BinaryPlus', '+', 1), - hl('Register', '@e', 1), - hl('BinaryPlus', '+', 1), - hl('NestingParenthesis', '(', 1), - hl('Lambda', '{'), - hl('IdentifierName', 'f'), - hl('Arrow', '->', 1), - hl('IdentifierName', 'g', 1), - hl('Lambda', '}'), - hl('NestingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('Register', '@h'), - hl('CallingParenthesis', ')'), - hl('Lambda', '}'), - hl('CallingParenthesis', '('), - hl('Register', '@i'), - hl('CallingParenthesis', ')'), - hl('Curly', '}'), - hl('IdentifierName', 'j'), - }) - check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', { - -- 01234567890123456789012345678901234567 - -- 0 1 2 3 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Comma:0:18:,', - children = { - { - 'Colon:0:8: :', - children = { - { - 'BinaryPlus:0:3: +', - children = { - 'Register(name=a):0:1:@a', - 'Register(name=b):0:5: @b', - }, - }, - { - 'BinaryPlus:0:13: +', - children = { - 'Register(name=c):0:10: @c', - 'Register(name=d):0:15: @d', - }, - }, - }, - }, - { - 'Colon:0:27: :', - children = { - { - 'BinaryPlus:0:22: +', - children = { - 'Register(name=e):0:19: @e', - 'Register(name=f):0:24: @f', - }, - }, - { - 'BinaryPlus:0:32: +', - children = { - 'Register(name=g):0:29: @g', - 'Register(name=i):0:34: @i', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Dict', '{'), - hl('Register', '@a'), - hl('BinaryPlus', '+', 1), - hl('Register', '@b', 1), - hl('Colon', ':', 1), - hl('Register', '@c', 1), - hl('BinaryPlus', '+', 1), - hl('Register', '@d', 1), - hl('Comma', ','), - hl('Register', '@e', 1), - hl('BinaryPlus', '+', 1), - hl('Register', '@f', 1), - hl('Colon', ':', 1), - hl('Register', '@g', 1), - hl('BinaryPlus', '+', 1), - hl('Register', '@i', 1), - hl('Dict', '}'), - }) - check_parsing('-> -> ->', { - -- 01234567 - ast = { - { - 'Arrow:0:0:->', - children = { - 'Missing:0:0:', - { - 'Arrow:0:2: ->', - children = { - 'Missing:0:2:', - { - 'Arrow:0:5: ->', - children = { - 'Missing:0:5:', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '-> -> ->', - msg = 'E15: Unexpected arrow: %.*s', - }, - }, { - hl('InvalidArrow', '->'), - hl('InvalidArrow', '->', 1), - hl('InvalidArrow', '->', 1), - }) - check_parsing('a -> b -> c -> d', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'Arrow:0:1: ->', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Arrow:0:6: ->', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4: b', - { - 'Arrow:0:11: ->', - children = { - 'PlainIdentifier(scope=0,ident=c):0:9: c', - 'PlainIdentifier(scope=0,ident=d):0:14: d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '-> b -> c -> d', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'c', 1), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'd', 1), - }) - check_parsing('{a -> b -> c}', { - -- 0123456789012 - -- 0 1 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Arrow:0:2: ->', - children = { - { - 'Arrow:0:7: ->', - children = { - 'PlainIdentifier(scope=0,ident=b):0:5: b', - 'PlainIdentifier(scope=0,ident=c):0:10: c', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '-> c}', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Arrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'c', 1), - hl('Lambda', '}'), - }) - check_parsing('{a: -> b}', { - -- 012345678 - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - { - 'Arrow:0:3: ->', - children = { - 'PlainIdentifier(scope=a,ident=):0:1:a:', - 'PlainIdentifier(scope=0,ident=b):0:6: b', - }, - }, - }, - }, - }, - err = { - arg = '-> b}', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('Curly', '{'), - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('Curly', '}'), - }) - - check_parsing('{a:b -> b}', { - -- 0123456789 - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - { - 'Arrow:0:4: ->', - children = { - 'PlainIdentifier(scope=a,ident=b):0:1:a:b', - 'PlainIdentifier(scope=0,ident=b):0:7: b', - }, - }, - }, - }, - }, - err = { - arg = '-> b}', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('Curly', '{'), - hl('IdentifierScope', 'a'), - hl('IdentifierScopeDelimiter', ':'), - hl('IdentifierName', 'b'), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('Curly', '}'), - }) - - check_parsing('{a#b -> b}', { - -- 0123456789 - ast = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - { - 'Arrow:0:4: ->', - children = { - 'PlainIdentifier(scope=0,ident=a#b):0:1:a#b', - 'PlainIdentifier(scope=0,ident=b):0:7: b', - }, - }, - }, - }, - }, - err = { - arg = '-> b}', - msg = 'E15: Arrow outside of lambda: %.*s', - }, - }, { - hl('Curly', '{'), - hl('IdentifierName', 'a#b'), - hl('InvalidArrow', '->', 1), - hl('IdentifierName', 'b', 1), - hl('Curly', '}'), - }) - check_parsing('{a : b : c}', { - -- 01234567890 - -- 0 1 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Colon:0:2: :', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Colon:0:6: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4: b', - 'PlainIdentifier(scope=0,ident=c):0:8: c', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ': c}', - msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', - }, - }, { - hl('Dict', '{'), - hl('IdentifierName', 'a'), - hl('Colon', ':', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidColon', ':', 1), - hl('IdentifierName', 'c', 1), - hl('Dict', '}'), - }) - check_parsing('{', { - -- 0 - ast = { - 'UnknownFigure(\\di):0:0:{', - }, - err = { - arg = '{', - msg = 'E15: Missing closing figure brace: %.*s', - }, - }, { - hl('FigureBrace', '{'), - }) - check_parsing('{a', { - -- 01 - ast = { - { - 'UnknownFigure(\\di):0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - err = { - arg = '{a', - msg = 'E15: Missing closing figure brace: %.*s', - }, - }, { - hl('FigureBrace', '{'), - hl('IdentifierName', 'a'), - }) - check_parsing('{a,b', { - -- 0123 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - }, - }, - }, - err = { - arg = '{a,b', - msg = 'E15: Missing closing figure brace for lambda: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - }) - check_parsing('{a,b->', { - -- 012345 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - 'Arrow:0:4:->', - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - }) - check_parsing('{a,b->c', { - -- 0123456 - ast = { - { - 'Lambda(\\di):0:0:{', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'Arrow:0:4:->', - children = { - 'PlainIdentifier(scope=0,ident=c):0:6:c', - }, - }, - }, - }, - }, - err = { - arg = '{a,b->c', - msg = 'E15: Missing closing figure brace for lambda: %.*s', - }, - }, { - hl('Lambda', '{'), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b'), - hl('Arrow', '->'), - hl('IdentifierName', 'c'), - }) - check_parsing('{a : b', { - -- 012345 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Colon:0:2: :', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, - }, - err = { - arg = '{a : b', - msg = 'E723: Missing end of Dictionary \'}\': %.*s', - }, - }, { - hl('Dict', '{'), - hl('IdentifierName', 'a'), - hl('Colon', ':', 1), - hl('IdentifierName', 'b', 1), - }) - check_parsing('{a : b,', { - -- 0123456 - ast = { - { - 'DictLiteral(-di):0:0:{', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:2: :', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('Dict', '{'), - hl('IdentifierName', 'a'), - hl('Colon', ':', 1), - hl('IdentifierName', 'b', 1), - hl('Comma', ','), - }) - end) - itp('works with ternary operator', function() - check_parsing('a ? b : c', { - -- 012345678 - ast = { - { - 'Ternary:0:1: ?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:5: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:7: c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?', 1), - hl('IdentifierName', 'b', 1), - hl('TernaryColon', ':', 1), - hl('IdentifierName', 'c', 1), - }) - check_parsing('@a?@b?@c:@d:@e', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'Ternary:0:2:?', - children = { - 'Register(name=a):0:0:@a', - { - 'TernaryValue:0:11::', - children = { - { - 'Ternary:0:5:?', - children = { - 'Register(name=b):0:3:@b', - { - 'TernaryValue:0:8::', - children = { - 'Register(name=c):0:6:@c', - 'Register(name=d):0:9:@d', - }, - }, - }, - }, - 'Register(name=e):0:12:@e', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('Ternary', '?'), - hl('Register', '@c'), - hl('TernaryColon', ':'), - hl('Register', '@d'), - hl('TernaryColon', ':'), - hl('Register', '@e'), - }) - check_parsing('@a?@b:@c?@d:@e', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'Ternary:0:2:?', - children = { - 'Register(name=a):0:0:@a', - { - 'TernaryValue:0:5::', - children = { - 'Register(name=b):0:3:@b', - { - 'Ternary:0:8:?', - children = { - 'Register(name=c):0:6:@c', - { - 'TernaryValue:0:11::', - children = { - 'Register(name=d):0:9:@d', - 'Register(name=e):0:12:@e', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('TernaryColon', ':'), - hl('Register', '@c'), - hl('Ternary', '?'), - hl('Register', '@d'), - hl('TernaryColon', ':'), - hl('Register', '@e'), - }) - check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', { - -- 01234567890123456789012345678901 - -- 0 1 2 3 - ast = { - { - 'Ternary:0:2:?', - children = { - 'Register(name=a):0:0:@a', - { - 'TernaryValue:0:29::', - children = { - { - 'Ternary:0:5:?', - children = { - 'Register(name=b):0:3:@b', - { - 'TernaryValue:0:20::', - children = { - { - 'Ternary:0:8:?', - children = { - 'Register(name=c):0:6:@c', - { - 'TernaryValue:0:11::', - children = { - 'Register(name=d):0:9:@d', - { - 'Ternary:0:14:?', - children = { - 'Register(name=e):0:12:@e', - { - 'TernaryValue:0:17::', - children = { - 'Register(name=f):0:15:@f', - 'Register(name=g):0:18:@g', - }, - }, - }, - }, - }, - }, - }, - }, - { - 'Ternary:0:23:?', - children = { - 'Register(name=h):0:21:@h', - { - 'TernaryValue:0:26::', - children = { - 'Register(name=i):0:24:@i', - 'Register(name=j):0:27:@j', - }, - }, - }, - }, - }, - }, - }, - }, - 'Register(name=k):0:30:@k', - }, - }, - }, - }, - }, - }, { - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('Ternary', '?'), - hl('Register', '@c'), - hl('Ternary', '?'), - hl('Register', '@d'), - hl('TernaryColon', ':'), - hl('Register', '@e'), - hl('Ternary', '?'), - hl('Register', '@f'), - hl('TernaryColon', ':'), - hl('Register', '@g'), - hl('TernaryColon', ':'), - hl('Register', '@h'), - hl('Ternary', '?'), - hl('Register', '@i'), - hl('TernaryColon', ':'), - hl('Register', '@j'), - hl('TernaryColon', ':'), - hl('Register', '@k'), - }) - check_parsing('?', { - -- 0 - ast = { - { - 'Ternary:0:0:?', - children = { - 'Missing:0:0:', - 'TernaryValue:0:0:?', - }, - }, - }, - err = { - arg = '?', - msg = 'E15: Expected value, got question mark: %.*s', - }, - }, { - hl('InvalidTernary', '?'), - }) - - check_parsing('?:', { - -- 01 - ast = { - { - 'Ternary:0:0:?', - children = { - 'Missing:0:0:', - { - 'TernaryValue:0:1::', - children = { - 'Missing:0:1:', - }, - }, - }, - }, - }, - err = { - arg = '?:', - msg = 'E15: Expected value, got question mark: %.*s', - }, - }, { - hl('InvalidTernary', '?'), - hl('InvalidTernaryColon', ':'), - }) - - check_parsing('?::', { - -- 012 - ast = { - { - 'Colon:0:2::', - children = { - { - 'Ternary:0:0:?', - children = { - 'Missing:0:0:', - { - 'TernaryValue:0:1::', - children = { - 'Missing:0:1:', - 'Missing:0:2:', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '?::', - msg = 'E15: Expected value, got question mark: %.*s', - }, - }, { - hl('InvalidTernary', '?'), - hl('InvalidTernaryColon', ':'), - hl('InvalidColon', ':'), - }) - - check_parsing('a?b', { - -- 012 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - }, - }, - }, - err = { - arg = '?b', - msg = 'E109: Missing \':\' after \'?\': %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierName', 'b'), - }) - check_parsing('a?b:', { - -- 0123 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:1:?', - children = { - 'PlainIdentifier(scope=b,ident=):0:2:b:', - }, - }, - }, - }, - }, - err = { - arg = '?b:', - msg = 'E109: Missing \':\' after \'?\': %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierScope', 'b'), - hl('IdentifierScopeDelimiter', ':'), - }) - - check_parsing('a?b::c', { - -- 012345 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:4::', - children = { - 'PlainIdentifier(scope=b,ident=):0:2:b:', - 'PlainIdentifier(scope=0,ident=c):0:5:c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierScope', 'b'), - hl('IdentifierScopeDelimiter', ':'), - hl('TernaryColon', ':'), - hl('IdentifierName', 'c'), - }) - - check_parsing('a?b :', { - -- 01234 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:3: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierName', 'b'), - hl('TernaryColon', ':', 1), - }) - - check_parsing('(@a?@b:@c)?@d:@e', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'Ternary:0:10:?', - children = { - { - 'Nested:0:0:(', - children = { - { - 'Ternary:0:3:?', - children = { - 'Register(name=a):0:1:@a', - { - 'TernaryValue:0:6::', - children = { - 'Register(name=b):0:4:@b', - 'Register(name=c):0:7:@c', - }, - }, - }, - }, - }, - }, - { - 'TernaryValue:0:13::', - children = { - 'Register(name=d):0:11:@d', - 'Register(name=e):0:14:@e', - }, - }, - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('TernaryColon', ':'), - hl('Register', '@c'), - hl('NestingParenthesis', ')'), - hl('Ternary', '?'), - hl('Register', '@d'), - hl('TernaryColon', ':'), - hl('Register', '@e'), - }) - - check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', { - -- 01234567890123456789012345678901 - -- 0 1 2 3 - ast = { - { - 'Ternary:0:10:?', - children = { - { - 'Nested:0:0:(', - children = { - { - 'Ternary:0:3:?', - children = { - 'Register(name=a):0:1:@a', - { - 'TernaryValue:0:6::', - children = { - 'Register(name=b):0:4:@b', - 'Register(name=c):0:7:@c', - }, - }, - }, - }, - }, - }, - { - 'TernaryValue:0:21::', - children = { - { - 'Nested:0:11:(', - children = { - { - 'Ternary:0:14:?', - children = { - 'Register(name=d):0:12:@d', - { - 'TernaryValue:0:17::', - children = { - 'Register(name=e):0:15:@e', - 'Register(name=f):0:18:@f', - }, - }, - }, - }, - }, - }, - { - 'Nested:0:22:(', - children = { - { - 'Ternary:0:25:?', - children = { - 'Register(name=g):0:23:@g', - { - 'TernaryValue:0:28::', - children = { - 'Register(name=h):0:26:@h', - 'Register(name=i):0:29:@i', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('TernaryColon', ':'), - hl('Register', '@c'), - hl('NestingParenthesis', ')'), - hl('Ternary', '?'), - hl('NestingParenthesis', '('), - hl('Register', '@d'), - hl('Ternary', '?'), - hl('Register', '@e'), - hl('TernaryColon', ':'), - hl('Register', '@f'), - hl('NestingParenthesis', ')'), - hl('TernaryColon', ':'), - hl('NestingParenthesis', '('), - hl('Register', '@g'), - hl('Ternary', '?'), - hl('Register', '@h'), - hl('TernaryColon', ':'), - hl('Register', '@i'), - hl('NestingParenthesis', ')'), - }) - - check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', { - -- 0123456789012345678901234567 - -- 0 1 2 - ast = { - { - 'Ternary:0:10:?', - children = { - { - 'Nested:0:0:(', - children = { - { - 'Ternary:0:3:?', - children = { - 'Register(name=a):0:1:@a', - { - 'TernaryValue:0:6::', - children = { - 'Register(name=b):0:4:@b', - 'Register(name=c):0:7:@c', - }, - }, - }, - }, - }, - }, - { - 'TernaryValue:0:19::', - children = { - { - 'Ternary:0:13:?', - children = { - 'Register(name=d):0:11:@d', - { - 'TernaryValue:0:16::', - children = { - 'Register(name=e):0:14:@e', - 'Register(name=f):0:17:@f', - }, - }, - }, - }, - { - 'Ternary:0:22:?', - children = { - 'Register(name=g):0:20:@g', - { - 'TernaryValue:0:25::', - children = { - 'Register(name=h):0:23:@h', - 'Register(name=i):0:26:@i', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('NestingParenthesis', '('), - hl('Register', '@a'), - hl('Ternary', '?'), - hl('Register', '@b'), - hl('TernaryColon', ':'), - hl('Register', '@c'), - hl('NestingParenthesis', ')'), - hl('Ternary', '?'), - hl('Register', '@d'), - hl('Ternary', '?'), - hl('Register', '@e'), - hl('TernaryColon', ':'), - hl('Register', '@f'), - hl('TernaryColon', ':'), - hl('Register', '@g'), - hl('Ternary', '?'), - hl('Register', '@h'), - hl('TernaryColon', ':'), - hl('Register', '@i'), - }) - check_parsing('a?b{cdef}g:h', { - -- 012345678901 - -- 0 1 - ast = { - { - 'Ternary:0:1:?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:10::', - children = { - { - 'ComplexIdentifier:0:3:', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - { - 'ComplexIdentifier:0:9:', - children = { - { - 'CurlyBracesIdentifier(--i):0:3:{', - children = { - 'PlainIdentifier(scope=0,ident=cdef):0:4:cdef', - }, - }, - 'PlainIdentifier(scope=0,ident=g):0:9:g', - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=h):0:11:h', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?'), - hl('IdentifierName', 'b'), - hl('Curly', '{'), - hl('IdentifierName', 'cdef'), - hl('Curly', '}'), - hl('IdentifierName', 'g'), - hl('TernaryColon', ':'), - hl('IdentifierName', 'h'), - }) - check_parsing('a ? b : c : d', { - -- 0123456789012 - -- 0 1 - ast = { - { - 'Colon:0:9: :', - children = { - { - 'Ternary:0:1: ?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'TernaryValue:0:5: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:7: c', - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=d):0:11: d', - }, - }, - }, - err = { - arg = ': d', - msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Ternary', '?', 1), - hl('IdentifierName', 'b', 1), - hl('TernaryColon', ':', 1), - hl('IdentifierName', 'c', 1), - hl('InvalidColon', ':', 1), - hl('IdentifierName', 'd', 1), - }) - end) - itp('works with comparison operators', function() - check_parsing('a == b', { - -- 012345 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=UseOption):0:1: ==', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '==', 1), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a ==? b', { - -- 0123456 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=IgnoreCase):0:1: ==?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '==', 1), - hl('ComparisonModifier', '?'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a ==# b', { - -- 0123456 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=MatchCase):0:1: ==#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '==', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a !=# b', { - -- 0123456 - ast = { - { - 'Comparison(type=Equal,inv=1,ccs=MatchCase):0:1: !=#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '!=', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a <=# b', { - -- 0123456 - ast = { - { - 'Comparison(type=Greater,inv=1,ccs=MatchCase):0:1: <=#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '<=', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a >=# b', { - -- 0123456 - ast = { - { - 'Comparison(type=GreaterOrEqual,inv=0,ccs=MatchCase):0:1: >=#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '>=', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a ># b', { - -- 012345 - ast = { - { - 'Comparison(type=Greater,inv=0,ccs=MatchCase):0:1: >#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '>', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a <# b', { - -- 012345 - ast = { - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:1: <#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '<', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a is#b', { - -- 012345 - ast = { - { - 'Comparison(type=Identical,inv=0,ccs=MatchCase):0:1: is#', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5:b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', 'is', 1), - hl('ComparisonModifier', '#'), - hl('IdentifierName', 'b'), - }) - - check_parsing('a is?b', { - -- 012345 - ast = { - { - 'Comparison(type=Identical,inv=0,ccs=IgnoreCase):0:1: is?', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:5:b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', 'is', 1), - hl('ComparisonModifier', '?'), - hl('IdentifierName', 'b'), - }) - - check_parsing('a isnot b', { - -- 012345678 - ast = { - { - 'Comparison(type=Identical,inv=1,ccs=UseOption):0:1: isnot', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:7: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', 'isnot', 1), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a < b < c', { - -- 012345678 - ast = { - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:5: <', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:7: c', - }, - }, - }, - }, - }, - err = { - arg = ' < c', - msg = 'E15: Operator is not associative: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '<', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidComparison', '<', 1), - hl('IdentifierName', 'c', 1), - }) - - check_parsing('a < b <# c', { - -- 012345678 - ast = { - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:5: <#', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:8: c', - }, - }, - }, - }, - }, - err = { - arg = ' <# c', - msg = 'E15: Operator is not associative: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('Comparison', '<', 1), - hl('IdentifierName', 'b', 1), - hl('InvalidComparison', '<', 1), - hl('InvalidComparisonModifier', '#'), - hl('IdentifierName', 'c', 1), - }) - - check_parsing('a += b', { - -- 012345 - ast = { - { - 'Assignment(Add):0:1: +=', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - }, - err = { - arg = '+= b', - msg = 'E15: Misplaced assignment: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidAssignmentWithAddition', '+=', 1), - hl('IdentifierName', 'b', 1), - }) - check_parsing('a + b == c + d', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=UseOption):0:5: ==', - children = { - { - 'BinaryPlus:0:1: +', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:3: b', - }, - }, - { - 'BinaryPlus:0:10: +', - children = { - 'PlainIdentifier(scope=0,ident=c):0:8: c', - 'PlainIdentifier(scope=0,ident=d):0:12: d', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('BinaryPlus', '+', 1), - hl('IdentifierName', 'b', 1), - hl('Comparison', '==', 1), - hl('IdentifierName', 'c', 1), - hl('BinaryPlus', '+', 1), - hl('IdentifierName', 'd', 1), - }) - check_parsing('+ a == + b', { - -- 0123456789 - ast = { - { - 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3: ==', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1: a', - }, - }, - { - 'UnaryPlus:0:6: +', - children = { - 'PlainIdentifier(scope=0,ident=b):0:8: b', - }, - }, - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('IdentifierName', 'a', 1), - hl('Comparison', '==', 1), - hl('UnaryPlus', '+', 1), - hl('IdentifierName', 'b', 1), - }) - end) - itp('works with concat/subscript', function() - check_parsing('.', { - -- 0 - ast = { - { - 'ConcatOrSubscript:0:0:.', - children = { - 'Missing:0:0:', - }, - }, - }, - err = { - arg = '.', - msg = 'E15: Unexpected dot: %.*s', - }, - }, { - hl('InvalidConcatOrSubscript', '.'), - }) - - check_parsing('a.', { - -- 01 - ast = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - }, - }, - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - }) - - check_parsing('a.b', { - -- 012 - ast = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainKey(key=b):0:2:b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', 'b'), - }) - - check_parsing('1.2', { - -- 012 - ast = { - 'Float(val=1.200000e+00):0:0:1.2', - }, - }, { - hl('Float', '1.2'), - }) - - check_parsing('1.2 + 1.3e-5', { - -- 012345678901 - -- 0 1 - ast = { - { - 'BinaryPlus:0:3: +', - children = { - 'Float(val=1.200000e+00):0:0:1.2', - 'Float(val=1.300000e-05):0:5: 1.3e-5', - }, - }, - }, - }, { - hl('Float', '1.2'), - hl('BinaryPlus', '+', 1), - hl('Float', '1.3e-5', 1), - }) - - check_parsing('a . 1.2 + 1.3e-5', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'BinaryPlus:0:7: +', - children = { - { - 'Concat:0:1: .', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'ConcatOrSubscript:0:5:.', - children = { - 'Integer(val=1):0:3: 1', - 'PlainKey(key=2):0:6:2', - }, - }, - }, - }, - 'Float(val=1.300000e-05):0:9: 1.3e-5', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Concat', '.', 1), - hl('Number', '1', 1), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - hl('BinaryPlus', '+', 1), - hl('Float', '1.3e-5', 1), - }) - - check_parsing('1.3e-5 + 1.2 . a', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'Concat:0:12: .', - children = { - { - 'BinaryPlus:0:6: +', - children = { - 'Float(val=1.300000e-05):0:0:1.3e-5', - 'Float(val=1.200000e+00):0:8: 1.2', - }, - }, - 'PlainIdentifier(scope=0,ident=a):0:14: a', - }, - }, - }, - }, { - hl('Float', '1.3e-5'), - hl('BinaryPlus', '+', 1), - hl('Float', '1.2', 1), - hl('Concat', '.', 1), - hl('IdentifierName', 'a', 1), - }) - - check_parsing('1.3e-5 + a . 1.2', { - -- 0123456789012345 - -- 0 1 - ast = { - { - 'Concat:0:10: .', - children = { - { - 'BinaryPlus:0:6: +', - children = { - 'Float(val=1.300000e-05):0:0:1.3e-5', - 'PlainIdentifier(scope=0,ident=a):0:8: a', - }, - }, - { - 'ConcatOrSubscript:0:14:.', - children = { - 'Integer(val=1):0:12: 1', - 'PlainKey(key=2):0:15:2', - }, - }, - }, - }, - }, - }, { - hl('Float', '1.3e-5'), - hl('BinaryPlus', '+', 1), - hl('IdentifierName', 'a', 1), - hl('Concat', '.', 1), - hl('Number', '1', 1), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - }) - - check_parsing('1.2.3', { - -- 01234 - ast = { - { - 'ConcatOrSubscript:0:3:.', - children = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'Integer(val=1):0:0:1', - 'PlainKey(key=2):0:2:2', - }, - }, - 'PlainKey(key=3):0:4:3', - }, - }, - }, - }, { - hl('Number', '1'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '3'), - }) - - check_parsing('a.1.2', { - -- 01234 - ast = { - { - 'ConcatOrSubscript:0:3:.', - children = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainKey(key=1):0:2:1', - }, - }, - 'PlainKey(key=2):0:4:2', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '1'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - }) - - check_parsing('a . 1.2', { - -- 0123456 - ast = { - { - 'Concat:0:1: .', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'ConcatOrSubscript:0:5:.', - children = { - 'Integer(val=1):0:3: 1', - 'PlainKey(key=2):0:6:2', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Concat', '.', 1), - hl('Number', '1', 1), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - }) - - check_parsing('+a . +b', { - -- 0123456 - ast = { - { - 'Concat:0:2: .', - children = { - { - 'UnaryPlus:0:0:+', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - { - 'UnaryPlus:0:4: +', - children = { - 'PlainIdentifier(scope=0,ident=b):0:6:b', - }, - }, - }, - }, - }, - }, { - hl('UnaryPlus', '+'), - hl('IdentifierName', 'a'), - hl('Concat', '.', 1), - hl('UnaryPlus', '+', 1), - hl('IdentifierName', 'b'), - }) - - check_parsing('a. b', { - -- 0123 - ast = { - { - 'Concat:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:2: b', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierName', 'b', 1), - }) - - check_parsing('a. 1', { - -- 0123 - ast = { - { - 'Concat:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'Integer(val=1):0:2: 1', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('Number', '1', 1), - }) - end) - itp('works with bracket subscripts', function() - check_parsing(':', { - -- 0 - ast = { - { - 'Colon:0:0::', - children = { - 'Missing:0:0:', - }, - }, - }, - err = { - arg = ':', - msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', - }, - }, { - hl('InvalidColon', ':'), - }) - check_parsing('a[]', { - -- 012 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - }, - }, - }, - err = { - arg = ']', - msg = 'E15: Expected value, got closing bracket: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('InvalidSubscriptBracket', ']'), - }) - check_parsing('a[b:]', { - -- 01234 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=b,ident=):0:2:b:', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierScope', 'b'), - hl('IdentifierScopeDelimiter', ':'), - hl('SubscriptBracket', ']'), - }) - - check_parsing('a[b:c]', { - -- 012345 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=b,ident=c):0:2:b:c', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierScope', 'b'), - hl('IdentifierScopeDelimiter', ':'), - hl('IdentifierName', 'c'), - hl('SubscriptBracket', ']'), - }) - check_parsing('a[b : c]', { - -- 01234567 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Colon:0:3: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - 'PlainIdentifier(scope=0,ident=c):0:5: c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'b'), - hl('SubscriptColon', ':', 1), - hl('IdentifierName', 'c', 1), - hl('SubscriptBracket', ']'), - }) - - check_parsing('a[: b]', { - -- 012345 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Colon:0:2::', - children = { - 'Missing:0:2:', - 'PlainIdentifier(scope=0,ident=b):0:3: b', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('SubscriptColon', ':'), - hl('IdentifierName', 'b', 1), - hl('SubscriptBracket', ']'), - }) - - check_parsing('a[b :]', { - -- 012345 - ast = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Colon:0:3: :', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'b'), - hl('SubscriptColon', ':', 1), - hl('SubscriptBracket', ']'), - }) - check_parsing('a[b][c][d](e)(f)(g)', { - -- 0123456789012345678 - -- 0 1 - ast = { - { - 'Call:0:16:(', - children = { - { - 'Call:0:13:(', - children = { - { - 'Call:0:10:(', - children = { - { - 'Subscript:0:7:[', - children = { - { - 'Subscript:0:4:[', - children = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - 'PlainIdentifier(scope=0,ident=c):0:5:c', - }, - }, - 'PlainIdentifier(scope=0,ident=d):0:8:d', - }, - }, - 'PlainIdentifier(scope=0,ident=e):0:11:e', - }, - }, - 'PlainIdentifier(scope=0,ident=f):0:14:f', - }, - }, - 'PlainIdentifier(scope=0,ident=g):0:17:g', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'b'), - hl('SubscriptBracket', ']'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'c'), - hl('SubscriptBracket', ']'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'd'), - hl('SubscriptBracket', ']'), - hl('CallingParenthesis', '('), - hl('IdentifierName', 'e'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('IdentifierName', 'f'), - hl('CallingParenthesis', ')'), - hl('CallingParenthesis', '('), - hl('IdentifierName', 'g'), - hl('CallingParenthesis', ')'), - }) - check_parsing('{a}{b}{c}[d][e][f]', { - -- 012345678901234567 - -- 0 1 - ast = { - { - 'Subscript:0:15:[', - children = { - { - 'Subscript:0:12:[', - children = { - { - 'Subscript:0:9:[', - children = { - { - 'ComplexIdentifier:0:3:', - children = { - { - 'CurlyBracesIdentifier(-di):0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - { - 'ComplexIdentifier:0:6:', - children = { - { - 'CurlyBracesIdentifier(--i):0:3:{', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4:b', - }, - }, - { - 'CurlyBracesIdentifier(--i):0:6:{', - children = { - 'PlainIdentifier(scope=0,ident=c):0:7:c', - }, - }, - }, - }, - }, - }, - 'PlainIdentifier(scope=0,ident=d):0:10:d', - }, - }, - 'PlainIdentifier(scope=0,ident=e):0:13:e', - }, - }, - 'PlainIdentifier(scope=0,ident=f):0:16:f', - }, - }, - }, - }, { - hl('Curly', '{'), - hl('IdentifierName', 'a'), - hl('Curly', '}'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - hl('Curly', '{'), - hl('IdentifierName', 'c'), - hl('Curly', '}'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'd'), - hl('SubscriptBracket', ']'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'e'), - hl('SubscriptBracket', ']'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'f'), - hl('SubscriptBracket', ']'), - }) - end) - itp('supports list literals', function() - check_parsing('[]', { - -- 01 - ast = { - 'ListLiteral:0:0:[', - }, - }, { - hl('List', '['), - hl('List', ']'), - }) - - check_parsing('[a]', { - -- 012 - ast = { - { - 'ListLiteral:0:0:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('List', ']'), - }) - - check_parsing('[a, b]', { - -- 012345 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3: b', - }, - }, - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b', 1), - hl('List', ']'), - }) - - check_parsing('[a, b, c]', { - -- 012345678 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - 'PlainIdentifier(scope=0,ident=c):0:6: c', - }, - }, - }, - }, - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b', 1), - hl('Comma', ','), - hl('IdentifierName', 'c', 1), - hl('List', ']'), - }) - - check_parsing('[a, b, c, ]', { - -- 01234567890 - -- 0 1 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:2:,', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'Comma:0:5:,', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3: b', - { - 'Comma:0:8:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:6: c', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('Comma', ','), - hl('IdentifierName', 'b', 1), - hl('Comma', ','), - hl('IdentifierName', 'c', 1), - hl('Comma', ','), - hl('List', ']', 1), - }) - - check_parsing('[a : b, c : d]', { - -- 01234567890123 - -- 0 1 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:6:,', - children = { - { - 'Colon:0:2: :', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:4: b', - }, - }, - { - 'Colon:0:9: :', - children = { - 'PlainIdentifier(scope=0,ident=c):0:7: c', - 'PlainIdentifier(scope=0,ident=d):0:11: d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = ': b, c : d]', - msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('InvalidColon', ':', 1), - hl('IdentifierName', 'b', 1), - hl('Comma', ','), - hl('IdentifierName', 'c', 1), - hl('InvalidColon', ':', 1), - hl('IdentifierName', 'd', 1), - hl('List', ']'), - }) - - check_parsing(']', { - -- 0 - ast = { - 'ListLiteral:0:0:', - }, - err = { - arg = ']', - msg = 'E15: Unexpected closing figure brace: %.*s', - }, - }, { - hl('InvalidList', ']'), - }) - - check_parsing('a]', { - -- 01 - ast = { - { - 'ListLiteral:0:1:', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - }, - }, - }, - err = { - arg = ']', - msg = 'E15: Unexpected closing figure brace: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('InvalidList', ']'), - }) - - check_parsing('[] []', { - -- 01234 - ast = { - { - 'OpMissing:0:2:', - children = { - 'ListLiteral:0:0:[', - 'ListLiteral:0:2: [', - }, - }, - }, - err = { - arg = '[]', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('List', '['), - hl('List', ']'), - hl('InvalidSpacing', ' '), - hl('List', '['), - hl('List', ']'), - }, { - [1] = { - ast = { - err = REMOVE_THIS, - ast = { - 'ListLiteral:0:0:[', - }, - }, - hl_fs = { - [3] = REMOVE_THIS, - [4] = REMOVE_THIS, - [5] = REMOVE_THIS, - }, - }, - }) - - check_parsing('[][]', { - -- 0123 - ast = { - { - 'Subscript:0:2:[', - children = { - 'ListLiteral:0:0:[', - }, - }, - }, - err = { - arg = ']', - msg = 'E15: Expected value, got closing bracket: %.*s', - }, - }, { - hl('List', '['), - hl('List', ']'), - hl('SubscriptBracket', '['), - hl('InvalidSubscriptBracket', ']'), - }) - - check_parsing('[', { - -- 0 - ast = { - 'ListLiteral:0:0:[', - }, - err = { - arg = '', - msg = 'E15: Expected value, got EOC: %.*s', - }, - }, { - hl('List', '['), - }) - - check_parsing('[1', { - -- 01 - ast = { - { - 'ListLiteral:0:0:[', - children = { - 'Integer(val=1):0:1:1', - }, - }, - }, - err = { - arg = '[1', - msg = 'E697: Missing end of List \']\': %.*s', - }, - }, { - hl('List', '['), - hl('Number', '1'), - }) - end) - itp('works with strings', function() - check_parsing('\'abc\'', { - -- 01234 - ast = { - 'SingleQuotedString(val="abc"):0:0:\'abc\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuotedBody', 'abc'), - hl('SingleQuote', '\''), - }) - check_parsing('"abc"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="abc"):0:0:"abc"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedBody', 'abc'), - hl('DoubleQuote', '"'), - }) - check_parsing('\'\'', { - -- 01 - ast = { - 'SingleQuotedString(val=NULL):0:0:\'\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuote', '\''), - }) - check_parsing('""', { - -- 01 - ast = { - 'DoubleQuotedString(val=NULL):0:0:""', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuote', '"'), - }) - check_parsing('"', { - -- 0 - ast = { - 'DoubleQuotedString(val=NULL):0:0:"', - }, - err = { - arg = '"', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - }) - check_parsing('\'', { - -- 0 - ast = { - 'SingleQuotedString(val=NULL):0:0:\'', - }, - err = { - arg = '\'', - msg = 'E115: Missing single quote: %.*s', - }, - }, { - hl('InvalidSingleQuote', '\''), - }) - check_parsing('"a', { - -- 01 - ast = { - 'DoubleQuotedString(val="a"):0:0:"a', - }, - err = { - arg = '"a', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedBody', 'a'), - }) - check_parsing('\'a', { - -- 01 - ast = { - 'SingleQuotedString(val="a"):0:0:\'a', - }, - err = { - arg = '\'a', - msg = 'E115: Missing single quote: %.*s', - }, - }, { - hl('InvalidSingleQuote', '\''), - hl('InvalidSingleQuotedBody', 'a'), - }) - check_parsing('\'abc\'\'def\'', { - -- 0123456789 - ast = { - 'SingleQuotedString(val="abc\'def"):0:0:\'abc\'\'def\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuotedBody', 'abc'), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedBody', 'def'), - hl('SingleQuote', '\''), - }) - check_parsing('\'abc\'\'', { - -- 012345 - ast = { - 'SingleQuotedString(val="abc\'"):0:0:\'abc\'\'', - }, - err = { - arg = '\'abc\'\'', - msg = 'E115: Missing single quote: %.*s', - }, - }, { - hl('InvalidSingleQuote', '\''), - hl('InvalidSingleQuotedBody', 'abc'), - hl('InvalidSingleQuotedQuote', '\'\''), - }) - check_parsing('\'\'\'\'\'\'\'\'', { - -- 01234567 - ast = { - 'SingleQuotedString(val="\'\'\'"):0:0:\'\'\'\'\'\'\'\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuote', '\''), - }) - check_parsing('\'\'\'a\'\'\'\'bc\'', { - -- 01234567890 - -- 0 1 - ast = { - 'SingleQuotedString(val="\'a\'\'bc"):0:0:\'\'\'a\'\'\'\'bc\'', - }, - }, { - hl('SingleQuote', '\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedBody', 'a'), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedQuote', '\'\''), - hl('SingleQuotedBody', 'bc'), - hl('SingleQuote', '\''), - }) - check_parsing('"\\"\\"\\"\\""', { - -- 0123456789 - ast = { - 'DoubleQuotedString(val="\\"\\"\\"\\""):0:0:"\\"\\"\\"\\""', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuote', '"'), - }) - check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', { - -- 0123456789012345678901234 - -- 0 1 2 - ast = { - 'DoubleQuotedString(val="abc\\"def\\"ghi\\"jkl\\"mno"):0:0:"abc\\"def\\"ghi\\"jkl\\"mno"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedBody', 'abc'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedBody', 'def'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedBody', 'ghi'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedBody', 'jkl'), - hl('DoubleQuotedEscape', '\\"'), - hl('DoubleQuotedBody', 'mno'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\b\\e\\f\\r\\t\\\\"', { - -- 0123456789012345 - -- 0 1 - ast = { - [[DoubleQuotedString(val="\8\27\12\13\9\\"):0:0:"\b\e\f\r\t\\"]], - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\b'), - hl('DoubleQuotedEscape', '\\e'), - hl('DoubleQuotedEscape', '\\f'), - hl('DoubleQuotedEscape', '\\r'), - hl('DoubleQuotedEscape', '\\t'), - hl('DoubleQuotedEscape', '\\\\'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\n\n"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="\\\n\\\n"):0:0:"\\n\n"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\n'), - hl('DoubleQuotedBody', '\n'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\x00"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0"):0:0:"\\x00"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\x00'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\xFF"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\255"):0:0:"\\xFF"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\xFF'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\xF"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\15"):0:0:"\\xF"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\xF'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\u00AB"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="«"):0:0:"\\u00AB"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u00AB'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\U000000AB"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="«"):0:0:"\\U000000AB"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U000000AB'), - hl('DoubleQuote', '"'), - }) - check_parsing('"\\x"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="x"):0:0:"\\x"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\x'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\x', { - -- 012 - ast = { - 'DoubleQuotedString(val="x"):0:0:"\\x', - }, - err = { - arg = '"\\x', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedUnknownEscape', '\\x'), - }) - - check_parsing('"\\xF', { - -- 0123 - ast = { - 'DoubleQuotedString(val="\\15"):0:0:"\\xF', - }, - err = { - arg = '"\\xF', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedEscape', '\\xF'), - }) - - check_parsing('"\\u"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="u"):0:0:"\\u"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\u'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u', { - -- 012 - ast = { - 'DoubleQuotedString(val="u"):0:0:"\\u', - }, - err = { - arg = '"\\u', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedUnknownEscape', '\\u'), - }) - - check_parsing('"\\U', { - -- 012 - ast = { - 'DoubleQuotedString(val="U"):0:0:"\\U', - }, - err = { - arg = '"\\U', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedUnknownEscape', '\\U'), - }) - - check_parsing('"\\U"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="U"):0:0:"\\U"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\U'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\xFX"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\15X"):0:0:"\\xFX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\xF'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\XFX"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\15X"):0:0:"\\XFX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\XF'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\xX"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="xX"):0:0:"\\xX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\x'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\XX"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="XX"):0:0:"\\XX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\X'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\uX"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="uX"):0:0:"\\uX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\u'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\UX"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="UX"):0:0:"\\UX"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\U'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\x0X"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\x0X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\x0'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\X0X"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\X0X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\X0'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u0X"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\u0X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u0'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U0X"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U0X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U0'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\x00X"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\x00X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\x00'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\X00X"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\X00X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\X00'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u00X"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\u00X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u00'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U00X"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U00X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U00'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u000X"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\u000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U000X"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u0000X"', { - -- 012345678 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\u0000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u0000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U0000X"', { - -- 012345678 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U0000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U00000X"', { - -- 0123456789 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U00000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U000000X"', { - -- 01234567890 - -- 0 1 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U000000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U000000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U0000000X"', { - -- 012345678901 - -- 0 1 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U0000000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U0000000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U00000000X"', { - -- 0123456789012 - -- 0 1 - ast = { - 'DoubleQuotedString(val="\\0X"):0:0:"\\U00000000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U00000000'), - hl('DoubleQuotedBody', 'X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\x000X"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="\\0000X"):0:0:"\\x000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\x00'), - hl('DoubleQuotedBody', '0X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\X000X"', { - -- 01234567 - ast = { - 'DoubleQuotedString(val="\\0000X"):0:0:"\\X000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\X00'), - hl('DoubleQuotedBody', '0X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\u00000X"', { - -- 0123456789 - ast = { - 'DoubleQuotedString(val="\\0000X"):0:0:"\\u00000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\u0000'), - hl('DoubleQuotedBody', '0X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\U000000000X"', { - -- 01234567890123 - -- 0 1 - ast = { - 'DoubleQuotedString(val="\\0000X"):0:0:"\\U000000000X"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\U00000000'), - hl('DoubleQuotedBody', '0X'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\0"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="\\0"):0:0:"\\0"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\0'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\00"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="\\0"):0:0:"\\00"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\00'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\000"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0"):0:0:"\\000"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\000'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\0000"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0000"):0:0:"\\0000"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\000'), - hl('DoubleQuotedBody', '0'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\8"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="8"):0:0:"\\8"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\8'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\08"', { - -- 01234 - ast = { - 'DoubleQuotedString(val="\\0008"):0:0:"\\08"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\0'), - hl('DoubleQuotedBody', '8'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\008"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\0008"):0:0:"\\008"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\00'), - hl('DoubleQuotedBody', '8'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\0008"', { - -- 0123456 - ast = { - 'DoubleQuotedString(val="\\0008"):0:0:"\\0008"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\000'), - hl('DoubleQuotedBody', '8'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\777"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\255"):0:0:"\\777"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\777'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\050"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\40"):0:0:"\\050"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\050'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\"', { - -- 012345 - ast = { - 'DoubleQuotedString(val="\\21"):0:0:"\\"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedEscape', '\\'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\<', { - -- 012 - ast = { - 'DoubleQuotedString(val="<"):0:0:"\\<', - }, - err = { - arg = '"\\<', - msg = 'E114: Missing double quote: %.*s', - }, - }, { - hl('InvalidDoubleQuote', '"'), - hl('InvalidDoubleQuotedUnknownEscape', '\\<'), - }) - - check_parsing('"\\<"', { - -- 0123 - ast = { - 'DoubleQuotedString(val="<"):0:0:"\\<"', - }, - }, { - hl('DoubleQuote', '"'), - hl('DoubleQuotedUnknownEscape', '\\<'), - hl('DoubleQuote', '"'), - }) - - check_parsing('"\\ {b{3}: 4}[5]}()] += 6', { - -- 012345678901234567890123456 - -- 0 1 2 - ast = { - { - 'Assignment(Add):0:22: +=', - children = { - { - 'Subscript:0:1:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'Call:0:19:(', - children = { - { - 'Lambda(\\di):0:2:{', - children = { - { - 'Arrow:0:3:->', - children = { - { - 'Subscript:0:15:[', - children = { - { - 'DictLiteral(-di):0:5: {', - children = { - { - 'Colon:0:11::', - children = { - { - 'ComplexIdentifier:0:8:', - children = { - 'PlainIdentifier(scope=0,ident=b):0:7:b', - { - 'CurlyBracesIdentifier(--i):0:8:{', - children = { - 'Integer(val=3):0:9:3', - }, - }, - }, - }, - 'Integer(val=4):0:12: 4', - }, - }, - }, - }, - 'Integer(val=5):0:16:5', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - 'Integer(val=6):0:25: 6', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('Lambda', '{'), - hl('Arrow', '->'), - hl('Dict', '{', 1), - hl('IdentifierName', 'b'), - hl('Curly', '{'), - hl('Number', '3'), - hl('Curly', '}'), - hl('Colon', ':'), - hl('Number', '4', 1), - hl('Dict', '}'), - hl('SubscriptBracket', '['), - hl('Number', '5'), - hl('SubscriptBracket', ']'), - hl('Lambda', '}'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - hl('SubscriptBracket', ']'), - hl('AssignmentWithAddition', '+=', 1), - hl('Number', '6', 1), - }) - - check_asgn_parsing('a{1}.2[{-> {b{3}: 4}[5]}()]', { - -- 012345678901234567890123456 - -- 0 1 2 - ast = { - { - 'Subscript:0:6:[', - children = { - { - 'ConcatOrSubscript:0:4:.', - children = { - { - 'ComplexIdentifier:0:1:', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'CurlyBracesIdentifier(--i):0:1:{', - children = { - 'Integer(val=1):0:2:1', - }, - }, - }, - }, - 'PlainKey(key=2):0:5:2', - }, - }, - { - 'Call:0:24:(', - children = { - { - 'Lambda(\\di):0:7:{', - children = { - { - 'Arrow:0:8:->', - children = { - { - 'Subscript:0:20:[', - children = { - { - 'DictLiteral(-di):0:10: {', - children = { - { - 'Colon:0:16::', - children = { - { - 'ComplexIdentifier:0:13:', - children = { - 'PlainIdentifier(scope=0,ident=b):0:12:b', - { - 'CurlyBracesIdentifier(--i):0:13:{', - children = { - 'Integer(val=3):0:14:3', - }, - }, - }, - }, - 'Integer(val=4):0:17: 4', - }, - }, - }, - }, - 'Integer(val=5):0:21:5', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Curly', '{'), - hl('Number', '1'), - hl('Curly', '}'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '2'), - hl('SubscriptBracket', '['), - hl('Lambda', '{'), - hl('Arrow', '->'), - hl('Dict', '{', 1), - hl('IdentifierName', 'b'), - hl('Curly', '{'), - hl('Number', '3'), - hl('Curly', '}'), - hl('Colon', ':'), - hl('Number', '4', 1), - hl('Dict', '}'), - hl('SubscriptBracket', '['), - hl('Number', '5'), - hl('SubscriptBracket', ']'), - hl('Lambda', '}'), - hl('CallingParenthesis', '('), - hl('CallingParenthesis', ')'), - hl('SubscriptBracket', ']'), - }) - - check_asgn_parsing('a', { - -- 0 - ast = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - }, - }, { - hl('IdentifierName', 'a'), - }) - - check_asgn_parsing('{a}', { - -- 012 - ast = { - { - 'CurlyBracesIdentifier(--i):0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - }, - }, { - hl('FigureBrace', '{'), - hl('IdentifierName', 'a'), - hl('Curly', '}'), - }) - - check_asgn_parsing('{a}b', { - -- 0123 - ast = { - { - 'ComplexIdentifier:0:3:', - children = { - { - 'CurlyBracesIdentifier(--i):0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - }, - }, { - hl('FigureBrace', '{'), - hl('IdentifierName', 'a'), - hl('Curly', '}'), - hl('IdentifierName', 'b'), - }) - - check_asgn_parsing('a{b}c', { - -- 01234 - ast = { - { - 'ComplexIdentifier:0:1:', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'ComplexIdentifier:0:4:', - children = { - { - 'CurlyBracesIdentifier(--i):0:1:{', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - 'PlainIdentifier(scope=0,ident=c):0:4:c', - }, - }, - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - hl('IdentifierName', 'c'), - }) - - check_asgn_parsing('a{b}c[0]', { - -- 01234567 - ast = { - { - 'Subscript:0:5:[', - children = { - { - 'ComplexIdentifier:0:1:', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'ComplexIdentifier:0:4:', - children = { - { - 'CurlyBracesIdentifier(--i):0:1:{', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - 'PlainIdentifier(scope=0,ident=c):0:4:c', - }, - }, - }, - }, - 'Integer(val=0):0:6:0', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - hl('IdentifierName', 'c'), - hl('SubscriptBracket', '['), - hl('Number', '0'), - hl('SubscriptBracket', ']'), - }) - - check_asgn_parsing('a{b}c.0', { - -- 0123456 - ast = { - { - 'ConcatOrSubscript:0:5:.', - children = { - { - 'ComplexIdentifier:0:1:', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - { - 'ComplexIdentifier:0:4:', - children = { - { - 'CurlyBracesIdentifier(--i):0:1:{', - children = { - 'PlainIdentifier(scope=0,ident=b):0:2:b', - }, - }, - 'PlainIdentifier(scope=0,ident=c):0:4:c', - }, - }, - }, - }, - 'PlainKey(key=0):0:6:0', - }, - }, - }, - }, { - hl('IdentifierName', 'a'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - hl('IdentifierName', 'c'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '0'), - }) - - check_asgn_parsing('[a{b}c[0].0]', { - -- 012345678901 - -- 0 1 - ast = { - { - 'ListLiteral:0:0:[', - children = { - { - 'ConcatOrSubscript:0:9:.', - children = { - { - 'Subscript:0:6:[', - children = { - { - 'ComplexIdentifier:0:2:', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - { - 'ComplexIdentifier:0:5:', - children = { - { - 'CurlyBracesIdentifier(--i):0:2:{', - children = { - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - 'PlainIdentifier(scope=0,ident=c):0:5:c', - }, - }, - }, - }, - 'Integer(val=0):0:7:0', - }, - }, - 'PlainKey(key=0):0:10:0', - }, - }, - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - hl('IdentifierName', 'c'), - hl('SubscriptBracket', '['), - hl('Number', '0'), - hl('SubscriptBracket', ']'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', '0'), - hl('List', ']'), - }) - - check_asgn_parsing('{a}{b}', { - -- 012345 - ast = { - { - 'ComplexIdentifier:0:3:', - children = { - { - 'CurlyBracesIdentifier(--i):0:0:{', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - { - 'CurlyBracesIdentifier(--i):0:3:{', - children = { - 'PlainIdentifier(scope=0,ident=b):0:4:b', - }, - }, - }, - }, - }, - }, { - hl('FigureBrace', '{'), - hl('IdentifierName', 'a'), - hl('Curly', '}'), - hl('Curly', '{'), - hl('IdentifierName', 'b'), - hl('Curly', '}'), - }) - - check_asgn_parsing('a.b{c}{d}', { - -- 012345678 - ast = { - { - 'OpMissing:0:3:', - children = { - { - 'ConcatOrSubscript:0:1:.', - children = { - 'PlainIdentifier(scope=0,ident=a):0:0:a', - 'PlainKey(key=b):0:2:b', - }, - }, - { - 'ComplexIdentifier:0:6:', - children = { - { - 'CurlyBracesIdentifier(--i):0:3:{', - children = { - 'PlainIdentifier(scope=0,ident=c):0:4:c', - }, - }, - { - 'CurlyBracesIdentifier(--i):0:6:{', - children = { - 'PlainIdentifier(scope=0,ident=d):0:7:d', - }, - }, - }, - }, - }, - }, - }, - err = { - arg = '{c}{d}', - msg = 'E15: Missing operator: %.*s', - }, - }, { - hl('IdentifierName', 'a'), - hl('ConcatOrSubscript', '.'), - hl('IdentifierKey', 'b'), - hl('InvalidFigureBrace', '{'), - hl('IdentifierName', 'c'), - hl('Curly', '}'), - hl('Curly', '{'), - hl('IdentifierName', 'd'), - hl('Curly', '}'), - }) - - check_asgn_parsing('[a] = 1', { - -- 0123456 - ast = { - { - 'Assignment(Plain):0:3: =', - children = { - { - 'ListLiteral:0:0:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - }, - }, - 'Integer(val=1):0:5: 1', - }, - }, - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('List', ']'), - hl('PlainAssignment', '=', 1), - hl('Number', '1', 1), - }) - - check_asgn_parsing('[a[b], [c, [d, [e]]]] = 1', { - -- 0123456789012345678901234 - -- 0 1 2 - ast = { - { - 'Assignment(Plain):0:21: =', - children = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:5:,', - children = { - { - 'Subscript:0:2:[', - children = { - 'PlainIdentifier(scope=0,ident=a):0:1:a', - 'PlainIdentifier(scope=0,ident=b):0:3:b', - }, - }, - { - 'ListLiteral:0:6: [', - children = { - { - 'Comma:0:9:,', - children = { - 'PlainIdentifier(scope=0,ident=c):0:8:c', - { - 'ListLiteral:0:10: [', - children = { - { - 'Comma:0:13:,', - children = { - 'PlainIdentifier(scope=0,ident=d):0:12:d', - { - 'ListLiteral:0:14: [', - children = { - 'PlainIdentifier(scope=0,ident=e):0:16:e', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - 'Integer(val=1):0:23: 1', - }, - }, - }, - err = { - arg = '[c, [d, [e]]]] = 1', - msg = 'E475: Nested lists not allowed when assigning: %.*s', - }, - }, { - hl('List', '['), - hl('IdentifierName', 'a'), - hl('SubscriptBracket', '['), - hl('IdentifierName', 'b'), - hl('SubscriptBracket', ']'), - hl('Comma', ','), - hl('InvalidList', '[', 1), - hl('IdentifierName', 'c'), - hl('Comma', ','), - hl('InvalidList', '[', 1), - hl('IdentifierName', 'd'), - hl('Comma', ','), - hl('InvalidList', '[', 1), - hl('IdentifierName', 'e'), - hl('List', ']'), - hl('List', ']'), - hl('List', ']'), - hl('List', ']'), - hl('PlainAssignment', '=', 1), - hl('Number', '1', 1), - }) - - check_asgn_parsing('$X += 1', { - -- 0123456 - ast = { - { - 'Assignment(Add):0:2: +=', - children = { - 'Environment(ident=X):0:0:$X', - 'Integer(val=1):0:5: 1', - }, - }, - }, - }, { - hl('EnvironmentSigil', '$'), - hl('EnvironmentName', 'X'), - hl('AssignmentWithAddition', '+=', 1), - hl('Number', '1', 1), - }) - - check_asgn_parsing('@a .= 1', { - -- 0123456 - ast = { - { - 'Assignment(Concat):0:2: .=', - children = { - 'Register(name=a):0:0:@a', - 'Integer(val=1):0:5: 1', - }, - }, - }, - }, { - hl('Register', '@a'), - hl('AssignmentWithConcatenation', '.=', 1), - hl('Number', '1', 1), - }) - - check_asgn_parsing('&option -= 1', { - -- 012345678901 - -- 0 1 - ast = { - { - 'Assignment(Subtract):0:7: -=', - children = { - 'Option(scope=0,ident=option):0:0:&option', - 'Integer(val=1):0:10: 1', - }, - }, - }, - }, { - hl('OptionSigil', '&'), - hl('OptionName', 'option'), - hl('AssignmentWithSubtraction', '-=', 1), - hl('Number', '1', 1), - }) - - check_asgn_parsing('[$X, @a, &l:option] = [1, 2, 3]', { - -- 0123456789012345678901234567890 - -- 0 1 2 3 - ast = { - { - 'Assignment(Plain):0:19: =', - children = { - { - 'ListLiteral:0:0:[', - children = { - { - 'Comma:0:3:,', - children = { - 'Environment(ident=X):0:1:$X', - { - 'Comma:0:7:,', - children = { - 'Register(name=a):0:4: @a', - 'Option(scope=l,ident=option):0:8: &l:option', - }, - }, - }, - }, - }, - }, - { - 'ListLiteral:0:21: [', - children = { - { - 'Comma:0:24:,', - children = { - 'Integer(val=1):0:23:1', - { - 'Comma:0:27:,', - children = { - 'Integer(val=2):0:25: 2', - 'Integer(val=3):0:28: 3', - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, { - hl('List', '['), - hl('EnvironmentSigil', '$'), - hl('EnvironmentName', 'X'), - hl('Comma', ','), - hl('Register', '@a', 1), - hl('Comma', ','), - hl('OptionSigil', '&', 1), - hl('OptionScope', 'l'), - hl('OptionScopeDelimiter', ':'), - hl('OptionName', 'option'), - hl('List', ']'), - hl('PlainAssignment', '=', 1), - hl('List', '[', 1), - hl('Number', '1'), - hl('Comma', ','), - hl('Number', '2', 1), - hl('Comma', ','), - hl('Number', '3', 1), - hl('List', ']'), - }) - end) - -- FIXME: Somehow make functional tests use the same code. Or, at least, - -- create an automated script which will do the import. + local function fmtn(typ, args, rest) + return ('%s(%s)%s'):format(typ, args, rest) + end + require('test.unit.viml.expressions.parser_tests')( + itp, _check_parsing, hl, fmtn) end) diff --git a/test/unit/viml/expressions/parser_tests.lua b/test/unit/viml/expressions/parser_tests.lua new file mode 100644 index 0000000000..f0c2723a38 --- /dev/null +++ b/test/unit/viml/expressions/parser_tests.lua @@ -0,0 +1,8185 @@ +local global_helpers = require('test.helpers') + +local REMOVE_THIS = global_helpers.REMOVE_THIS + +return function(itp, _check_parsing, hl, fmtn) + local function check_parsing(...) + return _check_parsing({flags={0, 1, 2, 3}, funcname='check_parsing'}, ...) + end + local function check_asgn_parsing(...) + return _check_parsing({ + flags={4, 5, 6, 7}, + funcname='check_asgn_parsing', + }, ...) + end + itp('works with + and @a', function() + check_parsing('@a', { + ast = { + 'Register(name=a):0:0:@a', + }, + }, { + hl('Register', '@a'), + }) + check_parsing('+@a', { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + }) + check_parsing('@a+@b', { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+@b+@c', { + ast = { + { + 'BinaryPlus:0:5:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + 'Register(name=c):0:6:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing('+@a+@b', { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:4:@b', + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('+@a++@b', { + ast = { + { + 'BinaryPlus:0:3:+', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'UnaryPlus:0:4:+', + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a@b', { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:2:@b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidRegister', '@b'), + }, { + [1] = { + ast = { + len = 2, + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0:@a' + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + }, + }, + }) + check_parsing(' @a \t @b', { + ast = { + { + 'OpMissing:0:3:', + children = { + 'Register(name=a):0:0: @a', + 'Register(name=b):0:3: \t @b', + }, + }, + }, + err = { + arg = '@b', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a', 1), + hl('InvalidSpacing', ' \t '), + hl('Register', '@b'), + }, { + [1] = { + ast = { + len = 6, + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0: @a' + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + }, + }, + }) + check_parsing('+', { + ast = { + 'UnaryPlus:0:0:+', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + }) + check_parsing(' +', { + ast = { + 'UnaryPlus:0:0: +', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('UnaryPlus', '+', 1), + }) + check_parsing('@a+ ', { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + }) + end) + itp('works with @a, + and parenthesis', function() + check_parsing('(@a)', { + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + }) + check_parsing('()', { + ast = { + { + 'Nested:0:0:(', + children = { + 'Missing:0:1:', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing(')', { + ast = { + { + 'Nested:0:0:', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+)', { + ast = { + { + 'Nested:0:1:', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('UnaryPlus', '+'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('+@a(@b)', { + ast = { + { + 'UnaryPlus:0:0:+', + children = { + { + 'Call:0:3:(', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+@b(@c)', { + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a()', { + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a ()', { + ast = { + { + 'OpMissing:0:2:', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:2: (', + children = { + 'Missing:0:4:', + }, + }, + }, + }, + }, + err = { + arg = '()', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidSpacing', ' '), + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + }, { + [1] = { + ast = { + len = 3, + err = REMOVE_THIS, + ast = { + 'Register(name=a):0:0:@a', + }, + }, + hl_fs = { + [2] = REMOVE_THIS, + [3] = REMOVE_THIS, + [4] = REMOVE_THIS, + }, + }, + }) + check_parsing('@a + (@b)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing('@a + (+@b)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'UnaryPlus:0:6:+', + children = { + 'Register(name=b):0:7:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }) + check_parsing('@a + (@b + @c)', { + ast = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + }) + check_parsing('(@a)+@b', { + ast = { + { + 'BinaryPlus:0:4:+', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:5:@b', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + }) + check_parsing('@a+(@b)(@c)', { + -- 01234567890 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:7:(', + children = { + { + 'Nested:0:3:(', + children = { 'Register(name=b):0:4:@b' }, + }, + 'Register(name=c):0:8:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))(@c)', { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:9:(', + children = { + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a+((@b))+@c', { + -- 01234567890123456890123456789 + -- 0 1 2 + ast = { + { + 'BinaryPlus:0:9:+', + children = { + { + 'BinaryPlus:0:2:+', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:3:(', + children = { + { + 'Nested:0:4:(', + children = { 'Register(name=b):0:5:@b' } + }, + }, + }, + }, + }, + 'Register(name=c):0:10:@c', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('NestingParenthesis', '('), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+'), + hl('Register', '@c'), + }) + check_parsing( + '@a + (@b + @c) + @d(@e) + (+@f) + ((+@g(@h))(@j)(@k))(@l)', {--[[ + | | | | | | | | || | | || | | ||| || || || || + 000000000011111111112222222222333333333344444444445555555 + 012345678901234567890123456789012345678901234567890123456 + ]] + ast = {{ + 'BinaryPlus:0:31: +', + children = { + { + 'BinaryPlus:0:23: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + { + 'BinaryPlus:0:2: +', + children = { + 'Register(name=a):0:0:@a', + { + 'Nested:0:4: (', + children = { + { + 'BinaryPlus:0:8: +', + children = { + 'Register(name=b):0:6:@b', + 'Register(name=c):0:10: @c', + }, + }, + }, + }, + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=d):0:16: @d', + 'Register(name=e):0:20:@e', + }, + }, + }, + }, + { + 'Nested:0:25: (', + children = { + { + 'UnaryPlus:0:27:+', + children = { + 'Register(name=f):0:28:@f', + }, + }, + }, + }, + }, + }, + { + 'Call:0:53:(', + children = { + { + 'Nested:0:33: (', + children = { + { + 'Call:0:48:(', + children = { + { + 'Call:0:44:(', + children = { + { + 'Nested:0:35:(', + children = { + { + 'UnaryPlus:0:36:+', + children = { + { + 'Call:0:39:(', + children = { + 'Register(name=g):0:37:@g', + 'Register(name=h):0:40:@h', + }, + }, + }, + }, + }, + }, + 'Register(name=j):0:45:@j', + }, + }, + 'Register(name=k):0:49:@k', + }, + }, + }, + }, + 'Register(name=l):0:54:@l', + }, + }, + }, + }}, + }, { + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Register', '@b'), + hl('BinaryPlus', '+', 1), + hl('Register', '@c', 1), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('UnaryPlus', '+'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('NestingParenthesis', '('), + hl('UnaryPlus', '+'), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@k'), + hl('CallingParenthesis', ')'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@l'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a)', { + -- 012 + ast = { + { + 'Nested:0:2:', + children = { + 'Register(name=a):0:0:@a', + }, + }, + }, + err = { + arg = ')', + msg = 'E15: Unexpected closing parenthesis: %.*s', + }, + }, { + hl('Register', '@a'), + hl('InvalidNestingParenthesis', ')'), + }) + check_parsing('(@a', { + -- 012 + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '(@a', + msg = 'E110: Missing closing parenthesis for nested expression: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + }) + check_parsing('@a(@b', { + -- 01234 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + 'Register(name=b):0:3:@b', + }, + }, + }, + err = { + arg = '(@b', + msg = 'E116: Missing closing parenthesis for function call: %.*s', + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + }) + check_parsing('@a(@b, @c, @d, @e)', { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Comma:0:5:,', + children = { + 'Register(name=b):0:3:@b', + { + 'Comma:0:9:,', + children = { + 'Register(name=c):0:6: @c', + { + 'Comma:0:13:,', + children = { + 'Register(name=d):0:10: @d', + 'Register(name=e):0:14: @e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c', 1), + hl('Comma', ','), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c))', { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + 'Register(name=c):0:6:@c', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + check_parsing('@a(@b(@c(@d(@e), @f(@g(@h), @i(@j)))))', { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + 'Call:0:2:(', + children = { + 'Register(name=a):0:0:@a', + { + 'Call:0:5:(', + children = { + 'Register(name=b):0:3:@b', + { + 'Call:0:8:(', + children = { + 'Register(name=c):0:6:@c', + { + 'Comma:0:15:,', + children = { + { + 'Call:0:11:(', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + { + 'Call:0:19:(', + children = { + 'Register(name=f):0:16: @f', + { + 'Comma:0:26:,', + children = { + { + 'Call:0:22:(', + children = { + 'Register(name=g):0:20:@g', + 'Register(name=h):0:23:@h', + }, + }, + { + 'Call:0:30:(', + children = { + 'Register(name=i):0:27: @i', + 'Register(name=j):0:31:@j', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', '('), + hl('Register', '@c'), + hl('CallingParenthesis', '('), + hl('Register', '@d'), + hl('CallingParenthesis', '('), + hl('Register', '@e'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@f', 1), + hl('CallingParenthesis', '('), + hl('Register', '@g'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Comma', ','), + hl('Register', '@i', 1), + hl('CallingParenthesis', '('), + hl('Register', '@j'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', ')'), + }) + check_parsing('()()', { + -- 0123 + ast = { + { + 'Call:0:2:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = ')()', + msg = 'E15: Expected value, got parenthesis: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('InvalidNestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a)()', { + -- 012345 + ast = { + { + 'Call:0:4:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a)(@b)', { + -- 01234567 + ast = { + { + 'Call:0:4:(', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'Register(name=b):0:5:@b', + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@b'), + hl('CallingParenthesis', ')'), + }) + check_parsing('(@a) (@b)', { + -- 012345678 + ast = { + { + 'OpMissing:0:4:', + children = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + 'Nested:0:4: (', + children = { + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + err = { + arg = '(@b)', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('NestingParenthesis', ')'), + hl('InvalidSpacing', ' '), + hl('NestingParenthesis', '('), + hl('Register', '@b'), + hl('NestingParenthesis', ')'), + }, { + [1] = { + ast = { + len = 5, + ast = { + { + 'Nested:0:0:(', + children = { + 'Register(name=a):0:1:@a', + REMOVE_THIS, + }, + }, + }, + err = REMOVE_THIS, + }, + hl_fs = { + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + [6] = REMOVE_THIS, + [7] = REMOVE_THIS, + }, + }, + }) + end) + itp('works with variable names, including curly braces ones', function() + check_parsing('var', { + ast = { + 'PlainIdentifier(scope=0,ident=var):0:0:var', + }, + }, { + hl('IdentifierName', 'var'), + }) + check_parsing('g:var', { + ast = { + 'PlainIdentifier(scope=g,ident=var):0:0:g:var', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'var'), + }) + check_parsing('g:', { + ast = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + }) + check_parsing('{a}', { + -- 012 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + }) + check_parsing('{a:b}', { + -- 012 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + }) + check_parsing('{a:@b}', { + -- 012345 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'OpMissing:0:3:', + children={ + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'Register(name=b):0:3:@b', + }, + }, + }, + }, + }, + err = { + arg = '@b}', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidRegister', '@b'), + hl('Curly', '}'), + }) + check_parsing('{@a}', { + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}{@b}', { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + { + fmtn('CurlyBracesIdentifier', '--i', ':0:4:{'), + children = { + 'Register(name=b):0:5:@b', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('Register', '@b'), + hl('Curly', '}'), + }) + check_parsing('g:{@a}', { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('{@a}_test', { + -- 012345678 + ast = { + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:4:_test', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + }) + check_parsing('g:{@a}_test', { + -- 01234567890 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + }) + check_parsing('g:{@a}_test()', { + -- 0123456789012 + ast = { + { + 'Call:0:11:(', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + 'ComplexIdentifier:0:6:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'Register(name=a):0:3:@a', + }, + }, + 'PlainIdentifier(scope=0,ident=_test):0:6:_test', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('IdentifierName', '_test'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a} ()', { + -- 0123456789012 + ast = { + { + 'Call:0:4: (', + children = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('g:{@a} ()', { + -- 0123456789012 + ast = { + { + 'Call:0:6: (', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=g,ident=):0:0:g:', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'g'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Register', '@a'), + hl('Curly', '}'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + check_parsing('{@a', { + -- 012 + ast = { + { + fmtn('UnknownFigure', '-di', ':0:0:{'), + children = { + 'Register(name=a):0:1:@a', + }, + }, + }, + err = { + arg = '{@a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('Register', '@a'), + }) + check_parsing('a ()', { + -- 0123 + ast = { + { + 'Call:0:1: (', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('CallingParenthesis', '(', 1), + hl('CallingParenthesis', ')'), + }) + end) + itp('works with lambdas and dictionaries', function() + check_parsing('{}', { + ast = { + fmtn('DictLiteral', '-di', ':0:0:{'), + }, + }, { + hl('Dict', '{'), + hl('Dict', '}'), + }) + check_parsing('{->@a}', { + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Arrow:0:1:->', + children = { + 'Register(name=a):0:3:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{->@a+@b}', { + -- 012345678 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Arrow:0:1:->', + children = { + { + 'BinaryPlus:0:5:+', + children = { + 'Register(name=a):0:3:@a', + 'Register(name=b):0:6:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('BinaryPlus', '+'), + hl('Register', '@b'), + hl('Lambda', '}'), + }) + check_parsing('{a->@a}', { + -- 012345678 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2:->', + children = { + 'Register(name=a):0:4:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->@a}', { + -- 012345678 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'Register(name=a):0:6:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c->@a}', { + -- 01234567890 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + { + 'Arrow:0:6:->', + children = { + 'Register(name=a):0:8:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d->@a}', { + -- 0123456789012 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:8:->', + children = { + 'Register(name=a):0:10:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b,c,d,->@a}', { + -- 01234567890123 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:4:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + { + 'Comma:0:6:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:5:c', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Arrow:0:9:->', + children = { + 'Register(name=a):0:11:@a', + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Comma', ','), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->{c,d->{e,f->@a}}}', { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + fmtn('Lambda', '\\di', ':0:6:{'), + children = { + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + 'PlainIdentifier(scope=0,ident=d):0:9:d', + }, + }, + { + 'Arrow:0:10:->', + children = { + { + fmtn('Lambda', '\\di', ':0:12:{'), + children = { + { + 'Comma:0:14:,', + children = { + 'PlainIdentifier(scope=0,ident=e):0:13:e', + 'PlainIdentifier(scope=0,ident=f):0:15:f', + }, + }, + { + 'Arrow:0:16:->', + children = { + 'Register(name=a):0:18:@a', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('IdentifierName', 'd'), + hl('Arrow', '->'), + hl('Lambda', '{'), + hl('IdentifierName', 'e'), + hl('Comma', ','), + hl('IdentifierName', 'f'), + hl('Arrow', '->'), + hl('Register', '@a'), + hl('Lambda', '}'), + hl('Lambda', '}'), + hl('Lambda', '}'), + }) + check_parsing('{a,b->c,d}', { + -- 0123456789 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',d}', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + hl('Lambda', '}'), + }) + check_parsing('a,b,c,d', { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidComma', ','), + hl('IdentifierName', 'b'), + hl('InvalidComma', ','), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + }) + check_parsing('a,b,c,d,', { + -- 0123456789 + ast = { + { + 'Comma:0:1:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comma:0:3:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + { + 'Comma:0:7:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:6:d', + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ',b,c,d,', + msg = 'E15: Comma outside of call, lambda or literal: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidComma', ','), + hl('IdentifierName', 'b'), + hl('InvalidComma', ','), + hl('IdentifierName', 'c'), + hl('InvalidComma', ','), + hl('IdentifierName', 'd'), + hl('InvalidComma', ','), + }) + check_parsing(',', { + -- 0123456789 + ast = { + { + 'Comma:0:0:,', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ',', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('InvalidComma', ','), + }) + check_parsing('{,a->@a}', { + -- 0123456789 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Arrow:0:3:->', + children = { + { + 'Comma:0:1:,', + children = { + 'Missing:0:1:', + 'PlainIdentifier(scope=0,ident=a):0:2:a', + }, + }, + 'Register(name=a):0:5:@a', + }, + }, + }, + }, + }, + err = { + arg = ',a->@a}', + msg = 'E15: Expected value, got comma: %.*s', + }, + }, { + hl('Curly', '{'), + hl('InvalidComma', ','), + hl('IdentifierName', 'a'), + hl('InvalidArrow', '->'), + hl('Register', '@a'), + hl('Curly', '}'), + }) + check_parsing('}', { + -- 0123456789 + ast = { + fmtn('UnknownFigure', '---', ':0:0:'), + }, + err = { + arg = '}', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidFigureBrace', '}'), + }) + check_parsing('{->}', { + -- 0123456789 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + 'Arrow:0:1:->', + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,b}', { + -- 0123456789 + ast = { + { + fmtn('Lambda', '-di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('InvalidLambda', '}'), + }) + check_parsing('{a,}', { + -- 0123456789 + ast = { + { + fmtn('Lambda', '-di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected lambda arguments list or arrow: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('InvalidLambda', '}'), + }) + check_parsing('{@a:@b}', { + -- 0123456789 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d}', { + -- 0123456789012 + -- 0 1 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,}', { + -- 01234567890123456789 + -- 0 1 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{@a:@b,@c:@d,@e:@f,@g:}', { + -- 01234567890123456789012 + -- 0 1 2 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + { + 'Comma:0:12:,', + children = { + { + 'Colon:0:9::', + children = { + 'Register(name=c):0:7:@c', + 'Register(name=d):0:10:@d', + }, + }, + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:15::', + children = { + 'Register(name=e):0:13:@e', + 'Register(name=f):0:16:@f', + }, + }, + { + 'Colon:0:21::', + children = { + 'Register(name=g):0:19:@g', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '}', + msg = 'E15: Expected value, got closing figure brace: %.*s', + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Register', '@c'), + hl('Colon', ':'), + hl('Register', '@d'), + hl('Comma', ','), + hl('Register', '@e'), + hl('Colon', ':'), + hl('Register', '@f'), + hl('Comma', ','), + hl('Register', '@g'), + hl('Colon', ':'), + hl('InvalidDict', '}'), + }) + check_parsing('{@a:@b,}', { + -- 01234567890123 + -- 0 1 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:3::', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:4:@b', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('Colon', ':'), + hl('Register', '@b'), + hl('Comma', ','), + hl('Dict', '}'), + }) + check_parsing('{({f -> g})(@h)(@i)}', { + -- 01234567890123456789 + -- 0 1 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Call:0:15:(', + children = { + { + 'Call:0:11:(', + children = { + { + 'Nested:0:1:(', + children = { + { + fmtn('Lambda', '\\di', ':0:2:{'), + children = { + 'PlainIdentifier(scope=0,ident=f):0:3:f', + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:7: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:12:@h', + }, + }, + 'Register(name=i):0:16:@i', + }, + }, + }, + }, + }, + }, { + hl('Curly', '{'), + hl('NestingParenthesis', '('), + hl('Lambda', '{'), + hl('IdentifierName', 'f'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + }) + check_parsing('a:{b()}c', { + -- 01234567 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:7:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + { + 'Call:0:4:(', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + }) + check_parsing('a:{{b, c -> @d + @e + ({f -> g})(@h)}(@i)}j', { + -- 01234567890123456789012345678901234567890123456 + -- 0 1 2 3 4 + ast = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=a,ident=):0:0:a:', + { + 'ComplexIdentifier:0:42:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + { + 'Call:0:37:(', + children = { + { + fmtn('Lambda', '\\di', ':0:3:{'), + children = { + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + { + 'Arrow:0:8: ->', + children = { + { + 'BinaryPlus:0:19: +', + children = { + { + 'BinaryPlus:0:14: +', + children = { + 'Register(name=d):0:11: @d', + 'Register(name=e):0:16: @e', + }, + }, + { + 'Call:0:32:(', + children = { + { + 'Nested:0:21: (', + children = { + { + fmtn('Lambda', '\\di', ':0:23:{'), + children = { + 'PlainIdentifier(scope=0,ident=f):0:24:f', + { + 'Arrow:0:25: ->', + children = { + 'PlainIdentifier(scope=0,ident=g):0:28: g', + }, + }, + }, + }, + }, + }, + 'Register(name=h):0:33:@h', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=i):0:38:@i', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=j):0:42:j', + }, + }, + }, + }, + }, + }, { + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('Curly', '{'), + hl('Lambda', '{'), + hl('IdentifierName', 'b'), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('Arrow', '->', 1), + hl('Register', '@d', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('NestingParenthesis', '(', 1), + hl('Lambda', '{'), + hl('IdentifierName', 'f'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'g', 1), + hl('Lambda', '}'), + hl('NestingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('Register', '@h'), + hl('CallingParenthesis', ')'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('Register', '@i'), + hl('CallingParenthesis', ')'), + hl('Curly', '}'), + hl('IdentifierName', 'j'), + }) + check_parsing('{@a + @b : @c + @d, @e + @f : @g + @i}', { + -- 01234567890123456789012345678901234567 + -- 0 1 2 3 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:18:,', + children = { + { + 'Colon:0:8: :', + children = { + { + 'BinaryPlus:0:3: +', + children = { + 'Register(name=a):0:1:@a', + 'Register(name=b):0:5: @b', + }, + }, + { + 'BinaryPlus:0:13: +', + children = { + 'Register(name=c):0:10: @c', + 'Register(name=d):0:15: @d', + }, + }, + }, + }, + { + 'Colon:0:27: :', + children = { + { + 'BinaryPlus:0:22: +', + children = { + 'Register(name=e):0:19: @e', + 'Register(name=f):0:24: @f', + }, + }, + { + 'BinaryPlus:0:32: +', + children = { + 'Register(name=g):0:29: @g', + 'Register(name=i):0:34: @i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Dict', '{'), + hl('Register', '@a'), + hl('BinaryPlus', '+', 1), + hl('Register', '@b', 1), + hl('Colon', ':', 1), + hl('Register', '@c', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@d', 1), + hl('Comma', ','), + hl('Register', '@e', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@f', 1), + hl('Colon', ':', 1), + hl('Register', '@g', 1), + hl('BinaryPlus', '+', 1), + hl('Register', '@i', 1), + hl('Dict', '}'), + }) + check_parsing('-> -> ->', { + -- 01234567 + ast = { + { + 'Arrow:0:0:->', + children = { + 'Missing:0:0:', + { + 'Arrow:0:2: ->', + children = { + 'Missing:0:2:', + { + 'Arrow:0:5: ->', + children = { + 'Missing:0:5:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> -> ->', + msg = 'E15: Unexpected arrow: %.*s', + }, + }, { + hl('InvalidArrow', '->'), + hl('InvalidArrow', '->', 1), + hl('InvalidArrow', '->', 1), + }) + check_parsing('a -> b -> c -> d', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Arrow:0:1: ->', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Arrow:0:6: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + { + 'Arrow:0:11: ->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:9: c', + 'PlainIdentifier(scope=0,ident=d):0:14: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> b -> c -> d', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'c', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'd', 1), + }) + check_parsing('{a -> b -> c}', { + -- 0123456789012 + -- 0 1 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Arrow:0:2: ->', + children = { + { + 'Arrow:0:7: ->', + children = { + 'PlainIdentifier(scope=0,ident=b):0:5: b', + 'PlainIdentifier(scope=0,ident=c):0:10: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '-> c}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Arrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'c', 1), + hl('Lambda', '}'), + }) + check_parsing('{a: -> b}', { + -- 012345678 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Arrow:0:3: ->', + children = { + 'PlainIdentifier(scope=a,ident=):0:1:a:', + 'PlainIdentifier(scope=0,ident=b):0:6: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a:b -> b}', { + -- 0123456789 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=a,ident=b):0:1:a:b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierScope', 'a'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'b'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + + check_parsing('{a#b -> b}', { + -- 0123456789 + ast = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + { + 'Arrow:0:4: ->', + children = { + 'PlainIdentifier(scope=0,ident=a#b):0:1:a#b', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, + }, + err = { + arg = '-> b}', + msg = 'E15: Arrow outside of lambda: %.*s', + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a#b'), + hl('InvalidArrow', '->', 1), + hl('IdentifierName', 'b', 1), + hl('Curly', '}'), + }) + check_parsing('{a : b : c}', { + -- 01234567890 + -- 0 1 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Colon:0:6: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:4: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': c}', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('Dict', '}'), + }) + check_parsing('{', { + -- 0 + ast = { + fmtn('UnknownFigure', '\\di', ':0:0:{'), + }, + err = { + arg = '{', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + }) + check_parsing('{a', { + -- 01 + ast = { + { + fmtn('UnknownFigure', '\\di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + err = { + arg = '{a', + msg = 'E15: Missing closing figure brace: %.*s', + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + }) + check_parsing('{a,b', { + -- 0123 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, + }, + err = { + arg = '{a,b', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + }) + check_parsing('{a,b->', { + -- 012345 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + 'Arrow:0:4:->', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + }) + check_parsing('{a,b->c', { + -- 0123456 + ast = { + { + fmtn('Lambda', '\\di', ':0:0:{'), + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'Arrow:0:4:->', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6:c', + }, + }, + }, + }, + }, + err = { + arg = '{a,b->c', + msg = 'E15: Missing closing figure brace for lambda: %.*s', + }, + }, { + hl('Lambda', '{'), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b'), + hl('Arrow', '->'), + hl('IdentifierName', 'c'), + }) + check_parsing('{a : b', { + -- 012345 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + err = { + arg = '{a : b', + msg = 'E723: Missing end of Dictionary \'}\': %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + }) + check_parsing('{a : b,', { + -- 0123456 + ast = { + { + fmtn('DictLiteral', '-di', ':0:0:{'), + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('Dict', '{'), + hl('IdentifierName', 'a'), + hl('Colon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + }) + end) + itp('works with ternary operator', function() + check_parsing('a ? b : c', { + -- 012345678 + ast = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?', 1), + hl('IdentifierName', 'b', 1), + hl('TernaryColon', ':', 1), + hl('IdentifierName', 'c', 1), + }) + check_parsing('@a?@b?@c:@d:@e', { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:11::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:8::', + children = { + 'Register(name=c):0:6:@c', + 'Register(name=d):0:9:@d', + }, + }, + }, + }, + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('TernaryColon', ':'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b:@c?@d:@e', { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:5::', + children = { + 'Register(name=b):0:3:@b', + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + 'Register(name=e):0:12:@e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + check_parsing('@a?@b?@c?@d:@e?@f:@g:@h?@i:@j:@k', { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:2:?', + children = { + 'Register(name=a):0:0:@a', + { + 'TernaryValue:0:29::', + children = { + { + 'Ternary:0:5:?', + children = { + 'Register(name=b):0:3:@b', + { + 'TernaryValue:0:20::', + children = { + { + 'Ternary:0:8:?', + children = { + 'Register(name=c):0:6:@c', + { + 'TernaryValue:0:11::', + children = { + 'Register(name=d):0:9:@d', + { + 'Ternary:0:14:?', + children = { + 'Register(name=e):0:12:@e', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=f):0:15:@f', + 'Register(name=g):0:18:@g', + }, + }, + }, + }, + }, + }, + }, + }, + { + 'Ternary:0:23:?', + children = { + 'Register(name=h):0:21:@h', + { + 'TernaryValue:0:26::', + children = { + 'Register(name=i):0:24:@i', + 'Register(name=j):0:27:@j', + }, + }, + }, + }, + }, + }, + }, + }, + 'Register(name=k):0:30:@k', + }, + }, + }, + }, + }, + }, { + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('Ternary', '?'), + hl('Register', '@c'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + hl('Ternary', '?'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('TernaryColon', ':'), + hl('Register', '@h'), + hl('Ternary', '?'), + hl('Register', '@i'), + hl('TernaryColon', ':'), + hl('Register', '@j'), + hl('TernaryColon', ':'), + hl('Register', '@k'), + }) + check_parsing('?', { + -- 0 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + 'TernaryValue:0:0:?', + }, + }, + }, + err = { + arg = '?', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + }) + + check_parsing('?:', { + -- 01 + ast = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + }, + }, + }, + }, + }, + err = { + arg = '?:', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + }) + + check_parsing('?::', { + -- 012 + ast = { + { + 'Colon:0:2::', + children = { + { + 'Ternary:0:0:?', + children = { + 'Missing:0:0:', + { + 'TernaryValue:0:1::', + children = { + 'Missing:0:1:', + 'Missing:0:2:', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '?::', + msg = 'E15: Expected value, got question mark: %.*s', + }, + }, { + hl('InvalidTernary', '?'), + hl('InvalidTernaryColon', ':'), + hl('InvalidColon', ':'), + }) + + check_parsing('a?b', { + -- 012 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '?b', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + }) + check_parsing('a?b:', { + -- 0123 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:1:?', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, + }, + err = { + arg = '?b:', + msg = 'E109: Missing \':\' after \'?\': %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + }) + + check_parsing('a?b::c', { + -- 012345 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:4::', + children = { + 'PlainIdentifier(scope=b,ident=):0:2:b:', + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('TernaryColon', ':'), + hl('IdentifierName', 'c'), + }) + + check_parsing('a?b :', { + -- 01234 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + hl('TernaryColon', ':', 1), + }) + + check_parsing('(@a?@b:@c)?@d:@e', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:13::', + children = { + 'Register(name=d):0:11:@d', + 'Register(name=e):0:14:@e', + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('TernaryColon', ':'), + hl('Register', '@e'), + }) + + check_parsing('(@a?@b:@c)?(@d?@e:@f):(@g?@h:@i)', { + -- 01234567890123456789012345678901 + -- 0 1 2 3 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:21::', + children = { + { + 'Nested:0:11:(', + children = { + { + 'Ternary:0:14:?', + children = { + 'Register(name=d):0:12:@d', + { + 'TernaryValue:0:17::', + children = { + 'Register(name=e):0:15:@e', + 'Register(name=f):0:18:@f', + }, + }, + }, + }, + }, + }, + { + 'Nested:0:22:(', + children = { + { + 'Ternary:0:25:?', + children = { + 'Register(name=g):0:23:@g', + { + 'TernaryValue:0:28::', + children = { + 'Register(name=h):0:26:@h', + 'Register(name=i):0:29:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('NestingParenthesis', '('), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('NestingParenthesis', ')'), + hl('TernaryColon', ':'), + hl('NestingParenthesis', '('), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + hl('NestingParenthesis', ')'), + }) + + check_parsing('(@a?@b:@c)?@d?@e:@f:@g?@h:@i', { + -- 0123456789012345678901234567 + -- 0 1 2 + ast = { + { + 'Ternary:0:10:?', + children = { + { + 'Nested:0:0:(', + children = { + { + 'Ternary:0:3:?', + children = { + 'Register(name=a):0:1:@a', + { + 'TernaryValue:0:6::', + children = { + 'Register(name=b):0:4:@b', + 'Register(name=c):0:7:@c', + }, + }, + }, + }, + }, + }, + { + 'TernaryValue:0:19::', + children = { + { + 'Ternary:0:13:?', + children = { + 'Register(name=d):0:11:@d', + { + 'TernaryValue:0:16::', + children = { + 'Register(name=e):0:14:@e', + 'Register(name=f):0:17:@f', + }, + }, + }, + }, + { + 'Ternary:0:22:?', + children = { + 'Register(name=g):0:20:@g', + { + 'TernaryValue:0:25::', + children = { + 'Register(name=h):0:23:@h', + 'Register(name=i):0:26:@i', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('NestingParenthesis', '('), + hl('Register', '@a'), + hl('Ternary', '?'), + hl('Register', '@b'), + hl('TernaryColon', ':'), + hl('Register', '@c'), + hl('NestingParenthesis', ')'), + hl('Ternary', '?'), + hl('Register', '@d'), + hl('Ternary', '?'), + hl('Register', '@e'), + hl('TernaryColon', ':'), + hl('Register', '@f'), + hl('TernaryColon', ':'), + hl('Register', '@g'), + hl('Ternary', '?'), + hl('Register', '@h'), + hl('TernaryColon', ':'), + hl('Register', '@i'), + }) + check_parsing('a?b{cdef}g:h', { + -- 012345678901 + -- 0 1 + ast = { + { + 'Ternary:0:1:?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:10::', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + { + 'ComplexIdentifier:0:9:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:3:{'), + children = { + 'PlainIdentifier(scope=0,ident=cdef):0:4:cdef', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:9:g', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=h):0:11:h', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?'), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('IdentifierName', 'cdef'), + hl('Curly', '}'), + hl('IdentifierName', 'g'), + hl('TernaryColon', ':'), + hl('IdentifierName', 'h'), + }) + check_parsing('a ? b : c : d', { + -- 0123456789012 + -- 0 1 + ast = { + { + 'Colon:0:9: :', + children = { + { + 'Ternary:0:1: ?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'TernaryValue:0:5: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + err = { + arg = ': d', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Ternary', '?', 1), + hl('IdentifierName', 'b', 1), + hl('TernaryColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'd', 1), + }) + end) + itp('works with comparison operators', function() + check_parsing('a == b', { + -- 012345 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:1: ==', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ==? b', { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=IgnoreCase):0:1: ==?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ==# b', { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=MatchCase):0:1: ==#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '==', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a !=# b', { + -- 0123456 + ast = { + { + 'Comparison(type=Equal,inv=1,ccs=MatchCase):0:1: !=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '!=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a <=# b', { + -- 0123456 + ast = { + { + 'Comparison(type=Greater,inv=1,ccs=MatchCase):0:1: <=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a >=# b', { + -- 0123456 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=0,ccs=MatchCase):0:1: >=#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '>=', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a ># b', { + -- 012345 + ast = { + { + 'Comparison(type=Greater,inv=0,ccs=MatchCase):0:1: >#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '>', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a <# b', { + -- 012345 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:1: <#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a is#b', { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=MatchCase):0:1: is#', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '#'), + hl('IdentifierName', 'b'), + }) + + check_parsing('a is?b', { + -- 012345 + ast = { + { + 'Comparison(type=Identical,inv=0,ccs=IgnoreCase):0:1: is?', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:5:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'is', 1), + hl('ComparisonModifier', '?'), + hl('IdentifierName', 'b'), + }) + + check_parsing('a isnot b', { + -- 012345678 + ast = { + { + 'Comparison(type=Identical,inv=1,ccs=UseOption):0:1: isnot', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:7: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', 'isnot', 1), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a < b < c', { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:5: <', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:7: c', + }, + }, + }, + }, + }, + err = { + arg = ' < c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('IdentifierName', 'c', 1), + }) + + check_parsing('a < b <# c', { + -- 012345678 + ast = { + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=UseOption):0:1: <', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Comparison(type=GreaterOrEqual,inv=1,ccs=MatchCase):0:5: <#', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:8: c', + }, + }, + }, + }, + }, + err = { + arg = ' <# c', + msg = 'E15: Operator is not associative: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('Comparison', '<', 1), + hl('IdentifierName', 'b', 1), + hl('InvalidComparison', '<', 1), + hl('InvalidComparisonModifier', '#'), + hl('IdentifierName', 'c', 1), + }) + + check_parsing('a += b', { + -- 012345 + ast = { + { + 'Assignment(Add):0:1: +=', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + }, + err = { + arg = '+= b', + msg = 'E15: Misplaced assignment: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidAssignmentWithAddition', '+=', 1), + hl('IdentifierName', 'b', 1), + }) + check_parsing('a + b == c + d', { + -- 01234567890123 + -- 0 1 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:5: ==', + children = { + { + 'BinaryPlus:0:1: +', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + { + 'BinaryPlus:0:10: +', + children = { + 'PlainIdentifier(scope=0,ident=c):0:8: c', + 'PlainIdentifier(scope=0,ident=d):0:12: d', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'b', 1), + hl('Comparison', '==', 1), + hl('IdentifierName', 'c', 1), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'd', 1), + }) + check_parsing('+ a == + b', { + -- 0123456789 + ast = { + { + 'Comparison(type=Equal,inv=0,ccs=UseOption):0:3: ==', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1: a', + }, + }, + { + 'UnaryPlus:0:6: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:8: b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('IdentifierName', 'a', 1), + hl('Comparison', '==', 1), + hl('UnaryPlus', '+', 1), + hl('IdentifierName', 'b', 1), + }) + end) + itp('works with concat/subscript', function() + check_parsing('.', { + -- 0 + ast = { + { + 'ConcatOrSubscript:0:0:.', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = '.', + msg = 'E15: Unexpected dot: %.*s', + }, + }, { + hl('InvalidConcatOrSubscript', '.'), + }) + + check_parsing('a.', { + -- 01 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + }) + + check_parsing('a.b', { + -- 012 + ast = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=b):0:2:b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', 'b'), + }) + + check_parsing('1.2', { + -- 012 + ast = { + 'Float(val=1.200000e+00):0:0:1.2', + }, + }, { + hl('Float', '1.2'), + }) + + check_parsing('1.2 + 1.3e-5', { + -- 012345678901 + -- 0 1 + ast = { + { + 'BinaryPlus:0:3: +', + children = { + 'Float(val=1.200000e+00):0:0:1.2', + 'Float(val=1.300000e-05):0:5: 1.3e-5', + }, + }, + }, + }, { + hl('Float', '1.2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('a . 1.2 + 1.3e-5', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'BinaryPlus:0:7: +', + children = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + 'Float(val=1.300000e-05):0:9: 1.3e-5', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.3e-5', 1), + }) + + check_parsing('1.3e-5 + 1.2 . a', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:12: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'Float(val=1.200000e+00):0:8: 1.2', + }, + }, + 'PlainIdentifier(scope=0,ident=a):0:14: a', + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('Float', '1.2', 1), + hl('Concat', '.', 1), + hl('IdentifierName', 'a', 1), + }) + + check_parsing('1.3e-5 + a . 1.2', { + -- 0123456789012345 + -- 0 1 + ast = { + { + 'Concat:0:10: .', + children = { + { + 'BinaryPlus:0:6: +', + children = { + 'Float(val=1.300000e-05):0:0:1.3e-5', + 'PlainIdentifier(scope=0,ident=a):0:8: a', + }, + }, + { + 'ConcatOrSubscript:0:14:.', + children = { + 'Integer(val=1):0:12: 1', + 'PlainKey(key=2):0:15:2', + }, + }, + }, + }, + }, + }, { + hl('Float', '1.3e-5'), + hl('BinaryPlus', '+', 1), + hl('IdentifierName', 'a', 1), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('1.2.3', { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'Integer(val=1):0:0:1', + 'PlainKey(key=2):0:2:2', + }, + }, + 'PlainKey(key=3):0:4:3', + }, + }, + }, + }, { + hl('Number', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '3'), + }) + + check_parsing('a.1.2', { + -- 01234 + ast = { + { + 'ConcatOrSubscript:0:3:.', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=1):0:2:1', + }, + }, + 'PlainKey(key=2):0:4:2', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '1'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('a . 1.2', { + -- 0123456 + ast = { + { + 'Concat:0:1: .', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ConcatOrSubscript:0:5:.', + children = { + 'Integer(val=1):0:3: 1', + 'PlainKey(key=2):0:6:2', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('Number', '1', 1), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + }) + + check_parsing('+a . +b', { + -- 0123456 + ast = { + { + 'Concat:0:2: .', + children = { + { + 'UnaryPlus:0:0:+', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'UnaryPlus:0:4: +', + children = { + 'PlainIdentifier(scope=0,ident=b):0:6:b', + }, + }, + }, + }, + }, + }, { + hl('UnaryPlus', '+'), + hl('IdentifierName', 'a'), + hl('Concat', '.', 1), + hl('UnaryPlus', '+', 1), + hl('IdentifierName', 'b'), + }) + + check_parsing('a. b', { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2: b', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierName', 'b', 1), + }) + + check_parsing('a. 1', { + -- 0123 + ast = { + { + 'Concat:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2: 1', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('Number', '1', 1), + }) + end) + itp('works with bracket subscripts', function() + check_parsing(':', { + -- 0 + ast = { + { + 'Colon:0:0::', + children = { + 'Missing:0:0:', + }, + }, + }, + err = { + arg = ':', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('InvalidColon', ':'), + }) + check_parsing('a[]', { + -- 012 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), + }) + check_parsing('a[b:]', { + -- 01234 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=):0:2:b:', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[b:c]', { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=b,ident=c):0:2:b:c', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierScope', 'b'), + hl('IdentifierScopeDelimiter', ':'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + }) + check_parsing('a[b : c]', { + -- 01234567 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + 'PlainIdentifier(scope=0,ident=c):0:5: c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptColon', ':', 1), + hl('IdentifierName', 'c', 1), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[: b]', { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:2::', + children = { + 'Missing:0:2:', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('SubscriptColon', ':'), + hl('IdentifierName', 'b', 1), + hl('SubscriptBracket', ']'), + }) + + check_parsing('a[b :]', { + -- 012345 + ast = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Colon:0:3: :', + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptColon', ':', 1), + hl('SubscriptBracket', ']'), + }) + check_parsing('a[b][c][d](e)(f)(g)', { + -- 0123456789012345678 + -- 0 1 + ast = { + { + 'Call:0:16:(', + children = { + { + 'Call:0:13:(', + children = { + { + 'Call:0:10:(', + children = { + { + 'Subscript:0:7:[', + children = { + { + 'Subscript:0:4:[', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:8:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:11:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:14:f', + }, + }, + 'PlainIdentifier(scope=0,ident=g):0:17:g', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'e'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'f'), + hl('CallingParenthesis', ')'), + hl('CallingParenthesis', '('), + hl('IdentifierName', 'g'), + hl('CallingParenthesis', ')'), + }) + check_parsing('{a}{b}{c}[d][e][f]', { + -- 012345678901234567 + -- 0 1 + ast = { + { + 'Subscript:0:15:[', + children = { + { + 'Subscript:0:12:[', + children = { + { + 'Subscript:0:9:[', + children = { + { + 'ComplexIdentifier:0:3:', + children = { + { + fmtn('CurlyBracesIdentifier', '-di', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + 'ComplexIdentifier:0:6:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:3:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + }, + }, + { + fmtn('CurlyBracesIdentifier', '--i', ':0:6:{'), + children = { + 'PlainIdentifier(scope=0,ident=c):0:7:c', + }, + }, + }, + }, + }, + }, + 'PlainIdentifier(scope=0,ident=d):0:10:d', + }, + }, + 'PlainIdentifier(scope=0,ident=e):0:13:e', + }, + }, + 'PlainIdentifier(scope=0,ident=f):0:16:f', + }, + }, + }, + }, { + hl('Curly', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'c'), + hl('Curly', '}'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'd'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'e'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'f'), + hl('SubscriptBracket', ']'), + }) + end) + itp('supports list literals', function() + check_parsing('[]', { + -- 01 + ast = { + 'ListLiteral:0:0:[', + }, + }, { + hl('List', '['), + hl('List', ']'), + }) + + check_parsing('[a]', { + -- 012 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + }) + + check_parsing('[a, b]', { + -- 012345 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3: b', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c]', { + -- 012345678 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('List', ']'), + }) + + check_parsing('[a, b, c, ]', { + -- 01234567890 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:2:,', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'Comma:0:5:,', + children = { + 'PlainIdentifier(scope=0,ident=b):0:3: b', + { + 'Comma:0:8:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:6: c', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Comma', ','), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('Comma', ','), + hl('List', ']', 1), + }) + + check_parsing('[a : b, c : d]', { + -- 01234567890123 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:6:,', + children = { + { + 'Colon:0:2: :', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:4: b', + }, + }, + { + 'Colon:0:9: :', + children = { + 'PlainIdentifier(scope=0,ident=c):0:7: c', + 'PlainIdentifier(scope=0,ident=d):0:11: d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = ': b, c : d]', + msg = 'E15: Colon outside of dictionary or ternary operator: %.*s', + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'b', 1), + hl('Comma', ','), + hl('IdentifierName', 'c', 1), + hl('InvalidColon', ':', 1), + hl('IdentifierName', 'd', 1), + hl('List', ']'), + }) + + check_parsing(']', { + -- 0 + ast = { + 'ListLiteral:0:0:', + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('InvalidList', ']'), + }) + + check_parsing('a]', { + -- 01 + ast = { + { + 'ListLiteral:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Unexpected closing figure brace: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('InvalidList', ']'), + }) + + check_parsing('[] []', { + -- 01234 + ast = { + { + 'OpMissing:0:2:', + children = { + 'ListLiteral:0:0:[', + 'ListLiteral:0:2: [', + }, + }, + }, + err = { + arg = '[]', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('InvalidSpacing', ' '), + hl('List', '['), + hl('List', ']'), + }, { + [1] = { + ast = { + len = 3, + err = REMOVE_THIS, + ast = { + 'ListLiteral:0:0:[', + }, + }, + hl_fs = { + [3] = REMOVE_THIS, + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + }, + }, + }) + + check_parsing('[][]', { + -- 0123 + ast = { + { + 'Subscript:0:2:[', + children = { + 'ListLiteral:0:0:[', + }, + }, + }, + err = { + arg = ']', + msg = 'E15: Expected value, got closing bracket: %.*s', + }, + }, { + hl('List', '['), + hl('List', ']'), + hl('SubscriptBracket', '['), + hl('InvalidSubscriptBracket', ']'), + }) + + check_parsing('[', { + -- 0 + ast = { + 'ListLiteral:0:0:[', + }, + err = { + arg = '', + msg = 'E15: Expected value, got EOC: %.*s', + }, + }, { + hl('List', '['), + }) + + check_parsing('[1', { + -- 01 + ast = { + { + 'ListLiteral:0:0:[', + children = { + 'Integer(val=1):0:1:1', + }, + }, + }, + err = { + arg = '[1', + msg = 'E697: Missing end of List \']\': %.*s', + }, + }, { + hl('List', '['), + hl('Number', '1'), + }) + end) + itp('works with strings', function() + check_parsing('\'abc\'', { + -- 01234 + ast = { + fmtn('SingleQuotedString', 'val="abc"', ':0:0:\'abc\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuote', '\''), + }) + check_parsing('"abc"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="abc"', ':0:0:"abc"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuote', '"'), + }) + check_parsing('\'\'', { + -- 01 + ast = { + fmtn('SingleQuotedString', 'val=NULL', ':0:0:\'\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuote', '\''), + }) + check_parsing('""', { + -- 01 + ast = { + fmtn('DoubleQuotedString', 'val=NULL', ':0:0:""'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuote', '"'), + }) + check_parsing('"', { + -- 0 + ast = { + fmtn('DoubleQuotedString', 'val=NULL', ':0:0:"'), + }, + err = { + arg = '"', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + }) + check_parsing('\'', { + -- 0 + ast = { + fmtn('SingleQuotedString', 'val=NULL', ':0:0:\''), + }, + err = { + arg = '\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + }) + check_parsing('"a', { + -- 01 + ast = { + fmtn('DoubleQuotedString', 'val="a"', ':0:0:"a'), + }, + err = { + arg = '"a', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedBody', 'a'), + }) + check_parsing('\'a', { + -- 01 + ast = { + fmtn('SingleQuotedString', 'val="a"', ':0:0:\'a'), + }, + err = { + arg = '\'a', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + hl('InvalidSingleQuotedBody', 'a'), + }) + check_parsing('\'abc\'\'def\'', { + -- 0123456789 + ast = { + fmtn('SingleQuotedString', 'val="abc\'def"', ':0:0:\'abc\'\'def\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedBody', 'abc'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'def'), + hl('SingleQuote', '\''), + }) + check_parsing('\'abc\'\'', { + -- 012345 + ast = { + fmtn('SingleQuotedString', 'val="abc\'"', ':0:0:\'abc\'\''), + }, + err = { + arg = '\'abc\'\'', + msg = 'E115: Missing single quote: %.*s', + }, + }, { + hl('InvalidSingleQuote', '\''), + hl('InvalidSingleQuotedBody', 'abc'), + hl('InvalidSingleQuotedQuote', '\'\''), + }) + check_parsing('\'\'\'\'\'\'\'\'', { + -- 01234567 + ast = { + fmtn('SingleQuotedString', 'val="\'\'\'"', ':0:0:\'\'\'\'\'\'\'\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuote', '\''), + }) + check_parsing('\'\'\'a\'\'\'\'bc\'', { + -- 01234567890 + -- 0 1 + ast = { + fmtn('SingleQuotedString', 'val="\'a\'\'bc"', ':0:0:\'\'\'a\'\'\'\'bc\''), + }, + }, { + hl('SingleQuote', '\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'a'), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedQuote', '\'\''), + hl('SingleQuotedBody', 'bc'), + hl('SingleQuote', '\''), + }) + check_parsing('"\\"\\"\\"\\""', { + -- 0123456789 + ast = { + fmtn('DoubleQuotedString', 'val="\\"\\"\\"\\""', ':0:0:"\\"\\"\\"\\""'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuote', '"'), + }) + check_parsing('"abc\\"def\\"ghi\\"jkl\\"mno"', { + -- 0123456789012345678901234 + -- 0 1 2 + ast = { + fmtn('DoubleQuotedString', 'val="abc\\"def\\"ghi\\"jkl\\"mno"', ':0:0:"abc\\"def\\"ghi\\"jkl\\"mno"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', 'abc'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'def'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'ghi'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'jkl'), + hl('DoubleQuotedEscape', '\\"'), + hl('DoubleQuotedBody', 'mno'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\b\\e\\f\\r\\t\\\\"', { + -- 0123456789012345 + -- 0 1 + ast = { + [[DoubleQuotedString(val="\8\27\12\13\9\\"):0:0:"\b\e\f\r\t\\"]], + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\b'), + hl('DoubleQuotedEscape', '\\e'), + hl('DoubleQuotedEscape', '\\f'), + hl('DoubleQuotedEscape', '\\r'), + hl('DoubleQuotedEscape', '\\t'), + hl('DoubleQuotedEscape', '\\\\'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\n\n"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="\\\n\\\n"', ':0:0:"\\n\n"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\n'), + hl('DoubleQuotedBody', '\n'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\x00"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0"', ':0:0:"\\x00"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\xFF"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\255"', ':0:0:"\\xFF"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xFF'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\xF"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\15"', ':0:0:"\\xF"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\u00AB"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="«"', ':0:0:"\\u00AB"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u00AB'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\U000000AB"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="«"', ':0:0:"\\U000000AB"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000000AB'), + hl('DoubleQuote', '"'), + }) + check_parsing('"\\x"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="x"', ':0:0:"\\x"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="x"', ':0:0:"\\x'), + }, + err = { + arg = '"\\x', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\x'), + }) + + check_parsing('"\\xF', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="\\15"', ':0:0:"\\xF'), + }, + err = { + arg = '"\\xF', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedEscape', '\\xF'), + }) + + check_parsing('"\\u"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="u"', ':0:0:"\\u"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="u"', ':0:0:"\\u'), + }, + err = { + arg = '"\\u', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\u'), + }) + + check_parsing('"\\U', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="U"', ':0:0:"\\U'), + }, + err = { + arg = '"\\U', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\U'), + }) + + check_parsing('"\\U"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="U"', ':0:0:"\\U"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\xFX"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\15X"', ':0:0:"\\xFX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\xF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\XFX"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\15X"', ':0:0:"\\XFX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\XF'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\xX"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="xX"', ':0:0:"\\xX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\x'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\XX"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="XX"', ':0:0:"\\XX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\X'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\uX"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="uX"', ':0:0:"\\uX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\u'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\UX"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="UX"', ':0:0:"\\UX"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\U'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x0X"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\x0X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X0X"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\X0X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u0X"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\u0X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0X"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U0X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x00X"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\x00X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X00X"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\X00X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u00X"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\u00X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00X"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U00X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u000X"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\u000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000X"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u0000X"', { + -- 012345678 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\u0000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0000X"', { + -- 012345678 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U0000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00000X"', { + -- 0123456789 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U00000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000000X"', { + -- 01234567890 + -- 0 1 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U000000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U0000000X"', { + -- 012345678901 + -- 0 1 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U0000000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U0000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U00000000X"', { + -- 0123456789012 + -- 0 1 + ast = { + fmtn('DoubleQuotedString', 'val="\\0X"', ':0:0:"\\U00000000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', 'X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\x000X"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000X"', ':0:0:"\\x000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\x00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\X000X"', { + -- 01234567 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000X"', ':0:0:"\\X000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\X00'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\u00000X"', { + -- 0123456789 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000X"', ':0:0:"\\u00000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\u0000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\U000000000X"', { + -- 01234567890123 + -- 0 1 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000X"', ':0:0:"\\U000000000X"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\U00000000'), + hl('DoubleQuotedBody', '0X'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="\\0"', ':0:0:"\\0"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\00"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="\\0"', ':0:0:"\\00"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\000"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0"', ':0:0:"\\000"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0000"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0000"', ':0:0:"\\0000"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '0'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\8"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="8"', ':0:0:"\\8"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\08"', { + -- 01234 + ast = { + fmtn('DoubleQuotedString', 'val="\\0008"', ':0:0:"\\08"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\0'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\008"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\0008"', ':0:0:"\\008"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\00'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\0008"', { + -- 0123456 + ast = { + fmtn('DoubleQuotedString', 'val="\\0008"', ':0:0:"\\0008"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\000'), + hl('DoubleQuotedBody', '8'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\777"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\255"', ':0:0:"\\777"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\777'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\050"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\40"', ':0:0:"\\050"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\050'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\"', { + -- 012345 + ast = { + fmtn('DoubleQuotedString', 'val="\\21"', ':0:0:"\\"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedEscape', '\\'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\<', { + -- 012 + ast = { + fmtn('DoubleQuotedString', 'val="<"', ':0:0:"\\<'), + }, + err = { + arg = '"\\<', + msg = 'E114: Missing double quote: %.*s', + }, + }, { + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedUnknownEscape', '\\<'), + }) + + check_parsing('"\\<"', { + -- 0123 + ast = { + fmtn('DoubleQuotedString', 'val="<"', ':0:0:"\\<"'), + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedUnknownEscape', '\\<'), + hl('DoubleQuote', '"'), + }) + + check_parsing('"\\ {b{3}: 4}[5]}()] += 6', { + -- 012345678901234567890123456 + -- 0 1 2 + ast = { + { + 'Assignment(Add):0:22: +=', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'Call:0:19:(', + children = { + { + fmtn('Lambda', '\\di', ':0:2:{'), + children = { + { + 'Arrow:0:3:->', + children = { + { + 'Subscript:0:15:[', + children = { + { + fmtn('DictLiteral', '-di', ':0:5: {'), + children = { + { + 'Colon:0:11::', + children = { + { + 'ComplexIdentifier:0:8:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:7:b', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:8:{'), + children = { + 'Integer(val=3):0:9:3', + }, + }, + }, + }, + 'Integer(val=4):0:12: 4', + }, + }, + }, + }, + 'Integer(val=5):0:16:5', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + 'Integer(val=6):0:25: 6', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Dict', '{', 1), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('Number', '3'), + hl('Curly', '}'), + hl('Colon', ':'), + hl('Number', '4', 1), + hl('Dict', '}'), + hl('SubscriptBracket', '['), + hl('Number', '5'), + hl('SubscriptBracket', ']'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('SubscriptBracket', ']'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '6', 1), + }) + + check_asgn_parsing('a{1}.2[{-> {b{3}: 4}[5]}()]', { + -- 012345678901234567890123456 + -- 0 1 2 + ast = { + { + 'Subscript:0:6:[', + children = { + { + 'ConcatOrSubscript:0:4:.', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:1:{'), + children = { + 'Integer(val=1):0:2:1', + }, + }, + }, + }, + 'PlainKey(key=2):0:5:2', + }, + }, + { + 'Call:0:24:(', + children = { + { + fmtn('Lambda', '\\di', ':0:7:{'), + children = { + { + 'Arrow:0:8:->', + children = { + { + 'Subscript:0:20:[', + children = { + { + fmtn('DictLiteral', '-di', ':0:10: {'), + children = { + { + 'Colon:0:16::', + children = { + { + 'ComplexIdentifier:0:13:', + children = { + 'PlainIdentifier(scope=0,ident=b):0:12:b', + { + fmtn('CurlyBracesIdentifier', '--i', ':0:13:{'), + children = { + 'Integer(val=3):0:14:3', + }, + }, + }, + }, + 'Integer(val=4):0:17: 4', + }, + }, + }, + }, + 'Integer(val=5):0:21:5', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('Number', '1'), + hl('Curly', '}'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '2'), + hl('SubscriptBracket', '['), + hl('Lambda', '{'), + hl('Arrow', '->'), + hl('Dict', '{', 1), + hl('IdentifierName', 'b'), + hl('Curly', '{'), + hl('Number', '3'), + hl('Curly', '}'), + hl('Colon', ':'), + hl('Number', '4', 1), + hl('Dict', '}'), + hl('SubscriptBracket', '['), + hl('Number', '5'), + hl('SubscriptBracket', ']'), + hl('Lambda', '}'), + hl('CallingParenthesis', '('), + hl('CallingParenthesis', ')'), + hl('SubscriptBracket', ']'), + }) + + check_asgn_parsing('a', { + -- 0 + ast = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + }, + }, { + hl('IdentifierName', 'a'), + }) + + check_asgn_parsing('{a}', { + -- 012 + ast = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + }) + + check_asgn_parsing('{a}b', { + -- 0123 + ast = { + { + 'ComplexIdentifier:0:3:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('IdentifierName', 'b'), + }) + + check_asgn_parsing('a{b}c', { + -- 01234 + ast = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:1:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + }) + + check_asgn_parsing('a{b}c[0]', { + -- 01234567 + ast = { + { + 'Subscript:0:5:[', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:1:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + 'Integer(val=0):0:6:0', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', '['), + hl('Number', '0'), + hl('SubscriptBracket', ']'), + }) + + check_asgn_parsing('a{b}c.0', { + -- 0123456 + ast = { + { + 'ConcatOrSubscript:0:5:.', + children = { + { + 'ComplexIdentifier:0:1:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + { + 'ComplexIdentifier:0:4:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:1:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:2:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + }, + }, + 'PlainKey(key=0):0:6:0', + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '0'), + }) + + check_asgn_parsing('[a{b}c[0].0]', { + -- 012345678901 + -- 0 1 + ast = { + { + 'ListLiteral:0:0:[', + children = { + { + 'ConcatOrSubscript:0:9:.', + children = { + { + 'Subscript:0:6:[', + children = { + { + 'ComplexIdentifier:0:2:', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + { + 'ComplexIdentifier:0:5:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:2:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + 'PlainIdentifier(scope=0,ident=c):0:5:c', + }, + }, + }, + }, + 'Integer(val=0):0:7:0', + }, + }, + 'PlainKey(key=0):0:10:0', + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + hl('IdentifierName', 'c'), + hl('SubscriptBracket', '['), + hl('Number', '0'), + hl('SubscriptBracket', ']'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', '0'), + hl('List', ']'), + }) + + check_asgn_parsing('{a}{b}', { + -- 012345 + ast = { + { + 'ComplexIdentifier:0:3:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:0:{'), + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + { + fmtn('CurlyBracesIdentifier', '--i', ':0:3:{'), + children = { + 'PlainIdentifier(scope=0,ident=b):0:4:b', + }, + }, + }, + }, + }, + }, { + hl('FigureBrace', '{'), + hl('IdentifierName', 'a'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'b'), + hl('Curly', '}'), + }) + + check_asgn_parsing('a.b{c}{d}', { + -- 012345678 + ast = { + { + 'OpMissing:0:3:', + children = { + { + 'ConcatOrSubscript:0:1:.', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'PlainKey(key=b):0:2:b', + }, + }, + { + 'ComplexIdentifier:0:6:', + children = { + { + fmtn('CurlyBracesIdentifier', '--i', ':0:3:{'), + children = { + 'PlainIdentifier(scope=0,ident=c):0:4:c', + }, + }, + { + fmtn('CurlyBracesIdentifier', '--i', ':0:6:{'), + children = { + 'PlainIdentifier(scope=0,ident=d):0:7:d', + }, + }, + }, + }, + }, + }, + }, + err = { + arg = '{c}{d}', + msg = 'E15: Missing operator: %.*s', + }, + }, { + hl('IdentifierName', 'a'), + hl('ConcatOrSubscript', '.'), + hl('IdentifierKey', 'b'), + hl('InvalidFigureBrace', '{'), + hl('IdentifierName', 'c'), + hl('Curly', '}'), + hl('Curly', '{'), + hl('IdentifierName', 'd'), + hl('Curly', '}'), + }) + + check_asgn_parsing('[a] = 1', { + -- 0123456 + ast = { + { + 'Assignment(Plain):0:3: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + }, + }, + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('[a[b], [c, [d, [e]]]] = 1', { + -- 0123456789012345678901234 + -- 0 1 2 + ast = { + { + 'Assignment(Plain):0:21: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:5:,', + children = { + { + 'Subscript:0:2:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:1:a', + 'PlainIdentifier(scope=0,ident=b):0:3:b', + }, + }, + { + 'ListLiteral:0:6: [', + children = { + { + 'Comma:0:9:,', + children = { + 'PlainIdentifier(scope=0,ident=c):0:8:c', + { + 'ListLiteral:0:10: [', + children = { + { + 'Comma:0:13:,', + children = { + 'PlainIdentifier(scope=0,ident=d):0:12:d', + { + 'ListLiteral:0:14: [', + children = { + 'PlainIdentifier(scope=0,ident=e):0:16:e', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + 'Integer(val=1):0:23: 1', + }, + }, + }, + err = { + arg = '[c, [d, [e]]]] = 1', + msg = 'E475: Nested lists not allowed when assigning: %.*s', + }, + }, { + hl('List', '['), + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('IdentifierName', 'b'), + hl('SubscriptBracket', ']'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'c'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'd'), + hl('Comma', ','), + hl('InvalidList', '[', 1), + hl('IdentifierName', 'e'), + hl('List', ']'), + hl('List', ']'), + hl('List', ']'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('$X += 1', { + -- 0123456 + ast = { + { + 'Assignment(Add):0:2: +=', + children = { + 'Environment(ident=X):0:0:$X', + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'X'), + hl('AssignmentWithAddition', '+=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('@a .= 1', { + -- 0123456 + ast = { + { + 'Assignment(Concat):0:2: .=', + children = { + 'Register(name=a):0:0:@a', + 'Integer(val=1):0:5: 1', + }, + }, + }, + }, { + hl('Register', '@a'), + hl('AssignmentWithConcatenation', '.=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('&option -= 1', { + -- 012345678901 + -- 0 1 + ast = { + { + 'Assignment(Subtract):0:7: -=', + children = { + 'Option(scope=0,ident=option):0:0:&option', + 'Integer(val=1):0:10: 1', + }, + }, + }, + }, { + hl('OptionSigil', '&'), + hl('OptionName', 'option'), + hl('AssignmentWithSubtraction', '-=', 1), + hl('Number', '1', 1), + }) + + check_asgn_parsing('[$X, @a, &l:option] = [1, 2, 3]', { + -- 0123456789012345678901234567890 + -- 0 1 2 3 + ast = { + { + 'Assignment(Plain):0:19: =', + children = { + { + 'ListLiteral:0:0:[', + children = { + { + 'Comma:0:3:,', + children = { + 'Environment(ident=X):0:1:$X', + { + 'Comma:0:7:,', + children = { + 'Register(name=a):0:4: @a', + 'Option(scope=l,ident=option):0:8: &l:option', + }, + }, + }, + }, + }, + }, + { + 'ListLiteral:0:21: [', + children = { + { + 'Comma:0:24:,', + children = { + 'Integer(val=1):0:23:1', + { + 'Comma:0:27:,', + children = { + 'Integer(val=2):0:25: 2', + 'Integer(val=3):0:28: 3', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + hl('List', '['), + hl('EnvironmentSigil', '$'), + hl('EnvironmentName', 'X'), + hl('Comma', ','), + hl('Register', '@a', 1), + hl('Comma', ','), + hl('OptionSigil', '&', 1), + hl('OptionScope', 'l'), + hl('OptionScopeDelimiter', ':'), + hl('OptionName', 'option'), + hl('List', ']'), + hl('PlainAssignment', '=', 1), + hl('List', '[', 1), + hl('Number', '1'), + hl('Comma', ','), + hl('Number', '2', 1), + hl('Comma', ','), + hl('Number', '3', 1), + hl('List', ']'), + }) + end) +end From f20f97c936f1438589c8176f62ce69c26e255f85 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 21:13:27 +0300 Subject: [PATCH 091/109] *: Fix linter errors --- src/nvim/viml/parser/expressions.c | 19 +++++++++++-------- test/functional/api/vim_spec.lua | 1 - test/helpers.lua | 2 +- test/unit/viml/expressions/parser_spec.lua | 10 +++++----- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 0824b3ca7d..6c7c328b6d 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -2014,17 +2014,20 @@ viml_pexpr_parse_process_token: lambda_node->data.fig.type_guesses.allow_lambda = false; if (lambda_node->children != NULL && lambda_node->children->type == kExprNodeComma) { - // If lambda has comma child this means that parser has already seen at - // least "{arg1,", so node cannot possibly be anything, but lambda. + // If lambda has comma child this means that parser has already seen + // at least "{arg1,", so node cannot possibly be anything, but + // lambda. - // Vim may give E121 or E720 in this case, but it does not look right to - // have either because both are results of reevaluation possibly-lambda - // node as a dictionary and here this is not going to happen. + // Vim may give E121 or E720 in this case, but it does not look + // right to have either because both are results of reevaluation + // possibly-lambda node as a dictionary and here this is not going + // to happen. ERROR_FROM_TOKEN_AND_MSG( - cur_token, _("E15: Expected lambda arguments list or arrow: %.*s")); + cur_token, + _("E15: Expected lambda arguments list or arrow: %.*s")); } else { - // Else it may appear that possibly-lambda node is actually a dictionary - // or curly-braces-name identifier. + // Else it may appear that possibly-lambda node is actually + // a dictionary or curly-braces-name identifier. lambda_node = NULL; kv_drop(pt_stack, 1); } diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index 3939bc9b52..2572675a58 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -12,7 +12,6 @@ local request = helpers.request local meth_pcall = helpers.meth_pcall local command = helpers.command -local REMOVE_THIS = global_helpers.REMOVE_THIS local intchar2lua = global_helpers.intchar2lua local format_string = global_helpers.format_string local mergedicts_copy = global_helpers.mergedicts_copy diff --git a/test/helpers.lua b/test/helpers.lua index ada690a4d2..fc3936a4ce 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -522,7 +522,7 @@ local function fixtbl(tbl) end local function fixtbl_rec(tbl) - for k, v in pairs(tbl) do + for _, v in pairs(tbl) do if type(v) == 'table' then fixtbl_rec(v) end diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 24f681f438..2ae235ca86 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -7,7 +7,6 @@ local make_enum_conv_tab = helpers.make_enum_conv_tab local child_call_once = helpers.child_call_once local alloc_log_new = helpers.alloc_log_new local kvi_destroy = helpers.kvi_destroy -local array_size = helpers.array_size local conv_enum = helpers.conv_enum local debug_log = helpers.debug_log local ptr2key = helpers.ptr2key @@ -26,7 +25,6 @@ local mergedicts_copy = global_helpers.mergedicts_copy local format_string = global_helpers.format_string local format_luav = global_helpers.format_luav local intchar2lua = global_helpers.intchar2lua -local REMOVE_THIS = global_helpers.REMOVE_THIS local dictdiff = global_helpers.dictdiff local lib = cimport('./src/nvim/viml/parser/expressions.h', @@ -147,12 +145,14 @@ child_call_once(function() -- nvim_hl_defs. eq(true, not not (nvim_hl_defs[grp_link] or predefined_hl_defs[grp_link])) + eq(false, not not (nvim_hl_defs[new_grp] + or predefined_hl_defs[new_grp])) nvim_hl_defs[new_grp] = {'link', grp_link} else local new_grp, grp_args = s:match('^(%w+) (.*)') neq(nil, new_grp) - eq(false, not not (nvim_hl_defs[grp_link] - or predefined_hl_defs[grp_link])) + eq(false, not not (nvim_hl_defs[new_grp] + or predefined_hl_defs[new_grp])) nvim_hl_defs[new_grp] = {'definition', grp_args} end end) @@ -206,7 +206,7 @@ local function format_check(expr, format_check_data, opts) -- That forces specific order. local zflags = opts.flags[1] local zdata = format_check_data[zflags] - local dig_len = 0 + local dig_len if opts.funcname then print(format_string('\n%s(%r, {', opts.funcname, expr)) dig_len = #opts.funcname + 2 From 731dc82f8c04e3019ecc3243b6b512535998635b Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 21:21:45 +0300 Subject: [PATCH 092/109] ex_getln: Fix memory leak in color_expr_cmdline --- src/nvim/ex_getln.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nvim/ex_getln.c b/src/nvim/ex_getln.c index e6fb2571c4..cabdda28cf 100644 --- a/src/nvim/ex_getln.c +++ b/src/nvim/ex_getln.c @@ -2419,6 +2419,7 @@ static void color_expr_cmdline(const CmdlineInfo *const colored_ccline, .attr = 0, })); } + kvi_destroy(colors); } /// Color command-line From ebb33eddd9ad0e9cec5013be2e37c8f9b0546c77 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 21:40:34 +0300 Subject: [PATCH 093/109] tests: Stabilize float format and %e in format_luav and format_string --- test/functional/api/vim_spec.lua | 2 +- test/helpers.lua | 15 +++++++++++++-- test/unit/viml/expressions/parser_spec.lua | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index 2572675a58..5b5340d9e2 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -750,7 +750,7 @@ describe('api', function() typ = typ .. ('(val=%u)'):format(east_api_node.ivalue) east_api_node.ivalue = nil elseif typ == 'Float' then - typ = typ .. ('(val=%e)'):format(east_api_node.fvalue) + typ = typ .. format_string('(val=%e)', east_api_node.fvalue) east_api_node.fvalue = nil elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then typ = format_string('%s(val=%q)', typ, east_api_node.svalue) diff --git a/test/helpers.lua b/test/helpers.lua index fc3936a4ce..cef05046d8 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -395,6 +395,13 @@ local function dedent(str, leave_indent) return str end +local function format_float(v) + -- On windows exponent appears to have three digits and not two + local ret = ('%.6e'):format(v) + local l, f, es, e = ret:match('^(%-?%d)%.(%d+)e([+%-])0*(%d%d+)$') + return l .. '.' .. f .. 'e' .. es .. e +end + local SUBTBL = { '\\000', '\\001', '\\002', '\\003', '\\004', '\\005', '\\006', '\\007', '\\008', '\\t', @@ -468,7 +475,7 @@ format_luav = function(v, indent, opts) if v % 1 == 0 then ret = ('%d'):format(v) else - ret = ('%e'):format(v) + ret = format_float(v) end elseif type(v) == 'nil' then ret = 'nil' @@ -501,7 +508,11 @@ local function format_string(fmt, ...) subfmt = subfmt:sub(1, -2) .. 's' arg = format_luav(arg) end - return subfmt:format(arg) + if subfmt == '%e' then + return format_float(arg) + else + return subfmt:format(arg) + end end) return ret end diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 2ae235ca86..79b19de833 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -375,7 +375,7 @@ local function eastnode2lua(pstate, eastnode, checked_nodes) elseif typ == 'Integer' then typ = typ .. ('(val=%u)'):format(tonumber(eastnode.data.num.value)) elseif typ == 'Float' then - typ = typ .. ('(val=%e)'):format(tonumber(eastnode.data.flt.value)) + typ = typ .. format_string('(val=%e)', tonumber(eastnode.data.flt.value)) elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then if eastnode.data.str.value == nil then typ = typ .. '(val=NULL)' From 7c20f60b88b1df97af8ea4a6a5b9727c72748ec1 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 21:46:54 +0300 Subject: [PATCH 094/109] unittests: Avoid infinite cycle somewhere because of init failure --- test/unit/helpers.lua | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index 2c148630dd..7689c1824b 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -644,13 +644,15 @@ local function itp_child(wr, func) s = s:sub(1, hook_msglen - 2) sc.write(wr, '>' .. s .. (' '):rep(hook_msglen - 2 - #s) .. '\n') end - init() - collectgarbage('stop') - child_sethook(wr) - local err, emsg = pcall(func) - collectgarbage('restart') - collectgarbage() - debug.sethook() + local err, emsg = pcall(init) + if err then + collectgarbage('stop') + child_sethook(wr) + err, emsg = pcall(func) + collectgarbage('restart') + collectgarbage() + debug.sethook() + end emsg = tostring(emsg) sc.write(wr, trace_end_msg) if not err then From 6ea3a08fdbb276fe64dda60c5fb934360327ed39 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 21:55:36 +0300 Subject: [PATCH 095/109] syntax: Fix duplicate group definitions --- src/nvim/syntax.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index c9e99d82f8..9a537130fd 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -6067,7 +6067,6 @@ const char *const highlight_init_cmdline[] = { "default link NVimSubscript NVimParenthesis", "default link NVimSubscriptBracket NVimSubscript", "default link NVimSubscriptColon NVimSubscript", - "default link NVimSubscriptColon NVimSubscript", "default link NVimCurly NVimSubscript", "default link NVimContainer NVimParenthesis", @@ -6164,7 +6163,6 @@ const char *const highlight_init_cmdline[] = { "default link NVimInvalidSubscript NVimInvalidParenthesis", "default link NVimInvalidSubscriptBracket NVimInvalidSubscript", "default link NVimInvalidSubscriptColon NVimInvalidSubscript", - "default link NVimInvalidSubscriptColon NVimInvalidSubscript", "default link NVimInvalidCurly NVimInvalidSubscript", "default link NVimInvalidContainer NVimInvalidParenthesis", From fe3a58273e7665e795e0402c961fad869f4e34f9 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 22:24:26 +0300 Subject: [PATCH 096/109] cmake: Fix api/version test failure --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cad3ea6786..fb0cc262b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,9 +68,9 @@ set(NVIM_VERSION_PATCH 3) set(NVIM_VERSION_PRERELEASE "-dev") # for package maintainers # API level -set(NVIM_API_LEVEL 3) # Bump this after any API change. +set(NVIM_API_LEVEL 4) # Bump this after any API change. set(NVIM_API_LEVEL_COMPAT 0) # Adjust this after a _breaking_ API change. -set(NVIM_API_PRERELEASE false) +set(NVIM_API_PRERELEASE true) file(TO_CMAKE_PATH ${CMAKE_CURRENT_LIST_DIR}/.git FORCED_GIT_DIR) include(GetGitRevisionDescription) From 64158f2b0b8f0d2e058f7be6f0a0c3faa7de5f22 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 22:32:02 +0300 Subject: [PATCH 097/109] unittests: Populate ARGTYPES in child process only --- test/unit/charset/vim_str2nr_spec.lua | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/unit/charset/vim_str2nr_spec.lua b/test/unit/charset/vim_str2nr_spec.lua index 22504649f6..1a0a000abb 100644 --- a/test/unit/charset/vim_str2nr_spec.lua +++ b/test/unit/charset/vim_str2nr_spec.lua @@ -3,6 +3,7 @@ local global_helpers = require('test.helpers') local itp = helpers.gen_itp(it) +local child_call_once = helpers.child_call_once local cimport = helpers.cimport local ffi = helpers.ffi @@ -11,12 +12,16 @@ local updated = global_helpers.updated local lib = cimport('./src/nvim/charset.h') -local ARGTYPES = { - num = ffi.typeof('varnumber_T[1]'), - unum = ffi.typeof('uvarnumber_T[1]'), - pre = ffi.typeof('int[1]'), - len = ffi.typeof('int[1]'), -} +local ARGTYPES + +child_call_once(function() + ARGTYPES = { + num = ffi.typeof('varnumber_T[1]'), + unum = ffi.typeof('uvarnumber_T[1]'), + pre = ffi.typeof('int[1]'), + len = ffi.typeof('int[1]'), + } +end) local icnt = -42 local ucnt = 4242 From 1ffa4e50477eb8a95c7097973c4d13d97551d784 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 23:33:02 +0300 Subject: [PATCH 098/109] doc: Update documentation --- runtime/doc/eval.txt | 121 ++++++++++++++++++++++++++++++++++++++- runtime/doc/vim_diff.txt | 13 +++-- 2 files changed, 127 insertions(+), 7 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index b752667d9a..090eb022e2 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -10506,7 +10506,7 @@ option will still be marked as it was set in the sandbox. In a few situations it is not allowed to change the text in the buffer, jump to another window and some other things that might confuse or break what Vim is currently doing. This mostly applies to things that happen when Vim is -actually doing something else. For example, evaluating the 'balloonexpr' may +actually doing something else. For example, evaluating the 'balloonexpr' may happen any moment the mouse cursor is resting at some position. This is not allowed when the textlock is active: @@ -10516,5 +10516,124 @@ This is not allowed when the textlock is active: - closing a window or quitting Vim - etc. +============================================================================== +13. Command-line expressions coloring *expr-coloring* + +Expressions entered by user in |i_CTRL-R_=|, |c_CTRL-\_e|, |quote=| are +colored by built-in expressions parser. It uses highlight groups described in +the below table, which may be overriden by user colorschemes, all linked to +some other highlighting group. + *hl-NVimInvalid* +In addition to highlighting groups prefixed with NVim described below there +are highlighting groups prefixed with NVimInvalid which have just the same +meaning, but used to indicate that relevant token contains an error or that +error had occurred just before it. They have mostly the same hierarchy, +except that by default in place of any non-NVim-prefixed group NVimInvalid +linking to `Error` is used and some other intermediate groups are present. + +Group Default link Colored expression ~ +*hl-NVimInternalError* None, red/red Parser bug + +*hl-NVimAssignment* Operator Generic assignment +*hl-NVimPlainAssignment* NVimAssignment `=` in |:let| +*hl-NVimAugmentedAssignment* NVimAssignment Generic, `+=`/`-=`/`.=` +*hl-NVimAssignmentWithAddition* NVimAugmentedAssignment `+=` in |:let+=| +*hl-NVimAssignmentWithSubtraction* NVimAugmentedAssignment `-=` in |:let-=| +*hl-NVimAssignmentWithConcatenation* NVimAugmentedAssignment `.=` in |:let.=| + +*hl-NVimOperator* Operator Generic operator + +*hl-NVimUnaryOperator* NVimOperator Generic unary op +*hl-NVimUnaryPlus* NVimUnaryOperator |expr-unary-+| +*hl-NVimUnaryMinus* NVimUnaryOperator |expr-unary--| +*hl-NVimNot* NVimUnaryOperator |expr-!| + +*hl-NVimBinaryOperator* NVimOperator Generic binary op +*hl-NVimComparison* NVimBinaryOperator Any |expr4| operator +*hl-NVimComparisonModifier* NVimComparison `#`/`?` near |expr4| op +*hl-NVimBinaryPlus* NVimBinaryOperator |expr-+| +*hl-NVimBinaryMinus* NVimBinaryOperator |expr--| +*hl-NVimConcat* NVimBinaryOperator |expr-.| +*hl-NVimConcatOrSubscript* NVimConcat |expr-.| or |expr-entry| +*hl-NVimOr* NVimBinaryOperator |expr-barbar| +*hl-NVimAnd* NVimBinaryOperator |expr-&&| +*hl-NVimMultiplication* NVimBinaryOperator |expr-star| +*hl-NVimDivision* NVimBinaryOperator |expr-/| +*hl-NVimMod* NVimBinaryOperator |expr-%| + +*hl-NVimTernary* NVimOperator `?` in |expr1| +*hl-NVimTernaryColon* NVimTernary `:` in |expr1| + +*hl-NVimParenthesis* Delimiter Generic bracket +*hl-NVimLambda* NVimParenthesis `{`/`}` in |lambda| +*hl-NVimNestingParenthesis* NVimParenthesis `(`/`)` in |expr-nesting| +*hl-NVimCallingParenthesis* NVimParenthesis `(`/`)` in |expr-function| + +*hl-NVimSubscript* NVimParenthesis Generic subscript +*hl-NVimSubscriptBracket* NVimSubscript `[`/`]` in |expr-[]| +*hl-NVimSubscriptColon* NVimSubscript `:` in |expr-[:]| +*hl-NVimCurly* NVimSubscript `{`/`}` in + |curly-braces-names| + +*hl-NVimContainer* NVimParenthesis Generic container +*hl-NVimDict* NVimContainer `{`/`}` in |dict| literal +*hl-NVimList* NVimContainer `[`/`]` in |list| literal + +*hl-NVimIdentifier* Identifier Generic identifier +*hl-NVimIdentifierScope* NVimIdentifier Namespace: letter + before `:` in + |internal-variables| +*hl-NVimIdentifierScopeDelimiter* NVimIdentifier `:` after namespace + letter +*hl-NVimIdentifierName* NVimIdentifier Rest of the ident +*hl-NVimIdentifierKey* NVimIdentifier Identifier after + |expr-entry| + +*hl-NVimColon* Delimiter `:` in |dict| literal +*hl-NVimComma* Delimiter `,` in |dict|/|list| + literal or + |expr-function| +*hl-NVimArrow* Delimiter `->` in |lambda| + +*hl-NVimRegister* SpecialChar |expr-register| +*hl-NVimNumber* Number Non-prefix digits + in integer + |expr-number| +*hl-NVimNumberPrefix* Type `0` for |octal-number| + `0x` for |hex-number| + `0b` for |binary-number| +*hl-NVimFloat* NVimNumber Floating-point + number + +*hl-NVimOptionSigil* Type `&` in |expr-option| +*hl-NVimOptionScope* NVimIdentifierScope Option scope if any +*hl-NVimOptionScopeDelimiter* NVimIdentifierScopeDelimiter + `:` after option scope +*hl-NVimOptionName* NVimIdentifier Option name + +*hl-NVimEnvironmentSigil* NVimOptionSigil `$` in |expr-env| +*hl-NVimEnvironmentName* NVimIdentifier Env variable name + +*hl-NVimString* String Generic string +*hl-NVimStringBody* NVimString Generic string + literal body +*hl-NVimStringQuote* NVimString Generic string quote +*hl-NVimStringSpecial* SpecialChar Generic string + non-literal body + +*hl-NVimSingleQuote* NVimStringQuote `'` in |expr-'| +*hl-NVimSingleQuotedBody* NVimStringBody Literal part of + |expr-'| string body +*hl-NVimSingleQuotedQuote* NVimStringSpecial `''` inside |expr-'| + string body + +*hl-NVimDoubleQuote* NVimStringQuote `"` in |expr-quote| +*hl-NVimDoubleQuotedBody* NVimStringBody Literal part of + |expr-quote| body +*hl-NVimDoubleQuotedEscape* NVimStringSpecial Valid |expr-quote| + escape sequence +*hl-NVimDoubleQuotedUnknownEscape* NVimInvalidValue Unrecognized + |expr-quote| escape + sequence vim:tw=78:ts=8:noet:ft=help:norl: diff --git a/runtime/doc/vim_diff.txt b/runtime/doc/vim_diff.txt index 026ff6a0fb..22e6f11bb5 100644 --- a/runtime/doc/vim_diff.txt +++ b/runtime/doc/vim_diff.txt @@ -167,12 +167,13 @@ Highlight groups: |hl-Whitespace| highlights 'listchars' whitespace UI: - *E5408* *E5409* *g:Nvim_color_expr* *g:Nvim_color_cmdline* - Command-line coloring is supported. Only |input()| and |inputdialog()| may - be colored. For testing purposes expressions (e.g. |i_CTRL-R_=|) and regular - command-line (|:|) are colored by callbacks defined in `g:Nvim_color_expr` - and `g:Nvim_color_cmdline` respectively (these callbacks are for testing - only, and will be removed in a future version). + *E5408* *E5409* *g:Nvim_color_cmdline* + Command-line coloring is supported. Only |input()| and |inputdialog()| may + be colored by user. For testing purposes regular command-line (|:|) is + colored by callback defined in `g:Nvim_color_cmdline` (this callback is for + testing only, and will be removed in a future version). Additionally + expression command-line is colored using built-in expressions parser (it is + not yet used for actually parsing expressions though), see |expr-coloring|. ============================================================================== 4. Changed features *nvim-features-changed* From 05a3c12118a6dae0ac8f3603f9ee4d9fd9450cce Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 19 Nov 2017 23:36:40 +0300 Subject: [PATCH 099/109] unittests: Run vim_str2nr tests with GC enabled --- test/unit/charset/vim_str2nr_spec.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unit/charset/vim_str2nr_spec.lua b/test/unit/charset/vim_str2nr_spec.lua index 1a0a000abb..394934c339 100644 --- a/test/unit/charset/vim_str2nr_spec.lua +++ b/test/unit/charset/vim_str2nr_spec.lua @@ -56,6 +56,12 @@ local function test_vim_str2nr(s, what, exp, maxlen) end end +local _itp = itp +itp = function(...) + collectgarbage('restart') + _itp(...) +end + describe('vim_str2nr()', function() itp('works fine when it has nothing to do', function() test_vim_str2nr('', 0, {len = 0, num = 0, unum = 0, pre = 0}, 0) From 11a05e778fd5f33f791ca8047f9839caa15b8da5 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 26 Nov 2017 15:56:27 +0300 Subject: [PATCH 100/109] doc: Some small fixes --- runtime/doc/eval.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 2e04c1d96e..46a7d92730 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -10578,13 +10578,13 @@ This is not allowed when the textlock is active: Expressions entered by user in |i_CTRL-R_=|, |c_CTRL-\_e|, |quote=| are colored by built-in expressions parser. It uses highlight groups described in -the below table, which may be overriden by user colorschemes, all linked to +the table below, which may be overriden by user colorschemes, all linked to some other highlighting group. *hl-NVimInvalid* In addition to highlighting groups prefixed with NVim described below there are highlighting groups prefixed with NVimInvalid which have just the same -meaning, but used to indicate that relevant token contains an error or that -error had occurred just before it. They have mostly the same hierarchy, +meaning, but used to indicate that the relevant token contains an error or +that error had occurred just before it. They have mostly the same hierarchy, except that by default in place of any non-NVim-prefixed group NVimInvalid linking to `Error` is used and some other intermediate groups are present. From 17077b68133a62d0dc1b84cb48779464c117e028 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 26 Nov 2017 16:08:53 +0300 Subject: [PATCH 101/109] viml/parser/expressions: Make $ENV not depend on &isident --- src/nvim/api/vim.c | 2 +- src/nvim/viml/parser/expressions.c | 19 +++++++++++-------- test/functional/api/vim_spec.lua | 3 +++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 7838f9faa3..a0816dbc8e 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -983,7 +983,7 @@ typedef kvec_withinit_t(ExprASTConvStackItem, 16) ExprASTConvStack; /// "DoubleQuotedString" nodes. Dictionary nvim_parse_expression(String expr, String flags, Boolean highlight, Error *err) - FUNC_API_SINCE(4) + FUNC_API_SINCE(4) FUNC_API_ASYNC { int pflags = 0; for (size_t i = 0 ; i < flags.size ; i++) { diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 6c7c328b6d..63ad6bab35 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -47,6 +47,8 @@ // type of what is in the first expression is generally not known when // parsing, so to have separate expressions like this separate them with // spaces. +// 7. 'isident' no longer applies to environment variables, they always include +// ASCII alphanumeric characters and underscore and nothing except this. #include #include @@ -383,10 +385,14 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) break; } +#define ISWORD_OR_AUTOLOAD(x) \ + (ASCII_ISALNUM(x) || (x) == AUTOLOAD_CHAR || (x) == '_') +#define ISWORD(x) \ + (ASCII_ISALNUM(x) || (x) == '_') + // Environment variable. case '$': { - // FIXME: Parser function can’t be thread-safe with vim_isIDc. - CHARREG(kExprLexEnv, vim_isIDc); + CHARREG(kExprLexEnv, ISWORD); break; } @@ -400,10 +406,6 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': { -#define ISWORD_OR_AUTOLOAD(x) \ - (ASCII_ISALNUM(x) || (x) == AUTOLOAD_CHAR || (x) == '_') -#define ISWORD(x) \ - (ASCII_ISALNUM(x) || (x) == '_') ret.data.var.scope = 0; ret.data.var.autoload = false; CHARREG(kExprLexPlainIdentifier, ISWORD); @@ -441,9 +443,10 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) CHARREG(kExprLexPlainIdentifier, ISWORD_OR_AUTOLOAD); } break; -#undef ISWORD_OR_AUTOLOAD -#undef ISWORD } + +#undef ISWORD +#undef ISWORD_OR_AUTOLOAD #undef CHARREG // Option. diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index 5b5340d9e2..841b7a584a 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -717,6 +717,9 @@ describe('api', function() end) describe('nvim_parse_expression', function() + before_each(function() + meths.set_option('isident', '') + end) local function simplify_east_api_node(line, east_api_node) if east_api_node == NIL then return nil From cddf84c3982b8225f1592b6a61b63f8d1883ca94 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 26 Nov 2017 16:45:29 +0300 Subject: [PATCH 102/109] functests: Add some more tests --- src/nvim/syntax.c | 16 ++-- test/functional/ui/cmdline_highlight_spec.lua | 87 +++++++++++++++++- test/unit/viml/expressions/parser_tests.lua | 88 +++++++++++++++++++ 3 files changed, 179 insertions(+), 12 deletions(-) diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index 8e5a119b1f..55ff49d160 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -6170,18 +6170,18 @@ const char *const highlight_init_cmdline[] = { "default link NVimInvalidDict NVimInvalidContainer", "default link NVimInvalidList NVimInvalidContainer", - "default link NVimInvalidIdentifier Identifier", - "default link NVimInvalidIdentifierScope NVimIdentifier", - "default link NVimInvalidIdentifierScopeDelimiter NVimIdentifier", - "default link NVimInvalidIdentifierName NVimIdentifier", - "default link NVimInvalidIdentifierKey NVimIdentifier", + "default link NVimInvalidValue NVimInvalid", + + "default link NVimInvalidIdentifier NVimInvalidValue", + "default link NVimInvalidIdentifierScope NVimInvalidIdentifier", + "default link NVimInvalidIdentifierScopeDelimiter NVimInvalidIdentifier", + "default link NVimInvalidIdentifierName NVimInvalidIdentifier", + "default link NVimInvalidIdentifierKey NVimInvalidIdentifier", "default link NVimInvalidColon NVimInvalidDelimiter", "default link NVimInvalidComma NVimInvalidDelimiter", "default link NVimInvalidArrow NVimInvalidDelimiter", - "default link NVimInvalidValue NVimInvalid", - "default link NVimInvalidRegister NVimInvalidValue", "default link NVimInvalidNumber NVimInvalidValue", "default link NVimInvalidFloat NVimInvalidNumber", @@ -6199,7 +6199,7 @@ const char *const highlight_init_cmdline[] = { // Invalid string bodies and specials are still highlighted as valid ones to // minimize the red area. "default link NVimInvalidString NVimInvalidValue", - "default link NVimInvalidStringBody NVimString", + "default link NVimInvalidStringBody NVimStringBody", "default link NVimInvalidStringQuote NVimInvalidString", "default link NVimInvalidStringSpecial NVimStringSpecial", diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua index 023673738d..54d27723f0 100644 --- a/test/functional/ui/cmdline_highlight_spec.lua +++ b/test/functional/ui/cmdline_highlight_spec.lua @@ -147,6 +147,10 @@ before_each(function() PE={bold = true, foreground = Screen.colors.SeaGreen4}, NUM={foreground = Screen.colors.Blue2}, NPAR={foreground = Screen.colors.Yellow}, + SQ={foreground = Screen.colors.Blue3}, + SB={foreground = Screen.colors.Blue4}, + E={foreground = Screen.colors.Red, background = Screen.colors.Blue}, + M={bold = true}, }) end) @@ -898,8 +902,83 @@ describe('Expressions coloring support', function() ={NUM:1}^ | ]]) end) - -- FIXME: Test expr coloring when using -u NORC and -u NONE. - -- FIXME: Test different ways of triggering expression highlighting (:=, - -- i=, :e, "=). - -- FIXME: Test with various invalid unicode and multibyte characters. + it('works correctly with non-ASCII and control characters', function() + meths.command('hi clear NVimStringBody') + meths.command('hi clear NVimStringQuote') + meths.command('hi clear NVimInvalid') + meths.command('hi NVimStringQuote guifg=Blue3') + meths.command('hi NVimStringBody guifg=Blue4') + meths.command('hi NVimInvalid guifg=Red guibg=Blue') + feed('i="«»"«»') + screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + ={SQ:"}{SB:«»}{SQ:"}{E:«»}^ | + ]]) + feed('') + screen:expect([[ + ^ | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {M:-- INSERT --} | + ]]) + feed('') + screen:expect([[ + ^ | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + | + ]]) + feed(':e""') + -- TODO(ZyX-I): Parser highlighting should not override special character + -- highlighting. + screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + ={SQ:"}{SB:^X}{SQ:"}{ERR:^X}^ | + ]]) + feed('') + screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + :^ | + ]]) + funcs.setreg('a', {'\192'}) + feed('="a"a"foo"') + -- TODO(ZyX-I): Parser highlighting should not override special character + -- highlighting. + screen:expect([[ + | + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + {EOB:~ }| + ={SQ:"}{SB:}{SQ:"}{E:"}{SB:foo}{E:"}^ | + ]]) + end) end) diff --git a/test/unit/viml/expressions/parser_tests.lua b/test/unit/viml/expressions/parser_tests.lua index f0c2723a38..4700b6ee42 100644 --- a/test/unit/viml/expressions/parser_tests.lua +++ b/test/unit/viml/expressions/parser_tests.lua @@ -8182,4 +8182,92 @@ return function(itp, _check_parsing, hl, fmtn) hl('List', ']'), }) end) + itp('works with non-ASCII characters', function() + check_parsing('"«»"«»', { + -- 013568 + ast = { + { + 'OpMissing:0:6:', + children = { + 'DoubleQuotedString(val="«»"):0:0:"«»"', + { + 'ComplexIdentifier:0:8:', + children = { + 'PlainIdentifier(scope=0,ident=«):0:6:«', + 'PlainIdentifier(scope=0,ident=»):0:8:»', + }, + }, + }, + }, + }, + err = { + arg = '«»', + msg = 'E15: Unidentified character: %.*s', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', '«»'), + hl('DoubleQuote', '"'), + hl('InvalidIdentifierName', '«'), + hl('InvalidIdentifierName', '»'), + }, { + [1] = { + ast = { + ast = { + 'DoubleQuotedString(val="«»"):0:0:"«»"', + }, + len = 6, + }, + hl_fs = { + [5] = REMOVE_THIS, + [4] = REMOVE_THIS, + }, + }, + }) + check_parsing('"\192"\192"foo"', { + -- 01 23 45678 + ast = { + { + 'OpMissing:0:3:', + children = { + 'DoubleQuotedString(val="\192"):0:0:"\192"', + { + 'OpMissing:0:4:', + children = { + 'PlainIdentifier(scope=0,ident=\192):0:3:\192', + 'DoubleQuotedString(val="foo"):0:4:"foo"', + }, + }, + }, + }, + }, + err = { + arg = '\192"foo"', + msg = 'E15: Unidentified character: %.*s', + }, + }, { + hl('DoubleQuote', '"'), + hl('DoubleQuotedBody', '\192'), + hl('DoubleQuote', '"'), + hl('InvalidIdentifierName', '\192'), + hl('InvalidDoubleQuote', '"'), + hl('InvalidDoubleQuotedBody', 'foo'), + hl('InvalidDoubleQuote', '"'), + }, { + [1] = { + ast = { + ast = { + 'DoubleQuotedString(val="\192"):0:0:"\192"', + }, + len = 3, + }, + hl_fs = { + [4] = REMOVE_THIS, + [5] = REMOVE_THIS, + [6] = REMOVE_THIS, + [7] = REMOVE_THIS, + }, + }, + }) + end) end From 36a4f3a259ffa282129b18358cce4130397077c5 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 26 Nov 2017 16:57:42 +0300 Subject: [PATCH 103/109] viml/parser/expressions: Make sure that listed nodes may be present With the new test leaving `assert(false);` for any of the cases makes tests crash. --- src/nvim/viml/parser/expressions.c | 5 +-- test/unit/viml/expressions/parser_tests.lua | 44 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 63ad6bab35..9773e60bbd 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -3065,12 +3065,9 @@ viml_pexpr_parse_end: // to be caught later. break; } + case kExprNodeSubscript: case kExprNodeConcatOrSubscript: case kExprNodeComplexIdentifier: - case kExprNodeSubscript: { - // FIXME: Investigate whether above are OK to be present in the stack. - break; - } case kExprNodeAssignment: case kExprNodeMod: case kExprNodeDivision: diff --git a/test/unit/viml/expressions/parser_tests.lua b/test/unit/viml/expressions/parser_tests.lua index 4700b6ee42..e085d7e932 100644 --- a/test/unit/viml/expressions/parser_tests.lua +++ b/test/unit/viml/expressions/parser_tests.lua @@ -4228,6 +4228,50 @@ return function(itp, _check_parsing, hl, fmtn) hl('ConcatOrSubscript', '.'), hl('Number', '1', 1), }) + + check_parsing('a[1][2][3[4', { + -- 01234567890 + -- 0 1 + ast = { + { + 'Subscript:0:7:[', + children = { + { + 'Subscript:0:4:[', + children = { + { + 'Subscript:0:1:[', + children = { + 'PlainIdentifier(scope=0,ident=a):0:0:a', + 'Integer(val=1):0:2:1', + }, + }, + 'Integer(val=2):0:5:2', + }, + }, + { + 'Subscript:0:9:[', + children = { + 'Integer(val=3):0:8:3', + 'Integer(val=4):0:10:4', + }, + }, + }, + }, + }, + }, { + hl('IdentifierName', 'a'), + hl('SubscriptBracket', '['), + hl('Number', '1'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('Number', '2'), + hl('SubscriptBracket', ']'), + hl('SubscriptBracket', '['), + hl('Number', '3'), + hl('SubscriptBracket', '['), + hl('Number', '4'), + }) end) itp('works with bracket subscripts', function() check_parsing(':', { From de45ec0146486c49719ff6f6dcceb4914b471c7a Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 30 Nov 2017 02:01:49 +0300 Subject: [PATCH 104/109] keymap: Do not use vim_isIDc in keymap.c Note: there are three changes to ascii_isident. Reverting first two (in find_special_key and first in get_special_key_code) normally fails the new test with empty &isident, but reverting the third does not. Hence adding `>` to &isident. Ref vim/vim#2389. --- src/nvim/ascii.h | 13 +++++++++++++ src/nvim/keymap.c | 6 +++--- src/nvim/viml/parser/expressions.c | 9 +++------ test/functional/ex_cmds/map_spec.lua | 21 +++++++++++++++++++++ test/symbolic/klee/nvim/charset.c | 5 ----- test/symbolic/klee/nvim/keymap.c | 7 ++++--- 6 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 test/functional/ex_cmds/map_spec.lua diff --git a/src/nvim/ascii.h b/src/nvim/ascii.h index adde91f9ec..9ccb70764d 100644 --- a/src/nvim/ascii.h +++ b/src/nvim/ascii.h @@ -3,6 +3,7 @@ #include +#include "nvim/macros.h" #include "nvim/func_attr.h" #include "nvim/os/os_defs.h" @@ -98,6 +99,10 @@ static inline bool ascii_isxdigit(int) REAL_FATTR_CONST REAL_FATTR_ALWAYS_INLINE; +static inline bool ascii_isident(int) + REAL_FATTR_CONST + REAL_FATTR_ALWAYS_INLINE; + static inline bool ascii_isbdigit(int) REAL_FATTR_CONST REAL_FATTR_ALWAYS_INLINE; @@ -138,6 +143,14 @@ static inline bool ascii_isxdigit(int c) || (c >= 'A' && c <= 'F'); } +/// Checks if `c` is an “identifier” character +/// +/// That is, whether it is alphanumeric character or underscore. +static inline bool ascii_isident(const int c) +{ + return ASCII_ISALNUM(c) || c == '_'; +} + /// Checks if `c` is a binary digit, that is, 0-1. /// /// @see {ascii_isdigit} diff --git a/src/nvim/keymap.c b/src/nvim/keymap.c index aca21c20a5..0eb5a41b1e 100644 --- a/src/nvim/keymap.c +++ b/src/nvim/keymap.c @@ -567,7 +567,7 @@ int find_special_key(const char_u **srcp, const size_t src_len, int *const modp, // Find end of modifier list last_dash = src; - for (bp = src + 1; bp <= end && (*bp == '-' || vim_isIDc(*bp)); bp++) { + for (bp = src + 1; bp <= end && (*bp == '-' || ascii_isident(*bp)); bp++) { if (*bp == '-') { last_dash = bp; if (bp + 1 <= end) { @@ -721,12 +721,12 @@ int get_special_key_code(const char_u *name) for (int i = 0; key_names_table[i].name != NULL; i++) { const char *const table_name = key_names_table[i].name; int j; - for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) { + for (j = 0; ascii_isident(name[j]) && table_name[j] != NUL; j++) { if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { break; } } - if (!vim_isIDc(name[j]) && table_name[j] == NUL) { + if (!ascii_isident(name[j]) && table_name[j] == NUL) { return key_names_table[i].key; } } diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index 9773e60bbd..cfcde6bb38 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -386,13 +386,11 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) } #define ISWORD_OR_AUTOLOAD(x) \ - (ASCII_ISALNUM(x) || (x) == AUTOLOAD_CHAR || (x) == '_') -#define ISWORD(x) \ - (ASCII_ISALNUM(x) || (x) == '_') + (ascii_isident(x) || (x) == AUTOLOAD_CHAR) // Environment variable. case '$': { - CHARREG(kExprLexEnv, ISWORD); + CHARREG(kExprLexEnv, ascii_isident); break; } @@ -408,7 +406,7 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) case '_': { ret.data.var.scope = 0; ret.data.var.autoload = false; - CHARREG(kExprLexPlainIdentifier, ISWORD); + CHARREG(kExprLexPlainIdentifier, ascii_isident); // "is" and "isnot" operators. if (!(flags & kELFlagIsNotCmp) && ((ret.len == 2 && memcmp(pline.data, "is", 2) == 0) @@ -445,7 +443,6 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags) break; } -#undef ISWORD #undef ISWORD_OR_AUTOLOAD #undef CHARREG diff --git a/test/functional/ex_cmds/map_spec.lua b/test/functional/ex_cmds/map_spec.lua new file mode 100644 index 0000000000..b46f83405e --- /dev/null +++ b/test/functional/ex_cmds/map_spec.lua @@ -0,0 +1,21 @@ +local helpers = require("test.functional.helpers")(after_each) + +local eq = helpers.eq +local feed = helpers.feed +local meths = helpers.meths +local clear = helpers.clear +local command = helpers.command + +describe(':*map', function() + before_each(clear) + + it('are not affected by &isident', function() + meths.set_var('counter', 0) + command('nnoremap :let counter+=1') + meths.set_option('isident', ('%u'):format(('>'):byte())) + command('nnoremap :let counter+=1') + -- &isident used to disable keycode parsing here as well + feed('\24\25') + eq(4, meths.get_var('counter')) + end) +end) diff --git a/test/symbolic/klee/nvim/charset.c b/test/symbolic/klee/nvim/charset.c index 77f690f08d..95853a6834 100644 --- a/test/symbolic/klee/nvim/charset.c +++ b/test/symbolic/klee/nvim/charset.c @@ -6,11 +6,6 @@ #include "nvim/eval/typval.h" #include "nvim/vim.h" -bool vim_isIDc(int c) -{ - return ASCII_ISALNUM(c); -} - int hex2nr(int c) { if ((c >= 'a') && (c <= 'f')) { diff --git a/test/symbolic/klee/nvim/keymap.c b/test/symbolic/klee/nvim/keymap.c index ff5e46e75b..a341a73689 100644 --- a/test/symbolic/klee/nvim/keymap.c +++ b/test/symbolic/klee/nvim/keymap.c @@ -2,6 +2,7 @@ #include "nvim/types.h" #include "nvim/keymap.h" +#include "nvim/ascii.h" #include "nvim/eval/typval.h" #define MOD_KEYS_ENTRY_SIZE 5 @@ -294,12 +295,12 @@ int get_special_key_code(const char_u *name) for (int i = 0; key_names_table[i].name != NULL; i++) { const char *const table_name = key_names_table[i].name; int j; - for (j = 0; vim_isIDc(name[j]) && table_name[j] != NUL; j++) { + for (j = 0; ascii_isident(name[j]) && table_name[j] != NUL; j++) { if (TOLOWER_ASC(table_name[j]) != TOLOWER_ASC(name[j])) { break; } } - if (!vim_isIDc(name[j]) && table_name[j] == NUL) { + if (!ascii_isident(name[j]) && table_name[j] == NUL) { return key_names_table[i].key; } } @@ -386,7 +387,7 @@ int find_special_key(const char_u **srcp, const size_t src_len, int *const modp, // Find end of modifier list last_dash = src; - for (bp = src + 1; bp <= end && (*bp == '-' || vim_isIDc(*bp)); bp++) { + for (bp = src + 1; bp <= end && (*bp == '-' || ascii_isident(*bp)); bp++) { if (*bp == '-') { last_dash = bp; if (bp + 1 <= end) { From 0b4054e043257137ccfd3b2207da48910ce32a5f Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 30 Nov 2017 11:44:48 +0300 Subject: [PATCH 105/109] unittests: Reduce memory used by vim_str2nr test --- test/README.md | 3 +++ test/unit/charset/vim_str2nr_spec.lua | 32 ++++++++++++++++++++++----- test/unit/helpers.lua | 2 ++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/test/README.md b/test/README.md index 6ad70e2f45..1cb814d85b 100644 --- a/test/README.md +++ b/test/README.md @@ -344,3 +344,6 @@ function calls, returns and lua source lines exuecuted. `NVIM_TEST_TRACE_ON_ERROR` (U) (1): makes unit tests yield trace on error in addition to regular error message. + +`NVIM_TEST_MAXTRACE` (U) (N): specifies maximum number of trace lines to keep. +Default is 1024. diff --git a/test/unit/charset/vim_str2nr_spec.lua b/test/unit/charset/vim_str2nr_spec.lua index 394934c339..a6fe613563 100644 --- a/test/unit/charset/vim_str2nr_spec.lua +++ b/test/unit/charset/vim_str2nr_spec.lua @@ -1,5 +1,6 @@ local helpers = require("test.unit.helpers")(after_each) local global_helpers = require('test.helpers') +local bit = require('bit') local itp = helpers.gen_itp(it) @@ -36,15 +37,36 @@ local function arginit(arg) end end +local function argreset(arg, args) + if arg == 'unum' then + ucnt = ucnt + 1 + args[arg][0] = ucnt + else + icnt = icnt - 1 + args[arg][0] = icnt + end +end + local function test_vim_str2nr(s, what, exp, maxlen) - local comb = {[''] = {}} + local comb = {[{}] = true} + local bits = {} for k, _ in pairs(exp) do - for ck, cv in pairs(comb) do - comb[ck .. ',' .. k] = updated(shallowcopy(cv), { [k] = arginit(k) }) - end + bits[#bits + 1] = k end maxlen = maxlen or #s - for _, cv in pairs(comb) do + local args = {} + for k, _ in pairs(ARGTYPES) do + args[k] = arginit(k) + end + for case = 0, ((2 ^ (#bits)) - 1) do + local cv = {} + for b = 0, (#bits - 1) do + if bit.band(case, (2 ^ b)) == 0 then + local k = bits[b + 1] + argreset(k, args) + cv[k] = args[k] + end + end lib.vim_str2nr(s, cv.pre, cv.len, what, cv.num, cv.unum, maxlen) for cck, ccv in pairs(cv) do if exp[cck] ~= tonumber(ccv[0]) then diff --git a/test/unit/helpers.lua b/test/unit/helpers.lua index c26b1c1bcb..b1e709c444 100644 --- a/test/unit/helpers.lua +++ b/test/unit/helpers.lua @@ -675,6 +675,7 @@ end local function check_child_err(rd) local trace = {} local did_traceline = false + local maxtrace = tonumber(os.getenv('NVIM_TEST_MAXTRACE')) or 1024 while true do local traceline = sc.read(rd, hook_msglen) if #traceline ~= hook_msglen then @@ -689,6 +690,7 @@ local function check_child_err(rd) break end trace[#trace + 1] = traceline + table.remove(trace, maxtrace + 1) end local res = sc.read(rd, 2) if #res ~= 2 then From 5ab0f988caffad5e8c87a075cbd3f91f0f7e002c Mon Sep 17 00:00:00 2001 From: ZyX Date: Thu, 30 Nov 2017 11:53:25 +0300 Subject: [PATCH 106/109] *: Replace all occurrences of NVim with Nvim --- runtime/doc/eval.txt | 140 ++++----- src/nvim/syntax.c | 294 +++++++++--------- src/nvim/viml/parser/expressions.c | 4 +- test/functional/api/vim_spec.lua | 2 +- test/functional/ui/cmdline_highlight_spec.lua | 24 +- test/unit/viml/expressions/parser_spec.lua | 16 +- 6 files changed, 240 insertions(+), 240 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 46a7d92730..2b2fda25e9 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -10580,116 +10580,116 @@ Expressions entered by user in |i_CTRL-R_=|, |c_CTRL-\_e|, |quote=| are colored by built-in expressions parser. It uses highlight groups described in the table below, which may be overriden by user colorschemes, all linked to some other highlighting group. - *hl-NVimInvalid* -In addition to highlighting groups prefixed with NVim described below there -are highlighting groups prefixed with NVimInvalid which have just the same + *hl-NvimInvalid* +In addition to highlighting groups prefixed with Nvim described below there +are highlighting groups prefixed with NvimInvalid which have just the same meaning, but used to indicate that the relevant token contains an error or that error had occurred just before it. They have mostly the same hierarchy, -except that by default in place of any non-NVim-prefixed group NVimInvalid +except that by default in place of any non-Nvim-prefixed group NvimInvalid linking to `Error` is used and some other intermediate groups are present. Group Default link Colored expression ~ -*hl-NVimInternalError* None, red/red Parser bug +*hl-NvimInternalError* None, red/red Parser bug -*hl-NVimAssignment* Operator Generic assignment -*hl-NVimPlainAssignment* NVimAssignment `=` in |:let| -*hl-NVimAugmentedAssignment* NVimAssignment Generic, `+=`/`-=`/`.=` -*hl-NVimAssignmentWithAddition* NVimAugmentedAssignment `+=` in |:let+=| -*hl-NVimAssignmentWithSubtraction* NVimAugmentedAssignment `-=` in |:let-=| -*hl-NVimAssignmentWithConcatenation* NVimAugmentedAssignment `.=` in |:let.=| +*hl-NvimAssignment* Operator Generic assignment +*hl-NvimPlainAssignment* NvimAssignment `=` in |:let| +*hl-NvimAugmentedAssignment* NvimAssignment Generic, `+=`/`-=`/`.=` +*hl-NvimAssignmentWithAddition* NvimAugmentedAssignment `+=` in |:let+=| +*hl-NvimAssignmentWithSubtraction* NvimAugmentedAssignment `-=` in |:let-=| +*hl-NvimAssignmentWithConcatenation* NvimAugmentedAssignment `.=` in |:let.=| -*hl-NVimOperator* Operator Generic operator +*hl-NvimOperator* Operator Generic operator -*hl-NVimUnaryOperator* NVimOperator Generic unary op -*hl-NVimUnaryPlus* NVimUnaryOperator |expr-unary-+| -*hl-NVimUnaryMinus* NVimUnaryOperator |expr-unary--| -*hl-NVimNot* NVimUnaryOperator |expr-!| +*hl-NvimUnaryOperator* NvimOperator Generic unary op +*hl-NvimUnaryPlus* NvimUnaryOperator |expr-unary-+| +*hl-NvimUnaryMinus* NvimUnaryOperator |expr-unary--| +*hl-NvimNot* NvimUnaryOperator |expr-!| -*hl-NVimBinaryOperator* NVimOperator Generic binary op -*hl-NVimComparison* NVimBinaryOperator Any |expr4| operator -*hl-NVimComparisonModifier* NVimComparison `#`/`?` near |expr4| op -*hl-NVimBinaryPlus* NVimBinaryOperator |expr-+| -*hl-NVimBinaryMinus* NVimBinaryOperator |expr--| -*hl-NVimConcat* NVimBinaryOperator |expr-.| -*hl-NVimConcatOrSubscript* NVimConcat |expr-.| or |expr-entry| -*hl-NVimOr* NVimBinaryOperator |expr-barbar| -*hl-NVimAnd* NVimBinaryOperator |expr-&&| -*hl-NVimMultiplication* NVimBinaryOperator |expr-star| -*hl-NVimDivision* NVimBinaryOperator |expr-/| -*hl-NVimMod* NVimBinaryOperator |expr-%| +*hl-NvimBinaryOperator* NvimOperator Generic binary op +*hl-NvimComparison* NvimBinaryOperator Any |expr4| operator +*hl-NvimComparisonModifier* NvimComparison `#`/`?` near |expr4| op +*hl-NvimBinaryPlus* NvimBinaryOperator |expr-+| +*hl-NvimBinaryMinus* NvimBinaryOperator |expr--| +*hl-NvimConcat* NvimBinaryOperator |expr-.| +*hl-NvimConcatOrSubscript* NvimConcat |expr-.| or |expr-entry| +*hl-NvimOr* NvimBinaryOperator |expr-barbar| +*hl-NvimAnd* NvimBinaryOperator |expr-&&| +*hl-NvimMultiplication* NvimBinaryOperator |expr-star| +*hl-NvimDivision* NvimBinaryOperator |expr-/| +*hl-NvimMod* NvimBinaryOperator |expr-%| -*hl-NVimTernary* NVimOperator `?` in |expr1| -*hl-NVimTernaryColon* NVimTernary `:` in |expr1| +*hl-NvimTernary* NvimOperator `?` in |expr1| +*hl-NvimTernaryColon* NvimTernary `:` in |expr1| -*hl-NVimParenthesis* Delimiter Generic bracket -*hl-NVimLambda* NVimParenthesis `{`/`}` in |lambda| -*hl-NVimNestingParenthesis* NVimParenthesis `(`/`)` in |expr-nesting| -*hl-NVimCallingParenthesis* NVimParenthesis `(`/`)` in |expr-function| +*hl-NvimParenthesis* Delimiter Generic bracket +*hl-NvimLambda* NvimParenthesis `{`/`}` in |lambda| +*hl-NvimNestingParenthesis* NvimParenthesis `(`/`)` in |expr-nesting| +*hl-NvimCallingParenthesis* NvimParenthesis `(`/`)` in |expr-function| -*hl-NVimSubscript* NVimParenthesis Generic subscript -*hl-NVimSubscriptBracket* NVimSubscript `[`/`]` in |expr-[]| -*hl-NVimSubscriptColon* NVimSubscript `:` in |expr-[:]| -*hl-NVimCurly* NVimSubscript `{`/`}` in +*hl-NvimSubscript* NvimParenthesis Generic subscript +*hl-NvimSubscriptBracket* NvimSubscript `[`/`]` in |expr-[]| +*hl-NvimSubscriptColon* NvimSubscript `:` in |expr-[:]| +*hl-NvimCurly* NvimSubscript `{`/`}` in |curly-braces-names| -*hl-NVimContainer* NVimParenthesis Generic container -*hl-NVimDict* NVimContainer `{`/`}` in |dict| literal -*hl-NVimList* NVimContainer `[`/`]` in |list| literal +*hl-NvimContainer* NvimParenthesis Generic container +*hl-NvimDict* NvimContainer `{`/`}` in |dict| literal +*hl-NvimList* NvimContainer `[`/`]` in |list| literal -*hl-NVimIdentifier* Identifier Generic identifier -*hl-NVimIdentifierScope* NVimIdentifier Namespace: letter +*hl-NvimIdentifier* Identifier Generic identifier +*hl-NvimIdentifierScope* NvimIdentifier Namespace: letter before `:` in |internal-variables| -*hl-NVimIdentifierScopeDelimiter* NVimIdentifier `:` after namespace +*hl-NvimIdentifierScopeDelimiter* NvimIdentifier `:` after namespace letter -*hl-NVimIdentifierName* NVimIdentifier Rest of the ident -*hl-NVimIdentifierKey* NVimIdentifier Identifier after +*hl-NvimIdentifierName* NvimIdentifier Rest of the ident +*hl-NvimIdentifierKey* NvimIdentifier Identifier after |expr-entry| -*hl-NVimColon* Delimiter `:` in |dict| literal -*hl-NVimComma* Delimiter `,` in |dict|/|list| +*hl-NvimColon* Delimiter `:` in |dict| literal +*hl-NvimComma* Delimiter `,` in |dict|/|list| literal or |expr-function| -*hl-NVimArrow* Delimiter `->` in |lambda| +*hl-NvimArrow* Delimiter `->` in |lambda| -*hl-NVimRegister* SpecialChar |expr-register| -*hl-NVimNumber* Number Non-prefix digits +*hl-NvimRegister* SpecialChar |expr-register| +*hl-NvimNumber* Number Non-prefix digits in integer |expr-number| -*hl-NVimNumberPrefix* Type `0` for |octal-number| +*hl-NvimNumberPrefix* Type `0` for |octal-number| `0x` for |hex-number| `0b` for |binary-number| -*hl-NVimFloat* NVimNumber Floating-point +*hl-NvimFloat* NvimNumber Floating-point number -*hl-NVimOptionSigil* Type `&` in |expr-option| -*hl-NVimOptionScope* NVimIdentifierScope Option scope if any -*hl-NVimOptionScopeDelimiter* NVimIdentifierScopeDelimiter +*hl-NvimOptionSigil* Type `&` in |expr-option| +*hl-NvimOptionScope* NvimIdentifierScope Option scope if any +*hl-NvimOptionScopeDelimiter* NvimIdentifierScopeDelimiter `:` after option scope -*hl-NVimOptionName* NVimIdentifier Option name +*hl-NvimOptionName* NvimIdentifier Option name -*hl-NVimEnvironmentSigil* NVimOptionSigil `$` in |expr-env| -*hl-NVimEnvironmentName* NVimIdentifier Env variable name +*hl-NvimEnvironmentSigil* NvimOptionSigil `$` in |expr-env| +*hl-NvimEnvironmentName* NvimIdentifier Env variable name -*hl-NVimString* String Generic string -*hl-NVimStringBody* NVimString Generic string +*hl-NvimString* String Generic string +*hl-NvimStringBody* NvimString Generic string literal body -*hl-NVimStringQuote* NVimString Generic string quote -*hl-NVimStringSpecial* SpecialChar Generic string +*hl-NvimStringQuote* NvimString Generic string quote +*hl-NvimStringSpecial* SpecialChar Generic string non-literal body -*hl-NVimSingleQuote* NVimStringQuote `'` in |expr-'| -*hl-NVimSingleQuotedBody* NVimStringBody Literal part of +*hl-NvimSingleQuote* NvimStringQuote `'` in |expr-'| +*hl-NvimSingleQuotedBody* NvimStringBody Literal part of |expr-'| string body -*hl-NVimSingleQuotedQuote* NVimStringSpecial `''` inside |expr-'| +*hl-NvimSingleQuotedQuote* NvimStringSpecial `''` inside |expr-'| string body -*hl-NVimDoubleQuote* NVimStringQuote `"` in |expr-quote| -*hl-NVimDoubleQuotedBody* NVimStringBody Literal part of +*hl-NvimDoubleQuote* NvimStringQuote `"` in |expr-quote| +*hl-NvimDoubleQuotedBody* NvimStringBody Literal part of |expr-quote| body -*hl-NVimDoubleQuotedEscape* NVimStringSpecial Valid |expr-quote| +*hl-NvimDoubleQuotedEscape* NvimStringSpecial Valid |expr-quote| escape sequence -*hl-NVimDoubleQuotedUnknownEscape* NVimInvalidValue Unrecognized +*hl-NvimDoubleQuotedUnknownEscape* NvimInvalidValue Unrecognized |expr-quote| escape sequence diff --git a/src/nvim/syntax.c b/src/nvim/syntax.c index 55ff49d160..d1a5f0bd1c 100644 --- a/src/nvim/syntax.c +++ b/src/nvim/syntax.c @@ -6025,204 +6025,204 @@ const char *const highlight_init_cmdline[] = { // XXX When modifying a list modify it in both valid and invalid halfs. // TODO(ZyX-I): merge valid and invalid groups via a macros. - // NVimInternalError should appear only when highlighter has a bug. - "NVimInternalError ctermfg=Red ctermbg=Red guifg=Red guibg=Red", + // NvimInternalError should appear only when highlighter has a bug. + "NvimInternalError ctermfg=Red ctermbg=Red guifg=Red guibg=Red", // Highlight groups (links) used by parser: - "default link NVimAssignment Operator", - "default link NVimPlainAssignment NVimAssignment", - "default link NVimAugmentedAssignment NVimAssignment", - "default link NVimAssignmentWithAddition NVimAugmentedAssignment", - "default link NVimAssignmentWithSubtraction NVimAugmentedAssignment", - "default link NVimAssignmentWithConcatenation NVimAugmentedAssignment", + "default link NvimAssignment Operator", + "default link NvimPlainAssignment NvimAssignment", + "default link NvimAugmentedAssignment NvimAssignment", + "default link NvimAssignmentWithAddition NvimAugmentedAssignment", + "default link NvimAssignmentWithSubtraction NvimAugmentedAssignment", + "default link NvimAssignmentWithConcatenation NvimAugmentedAssignment", - "default link NVimOperator Operator", + "default link NvimOperator Operator", - "default link NVimUnaryOperator NVimOperator", - "default link NVimUnaryPlus NVimUnaryOperator", - "default link NVimUnaryMinus NVimUnaryOperator", - "default link NVimNot NVimUnaryOperator", + "default link NvimUnaryOperator NvimOperator", + "default link NvimUnaryPlus NvimUnaryOperator", + "default link NvimUnaryMinus NvimUnaryOperator", + "default link NvimNot NvimUnaryOperator", - "default link NVimBinaryOperator NVimOperator", - "default link NVimComparison NVimBinaryOperator", - "default link NVimComparisonModifier NVimComparison", - "default link NVimBinaryPlus NVimBinaryOperator", - "default link NVimBinaryMinus NVimBinaryOperator", - "default link NVimConcat NVimBinaryOperator", - "default link NVimConcatOrSubscript NVimConcat", - "default link NVimOr NVimBinaryOperator", - "default link NVimAnd NVimBinaryOperator", - "default link NVimMultiplication NVimBinaryOperator", - "default link NVimDivision NVimBinaryOperator", - "default link NVimMod NVimBinaryOperator", + "default link NvimBinaryOperator NvimOperator", + "default link NvimComparison NvimBinaryOperator", + "default link NvimComparisonModifier NvimComparison", + "default link NvimBinaryPlus NvimBinaryOperator", + "default link NvimBinaryMinus NvimBinaryOperator", + "default link NvimConcat NvimBinaryOperator", + "default link NvimConcatOrSubscript NvimConcat", + "default link NvimOr NvimBinaryOperator", + "default link NvimAnd NvimBinaryOperator", + "default link NvimMultiplication NvimBinaryOperator", + "default link NvimDivision NvimBinaryOperator", + "default link NvimMod NvimBinaryOperator", - "default link NVimTernary NVimOperator", - "default link NVimTernaryColon NVimTernary", + "default link NvimTernary NvimOperator", + "default link NvimTernaryColon NvimTernary", - "default link NVimParenthesis Delimiter", - "default link NVimLambda NVimParenthesis", - "default link NVimNestingParenthesis NVimParenthesis", - "default link NVimCallingParenthesis NVimParenthesis", + "default link NvimParenthesis Delimiter", + "default link NvimLambda NvimParenthesis", + "default link NvimNestingParenthesis NvimParenthesis", + "default link NvimCallingParenthesis NvimParenthesis", - "default link NVimSubscript NVimParenthesis", - "default link NVimSubscriptBracket NVimSubscript", - "default link NVimSubscriptColon NVimSubscript", - "default link NVimCurly NVimSubscript", + "default link NvimSubscript NvimParenthesis", + "default link NvimSubscriptBracket NvimSubscript", + "default link NvimSubscriptColon NvimSubscript", + "default link NvimCurly NvimSubscript", - "default link NVimContainer NVimParenthesis", - "default link NVimDict NVimContainer", - "default link NVimList NVimContainer", + "default link NvimContainer NvimParenthesis", + "default link NvimDict NvimContainer", + "default link NvimList NvimContainer", - "default link NVimIdentifier Identifier", - "default link NVimIdentifierScope NVimIdentifier", - "default link NVimIdentifierScopeDelimiter NVimIdentifier", - "default link NVimIdentifierName NVimIdentifier", - "default link NVimIdentifierKey NVimIdentifier", + "default link NvimIdentifier Identifier", + "default link NvimIdentifierScope NvimIdentifier", + "default link NvimIdentifierScopeDelimiter NvimIdentifier", + "default link NvimIdentifierName NvimIdentifier", + "default link NvimIdentifierKey NvimIdentifier", - "default link NVimColon Delimiter", - "default link NVimComma Delimiter", - "default link NVimArrow Delimiter", + "default link NvimColon Delimiter", + "default link NvimComma Delimiter", + "default link NvimArrow Delimiter", - "default link NVimRegister SpecialChar", - "default link NVimNumber Number", - "default link NVimFloat NVimNumber", - "default link NVimNumberPrefix Type", + "default link NvimRegister SpecialChar", + "default link NvimNumber Number", + "default link NvimFloat NvimNumber", + "default link NvimNumberPrefix Type", - "default link NVimOptionSigil Type", - "default link NVimOptionName NVimIdentifier", - "default link NVimOptionScope NVimIdentifierScope", - "default link NVimOptionScopeDelimiter NVimIdentifierScopeDelimiter", + "default link NvimOptionSigil Type", + "default link NvimOptionName NvimIdentifier", + "default link NvimOptionScope NvimIdentifierScope", + "default link NvimOptionScopeDelimiter NvimIdentifierScopeDelimiter", - "default link NVimEnvironmentSigil NVimOptionSigil", - "default link NVimEnvironmentName NVimIdentifier", + "default link NvimEnvironmentSigil NvimOptionSigil", + "default link NvimEnvironmentName NvimIdentifier", - "default link NVimString String", - "default link NVimStringBody NVimString", - "default link NVimStringQuote NVimString", - "default link NVimStringSpecial SpecialChar", + "default link NvimString String", + "default link NvimStringBody NvimString", + "default link NvimStringQuote NvimString", + "default link NvimStringSpecial SpecialChar", - "default link NVimSingleQuote NVimStringQuote", - "default link NVimSingleQuotedBody NVimStringBody", - "default link NVimSingleQuotedQuote NVimStringSpecial", + "default link NvimSingleQuote NvimStringQuote", + "default link NvimSingleQuotedBody NvimStringBody", + "default link NvimSingleQuotedQuote NvimStringSpecial", - "default link NVimDoubleQuote NVimStringQuote", - "default link NVimDoubleQuotedBody NVimStringBody", - "default link NVimDoubleQuotedEscape NVimStringSpecial", + "default link NvimDoubleQuote NvimStringQuote", + "default link NvimDoubleQuotedBody NvimStringBody", + "default link NvimDoubleQuotedEscape NvimStringSpecial", - "default link NVimFigureBrace NVimInternalError", - "default link NVimSingleQuotedUnknownEscape NVimInternalError", + "default link NvimFigureBrace NvimInternalError", + "default link NvimSingleQuotedUnknownEscape NvimInternalError", - "default link NVimSpacing Normal", + "default link NvimSpacing Normal", - // NVimInvalid groups: + // NvimInvalid groups: - "default link NVimInvalidSingleQuotedUnknownEscape NVimInternalError", + "default link NvimInvalidSingleQuotedUnknownEscape NvimInternalError", - "default link NVimInvalid Error", + "default link NvimInvalid Error", - "default link NVimInvalidAssignment NVimInvalid", - "default link NVimInvalidPlainAssignment NVimInvalidAssignment", - "default link NVimInvalidAugmentedAssignment NVimInvalidAssignment", - "default link NVimInvalidAssignmentWithAddition " - "NVimInvalidAugmentedAssignment", - "default link NVimInvalidAssignmentWithSubtraction " - "NVimInvalidAugmentedAssignment", - "default link NVimInvalidAssignmentWithConcatenation " - "NVimInvalidAugmentedAssignment", + "default link NvimInvalidAssignment NvimInvalid", + "default link NvimInvalidPlainAssignment NvimInvalidAssignment", + "default link NvimInvalidAugmentedAssignment NvimInvalidAssignment", + "default link NvimInvalidAssignmentWithAddition " + "NvimInvalidAugmentedAssignment", + "default link NvimInvalidAssignmentWithSubtraction " + "NvimInvalidAugmentedAssignment", + "default link NvimInvalidAssignmentWithConcatenation " + "NvimInvalidAugmentedAssignment", - "default link NVimInvalidOperator NVimInvalid", + "default link NvimInvalidOperator NvimInvalid", - "default link NVimInvalidUnaryOperator NVimInvalidOperator", - "default link NVimInvalidUnaryPlus NVimInvalidUnaryOperator", - "default link NVimInvalidUnaryMinus NVimInvalidUnaryOperator", - "default link NVimInvalidNot NVimInvalidUnaryOperator", + "default link NvimInvalidUnaryOperator NvimInvalidOperator", + "default link NvimInvalidUnaryPlus NvimInvalidUnaryOperator", + "default link NvimInvalidUnaryMinus NvimInvalidUnaryOperator", + "default link NvimInvalidNot NvimInvalidUnaryOperator", - "default link NVimInvalidBinaryOperator NVimInvalidOperator", - "default link NVimInvalidComparison NVimInvalidBinaryOperator", - "default link NVimInvalidComparisonModifier NVimInvalidComparison", - "default link NVimInvalidBinaryPlus NVimInvalidBinaryOperator", - "default link NVimInvalidBinaryMinus NVimInvalidBinaryOperator", - "default link NVimInvalidConcat NVimInvalidBinaryOperator", - "default link NVimInvalidConcatOrSubscript NVimInvalidConcat", - "default link NVimInvalidOr NVimInvalidBinaryOperator", - "default link NVimInvalidAnd NVimInvalidBinaryOperator", - "default link NVimInvalidMultiplication NVimInvalidBinaryOperator", - "default link NVimInvalidDivision NVimInvalidBinaryOperator", - "default link NVimInvalidMod NVimInvalidBinaryOperator", + "default link NvimInvalidBinaryOperator NvimInvalidOperator", + "default link NvimInvalidComparison NvimInvalidBinaryOperator", + "default link NvimInvalidComparisonModifier NvimInvalidComparison", + "default link NvimInvalidBinaryPlus NvimInvalidBinaryOperator", + "default link NvimInvalidBinaryMinus NvimInvalidBinaryOperator", + "default link NvimInvalidConcat NvimInvalidBinaryOperator", + "default link NvimInvalidConcatOrSubscript NvimInvalidConcat", + "default link NvimInvalidOr NvimInvalidBinaryOperator", + "default link NvimInvalidAnd NvimInvalidBinaryOperator", + "default link NvimInvalidMultiplication NvimInvalidBinaryOperator", + "default link NvimInvalidDivision NvimInvalidBinaryOperator", + "default link NvimInvalidMod NvimInvalidBinaryOperator", - "default link NVimInvalidTernary NVimInvalidOperator", - "default link NVimInvalidTernaryColon NVimInvalidTernary", + "default link NvimInvalidTernary NvimInvalidOperator", + "default link NvimInvalidTernaryColon NvimInvalidTernary", - "default link NVimInvalidDelimiter NVimInvalid", + "default link NvimInvalidDelimiter NvimInvalid", - "default link NVimInvalidParenthesis NVimInvalidDelimiter", - "default link NVimInvalidLambda NVimInvalidParenthesis", - "default link NVimInvalidNestingParenthesis NVimInvalidParenthesis", - "default link NVimInvalidCallingParenthesis NVimInvalidParenthesis", + "default link NvimInvalidParenthesis NvimInvalidDelimiter", + "default link NvimInvalidLambda NvimInvalidParenthesis", + "default link NvimInvalidNestingParenthesis NvimInvalidParenthesis", + "default link NvimInvalidCallingParenthesis NvimInvalidParenthesis", - "default link NVimInvalidSubscript NVimInvalidParenthesis", - "default link NVimInvalidSubscriptBracket NVimInvalidSubscript", - "default link NVimInvalidSubscriptColon NVimInvalidSubscript", - "default link NVimInvalidCurly NVimInvalidSubscript", + "default link NvimInvalidSubscript NvimInvalidParenthesis", + "default link NvimInvalidSubscriptBracket NvimInvalidSubscript", + "default link NvimInvalidSubscriptColon NvimInvalidSubscript", + "default link NvimInvalidCurly NvimInvalidSubscript", - "default link NVimInvalidContainer NVimInvalidParenthesis", - "default link NVimInvalidDict NVimInvalidContainer", - "default link NVimInvalidList NVimInvalidContainer", + "default link NvimInvalidContainer NvimInvalidParenthesis", + "default link NvimInvalidDict NvimInvalidContainer", + "default link NvimInvalidList NvimInvalidContainer", - "default link NVimInvalidValue NVimInvalid", + "default link NvimInvalidValue NvimInvalid", - "default link NVimInvalidIdentifier NVimInvalidValue", - "default link NVimInvalidIdentifierScope NVimInvalidIdentifier", - "default link NVimInvalidIdentifierScopeDelimiter NVimInvalidIdentifier", - "default link NVimInvalidIdentifierName NVimInvalidIdentifier", - "default link NVimInvalidIdentifierKey NVimInvalidIdentifier", + "default link NvimInvalidIdentifier NvimInvalidValue", + "default link NvimInvalidIdentifierScope NvimInvalidIdentifier", + "default link NvimInvalidIdentifierScopeDelimiter NvimInvalidIdentifier", + "default link NvimInvalidIdentifierName NvimInvalidIdentifier", + "default link NvimInvalidIdentifierKey NvimInvalidIdentifier", - "default link NVimInvalidColon NVimInvalidDelimiter", - "default link NVimInvalidComma NVimInvalidDelimiter", - "default link NVimInvalidArrow NVimInvalidDelimiter", + "default link NvimInvalidColon NvimInvalidDelimiter", + "default link NvimInvalidComma NvimInvalidDelimiter", + "default link NvimInvalidArrow NvimInvalidDelimiter", - "default link NVimInvalidRegister NVimInvalidValue", - "default link NVimInvalidNumber NVimInvalidValue", - "default link NVimInvalidFloat NVimInvalidNumber", - "default link NVimInvalidNumberPrefix NVimInvalidNumber", + "default link NvimInvalidRegister NvimInvalidValue", + "default link NvimInvalidNumber NvimInvalidValue", + "default link NvimInvalidFloat NvimInvalidNumber", + "default link NvimInvalidNumberPrefix NvimInvalidNumber", - "default link NVimInvalidOptionSigil NVimInvalidIdentifier", - "default link NVimInvalidOptionName NVimInvalidIdentifier", - "default link NVimInvalidOptionScope NVimInvalidIdentifierScope", - "default link NVimInvalidOptionScopeDelimiter " - "NVimInvalidIdentifierScopeDelimiter", + "default link NvimInvalidOptionSigil NvimInvalidIdentifier", + "default link NvimInvalidOptionName NvimInvalidIdentifier", + "default link NvimInvalidOptionScope NvimInvalidIdentifierScope", + "default link NvimInvalidOptionScopeDelimiter " + "NvimInvalidIdentifierScopeDelimiter", - "default link NVimInvalidEnvironmentSigil NVimInvalidOptionSigil", - "default link NVimInvalidEnvironmentName NVimInvalidIdentifier", + "default link NvimInvalidEnvironmentSigil NvimInvalidOptionSigil", + "default link NvimInvalidEnvironmentName NvimInvalidIdentifier", // Invalid string bodies and specials are still highlighted as valid ones to // minimize the red area. - "default link NVimInvalidString NVimInvalidValue", - "default link NVimInvalidStringBody NVimStringBody", - "default link NVimInvalidStringQuote NVimInvalidString", - "default link NVimInvalidStringSpecial NVimStringSpecial", + "default link NvimInvalidString NvimInvalidValue", + "default link NvimInvalidStringBody NvimStringBody", + "default link NvimInvalidStringQuote NvimInvalidString", + "default link NvimInvalidStringSpecial NvimStringSpecial", - "default link NVimInvalidSingleQuote NVimInvalidStringQuote", - "default link NVimInvalidSingleQuotedBody NVimInvalidStringBody", - "default link NVimInvalidSingleQuotedQuote NVimInvalidStringSpecial", + "default link NvimInvalidSingleQuote NvimInvalidStringQuote", + "default link NvimInvalidSingleQuotedBody NvimInvalidStringBody", + "default link NvimInvalidSingleQuotedQuote NvimInvalidStringSpecial", - "default link NVimInvalidDoubleQuote NVimInvalidStringQuote", - "default link NVimInvalidDoubleQuotedBody NVimInvalidStringBody", - "default link NVimInvalidDoubleQuotedEscape NVimInvalidStringSpecial", - "default link NVimInvalidDoubleQuotedUnknownEscape NVimInvalidValue", + "default link NvimInvalidDoubleQuote NvimInvalidStringQuote", + "default link NvimInvalidDoubleQuotedBody NvimInvalidStringBody", + "default link NvimInvalidDoubleQuotedEscape NvimInvalidStringSpecial", + "default link NvimInvalidDoubleQuotedUnknownEscape NvimInvalidValue", - "default link NVimInvalidFigureBrace NVimInvalidDelimiter", + "default link NvimInvalidFigureBrace NvimInvalidDelimiter", - "default link NVimInvalidSpacing ErrorMsg", + "default link NvimInvalidSpacing ErrorMsg", // Not actually invalid, but we highlight user that he is doing something // wrong. - "default link NVimDoubleQuotedUnknownEscape NVimInvalidValue", + "default link NvimDoubleQuotedUnknownEscape NvimInvalidValue", NULL, }; -/// Create default links for NVim* highlight groups used for cmdline coloring +/// Create default links for Nvim* highlight groups used for cmdline coloring void syn_init_cmdline_highlight(bool reset, bool init) { for (size_t i = 0 ; highlight_init_cmdline[i] != NULL ; i++) { diff --git a/src/nvim/viml/parser/expressions.c b/src/nvim/viml/parser/expressions.c index cfcde6bb38..4196ecb9d2 100644 --- a/src/nvim/viml/parser/expressions.c +++ b/src/nvim/viml/parser/expressions.c @@ -1326,7 +1326,7 @@ static inline ParserPosition recol_pos(const ParserPosition pos, } /// Get highlight group name -#define HL(g) (is_invalid ? "NVimInvalid" #g : "NVim" #g) +#define HL(g) (is_invalid ? "NvimInvalid" #g : "Nvim" #g) /// Highlight current token with the given group #define HL_CUR_TOKEN(g) \ @@ -2570,7 +2570,7 @@ viml_pexpr_parse_bracket_closing_error: new_top_node, _("E15: Don't know what figure brace means: %.*s")); if (pstate->colors) { - // Will reset to NVimInvalidFigureBrace. + // Will reset to NvimInvalidFigureBrace. kv_A(*pstate->colors, new_top_node->data.fig.opening_hl_idx).group = ( HL(FigureBrace)); diff --git a/test/functional/api/vim_spec.lua b/test/functional/api/vim_spec.lua index 841b7a584a..ff28e3d133 100644 --- a/test/functional/api/vim_spec.lua +++ b/test/functional/api/vim_spec.lua @@ -885,7 +885,7 @@ describe('api', function() return function(next_col) local col = next_col + (shift or 0) return (('%s:%u:%u:%s'):format( - 'NVim' .. group, + 'Nvim' .. group, 0, col, str)), (col + #str) diff --git a/test/functional/ui/cmdline_highlight_spec.lua b/test/functional/ui/cmdline_highlight_spec.lua index 54d27723f0..73fe94c056 100644 --- a/test/functional/ui/cmdline_highlight_spec.lua +++ b/test/functional/ui/cmdline_highlight_spec.lua @@ -869,10 +869,10 @@ describe('Ex commands coloring support', function() end) describe('Expressions coloring support', function() it('works', function() - meths.command('hi clear NVimNumber') - meths.command('hi clear NVimNestingParenthesis') - meths.command('hi NVimNumber guifg=Blue2') - meths.command('hi NVimNestingParenthesis guifg=Yellow') + meths.command('hi clear NvimNumber') + meths.command('hi clear NvimNestingParenthesis') + meths.command('hi NvimNumber guifg=Blue2') + meths.command('hi NvimNestingParenthesis guifg=Yellow') feed(':echo =(((1)))') screen:expect([[ | @@ -888,8 +888,8 @@ describe('Expressions coloring support', function() it('does not use Nvim_color_expr', function() meths.set_var('Nvim_color_expr', 42) -- Used to error out due to failing to get callback. - meths.command('hi clear NVimNumber') - meths.command('hi NVimNumber guifg=Blue2') + meths.command('hi clear NvimNumber') + meths.command('hi NvimNumber guifg=Blue2') feed(':=1') screen:expect([[ | @@ -903,12 +903,12 @@ describe('Expressions coloring support', function() ]]) end) it('works correctly with non-ASCII and control characters', function() - meths.command('hi clear NVimStringBody') - meths.command('hi clear NVimStringQuote') - meths.command('hi clear NVimInvalid') - meths.command('hi NVimStringQuote guifg=Blue3') - meths.command('hi NVimStringBody guifg=Blue4') - meths.command('hi NVimInvalid guifg=Red guibg=Blue') + meths.command('hi clear NvimStringBody') + meths.command('hi clear NvimStringQuote') + meths.command('hi clear NvimInvalid') + meths.command('hi NvimStringQuote guifg=Blue3') + meths.command('hi NvimStringBody guifg=Blue4') + meths.command('hi NvimInvalid guifg=Red guibg=Blue') feed('i="«»"«»') screen:expect([[ | diff --git a/test/unit/viml/expressions/parser_spec.lua b/test/unit/viml/expressions/parser_spec.lua index 79b19de833..73388e5dd2 100644 --- a/test/unit/viml/expressions/parser_spec.lua +++ b/test/unit/viml/expressions/parser_spec.lua @@ -164,14 +164,14 @@ child_call_once(function() i = i + 1 end for k, _ in ipairs(nvim_hl_defs) do - eq('NVim', k:sub(1, 4)) - -- NVimInvalid + eq('Nvim', k:sub(1, 4)) + -- NvimInvalid -- 12345678901 local err, msg = pcall(function() if k:sub(5, 11) == 'Invalid' then - neq(nil, nvim_hl_defs['NVim' .. k:sub(12)]) + neq(nil, nvim_hl_defs['Nvim' .. k:sub(12)]) else - neq(nil, nvim_hl_defs['NVimInvalid' .. k:sub(5)]) + neq(nil, nvim_hl_defs['NvimInvalid' .. k:sub(5)]) end end) if not err then @@ -185,7 +185,7 @@ local function hls_to_hl_fs(hls) local ret = {} local next_col = 0 for i, v in ipairs(hls) do - local group, line, col, str = v:match('^NVim([a-zA-Z]+):(%d+):(%d+):(.*)$') + local group, line, col, str = v:match('^Nvim([a-zA-Z]+):(%d+):(%d+):(.*)$') col = tonumber(col) line = tonumber(line) assert(line == 0) @@ -521,12 +521,12 @@ describe('Expressions parser', function() end local function hl(group, str, shift) return function(next_col) - if nvim_hl_defs['NVim' .. group] == nil then - error(('Unknown group: NVim%s'):format(group)) + if nvim_hl_defs['Nvim' .. group] == nil then + error(('Unknown group: Nvim%s'):format(group)) end local col = next_col + (shift or 0) return (('%s:%u:%u:%s'):format( - 'NVim' .. group, + 'Nvim' .. group, 0, col, str)), (col + #str) From 62993323494d846190590606d37aff1136421671 Mon Sep 17 00:00:00 2001 From: ZyX Date: Fri, 1 Dec 2017 00:34:16 +0300 Subject: [PATCH 107/109] fix! set lsan options --- .travis.yml | 1 + src/nvim/eval/encode.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2bab1635ad..a32532d21a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,7 @@ env: - ASAN_OPTIONS="detect_leaks=1:check_initialization_order=1:log_path=$LOG_DIR/asan" - TSAN_OPTIONS="log_path=$LOG_DIR/tsan" - UBSAN_OPTIONS="print_stacktrace=1 log_path=$LOG_DIR/ubsan" + - LSAN_OPTIONS="verbosity=1:log_threads=1" # Environment variables for Valgrind. - VALGRIND_LOG="$LOG_DIR/valgrind-%p.log" # Cache marker for third-party dependencies cache. diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index ef647b3ee4..650f11a989 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -316,7 +316,7 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, #define TYPVAL_ENCODE_CONV_FLOAT(tv, flt) \ do { \ const float_T flt_ = (flt); \ - switch (fpclassify(flt_)) { \ + switch (fpclassify((double)flt_)) { \ case FP_NAN: { \ ga_concat(gap, (char_u *) "str2float('nan')"); \ break; \ From 6bc54832efcb8c269875dfa4eecd89e1984ead69 Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 3 Dec 2017 16:53:29 +0300 Subject: [PATCH 108/109] Revert "fix! set lsan options" This reverts commit 62993323494d846190590606d37aff1136421671. --- .travis.yml | 1 - src/nvim/eval/encode.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a32532d21a..2bab1635ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,6 @@ env: - ASAN_OPTIONS="detect_leaks=1:check_initialization_order=1:log_path=$LOG_DIR/asan" - TSAN_OPTIONS="log_path=$LOG_DIR/tsan" - UBSAN_OPTIONS="print_stacktrace=1 log_path=$LOG_DIR/ubsan" - - LSAN_OPTIONS="verbosity=1:log_threads=1" # Environment variables for Valgrind. - VALGRIND_LOG="$LOG_DIR/valgrind-%p.log" # Cache marker for third-party dependencies cache. diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 650f11a989..ef647b3ee4 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -316,7 +316,7 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, #define TYPVAL_ENCODE_CONV_FLOAT(tv, flt) \ do { \ const float_T flt_ = (flt); \ - switch (fpclassify((double)flt_)) { \ + switch (fpclassify(flt_)) { \ case FP_NAN: { \ ga_concat(gap, (char_u *) "str2float('nan')"); \ break; \ From fbdc3ac4efbc97e2965b6083d429beabe261461c Mon Sep 17 00:00:00 2001 From: ZyX Date: Sun, 3 Dec 2017 20:22:09 +0300 Subject: [PATCH 109/109] tests: Fix linter errors --- test/helpers.lua | 1 + test/unit/charset/vim_str2nr_spec.lua | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/test/helpers.lua b/test/helpers.lua index 3ec4aa511f..faf5c8e7f2 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -273,6 +273,7 @@ local deepcopy_funcs = { for k, v in pairs(orig) do copy[deepcopy(k)] = deepcopy(v) end + return copy end, number = id, string = id, diff --git a/test/unit/charset/vim_str2nr_spec.lua b/test/unit/charset/vim_str2nr_spec.lua index a6fe613563..891e6def09 100644 --- a/test/unit/charset/vim_str2nr_spec.lua +++ b/test/unit/charset/vim_str2nr_spec.lua @@ -1,5 +1,4 @@ local helpers = require("test.unit.helpers")(after_each) -local global_helpers = require('test.helpers') local bit = require('bit') local itp = helpers.gen_itp(it) @@ -8,9 +7,6 @@ local child_call_once = helpers.child_call_once local cimport = helpers.cimport local ffi = helpers.ffi -local shallowcopy = global_helpers.shallowcopy -local updated = global_helpers.updated - local lib = cimport('./src/nvim/charset.h') local ARGTYPES @@ -48,7 +44,6 @@ local function argreset(arg, args) end local function test_vim_str2nr(s, what, exp, maxlen) - local comb = {[{}] = true} local bits = {} for k, _ in pairs(exp) do bits[#bits + 1] = k