IntenseFinance/neovim
3
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2025-02-25 18:55:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

viml/parser/expressions: Make lexer parse numbers, support non-decimal

Browse Source
This commit is contained in:
ZyX 2017-10-06 01:19:43 +03:00
parent 0bc4e22379
commit 163792e9b9
5 changed files with 362 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
src/nvim/viml/parser/expressions.c
View File

@ -15,10 +15,13 @@
#include "nvim/ascii.h"
#include "nvim/assert.h"
#include "nvim/lib/kvec.h"
#include "nvim/eval/typval.h"
#include "nvim/viml/parser/expressions.h"
#include "nvim/viml/parser/parser.h"
#define vim_str2nr(s, ...) vim_str2nr((const char_u *)(s), __VA_ARGS__)
typedef kvec_withinit_t(ExprASTNode **, 16) ExprASTStack;
/// Which nodes may be wanted
@ -72,6 +75,43 @@ typedef enum {
/// Character used as a separator in autoload function/variable names.
#define AUTOLOAD_CHAR '#'
/// Scale number by a given factor
///
/// Used to apply exponent to a number. Idea taken from uClibc.
///
/// @param[in] num Number to scale. Does not bother doing anything if it is
/// zero.
/// @param[in] base Base, should be 10 since non-decimal floating-point
/// numbers are not supported.
/// @param[in] exponent Exponent to scale by.
/// @param[in] exponent_negative True if exponent is negative.
static inline float_T scale_number(const float_T num,
const uint8_t base,
const uvarnumber_T exponent,
const bool exponent_negative)
FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_CONST
{
if (num == 0 || exponent == 0) {
return num;
}
assert(base);
uvarnumber_T exp = exponent;
float_T p_base = (float_T)base;
float_T ret = num;
while (exp) {
if (exp & 1) {
if (exponent_negative) {
ret /= p_base;
} else {
ret *= p_base;
}
}
exp >>= 1;
p_base *= p_base;
}
return ret;
}
/// Get next token for the VimL expression input
///
/// @param pstate Parser state.
@ -184,6 +224,11 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
case '0': case '1': case '2': case '3': case '4': case '5': case '6':
case '7': case '8': case '9': {
ret.data.num.is_float = false;
ret.data.num.base = 10;
size_t frac_start = 0;
size_t exp_start = 0;
size_t frac_end = 0;
bool exp_negative = false;
CHARREG(kExprLexNumber, ascii_isdigit);
if (flags & kELFlagAllowFloat) {
const LexExprToken non_float_ret = ret;
@ -191,8 +236,18 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
&& pline.data[ret.len] == '.'
&& ascii_isdigit(pline.data[ret.len + 1])) {
ret.len++;
frac_start = ret.len;
frac_end = ret.len;
ret.data.num.is_float = true;
CHARREG(kExprLexNumber, ascii_isdigit);
for (; ret.len < pline.size && ascii_isdigit(pline.data[ret.len])
; ret.len++) {
// A small optimization: trailing zeroes in fractional part do not
// add anything to significand, so it is useless to include them in
// frac_end.
if (pline.data[ret.len] != '0') {
frac_end = ret.len + 1;
}
}
if (pline.size > ret.len + 1
&& (pline.data[ret.len] == 'e'
|| pline.data[ret.len] == 'E')
@ -202,9 +257,11 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
&& ascii_isdigit(pline.data[ret.len + 2]))
|| ascii_isdigit(pline.data[ret.len + 1]))) {
ret.len++;
if (pline.data[ret.len] == '+' || pline.data[ret.len] == '-') {
if (pline.data[ret.len] == '+'
|| (exp_negative = (pline.data[ret.len] == '-'))) {
ret.len++;
}
exp_start = ret.len;
CHARREG(kExprLexNumber, ascii_isdigit);
}
}
@ -214,6 +271,58 @@ LexExprToken viml_pexpr_next_token(ParserState *const pstate, const int flags)
ret = non_float_ret;
}
}
// TODO(ZyX-I): detect overflows
if (ret.data.num.is_float) {
// Vim used to use string2float here which in turn uses strtod(). There
// are two problems with this approach:
// 1. strtod() is locale-dependent. Not sure how it is worked around so
// that I do not see relevant bugs, but it still does not look like
// a good idea.
// 2. strtod() does not accept length argument.
//
// The below variant of parsing floats was recognized as acceptable
// because it is basically how uClibc does the thing: it generates
// a number ignoring decimal point (but recording its position), then
// uses recorded position to scale number down when processing exponent.
float_T significand_part = 0;
uvarnumber_T exp_part = 0;
const size_t frac_size = (size_t)(frac_end - frac_start);
for (size_t i = 0; i < frac_end; i++) {
if (i == frac_start - 1) {
continue;
}
significand_part = significand_part * 10 + (pline.data[i] - '0');
}
if (exp_start) {
vim_str2nr(pline.data + exp_start, NULL, NULL, 0, NULL, &exp_part,
(int)(ret.len - exp_start));
}
if (exp_negative) {
exp_part += frac_size;
} else {
if (exp_part < frac_size) {
exp_negative = true;
exp_part = frac_size - exp_part;
} else {
exp_part -= frac_size;
}
}
ret.data.num.val.floating = scale_number(significand_part, 10, exp_part,
exp_negative);
} else {
int len;
int prep;
vim_str2nr(pline.data, &prep, &len, STR2NR_ALL, NULL,
&ret.data.num.val.integer, (int)pline.size);
ret.len = (size_t)len;
const uint8_t bases[] = {
[0] = 10,
['0'] = 8,
['x'] = 16, ['X'] = 16,
['b'] = 2, ['B'] = 2,
};
ret.data.num.base = bases[prep];
}
break;
}
@ -474,7 +583,6 @@ viml_pexpr_next_token_adv_return:
return ret;
}
#ifdef UNIT_TESTING
static const char *const eltkn_type_tab[] = {
[kExprLexInvalid] = "Invalid",
[kExprLexMissing] = "Missing",
@ -617,7 +725,12 @@ const char *viml_pexpr_repr_token(const ParserState *const pstate,
(int)token.data.opt.len, token.data.opt.name)
TKNARGS(kExprLexPlainIdentifier, "(scope=%s,autoload=%i)",
intchar2str(token.data.var.scope), (int)token.data.var.autoload)
TKNARGS(kExprLexNumber, "(is_float=%i)", (int)token.data.num.is_float)
TKNARGS(kExprLexNumber, "(is_float=%i,base=%i,val=%lg)",
(int)token.data.num.is_float,
(int)token.data.num.base,
(double)(token.data.num.is_float
? token.data.num.val.floating
: token.data.num.val.integer))
TKNARGS(kExprLexInvalid, "(msg=%s)", token.data.err.msg)
default: {
// No additional arguments.
@ -642,7 +755,6 @@ viml_pexpr_repr_token_end:
}
return ret;
}
#endif
#ifdef UNIT_TESTING
#include <stdio.h>
@ -776,8 +888,10 @@ static inline void viml_pexpr_debug_print_token(
// NVimOperator -> Operator
// NVimUnaryOperator -> NVimOperator
// NVimBinaryOperator -> NVimOperator
//
// NVimComparisonOperator -> NVimBinaryOperator
// NVimComparisonOperatorModifier -> NVimComparisonOperator
//
// NVimTernary -> NVimOperator
// NVimTernaryColon -> NVimTernary
//
@ -795,8 +909,21 @@ static inline void viml_pexpr_debug_print_token(
// NVimIdentifierScope -> NVimIdentifier
// NVimIdentifierScopeDelimiter -> NVimIdentifier
//
// NVimIdentifierKey -> Identifier
//
// NVimFigureBrace -> NVimInternalError
//
// NVimUnaryPlus -> NVimUnaryOperator
// NVimBinaryPlus -> NVimBinaryOperator
// NVimConcatOrSubscript -> NVimBinaryOperator
//
// NVimRegister -> SpecialChar
// NVimNumber -> Number
// NVimFloat -> NVimNumber
//
// NVimNestingParenthesis -> NVimParenthesis
// NVimCallingParenthesis -> NVimParenthesis
//
// NVimInvalidComma -> NVimInvalidDelimiter
// NVimInvalidSpacing -> NVimInvalid
// NVimInvalidTernary -> NVimInvalidOperator
@ -814,12 +941,9 @@ static inline void viml_pexpr_debug_print_token(
// NVimInvalidIdentifierScopeDelimiter -> NVimInvalidValue
// NVimInvalidComparisonOperator -> NVimInvalidOperator
// NVimInvalidComparisonOperatorModifier -> NVimInvalidComparisonOperator
//
// NVimUnaryPlus -> NVimUnaryOperator
// NVimBinaryPlus -> NVimBinaryOperator
// NVimRegister -> SpecialChar
// NVimNestingParenthesis -> NVimParenthesis
// NVimCallingParenthesis -> NVimParenthesis
// NVimInvalidNumber -> NVimInvalidValue
// NVimInvalidFloat -> NVimInvalidValue
// NVimInvalidIdentifierKey -> NVimInvalidIdentifier
/// Allocate a new node and set some of the values
///

6
src/nvim/viml/parser/expressions.h
View File

@ -7,6 +7,7 @@
#include "nvim/types.h"
#include "nvim/viml/parser/parser.h"
#include "nvim/eval/typval.h"
// Defines whether to ignore case:
// == kCCStrategyUseOption
@ -113,6 +114,11 @@ typedef struct {
} err; ///< For kExprLexInvalid
struct {
union {
float_T floating;
uvarnumber_T integer;
} val; ///< Number value.
uint8_t base; ///< Base: 2, 8, 10 or 16.
bool is_float; ///< True if number is a floating-point.
} num; ///< For kExprLexNumber
} data; ///< Additional data, if needed.

165
test/symbolic/klee/nvim/charset.c
View File

@ -3,8 +3,173 @@
#include "nvim/ascii.h"
#include "nvim/macros.h"
#include "nvim/charset.h"
#include "nvim/eval/typval.h"
#include "nvim/vim.h"
bool vim_isIDc(int c)
{
return ASCII_ISALNUM(c);
}
int hex2nr(int c)
{
if ((c >= 'a') && (c <= 'f')) {
return c - 'a' + 10;
}
if ((c >= 'A') && (c <= 'F')) {
return c - 'A' + 10;
}
return c - '0';
}
void vim_str2nr(const char_u *const start, int *const prep, int *const len,
const int what, varnumber_T *const nptr,
uvarnumber_T *const unptr, const int maxlen)
{
const char_u *ptr = start;
int pre = 0; // default is decimal
bool negative = false;
uvarnumber_T un = 0;
if (ptr[0] == '-') {
negative = true;
ptr++;
}
// Recognize hex, octal and bin.
if ((ptr[0] == '0') && (ptr[1] != '8') && (ptr[1] != '9')
&& (maxlen == 0 || maxlen > 1)) {
pre = ptr[1];
if ((what & STR2NR_HEX)
&& ((pre == 'X') || (pre == 'x'))
&& ascii_isxdigit(ptr[2])
&& (maxlen == 0 || maxlen > 2)) {
// hexadecimal
ptr += 2;
} else if ((what & STR2NR_BIN)
&& ((pre == 'B') || (pre == 'b'))
&& ascii_isbdigit(ptr[2])
&& (maxlen == 0 || maxlen > 2)) {
// binary
ptr += 2;
} else {
// decimal or octal, default is decimal
pre = 0;
if (what & STR2NR_OCT) {
// Don't interpret "0", "08" or "0129" as octal.
for (int n = 1; ascii_isdigit(ptr[n]); ++n) {
if (ptr[n] > '7') {
// can't be octal
pre = 0;
break;
}
if (ptr[n] >= '0') {
// assume octal
pre = '0';
}
if (n == maxlen) {
break;
}
}
}
}
}
// Do the string-to-numeric conversion "manually" to avoid sscanf quirks.
int n = 1;
if ((pre == 'B') || (pre == 'b') || what == STR2NR_BIN + STR2NR_FORCE) {
// bin
if (pre != 0) {
n += 2; // skip over "0b"
}
while ('0' <= *ptr && *ptr <= '1') {
// avoid ubsan error for overflow
if (un < UVARNUMBER_MAX / 2) {
un = 2 * un + (uvarnumber_T)(*ptr - '0');
} else {
un = UVARNUMBER_MAX;
}
ptr++;
if (n++ == maxlen) {
break;
}
}
} else if ((pre == '0') || what == STR2NR_OCT + STR2NR_FORCE) {
// octal
while ('0' <= *ptr && *ptr <= '7') {
// avoid ubsan error for overflow
if (un < UVARNUMBER_MAX / 8) {
un = 8 * un + (uvarnumber_T)(*ptr - '0');
} else {
un = UVARNUMBER_MAX;
}
ptr++;
if (n++ == maxlen) {
break;
}
}
} else if ((pre == 'X') || (pre == 'x')
|| what == STR2NR_HEX + STR2NR_FORCE) {
// hex
if (pre != 0) {
n += 2; // skip over "0x"
}
while (ascii_isxdigit(*ptr)) {
// avoid ubsan error for overflow
if (un < UVARNUMBER_MAX / 16) {
un = 16 * un + (uvarnumber_T)hex2nr(*ptr);
} else {
un = UVARNUMBER_MAX;
}
ptr++;
if (n++ == maxlen) {
break;
}
}
} else {
// decimal
while (ascii_isdigit(*ptr)) {
// avoid ubsan error for overflow
if (un < UVARNUMBER_MAX / 10) {
un = 10 * un + (uvarnumber_T)(*ptr - '0');
} else {
un = UVARNUMBER_MAX;
}
ptr++;
if (n++ == maxlen) {
break;
}
}
}
if (prep != NULL) {
*prep = pre;
}
if (len != NULL) {
*len = (int)(ptr - start);
}
if (nptr != NULL) {
if (negative) { // account for leading '-' for decimal numbers
// avoid ubsan error for overflow
if (un > VARNUMBER_MAX) {
*nptr = VARNUMBER_MIN;
} else {
*nptr = -(varnumber_T)un;
}
} else {
if (un > VARNUMBER_MAX) {
un = VARNUMBER_MAX;
}
*nptr = (varnumber_T)un;
}
}
if (unptr != NULL) {
*unptr = un;
}
}

6
test/symbolic/klee/viml_expressions_lexer.c
View File

@ -2,6 +2,7 @@
# include <klee/klee.h>
#else
# include <string.h>
# include <stdio.h>
#endif
#include <stddef.h>
#include <stdint.h>
@ -56,7 +57,7 @@ int main(const int argc, const char *const *const argv,
.data = &input[shift],
.size = sizeof(input) - shift,
#else
.data = (const char *)&argv[1],
.data = (const char *)argv[1],
.size = strlen(argv[1]),
#endif
.allocated = false,
@ -97,4 +98,7 @@ int main(const int argc, const char *const *const argv,
}
assert(allocated_memory == 0);
assert(ever_allocated_memory == 0);
#ifndef USE_KLEE
fprintf(stderr, "tkn: %s\n", viml_pexpr_repr_token(&pstate, token, NULL));
#endif
}

73
test/unit/viml/expressions/lexer_spec.lua
View File

@ -114,7 +114,11 @@ local function eltkn2lua(pstate, tkn)
elseif ret.type == 'Number' then
ret.data = {
is_float = (not not tkn.data.num.is_float),
base = tonumber(tkn.data.num.base),
}
ret.data.val = tonumber(tkn.data.num.is_float
and tkn.data.num.val.floating
or tkn.data.num.val.integer)
elseif ret.type == 'Invalid' then
ret.data = { error = ffi.string(tkn.data.err.msg) }
end
@ -204,9 +208,20 @@ describe('Expressions lexer', function()
singl_eltkn_test('Spacing', ' ')
singl_eltkn_test('Spacing', '\t')
singl_eltkn_test('Invalid', '\x01\x02\x03', {error='E15: Invalid control character present in input: %.*s'})
singl_eltkn_test('Number', '0123', {is_float=false})
singl_eltkn_test('Number', '0', {is_float=false})
singl_eltkn_test('Number', '9', {is_float=false})
singl_eltkn_test('Number', '0123', {is_float=false, base=8, val=83})
singl_eltkn_test('Number', '01234567', {is_float=false, base=8, val=342391})
singl_eltkn_test('Number', '012345678', {is_float=false, base=10, val=12345678})
singl_eltkn_test('Number', '0x123', {is_float=false, base=16, val=291})
singl_eltkn_test('Number', '0x56FF', {is_float=false, base=16, val=22271})
singl_eltkn_test('Number', '0xabcdef', {is_float=false, base=16, val=11259375})
singl_eltkn_test('Number', '0xABCDEF', {is_float=false, base=16, val=11259375})
singl_eltkn_test('Number', '0x0', {is_float=false, base=16, val=0})
singl_eltkn_test('Number', '00', {is_float=false, base=8, val=0})
singl_eltkn_test('Number', '0b0', {is_float=false, base=2, val=0})
singl_eltkn_test('Number', '0b010111', {is_float=false, base=2, val=23})
singl_eltkn_test('Number', '0b100111', {is_float=false, base=2, val=39})
singl_eltkn_test('Number', '0', {is_float=false, base=10, val=0})
singl_eltkn_test('Number', '9', {is_float=false, base=10, val=9})
singl_eltkn_test('Env', '$abc')
singl_eltkn_test('Env', '$')
singl_eltkn_test('PlainIdentifier', 'test', {autoload=false, scope=0})
@ -262,17 +277,21 @@ describe('Expressions lexer', function()
singl_eltkn_test('Invalid', '~', {error='E15: Unidentified character: %.*s'})
simple_test({{data=nil, size=0}}, 'EOC', 0, {error='start.col >= #pstr'})
simple_test({''}, 'EOC', 0, {error='start.col >= #pstr'})
simple_test({'2.'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.x'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.2.'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0x'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.2.'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e+'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e-'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e+x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e-x'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e+1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e-1a'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'0b102'}, 'Number', 4, {data={is_float=false, base=2, val=2}, str='0b10'})
simple_test({'10F'}, 'Number', 2, {data={is_float=false, base=10, val=10}, str='10'})
simple_test({'0x0123456789ABCDEFG'}, 'Number', 18, {data={is_float=false, base=16, val=81985529216486895}, str='0x0123456789ABCDEF'})
end
local function regular_scope_tests()
@ -304,10 +323,10 @@ describe('Expressions lexer', function()
end
local function regular_number_tests()
simple_test({'2.0'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false}, str='2'})
simple_test({'2.0'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e+5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({'2.0e-5'}, 'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
end
local function regular_eoc_tests()
@ -352,10 +371,20 @@ describe('Expressions lexer', function()
regular_scope_tests()
regular_is_tests()
simple_test({'2.0'}, 'Number', 3, {data={is_float=true}, str='2.0'})
simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true}, str='2.0e5'})
simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true}, str='2.0e+5'})
simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true}, str='2.0e-5'})
simple_test({'2.2'}, 'Number', 3, {data={is_float=true, base=10, val=2.2}, str='2.2'})
simple_test({'2.0e5'}, 'Number', 5, {data={is_float=true, base=10, val=2e5}, str='2.0e5'})
simple_test({'2.0e+5'}, 'Number', 6, {data={is_float=true, base=10, val=2e5}, str='2.0e+5'})
simple_test({'2.0e-5'}, 'Number', 6, {data={is_float=true, base=10, val=2e-5}, str='2.0e-5'})
simple_test({'2.500000e-5'}, 'Number', 11, {data={is_float=true, base=10, val=2.5e-5}, str='2.500000e-5'})
simple_test({'2.5555e2'}, 'Number', 8, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e2'})
simple_test({'2.5555e+2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e2}, str='2.5555e+2'})
simple_test({'2.5555e-2'}, 'Number', 9, {data={is_float=true, base=10, val=2.5555e-2}, str='2.5555e-2'})
simple_test({{data='2.5e-5', size=3}},
'Number', 3, {data={is_float=true, base=10, val=2.5}, str='2.5'})
simple_test({{data='2.5e5', size=4}},
'Number', 1, {data={is_float=false, base=10, val=2}, str='2'})
simple_test({{data='2.5e-50', size=6}},
'Number', 6, {data={is_float=true, base=10, val=2.5e-5}, str='2.5e-5'})
end)
itp('treats `is` as an identifier', function()
flags = tonumber(lib.kELFlagIsNotCmp)