From d746f5aa418f86828aef689a2c4f8d5b53c9f7de Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 6 Dec 2021 20:50:29 +0000 Subject: [PATCH 1/4] feat(eval): partially port v8.2.0878 Problem: No reduce() function. Solution: Add a reduce() function. (closes vim/vim#5481) https://github.com/vim/vim/commit/85629985b71035608a37ba3bde86968481490d46 Needs CHECK_LIST_MATERIALIZE from v8.2.0751 (and range_list_materialize from 8.2.0149). Move e_reduceempty to funcs.c, as it's only used there. Make it static. Use tv_blob_len, tv_list_len == 0 for empty checks. Replace vim_memset(&funcexe, 0, ...) with FUNCEXE_INIT. Leave li initially undefined (tv_list_first returns NULL if list is NULL). This patch has a memory leak fixed by v8.2.0882. --- runtime/doc/builtin.txt | 21 ++++++++ src/nvim/eval.lua | 1 + src/nvim/eval/funcs.c | 85 ++++++++++++++++++++++++++++++ src/nvim/testdir/test_listdict.vim | 31 +++++++++++ 4 files changed, 138 insertions(+) diff --git a/runtime/doc/builtin.txt b/runtime/doc/builtin.txt index a421de078b..92f33bd933 100644 --- a/runtime/doc/builtin.txt +++ b/runtime/doc/builtin.txt @@ -330,6 +330,8 @@ range({expr} [, {max} [, {stride}]]) readdir({dir} [, {expr}]) List file names in {dir} selected by {expr} readfile({fname} [, {type} [, {max}]]) List get list of lines from file {fname} +reduce({object}, {func} [, {initial}]) + any reduce {object} using {func} reg_executing() String get the executing register name reg_recorded() String get the last recorded register name reg_recording() String get the recording register name @@ -5589,6 +5591,25 @@ readfile({fname} [, {type} [, {max}]]) Can also be used as a |method|: > GetFileName()->readfile() +reduce({object}, {func} [, {initial}]) *reduce()* *E998* + {func} is called for every item in {object}, which can be a + |List| or a |Blob|. {func} is called with two arguments: the + result so far and current item. After processing all items + the result is returned. + + {initial} is the initial result. When omitted, the first item + in {object} is used and {func} is first called for the second + item. If {initial} is not given and {object} is empty no + result can be computed, an E998 error is given. + + Examples: > + echo reduce([1, 3, 5], { acc, val -> acc + val }) + echo reduce(['x', 'y'], { acc, val -> acc .. val }, 'a') + echo reduce(0z1122, { acc, val -> 2 * acc + val }) +< + Can also be used as a |method|: > + echo mylist->reduce({ acc, val -> acc + val }, 0) + reg_executing() *reg_executing()* Returns the single letter name of the register being executed. Returns an empty string when no register is being executed. diff --git a/src/nvim/eval.lua b/src/nvim/eval.lua index 18967b80f2..e00a14fca7 100644 --- a/src/nvim/eval.lua +++ b/src/nvim/eval.lua @@ -276,6 +276,7 @@ return { range={args={1, 3}, base=1}, readdir={args={1, 2}, base=1}, readfile={args={1, 3}, base=1}, + reduce={args={2, 3}, base=1}, reg_executing={}, reg_recording={}, reg_recorded={}, diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 29619f62e9..1a5f6e08bc 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -100,6 +100,7 @@ PRAGMA_DIAG_POP static char *e_listarg = N_("E686: Argument of %s must be a List"); static char *e_listblobarg = N_("E899: Argument of %s must be a List or Blob"); static char *e_invalwindow = N_("E957: Invalid window number"); +static char *e_reduceempty = N_("E998: Reduce of an empty %s with no initial value"); /// Dummy va_list for passing to vim_snprintf /// @@ -7855,6 +7856,90 @@ static void f_reverse(typval_T *argvars, typval_T *rettv, FunPtr fptr) } } +/// "reduce(list, { accumlator, element -> value } [, initial])" function +static void f_reduce(typval_T *argvars, typval_T *rettv, FunPtr fptr) +{ + if (argvars[0].v_type != VAR_LIST && argvars[0].v_type != VAR_BLOB) { + emsg(_(e_listblobreq)); + return; + } + + const char_u *func_name; + partial_T *partial = NULL; + if (argvars[1].v_type == VAR_FUNC) { + func_name = argvars[1].vval.v_string; + } else if (argvars[1].v_type == VAR_PARTIAL) { + partial = argvars[1].vval.v_partial; + func_name = partial_name(partial); + } else { + func_name = (const char_u *)tv_get_string(&argvars[1]); + } + if (*func_name == NUL) { + return; // type error or empty name + } + + funcexe_T funcexe = FUNCEXE_INIT; + funcexe.evaluate = true; + funcexe.partial = partial; + + typval_T accum; + typval_T argv[3]; + if (argvars[0].v_type == VAR_LIST) { + const list_T *const l = argvars[0].vval.v_list; + const listitem_T *li; + + if (argvars[2].v_type == VAR_UNKNOWN) { + if (tv_list_len(l) == 0) { + semsg(_(e_reduceempty), "List"); + return; + } + const listitem_T *const first = tv_list_first(l); + accum = *TV_LIST_ITEM_TV(first); + li = TV_LIST_ITEM_NEXT(l, first); + } else { + accum = argvars[2]; + li = tv_list_first(l); + } + + tv_copy(&accum, rettv); + for (; li != NULL; li = TV_LIST_ITEM_NEXT(l, li)) { + argv[0] = accum; + argv[1] = *TV_LIST_ITEM_TV(li); + if (call_func(func_name, -1, rettv, 2, argv, &funcexe) == FAIL) { + return; + } + accum = *rettv; + } + } else { + const blob_T *const b = argvars[0].vval.v_blob; + int i; + + if (argvars[2].v_type == VAR_UNKNOWN) { + if (tv_blob_len(b) == 0) { + semsg(_(e_reduceempty), "Blob"); + return; + } + accum.v_type = VAR_NUMBER; + accum.vval.v_number = tv_blob_get(b, 0); + i = 1; + } else { + accum = argvars[2]; + i = 0; + } + + tv_copy(&accum, rettv); + for (; i < tv_blob_len(b); i++) { + argv[0] = accum; + argv[1].v_type = VAR_NUMBER; + argv[1].vval.v_number = tv_blob_get(b, i); + if (call_func(func_name, -1, rettv, 2, argv, &funcexe) == FAIL) { + return; + } + accum = *rettv; + } + } +} + #define SP_NOMOVE 0x01 ///< don't move cursor #define SP_REPEAT 0x02 ///< repeat to find outer pair #define SP_RETCOUNT 0x04 ///< return matchcount diff --git a/src/nvim/testdir/test_listdict.vim b/src/nvim/testdir/test_listdict.vim index f6c404d390..42b46fc76c 100644 --- a/src/nvim/testdir/test_listdict.vim +++ b/src/nvim/testdir/test_listdict.vim @@ -620,6 +620,37 @@ func Test_reverse_sort_uniq() call assert_fails('call reverse("")', 'E899:') endfunc +" reduce a list or a blob +func Test_reduce() + call assert_equal(1, reduce([], { acc, val -> acc + val }, 1)) + call assert_equal(10, reduce([1, 3, 5], { acc, val -> acc + val }, 1)) + call assert_equal(2 * (2 * ((2 * 1) + 2) + 3) + 4, reduce([2, 3, 4], { acc, val -> 2 * acc + val }, 1)) + call assert_equal('a x y z', ['x', 'y', 'z']->reduce({ acc, val -> acc .. ' ' .. val}, 'a')) + call assert_equal(#{ x: 1, y: 1, z: 1 }, ['x', 'y', 'z']->reduce({ acc, val -> extend(acc, { val: 1 }) }, {})) + call assert_equal([0, 1, 2, 3], reduce([1, 2, 3], function('add'), [0])) + + let l = ['x', 'y', 'z'] + call assert_equal(42, reduce(l, function('get'), #{ x: #{ y: #{ z: 42 } } })) + call assert_equal(['x', 'y', 'z'], l) + + call assert_equal(1, reduce([1], { acc, val -> acc + val })) + call assert_equal('x y z', reduce(['x', 'y', 'z'], { acc, val -> acc .. ' ' .. val })) + call assert_equal(120, range(1, 5)->reduce({ acc, val -> acc * val })) + call assert_fails("call reduce([], { acc, val -> acc + val })", 'E998: Reduce of an empty List with no initial value') + + call assert_equal(1, reduce(0z, { acc, val -> acc + val }, 1)) + call assert_equal(1 + 0xaf + 0xbf + 0xcf, reduce(0zAFBFCF, { acc, val -> acc + val }, 1)) + call assert_equal(2 * (2 * 1 + 0xaf) + 0xbf, 0zAFBF->reduce({ acc, val -> 2 * acc + val }, 1)) + + call assert_equal(0xff, reduce(0zff, { acc, val -> acc + val })) + call assert_equal(2 * (2 * 0xaf + 0xbf) + 0xcf, reduce(0zAFBFCF, { acc, val -> 2 * acc + val })) + call assert_fails("call reduce(0z, { acc, val -> acc + val })", 'E998: Reduce of an empty Blob with no initial value') + + call assert_fails("call reduce({}, { acc, val -> acc + val }, 1)", 'E897:') + call assert_fails("call reduce(0, { acc, val -> acc + val }, 1)", 'E897:') + call assert_fails("call reduce('', { acc, val -> acc + val }, 1)", 'E897:') +endfunc + " splitting a string to a List func Test_str_split() call assert_equal(['aa', 'bb'], split(' aa bb ')) From af0bae38e286c7139f56307e318fa9818218c3d2 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 6 Dec 2021 22:34:02 +0000 Subject: [PATCH 2/4] vim-patch:8.2.0882: leaking memory when using reduce() Problem: Leaking memory when using reduce(). Solution: Free the intermediate value. https://github.com/vim/vim/commit/48b1c21809553d3463b5ed6c2b3bc6d335663bb6 --- src/nvim/eval/funcs.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 1a5f6e08bc..997096aad4 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -7882,7 +7882,7 @@ static void f_reduce(typval_T *argvars, typval_T *rettv, FunPtr fptr) funcexe.evaluate = true; funcexe.partial = partial; - typval_T accum; + typval_T initial; typval_T argv[3]; if (argvars[0].v_type == VAR_LIST) { const list_T *const l = argvars[0].vval.v_list; @@ -7894,21 +7894,23 @@ static void f_reduce(typval_T *argvars, typval_T *rettv, FunPtr fptr) return; } const listitem_T *const first = tv_list_first(l); - accum = *TV_LIST_ITEM_TV(first); + initial = *TV_LIST_ITEM_TV(first); li = TV_LIST_ITEM_NEXT(l, first); } else { - accum = argvars[2]; + initial = argvars[2]; li = tv_list_first(l); } - tv_copy(&accum, rettv); + tv_copy(&initial, rettv); for (; li != NULL; li = TV_LIST_ITEM_NEXT(l, li)) { - argv[0] = accum; + argv[0] = *rettv; argv[1] = *TV_LIST_ITEM_TV(li); - if (call_func(func_name, -1, rettv, 2, argv, &funcexe) == FAIL) { + rettv->v_type = VAR_UNKNOWN; + const int r = call_func(func_name, -1, rettv, 2, argv, &funcexe); + tv_clear(&argv[0]); + if (r == FAIL) { return; } - accum = *rettv; } } else { const blob_T *const b = argvars[0].vval.v_blob; @@ -7919,23 +7921,25 @@ static void f_reduce(typval_T *argvars, typval_T *rettv, FunPtr fptr) semsg(_(e_reduceempty), "Blob"); return; } - accum.v_type = VAR_NUMBER; - accum.vval.v_number = tv_blob_get(b, 0); + initial.v_type = VAR_NUMBER; + initial.vval.v_number = tv_blob_get(b, 0); i = 1; + } else if (argvars[2].v_type != VAR_NUMBER) { + emsg(_(e_number_exp)); + return; } else { - accum = argvars[2]; + initial = argvars[2]; i = 0; } - tv_copy(&accum, rettv); + tv_copy(&initial, rettv); for (; i < tv_blob_len(b); i++) { - argv[0] = accum; + argv[0] = *rettv; argv[1].v_type = VAR_NUMBER; argv[1].vval.v_number = tv_blob_get(b, i); if (call_func(func_name, -1, rettv, 2, argv, &funcexe) == FAIL) { return; } - accum = *rettv; } } } From 44a5875b24f033c472af50aa4eec4468c554b7c9 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 6 Dec 2021 22:43:59 +0000 Subject: [PATCH 3/4] vim-patch:8.2.1051: crash when changing a list while using reduce() on it Problem: Crash when changing a list while using reduce() on it. Solution: Lock the list. (closes vim/vim#6330) https://github.com/vim/vim/commit/ca275a05d8b79f6a9101604fdede2373d0dea44e --- src/nvim/eval/funcs.c | 11 ++++++++--- src/nvim/testdir/test_listdict.vim | 9 +++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 997096aad4..c80ff8f36a 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -7885,7 +7885,7 @@ static void f_reduce(typval_T *argvars, typval_T *rettv, FunPtr fptr) typval_T initial; typval_T argv[3]; if (argvars[0].v_type == VAR_LIST) { - const list_T *const l = argvars[0].vval.v_list; + list_T *const l = argvars[0].vval.v_list; const listitem_T *li; if (argvars[2].v_type == VAR_UNKNOWN) { @@ -7901,6 +7901,10 @@ static void f_reduce(typval_T *argvars, typval_T *rettv, FunPtr fptr) li = tv_list_first(l); } + const VarLockStatus prev_locked = tv_list_locked(l); + const int called_emsg_start = called_emsg; + + tv_list_set_lock(l, VAR_FIXED); // disallow the list changing here tv_copy(&initial, rettv); for (; li != NULL; li = TV_LIST_ITEM_NEXT(l, li)) { argv[0] = *rettv; @@ -7908,10 +7912,11 @@ static void f_reduce(typval_T *argvars, typval_T *rettv, FunPtr fptr) rettv->v_type = VAR_UNKNOWN; const int r = call_func(func_name, -1, rettv, 2, argv, &funcexe); tv_clear(&argv[0]); - if (r == FAIL) { - return; + if (r == FAIL || called_emsg != called_emsg_start) { + break; } } + tv_list_set_lock(l, prev_locked); } else { const blob_T *const b = argvars[0].vval.v_blob; int i; diff --git a/src/nvim/testdir/test_listdict.vim b/src/nvim/testdir/test_listdict.vim index 42b46fc76c..fdf9123d82 100644 --- a/src/nvim/testdir/test_listdict.vim +++ b/src/nvim/testdir/test_listdict.vim @@ -649,6 +649,15 @@ func Test_reduce() call assert_fails("call reduce({}, { acc, val -> acc + val }, 1)", 'E897:') call assert_fails("call reduce(0, { acc, val -> acc + val }, 1)", 'E897:') call assert_fails("call reduce('', { acc, val -> acc + val }, 1)", 'E897:') + + let g:lut = [1, 2, 3, 4] + func EvilRemove() + call remove(g:lut, 1) + return 1 + endfunc + call assert_fails("call reduce(g:lut, { acc, val -> EvilRemove() }, 1)", 'E742:') + unlet g:lut + delfunc EvilRemove endfunc " splitting a string to a List From 8d99f53f3dc0815d5515551473367d06669836e0 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 6 Dec 2021 22:51:08 +0000 Subject: [PATCH 4/4] vim-patch:8.2.1083: crash when using reduce() on a NULL list Problem: Crash when using reduce() on a NULL list. Solution: Only access the list when not NULL. https://github.com/vim/vim/commit/fda20c4cc59008264676a6deb6a3095ed0c248e0 CHECK_LIST_MATERIALIZE hasn't been ported yet, but presumably if it is ported it'll use tv_list_first to check for range_list_item, which already checks for NULL, so this should need no extra changes and can be a full port. We didn't actually crash here due to the use of Nvim's tv_list functions checking for NULL, but apply these changes to match Vim better anyway. --- src/nvim/eval/funcs.c | 29 ++++++++++++++++------------- src/nvim/testdir/test_listdict.vim | 3 +++ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index c80ff8f36a..bd790bfdd3 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -7901,22 +7901,25 @@ static void f_reduce(typval_T *argvars, typval_T *rettv, FunPtr fptr) li = tv_list_first(l); } - const VarLockStatus prev_locked = tv_list_locked(l); - const int called_emsg_start = called_emsg; - - tv_list_set_lock(l, VAR_FIXED); // disallow the list changing here tv_copy(&initial, rettv); - for (; li != NULL; li = TV_LIST_ITEM_NEXT(l, li)) { - argv[0] = *rettv; - argv[1] = *TV_LIST_ITEM_TV(li); - rettv->v_type = VAR_UNKNOWN; - const int r = call_func(func_name, -1, rettv, 2, argv, &funcexe); - tv_clear(&argv[0]); - if (r == FAIL || called_emsg != called_emsg_start) { - break; + + if (l != NULL) { + const VarLockStatus prev_locked = tv_list_locked(l); + const int called_emsg_start = called_emsg; + + tv_list_set_lock(l, VAR_FIXED); // disallow the list changing here + for (; li != NULL; li = TV_LIST_ITEM_NEXT(l, li)) { + argv[0] = *rettv; + argv[1] = *TV_LIST_ITEM_TV(li); + rettv->v_type = VAR_UNKNOWN; + const int r = call_func(func_name, -1, rettv, 2, argv, &funcexe); + tv_clear(&argv[0]); + if (r == FAIL || called_emsg != called_emsg_start) { + break; + } } + tv_list_set_lock(l, prev_locked); } - tv_list_set_lock(l, prev_locked); } else { const blob_T *const b = argvars[0].vval.v_blob; int i; diff --git a/src/nvim/testdir/test_listdict.vim b/src/nvim/testdir/test_listdict.vim index fdf9123d82..114cca7ec0 100644 --- a/src/nvim/testdir/test_listdict.vim +++ b/src/nvim/testdir/test_listdict.vim @@ -658,6 +658,9 @@ func Test_reduce() call assert_fails("call reduce(g:lut, { acc, val -> EvilRemove() }, 1)", 'E742:') unlet g:lut delfunc EvilRemove + + call assert_equal(42, reduce(v:_null_list, function('add'), 42)) + call assert_equal(42, reduce(v:_null_blob, function('add'), 42)) endfunc " splitting a string to a List