From 9095101743b2606fb1d5f7a5a1216f22d2fb2b4a Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Wed, 18 Nov 2020 03:11:00 +0000 Subject: [PATCH 01/37] vim-patch:8.1.0735: cannot handle binary data Problem: Cannot handle binary data. Solution: Add the Blob type. (Yasuhiro Matsumoto, closes vim/vim#3638) https://github.com/vim/vim/commit/6e5ea8d2a995b32bbc5972edc4f827b959f2702f Nvim-specific Blob conversions are implemented in future commits. Refactor write_blob() to use a FileDescriptor, as f_writefile() was refactored to use one (does not apply to read_blob()). Use var_check_lock() in f_add() for Blobs from v8.1.0897. Add a modeline to test_blob.vim and fix some doc typos. Include if_perl.txt's VIM::Blob() documentation. Interestingly, this function already worked before this port, as it just returns a Blob string literal, not an actual Blob object. N/A patches for version.c: vim-patch:8.1.0741: viminfo with Blob is not tested Problem: Viminfo with Blob is not tested. Solution: Extend the viminfo test. Fix reading a blob. Fixed storing a special variable value. https://github.com/vim/vim/commit/8c8b8bb56c724cc1bfc3d8520eec33f2d399697c vim-patch:8.1.1022: may use NULL pointer when out of memory Problem: May use NULL pointer when out of memory. (Coverity) Solution: Check for blob_alloc() returning NULL. https://github.com/vim/vim/commit/e142a9467a7f6845a426d8db6efedf246d3c13ac --- runtime/doc/eval.txt | 122 ++++++--- runtime/doc/if_perl.txt | 3 + src/nvim/api/private/helpers.c | 4 + src/nvim/eval.c | 331 ++++++++++++++++++++++-- src/nvim/eval.h | 2 + src/nvim/eval/encode.c | 31 +++ src/nvim/eval/executor.c | 14 + src/nvim/eval/funcs.c | 218 ++++++++++++++-- src/nvim/eval/typval.c | 130 +++++++++- src/nvim/eval/typval.h | 69 +++++ src/nvim/eval/typval_encode.c.h | 12 + src/nvim/globals.h | 2 + src/nvim/lua/converter.c | 4 + src/nvim/testdir/test_blob.vim | 176 +++++++++++++ src/nvim/testdir/test_method.vim | 11 +- src/nvim/vim.h | 1 + test/functional/eval/null_spec.lua | 2 +- test/functional/eval/writefile_spec.lua | 2 +- 18 files changed, 1029 insertions(+), 105 deletions(-) create mode 100644 src/nvim/testdir/test_blob.vim diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 9d15bd52a5..b9eeec9a6b 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -15,7 +15,7 @@ Using expressions is introduced in chapter 41 of the user manual |usr_41.txt|. 1.1 Variable types ~ *E712* -There are six types of variables: +There are seven types of variables: *Number* *Integer* Number A 32 or 64 bit signed number. |expr-number| @@ -43,6 +43,10 @@ Dictionary An associative, unordered array: Each entry has a key and a {'blue': "#0000ff", 'red': "#ff0000"} #{blue: "#0000ff", red: "#ff0000"} +Blob Binary Large Object. Stores any sequence of bytes. *Blob* + Example: 0zFF00ED015DAF + 0z is an empty Blob. + The Number and String types are converted automatically, depending on how they are used. @@ -97,6 +101,7 @@ Note that " " and "0" are also non-empty strings, thus considered to be TRUE. A List, Dictionary or Float is not a Number or String, thus evaluate to FALSE. *E745* *E728* *E703* *E729* *E730* *E731* + *E974* *E975* *E976* |List|, |Dictionary|, |Funcref|, and |Blob| types are not automatically converted. @@ -1012,6 +1017,12 @@ just above. Also see |sublist| below. Examples: > :let l = mylist[4:4] " List with one item :let l = mylist[:] " shallow copy of a List +If expr8 is a |Blob| this results in a new |Blob| with the bytes in the +indexes expr1a and expr1b, inclusive. Examples: > + :let b = 0zDEADBEEF + :let bs = b[1:2] " 0zADBE + :let bs = b[] " copy of 0zDEADBEEF + Using expr8[expr1] or expr8[expr1a : expr1b] on a |Funcref| results in an error. @@ -1180,6 +1191,14 @@ encodings. Use "\u00ff" to store character 255 correctly as UTF-8. Note that "\000" and "\x00" force the end of the string. +blob-literal *blob-literal* *E973* *E977* *E978* +------------ + +Hexadecimal starting with 0z or 0Z, with an arbitrary number of bytes. +The sequence must be an even number of hex characters. Example: > + :let b = 0zFF00ED015DAF + + literal-string *literal-string* *E115* --------------- 'string' string constant *expr-'* @@ -2020,6 +2039,8 @@ v:t_list Value of List type. Read-only. See: |type()| v:t_number Value of Number type. Read-only. See: |type()| *v:t_string* *t_string-variable* v:t_string Value of String type. Read-only. See: |type()| + *v:t_blob* *t_blob-variable* +v:t_blob Value of Blob type. Read-only. See: |type()| *v:termresponse* *termresponse-variable* v:termresponse The escape sequence returned by the terminal for the DA @@ -2315,8 +2336,8 @@ hlID({name}) Number syntax ID of highlight group {name} hostname() String name of the machine Vim is running on iconv({expr}, {from}, {to}) String convert encoding of {expr} indent({lnum}) Number indent of line {lnum} -index({list}, {expr} [, {start} [, {ic}]]) - Number index in {list} where {expr} appears +index({object}, {expr} [, {start} [, {ic}]]) + Number index in {object} where {expr} appears input({prompt} [, {text} [, {completion}]]) String get input from the user inputlist({textlist}) Number let the user pick from a choice list @@ -2324,8 +2345,8 @@ inputrestore() Number restore typeahead inputsave() Number save and clear typeahead inputsecret({prompt} [, {text}]) String like input() but hiding the text -insert({list}, {item} [, {idx}]) - List insert {item} in {list} [before {idx}] +insert({object}, {item} [, {idx}]) + List insert {item} in {object} [before {idx}] interrupt() none interrupt script execution invert({expr}) Number bitwise invert isdirectory({directory}) Number |TRUE| if {directory} is a directory @@ -2407,7 +2428,7 @@ pyxeval({expr}) any evaluate |python_x| expression range({expr} [, {max} [, {stride}]]) List items from {expr} to {max} readdir({dir} [, {expr}]) List file names in {dir} selected by {expr} -readfile({fname} [, {binary} [, {max}]]) +readfile({fname} [, {type} [, {max}]]) List get list of lines from file {fname} reg_executing() String get the executing register name reg_recording() String get the recording register name @@ -2612,8 +2633,8 @@ winrestview({dict}) none restore view of current window winsaveview() Dict save view of current window winwidth({nr}) Number width of window {nr} wordcount() Dict get byte/char/word statistics -writefile({list}, {fname} [, {flags}]) - Number write list of lines to file {fname} +writefile({object}, {fname} [, {flags}]) + Number write |Blob| or |List| of lines to file xor({expr}, {expr}) Number bitwise XOR @@ -5548,17 +5569,21 @@ indent({lnum}) The result is a Number, which is indent of line {lnum} in the When {lnum} is invalid -1 is returned. -index({list}, {expr} [, {start} [, {ic}]]) *index()* - Return the lowest index in |List| {list} where the item has a - value equal to {expr}. There is no automatic conversion, so - the String "4" is different from the Number 4. And the number - 4 is different from the Float 4.0. The value of 'ignorecase' - is not used here, case always matters. +index({object}, {expr} [, {start} [, {ic}]]) *index()* + If {object} is a |List| return the lowest index where the item + has a value equal to {expr}. There is no automatic + conversion, so the String "4" is different from the Number 4. + And the number 4 is different from the Float 4.0. The value + of 'ignorecase' is not used here, case always matters. + + If {object} is |Blob| return the lowest index where the byte + value is equal to {expr}. + If {start} is given then start looking at the item with index {start} (may be negative for an item relative to the end). When {ic} is given and it is |TRUE|, ignore case. Otherwise case must match. - -1 is returned when {expr} is not found in {list}. + -1 is returned when {expr} is not found in {object}. Example: > :let idx = index(words, "the") :if index(numbers, 123) >= 0 @@ -5717,13 +5742,16 @@ inputsecret({prompt} [, {text}]) *inputsecret()* typed on the command-line in response to the issued prompt. NOTE: Command-line completion is not supported. -insert({list}, {item} [, {idx}]) *insert()* - Insert {item} at the start of |List| {list}. +insert({object}, {item} [, {idx}]) *insert()* + When {object} is a |List| or a |Blob| insert {item} at the start + of it. + If {idx} is specified insert {item} before the item with index {idx}. If {idx} is zero it goes before the first item, just like omitting {idx}. A negative {idx} is also possible, see |list-index|. -1 inserts just before the last item. - Returns the resulting |List|. Examples: > + + Returns the resulting |List| or |Blob|. Examples: > :let mylist = insert([2, 3, 5], 1) :call insert(mylist, 4, -1) :call insert(mylist, 6, len(mylist)) @@ -5787,16 +5815,16 @@ islocked({expr}) *islocked()* *E786* id({expr}) *id()* Returns a |String| which is a unique identifier of the - container type (|List|, |Dict| and |Partial|). It is + container type (|List|, |Dict|, |Blob| and |Partial|). It is guaranteed that for the mentioned types `id(v1) ==# id(v2)` returns true iff `type(v1) == type(v2) && v1 is v2`. - Note that |v:_null_string|, |v:_null_list|, and |v:_null_dict| - have the same `id()` with different types because they are - internally represented as a NULL pointers. `id()` returns a - hexadecimal representanion of the pointers to the containers - (i.e. like `0x994a40`), same as `printf("%p", {expr})`, - but it is advised against counting on the exact format of - return value. + Note that |v:_null_string|, |v:_null_list|, |v:_null_dict| and + |v:_null_blob| have the same `id()` with different types + because they are internally represented as NULL pointers. + `id()` returns a hexadecimal representanion of the pointers to + the containers (i.e. like `0x994a40`), same as `printf("%p", + {expr})`, but it is advised against counting on the exact + format of the return value. It is not guaranteed that `id(no_longer_existing_container)` will not be equal to some other `id()`: new containers may @@ -7168,16 +7196,18 @@ readdir({directory} [, {expr}]) echo s:tree(".") < *readfile()* -readfile({fname} [, {binary} [, {max}]]) +readfile({fname} [, {type} [, {max}]]) Read file {fname} and return a |List|, each line of the file as an item. Lines are broken at NL characters. Macintosh files separated with CR will result in a single long line (unless a NL appears somewhere). All NUL characters are replaced with a NL character. - When {binary} contains "b" binary mode is used: + When {type} contains "b" binary mode is used: - When the last line ends in a NL an extra empty list item is added. - No CR characters are removed. + When {type} contains "B" a |Blob| is returned with the binary + data of the file unmodified. Otherwise: - CR characters that appear before a NL are removed. - Whether the last line ends in a NL or not does not matter. @@ -7357,6 +7387,17 @@ remove({list}, {idx} [, {end}]) *remove()* < Can also be used as a |method|: > mylist->remove(idx) +remove({blob}, {idx} [, {end}]) + Without {end}: Remove the byte at {idx} from |Blob| {blob} and + return the byte. + With {end}: Remove bytes from {idx} to {end} (inclusive) and + return a |Blob| with these bytes. When {idx} points to the same + byte as {end} a |Blob| with one byte is returned. When {end} + points to a byte before {idx} this is an error. + Example: > + :echo "last byte: " . remove(myblob, -1) + :call remove(mylist, 0, 9) + remove({dict}, {key}) Remove the entry from {dict} with key {key} and return it. Example: > @@ -7400,9 +7441,11 @@ resolve({filename}) *resolve()* *E655* path name) and also keeps a trailing path separator. *reverse()* -reverse({list}) Reverse the order of items in {list} in-place. Returns - {list}. - If you want a list to remain unmodified make a copy first: > +reverse({object}) + Reverse the order of items in {object} in-place. + {object} can be a |List| or a |Blob|. + Returns {object}. + If you want an object to remain unmodified make a copy first: > :let revlist = reverse(copy(mylist)) < Can also be used as a |method|: > mylist->reverse() @@ -9483,6 +9526,7 @@ type({expr}) *type()* Float: 5 (|v:t_float|) Boolean: 6 (|v:true| and |v:false|) Null: 7 (|v:null|) + Blob: 10 (|v:t_blob|) For backward compatibility, this method can be used: > :if type(myvar) == type(0) :if type(myvar) == type("") @@ -9927,14 +9971,17 @@ wordcount() *wordcount()* *writefile()* -writefile({list}, {fname} [, {flags}]) - Write |List| {list} to file {fname}. Each list item is - separated with a NL. Each list item must be a String or - Number. +writefile({object}, {fname} [, {flags}]) + When {object} is a |List| write it to file {fname}. Each list + item is separated with a NL. Each list item must be a String + or Number. When {flags} contains "b" then binary mode is used: There will not be a NL after the last list item. An empty item at the end does cause the last line in the file to end in a NL. + When {object} is a |Blob| write the bytes to file {fname} + unmodified. + When {flags} contains "a" then append mode is used, lines are appended to the file: > :call writefile(["foo"], "event.log", "a") @@ -10450,7 +10497,10 @@ This does NOT work: > This cannot be used to set a byte in a String. You can do that like this: > :let var = var[0:2] . 'X' . var[4:] -< +< When {var-name} is a |Blob| then {idx} can be the + length of the blob, in which case one byte is + appended. + *E711* *E719* :let {var-name}[{idx1}:{idx2}] = {expr1} *E708* *E709* *E710* Set a sequence of items in a |List| to the result of diff --git a/runtime/doc/if_perl.txt b/runtime/doc/if_perl.txt index ddcf220844..3787ca69ba 100644 --- a/runtime/doc/if_perl.txt +++ b/runtime/doc/if_perl.txt @@ -189,6 +189,9 @@ VIM::Eval({expr}) Evaluates {expr} and returns (success, value) in list A |List| is turned into a string by joining the items and inserting line breaks. + *perl-Blob* +VIM::Blob({expr}) Return Blob literal string 0zXXXX from scalar value. + ============================================================================== 3. VIM::Buffer objects *perl-buffer* diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c index 0ed5e6408b..6ece819afa 100644 --- a/src/nvim/api/private/helpers.c +++ b/src/nvim/api/private/helpers.c @@ -444,6 +444,9 @@ void set_option_to(uint64_t channel_id, void *to, int type, #define TYPVAL_ENCODE_CONV_EXT_STRING(tv, str, len, type) \ TYPVAL_ENCODE_CONV_NIL(tv) +#define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ + abort() /* TODO(seandewar) */ \ + #define TYPVAL_ENCODE_CONV_FUNC_START(tv, fun) \ do { \ TYPVAL_ENCODE_CONV_NIL(tv); \ @@ -584,6 +587,7 @@ static inline void typval_encode_dict_end(EncodedData *const edata) #undef TYPVAL_ENCODE_CONV_STRING #undef TYPVAL_ENCODE_CONV_STR_STRING #undef TYPVAL_ENCODE_CONV_EXT_STRING +#undef TYPVAL_ENCODE_CONV_BLOB #undef TYPVAL_ENCODE_CONV_NUMBER #undef TYPVAL_ENCODE_CONV_FLOAT #undef TYPVAL_ENCODE_CONV_FUNC_START diff --git a/src/nvim/eval.c b/src/nvim/eval.c index dfbb187b5b..c3a9c0b9ca 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -69,6 +69,7 @@ static char *e_nowhitespace = N_("E274: No white space allowed before parenthesis"); static char *e_invalwindow = N_("E957: Invalid window number"); static char *e_lock_unlock = N_("E940: Cannot lock or unlock variable %s"); +static char *e_write2 = N_("E80: Error while writing: %s"); // TODO(ZyX-I): move to eval/executor static char *e_letwrong = N_("E734: Wrong variable type for %s="); @@ -113,6 +114,8 @@ typedef struct { int fi_varcount; // nr of variables in the list listwatch_T fi_lw; // keep an eye on the item used. list_T *fi_list; // list being used + int fi_bi; // index of blob + blob_T *fi_blob; // blob being used } forinfo_T; // values for vv_flags: @@ -227,6 +230,7 @@ static struct vimvar { VV(VV_TYPE_DICT, "t_dict", VAR_NUMBER, VV_RO), VV(VV_TYPE_FLOAT, "t_float", VAR_NUMBER, VV_RO), VV(VV_TYPE_BOOL, "t_bool", VAR_NUMBER, VV_RO), + VV(VV_TYPE_BLOB, "t_blob", VAR_NUMBER, VV_RO), VV(VV_EVENT, "event", VAR_DICT, VV_RO), VV(VV_ECHOSPACE, "echospace", VAR_NUMBER, VV_RO), VV(VV_ARGV, "argv", VAR_LIST, VV_RO), @@ -251,6 +255,7 @@ static struct vimvar { #define vv_str vv_di.di_tv.vval.v_string #define vv_list vv_di.di_tv.vval.v_list #define vv_dict vv_di.di_tv.vval.v_dict +#define vv_blob vv_di.di_tv.vval.v_blob #define vv_partial vv_di.di_tv.vval.v_partial #define vv_tv vv_di.di_tv @@ -393,6 +398,7 @@ void eval_init(void) set_vim_var_nr(VV_TYPE_DICT, VAR_TYPE_DICT); set_vim_var_nr(VV_TYPE_FLOAT, VAR_TYPE_FLOAT); set_vim_var_nr(VV_TYPE_BOOL, VAR_TYPE_BOOL); + set_vim_var_nr(VV_TYPE_BLOB, VAR_TYPE_BLOB); set_vim_var_bool(VV_FALSE, kBoolVarFalse); set_vim_var_bool(VV_TRUE, kBoolVarTrue); @@ -2059,18 +2065,17 @@ char_u *get_lval(char_u *const name, typval_T *const rettv, return NULL; } - /* - * Loop until no more [idx] or .key is following. - */ + // Loop until no more [idx] or .key is following. lp->ll_tv = &v->di_tv; var1.v_type = VAR_UNKNOWN; var2.v_type = VAR_UNKNOWN; while (*p == '[' || (*p == '.' && lp->ll_tv->v_type == VAR_DICT)) { if (!(lp->ll_tv->v_type == VAR_LIST && lp->ll_tv->vval.v_list != NULL) - && !(lp->ll_tv->v_type == VAR_DICT - && lp->ll_tv->vval.v_dict != NULL)) { - if (!quiet) - EMSG(_("E689: Can only index a List or Dictionary")); + && !(lp->ll_tv->v_type == VAR_DICT && lp->ll_tv->vval.v_dict != NULL) + && !(lp->ll_tv->v_type == VAR_BLOB && lp->ll_tv->vval.v_blob != NULL)) { + if (!quiet) { + EMSG(_("E689: Can only index a List, Dictionary or Blob")); + } return NULL; } if (lp->ll_range) { @@ -2119,10 +2124,11 @@ char_u *get_lval(char_u *const name, typval_T *const rettv, tv_clear(&var1); return NULL; } - if (rettv != NULL && (rettv->v_type != VAR_LIST - || rettv->vval.v_list == NULL)) { + if (rettv != NULL + && !(rettv->v_type == VAR_LIST || rettv->vval.v_list != NULL) + && !(rettv->v_type == VAR_BLOB || rettv->vval.v_blob != NULL)) { if (!quiet) { - EMSG(_("E709: [:] requires a List value")); + EMSG(_("E709: [:] requires a List or Blob value")); } tv_clear(&var1); return NULL; @@ -2236,6 +2242,28 @@ char_u *get_lval(char_u *const name, typval_T *const rettv, tv_clear(&var1); lp->ll_tv = &lp->ll_di->di_tv; + } else if (lp->ll_tv->v_type == VAR_BLOB) { + // Get the number and item for the only or first index of the List. + if (empty1) { + lp->ll_n1 = 0; + } else { + // Is number or string. + lp->ll_n1 = (long)tv_get_number(&var1); + } + tv_clear(&var1); + + if (lp->ll_n1 < 0 || lp->ll_n1 > tv_blob_len(lp->ll_tv->vval.v_blob)) { + if (!quiet) { + EMSGN(_(e_listidx), lp->ll_n1); + } + return NULL; + } + if (lp->ll_range && !lp->ll_empty2) { + lp->ll_n2 = (long)tv_get_number(&var2); + tv_clear(&var2); + } + lp->ll_blob = lp->ll_tv->vval.v_blob; + lp->ll_tv = NULL; } else { // Get the number and item for the only or first index of the List. if (empty1) { @@ -2329,7 +2357,41 @@ static void set_var_lval(lval_T *lp, char_u *endp, typval_T *rettv, if (lp->ll_tv == NULL) { cc = *endp; *endp = NUL; - if (op != NULL && *op != '=') { + if (lp->ll_blob != NULL) { + if (op != NULL && *op != '=') { + EMSG2(_(e_letwrong), op); + return; + } + + if (lp->ll_range && rettv->v_type == VAR_BLOB) { + if (tv_blob_len(rettv->vval.v_blob) != tv_blob_len(lp->ll_blob)) { + EMSG(_("E972: Blob value has more items than target")); + return; + } + + for (int i = lp->ll_n1; i <= lp->ll_n2; i++) { + tv_blob_set(lp->ll_blob, i, tv_blob_get(rettv->vval.v_blob, i)); + } + } else { + bool error = false; + const char_u val = tv_get_number_chk(rettv, &error); + if (!error) { + garray_T *const gap = &lp->ll_blob->bv_ga; + + // Allow for appending a byte. Setting a byte beyond + // the end is an error otherwise. + if (lp->ll_n1 < gap->ga_len || lp->ll_n1 == gap->ga_len) { + ga_grow(&lp->ll_blob->bv_ga, 1); + tv_blob_set(lp->ll_blob, lp->ll_n1, val); + if (lp->ll_n1 == gap->ga_len) { + gap->ga_len++; + } + } else { + EMSG(_(e_invrange)); + } + } + } + } else if (op != NULL && *op != '=') { typval_T tv; if (is_const) { @@ -2508,19 +2570,29 @@ void *eval_for_line(const char_u *arg, bool *errp, char_u **nextcmdp, int skip) if (eval0(skipwhite(expr + 2), &tv, nextcmdp, !skip) == OK) { *errp = false; if (!skip) { - l = tv.vval.v_list; - if (tv.v_type != VAR_LIST) { + if (tv.v_type == VAR_LIST) { + l = tv.vval.v_list; + if (l == NULL) { + // a null list is like an empty list: do nothing + tv_clear(&tv); + } else { + // No need to increment the refcount, it's already set for + // the list being used in "tv". + fi->fi_list = l; + tv_list_watch_add(l, &fi->fi_lw); + fi->fi_lw.lw_item = tv_list_first(l); + } + } else if (tv.v_type == VAR_BLOB) { + blob_T *const b = tv.vval.v_blob; + if (b == NULL) { + tv_clear(&tv); + } else { + fi->fi_blob = b; + fi->fi_bi = 0; + } + } else { EMSG(_(e_listreq)); tv_clear(&tv); - } else if (l == NULL) { - // a null list is like an empty list: do nothing - tv_clear(&tv); - } else { - /* No need to increment the refcount, it's already set for the - * list being used in "tv". */ - fi->fi_list = l; - tv_list_watch_add(l, &fi->fi_lw); - fi->fi_lw.lw_item = tv_list_first(l); } } } @@ -2542,6 +2614,19 @@ bool next_for_item(void *fi_void, char_u *arg) { forinfo_T *fi = (forinfo_T *)fi_void; + if (fi->fi_blob != NULL) { + if (fi->fi_bi >= tv_blob_len(fi->fi_blob)) { + return false; + } + typval_T tv; + tv.v_type = VAR_NUMBER; + tv.v_lock = VAR_FIXED; + tv.vval.v_number = tv_blob_get(fi->fi_blob, fi->fi_bi); + fi->fi_bi++; + return ex_let_vars(arg, &tv, true, + fi->fi_semicolon, fi->fi_varcount, false, NULL) == OK; + } + listitem_T *item = fi->fi_lw.lw_item; if (item == NULL) { return false; @@ -3607,7 +3692,7 @@ static int eval5(char_u **arg, typval_T *rettv, int evaluate) if (op != '+' && op != '-' && op != '.') break; - if ((op != '+' || rettv->v_type != VAR_LIST) + if ((op != '+' || (rettv->v_type != VAR_LIST && rettv->v_type != VAR_BLOB)) && (op == '.' || rettv->v_type != VAR_FLOAT)) { // For "list + ...", an illegal use of the first operand as // a number cannot be determined before evaluating the 2nd @@ -3653,6 +3738,21 @@ static int eval5(char_u **arg, typval_T *rettv, int evaluate) tv_clear(rettv); rettv->v_type = VAR_STRING; rettv->vval.v_string = p; + } else if (op == '+' && rettv->v_type == VAR_BLOB + && var2.v_type == VAR_BLOB) { + const blob_T *const b1 = rettv->vval.v_blob; + const blob_T *const b2 = var2.vval.v_blob; + blob_T *const b = tv_blob_alloc(); + + for (int i = 0; i < tv_blob_len(b1); i++) { + ga_append(&b->bv_ga, tv_blob_get(b1, i)); + } + for (int i = 0; i < tv_blob_len(b2); i++) { + ga_append(&b->bv_ga, tv_blob_get(b2, i)); + } + + tv_clear(rettv); + tv_blob_set_ret(rettv, b); } else if (op == '+' && rettv->v_type == VAR_LIST && var2.v_type == VAR_LIST) { // Concatenate Lists. @@ -3850,6 +3950,7 @@ static int eval6(char_u **arg, typval_T *rettv, int evaluate, int want_string) // Handle sixth level expression: // number number constant +// 0zFFFFFFFF Blob constant // "string" string constant // 'string' literal string constant // &option-name option value @@ -3946,7 +4047,31 @@ static int eval7( rettv->v_type = VAR_FLOAT; rettv->vval.v_float = f; } + } else if (**arg == '0' && ((*arg)[1] == 'z' || (*arg)[1] == 'Z')) { + blob_T *blob = NULL; + // Blob constant: 0z0123456789abcdef + if (evaluate) { + blob = tv_blob_alloc(); + } + char_u *bp; + for (bp = *arg + 2; ascii_isxdigit(bp[0]); bp += 2) { + if (!ascii_isxdigit(bp[1])) { + EMSG(_("E973: Blob literal should have an even number of hex " + "characters")); + xfree(blob); + ret = FAIL; + break; + } + if (blob != NULL) { + ga_append(&blob->bv_ga, (hex2nr(*bp) << 4) + hex2nr(*(bp + 1))); + } + } + if (blob != NULL) { + tv_blob_set_ret(rettv, blob); + } + *arg = bp; } else { + // decimal, hex or octal number vim_str2nr(*arg, NULL, &len, STR2NR_ALL, &n, NULL, 0, true); if (len == 0) { EMSG2(_(e_invexpr2), *arg); @@ -4336,7 +4461,8 @@ eval_index( case VAR_STRING: case VAR_NUMBER: case VAR_LIST: - case VAR_DICT: { + case VAR_DICT: + case VAR_BLOB: { break; } } @@ -4462,6 +4588,51 @@ eval_index( rettv->vval.v_string = (char_u *)v; break; } + case VAR_BLOB: { + len = tv_blob_len(rettv->vval.v_blob); + if (range) { + // The resulting variable is a substring. If the indexes + // are out of range the result is empty. + if (n1 < 0) { + n1 = len + n1; + if (n1 < 0) { + n1 = 0; + } + } + if (n2 < 0) { + n2 = len + n2; + } else if (n2 >= len) { + n2 = len - 1; + } + if (n1 >= len || n2 < 0 || n1 > n2) { + tv_clear(rettv); + rettv->v_type = VAR_BLOB; + rettv->vval.v_blob = NULL; + } else { + blob_T *const blob = tv_blob_alloc(); + ga_grow(&blob->bv_ga, n2 - n1 + 1); + blob->bv_ga.ga_len = n2 - n1 + 1; + for (long i = n1; i <= n2; i++) { + tv_blob_set(blob, i - n1, tv_blob_get(rettv->vval.v_blob, i)); + } + tv_clear(rettv); + tv_blob_set_ret(rettv, blob); + } + } else { + // The resulting variable is a string of a single + // character. If the index is too big or negative the + // result is empty. + if (n1 < len && n1 >= 0) { + const int v = (int)tv_blob_get(rettv->vval.v_blob, n1); + tv_clear(rettv); + rettv->v_type = VAR_NUMBER; + rettv->vval.v_number = v; + } else { + EMSGN(_(e_blobidx), n1); + } + } + break; + } case VAR_LIST: { len = tv_list_len(rettv->vval.v_list); if (n1 < 0) { @@ -5398,7 +5569,8 @@ bool set_ref_in_item(typval_T *tv, int copyID, ht_stack_T **ht_stack, case VAR_SPECIAL: case VAR_FLOAT: case VAR_NUMBER: - case VAR_STRING: { + case VAR_STRING: + case VAR_BLOB: { break; } } @@ -6161,6 +6333,7 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map) dict_T *d = NULL; typval_T save_val; typval_T save_key; + blob_T *b = NULL; int rem = false; int todo; char_u *ermsg = (char_u *)(map ? "map()" : "filter()"); @@ -6170,7 +6343,12 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map) int save_did_emsg; int idx = 0; - if (argvars[0].v_type == VAR_LIST) { + if (argvars[0].v_type == VAR_BLOB) { + tv_copy(&argvars[0], rettv); + if ((b = argvars[0].vval.v_blob) == NULL) { + return; + } + } else if (argvars[0].v_type == VAR_LIST) { tv_copy(&argvars[0], rettv); if ((l = argvars[0].vval.v_list) == NULL || (!map @@ -6234,6 +6412,30 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map) } } hash_unlock(ht); + } else if (argvars[0].v_type == VAR_BLOB) { + vimvars[VV_KEY].vv_type = VAR_NUMBER; + + for (int i = 0; i < b->bv_ga.ga_len; i++) { + typval_T tv; + tv.v_type = VAR_NUMBER; + tv.vval.v_number = tv_blob_get(b, i); + vimvars[VV_KEY].vv_nr = idx; + if (filter_map_one(&tv, expr, map, &rem) == FAIL || did_emsg) { + break; + } + if (tv.v_type != VAR_NUMBER) { + EMSG(_(e_invalblob)); + return; + } + tv.v_type = VAR_NUMBER; + tv_blob_set(b, i, tv.vval.v_number); + if (!map && rem) { + char_u *const p = (char_u *)argvars[0].vval.v_blob->bv_ga.ga_data; + memmove(p + idx, p + i + 1, (size_t)b->bv_ga.ga_len - i - 1); + b->bv_ga.ga_len--; + i--; + } + } } else { assert(argvars[0].v_type == VAR_LIST); vimvars[VV_KEY].vv_type = VAR_NUMBER; @@ -7728,10 +7930,61 @@ bool write_list(FileDescriptor *const fp, const list_T *const list, } return true; write_list_error: - emsgf(_("E80: Error while writing: %s"), os_strerror(error)); + emsgf(_(e_write2), os_strerror(error)); return false; } +/// Write a blob to file with descriptor `fp`. +/// +/// @param[in] fp File to write to. +/// @param[in] blob Blob to write. +/// +/// @return true on success, or false on failure. +bool write_blob(FileDescriptor *const fp, const blob_T *const blob) + FUNC_ATTR_NONNULL_ARG(1) +{ + int error = 0; + const int len = tv_blob_len(blob); + if (len > 0) { + const ptrdiff_t written = file_write(fp, blob->bv_ga.ga_data, (size_t)len); + if (written < (ptrdiff_t)len) { + error = (int)written; + goto write_blob_error; + } + } + error = file_flush(fp); + if (error != 0) { + goto write_blob_error; + } + return true; +write_blob_error: + EMSG2(_(e_write2), os_strerror(error)); + return false; +} + +/// Read a blob from a file `fd`. +/// +/// @param[in] fd File to read from. +/// @param[in,out] blob Blob to write to. +/// +/// @return true on success, or false on failure. +bool read_blob(FILE *const fd, blob_T *const blob) + FUNC_ATTR_NONNULL_ALL +{ + FileInfo file_info; + if (!os_fileinfo_fd(fileno(fd), &file_info)) { + return false; + } + const int size = (int)os_fileinfo_size(&file_info); + ga_grow(&blob->bv_ga, size); + blob->bv_ga.ga_len = size; + if (fread(blob->bv_ga.ga_data, 1, blob->bv_ga.ga_len, fd) + < (size_t)blob->bv_ga.ga_len) { + return false; + } + return true; +} + /// Saves a typval_T as a string. /// /// For lists or buffers, replaces NLs with NUL and separates items with NLs. @@ -9423,6 +9676,7 @@ int var_item_copy(const vimconv_T *const conv, case VAR_PARTIAL: case VAR_BOOL: case VAR_SPECIAL: + case VAR_BLOB: tv_copy(from, to); break; case VAR_STRING: @@ -10815,6 +11069,29 @@ int typval_compare( // For "is" a different type always means false, for "notis" // it means true. n1 = type == EXPR_ISNOT; + } else if (typ1->v_type == VAR_BLOB || typ2->v_type == VAR_BLOB) { + if (type_is) { + n1 = typ1->v_type == typ2->v_type + && typ1->vval.v_blob == typ2->vval.v_blob; + if (type == EXPR_ISNOT) { + n1 = !n1; + } + } else if (typ1->v_type != typ2->v_type + || (type != EXPR_EQUAL && type != EXPR_NEQUAL)) { + if (typ1->v_type != typ2->v_type) { + EMSG(_("E977: Can only compare Blob with Blob")); + } else { + EMSG(_(e_invalblob)); + } + tv_clear(typ1); + return FAIL; + } else { + // Compare two Blobs for being equal or unequal. + n1 = tv_blob_equal(typ1->vval.v_blob, typ2->vval.v_blob); + if (type == EXPR_NEQUAL) { + n1 = !n1; + } + } } else if (typ1->v_type == VAR_LIST || typ2->v_type == VAR_LIST) { if (type_is) { n1 = typ1->v_type == typ2->v_type diff --git a/src/nvim/eval.h b/src/nvim/eval.h index 41120b3c78..e5a65122f1 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -63,6 +63,7 @@ typedef struct lval_S { dict_T *ll_dict; ///< The Dictionary or NULL. dictitem_T *ll_di; ///< The dictitem or NULL. char_u *ll_newkey; ///< New key for Dict in allocated memory or NULL. + blob_T *ll_blob; ///< The Blob or NULL. } lval_T; /// enum used by var_flavour() @@ -154,6 +155,7 @@ typedef enum { VV_TYPE_DICT, VV_TYPE_FLOAT, VV_TYPE_BOOL, + VV_TYPE_BLOB, VV_EVENT, VV_ECHOSPACE, VV_ARGV, diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index a4d7af7971..45927db0ac 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -319,6 +319,28 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, #define TYPVAL_ENCODE_CONV_EXT_STRING(tv, buf, len, type) +#define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ + do { \ + const blob_T *const blob_ = (blob); \ + const int len_ = (len); \ + if (len_ == 0) { \ + ga_concat(gap, "[]"); \ + } else { \ + ga_grow(gap, 1 + len_ * 5); \ + ga_append(gap, '['); \ + char numbuf[NUMBUFLEN]; \ + for (int i_ = 0; i_ < len_; i_++) { \ + if (i_ > 0) { \ + ga_append(gap, ','); \ + } \ + vim_snprintf((char *)numbuf, ARRAY_SIZE(numbuf), "0x%02X", \ + (int)tv_blob_get(blob_, i_)); \ + ga_concat(gap, numbuf); \ + } \ + ga_append(gap, ']'); \ + } \ + } while (0) + #define TYPVAL_ENCODE_CONV_NUMBER(tv, num) \ do { \ char numbuf[NUMBUFLEN]; \ @@ -705,6 +727,10 @@ static inline int convert_to_json_string(garray_T *const gap, return FAIL; \ } while (0) +#undef TYPVAL_ENCODE_CONV_BLOB +#define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ + abort() /* TODO(seandewar) */ \ + #undef TYPVAL_ENCODE_CONV_FUNC_START #define TYPVAL_ENCODE_CONV_FUNC_START(tv, fun) \ return conv_error(_("E474: Error while dumping %s, %s: " \ @@ -770,6 +796,7 @@ bool encode_check_json_key(const typval_T *const tv) #undef TYPVAL_ENCODE_CONV_STRING #undef TYPVAL_ENCODE_CONV_STR_STRING #undef TYPVAL_ENCODE_CONV_EXT_STRING +#undef TYPVAL_ENCODE_CONV_BLOB #undef TYPVAL_ENCODE_CONV_NUMBER #undef TYPVAL_ENCODE_CONV_FLOAT #undef TYPVAL_ENCODE_CONV_FUNC_START @@ -904,6 +931,9 @@ char *encode_tv2json(typval_T *tv, size_t *len) } \ } while (0) +#define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ + abort() /* TODO(seandewar) */ \ + #define TYPVAL_ENCODE_CONV_NUMBER(tv, num) \ msgpack_pack_int64(packer, (int64_t)(num)) @@ -982,6 +1012,7 @@ char *encode_tv2json(typval_T *tv, size_t *len) #undef TYPVAL_ENCODE_CONV_STRING #undef TYPVAL_ENCODE_CONV_STR_STRING #undef TYPVAL_ENCODE_CONV_EXT_STRING +#undef TYPVAL_ENCODE_CONV_BLOB #undef TYPVAL_ENCODE_CONV_NUMBER #undef TYPVAL_ENCODE_CONV_FLOAT #undef TYPVAL_ENCODE_CONV_FUNC_START diff --git a/src/nvim/eval/executor.c b/src/nvim/eval/executor.c index 8ac2c3b8eb..5ced2a0535 100644 --- a/src/nvim/eval/executor.c +++ b/src/nvim/eval/executor.c @@ -38,6 +38,20 @@ int eexe_mod_op(typval_T *const tv1, const typval_T *const tv2, case VAR_SPECIAL: { break; } + case VAR_BLOB: { + if (*op != '+' || tv2->v_type != VAR_BLOB) { + break; + } + // Blob += Blob + if (tv1->vval.v_blob != NULL && tv2->vval.v_blob != NULL) { + blob_T *const b1 = tv1->vval.v_blob; + blob_T *const b2 = tv2->vval.v_blob; + for (int i = 0; i < tv_blob_len(b2); i++) { + ga_append(&b1->bv_ga, (char)tv_blob_get(b2, i)); + } + } + return OK; + } case VAR_LIST: { if (*op != '+' || tv2->v_type != VAR_LIST) { break; diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 801b0f9d1c..dbb00da911 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -321,6 +321,13 @@ static void f_add(typval_T *argvars, typval_T *rettv, FunPtr fptr) tv_list_append_tv(l, &argvars[1]); tv_copy(&argvars[0], rettv); } + } else if (argvars[0].v_type == VAR_BLOB) { + blob_T *const b = argvars[0].vval.v_blob; + if (b != NULL + && !var_check_lock(b->bv_lock, N_("add() argument"), TV_TRANSLATE)) { + ga_append(&b->bv_ga, (char_u)tv_get_number(&argvars[1])); + tv_copy(&argvars[0], rettv); + } } else { EMSG(_(e_listreq)); } @@ -1874,6 +1881,10 @@ static void f_empty(typval_T *argvars, typval_T *rettv, FunPtr fptr) n = argvars[0].vval.v_special == kSpecialVarNull; break; } + case VAR_BLOB: { + n = (tv_blob_len(argvars[0].vval.v_blob) == 0); + break; + } case VAR_UNKNOWN: { internal_error("f_empty(UNKNOWN)"); break; @@ -2791,7 +2802,19 @@ static void f_get(typval_T *argvars, typval_T *rettv, FunPtr fptr) typval_T *tv = NULL; bool what_is_dict = false; - if (argvars[0].v_type == VAR_LIST) { + if (argvars[0].v_type == VAR_BLOB) { + bool error = false; + const int idx = tv_get_number_chk(&argvars[1], &error); + + if (!error) { + rettv->v_type = VAR_NUMBER; + if (idx >= tv_blob_len(argvars[0].vval.v_blob)) { + EMSGN(_(e_blobidx), idx); + } else { + rettv->vval.v_number = tv_blob_get(argvars[0].vval.v_blob, idx); + } + } + } else if (argvars[0].v_type == VAR_LIST) { if ((l = argvars[0].vval.v_list) != NULL) { bool error = false; @@ -4790,7 +4813,31 @@ static void f_index(typval_T *argvars, typval_T *rettv, FunPtr fptr) bool ic = false; rettv->vval.v_number = -1; - if (argvars[0].v_type != VAR_LIST) { + if (argvars[0].v_type == VAR_BLOB) { + bool error = false; + int start = 0; + + if (argvars[2].v_type != VAR_UNKNOWN) { + start = tv_get_number_chk(&argvars[2], &error); + if (error) { + return; + } + } + blob_T *const b = argvars[0].vval.v_blob; + if (b == NULL) { + return; + } + for (idx = start; idx < tv_blob_len(b); idx++) { + typval_T tv; + tv.v_type = VAR_NUMBER; + tv.vval.v_number = tv_blob_get(b, idx); + if (tv_equal(&tv, &argvars[1], ic, false)) { + rettv->vval.v_number = idx; + return; + } + } + return; + } else if (argvars[0].v_type != VAR_LIST) { EMSG(_(e_listreq)); return; } @@ -4920,7 +4967,37 @@ static void f_insert(typval_T *argvars, typval_T *rettv, FunPtr fptr) list_T *l; bool error = false; - if (argvars[0].v_type != VAR_LIST) { + if (argvars[0].v_type == VAR_BLOB) { + long before = 0; + const int len = tv_blob_len(argvars[0].vval.v_blob); + + if (argvars[2].v_type != VAR_UNKNOWN) { + before = (long)tv_get_number_chk(&argvars[2], &error); + if (error) { + return; // type error; errmsg already given + } + if (before < 0 || before > len) { + EMSG2(_(e_invarg2), tv_get_string(&argvars[2])); + return; + } + } + const int val = tv_get_number_chk(&argvars[1], &error); + if (error) { + return; + } + if (val < 0 || val > 255) { + EMSG2(_(e_invarg2), tv_get_string(&argvars[1])); + return; + } + + ga_grow(&argvars[0].vval.v_blob->bv_ga, 1); + char_u *const p = (char_u *)argvars[0].vval.v_blob->bv_ga.ga_data; + memmove(p + before + 1, p + before, (size_t)len - before); + *(p + before) = val; + argvars[0].vval.v_blob->bv_ga.ga_len++; + + tv_copy(&argvars[0], rettv); + } else if (argvars[0].v_type != VAR_LIST) { EMSG2(_(e_listarg), "insert()"); } else if (!var_check_lock(tv_list_locked((l = argvars[0].vval.v_list)), N_("insert() argument"), TV_TRANSLATE)) { @@ -5581,6 +5658,10 @@ static void f_len(typval_T *argvars, typval_T *rettv, FunPtr fptr) tv_get_string(&argvars[0])); break; } + case VAR_BLOB: { + rettv->vval.v_number = tv_blob_len(argvars[0].vval.v_blob); + break; + } case VAR_LIST: { rettv->vval.v_number = tv_list_len(argvars[0].vval.v_list); break; @@ -6894,6 +6975,7 @@ static void f_readdir(typval_T *argvars, typval_T *rettv, FunPtr fptr) static void f_readfile(typval_T *argvars, typval_T *rettv, FunPtr fptr) { bool binary = false; + bool blob = false; FILE *fd; char_u buf[(IOSIZE/256) * 256]; // rounded to avoid odd + 1 int io_size = sizeof(buf); @@ -6906,14 +6988,14 @@ static void f_readfile(typval_T *argvars, typval_T *rettv, FunPtr fptr) if (argvars[1].v_type != VAR_UNKNOWN) { if (strcmp(tv_get_string(&argvars[1]), "b") == 0) { binary = true; + } else if (strcmp(tv_get_string(&argvars[1]), "B") == 0) { + blob = true; } if (argvars[2].v_type != VAR_UNKNOWN) { maxline = tv_get_number(&argvars[2]); } } - list_T *const l = tv_list_alloc_ret(rettv, kListLenUnknown); - // Always open the file in binary mode, library functions have a mind of // their own about CR-LF conversion. const char *const fname = tv_get_string(&argvars[0]); @@ -6922,6 +7004,18 @@ static void f_readfile(typval_T *argvars, typval_T *rettv, FunPtr fptr) return; } + if (blob) { + tv_blob_alloc_ret(rettv); + if (!read_blob(fd, rettv->vval.v_blob)) { + EMSG("cannot read file"); + tv_blob_free(rettv->vval.v_blob); + } + fclose(fd); + return; + } + + list_T *const l = tv_list_alloc_ret(rettv, kListLenUnknown); + while (maxline < 0 || tv_list_len(l) < maxline) { readlen = (int)fread(buf, 1, io_size, fd); @@ -7190,6 +7284,55 @@ static void f_remove(typval_T *argvars, typval_T *rettv, FunPtr fptr) } } } + } else if (argvars[0].v_type == VAR_BLOB) { + bool error = false; + idx = (long)tv_get_number_chk(&argvars[1], &error); + + if (!error) { + blob_T *const b = argvars[0].vval.v_blob; + const int len = tv_blob_len(b); + + if (idx < 0) { + // count from the end + idx = len + idx; + } + if (idx < 0 || idx >= len) { + EMSGN(_(e_blobidx), idx); + return; + } + if (argvars[2].v_type == VAR_UNKNOWN) { + // Remove one item, return its value. + char_u *const p = (char_u *)b->bv_ga.ga_data; + rettv->vval.v_number = (varnumber_T)(*(p + idx)); + memmove(p + idx, p + idx + 1, (size_t)len - idx - 1); + b->bv_ga.ga_len--; + } else { + // Remove range of items, return list with values. + end = (long)tv_get_number_chk(&argvars[2], &error); + if (error) { + return; + } + if (end < 0) { + // count from the end + end = len + end; + } + if (end >= len || idx > end) { + EMSGN(_(e_blobidx), end); + return; + } + blob_T *const blob = tv_blob_alloc(); + blob->bv_ga.ga_len = end - idx + 1; + ga_grow(&blob->bv_ga, end - idx + 1); + + char_u *const p = (char_u *)b->bv_ga.ga_data; + memmove((char_u *)blob->bv_ga.ga_data, p + idx, + (size_t)(end - idx + 1)); + tv_blob_set_ret(rettv, blob); + + memmove(p + idx, p + end + 1, (size_t)(len - end)); + b->bv_ga.ga_len -= end - idx + 1; + } + } } else if (argvars[0].v_type != VAR_LIST) { EMSG2(_(e_listdictarg), "remove()"); } else if (!var_check_lock(tv_list_locked((l = argvars[0].vval.v_list)), @@ -7465,13 +7608,25 @@ static void f_resolve(typval_T *argvars, typval_T *rettv, FunPtr fptr) */ static void f_reverse(typval_T *argvars, typval_T *rettv, FunPtr fptr) { - list_T *l; - if (argvars[0].v_type != VAR_LIST) { + if (argvars[0].v_type == VAR_BLOB) { + blob_T *const b = argvars[0].vval.v_blob; + const int len = tv_blob_len(b); + + for (int i = 0; i < len / 2; i++) { + const char_u tmp = tv_blob_get(b, i); + tv_blob_set(b, i, tv_blob_get(b, len - i - 1)); + tv_blob_set(b, len - i - 1, tmp); + } + tv_blob_set_ret(rettv, b); + } else if (argvars[0].v_type != VAR_LIST) { EMSG2(_(e_listarg), "reverse()"); - } else if (!var_check_lock(tv_list_locked((l = argvars[0].vval.v_list)), - N_("reverse() argument"), TV_TRANSLATE)) { - tv_list_reverse(l); - tv_list_set_ret(rettv, l); + } else { + list_T *const l = argvars[0].vval.v_list; + if (!var_check_lock(tv_list_locked(l), N_("reverse() argument"), + TV_TRANSLATE)) { + tv_list_reverse(l); + tv_list_set_ret(rettv, l); + } } } @@ -11291,15 +11446,16 @@ static void f_type(typval_T *argvars, typval_T *rettv, FunPtr fptr) int n = -1; switch (argvars[0].v_type) { - case VAR_NUMBER: n = VAR_TYPE_NUMBER; break; - case VAR_STRING: n = VAR_TYPE_STRING; break; + case VAR_NUMBER: n = VAR_TYPE_NUMBER; break; + case VAR_STRING: n = VAR_TYPE_STRING; break; case VAR_PARTIAL: - case VAR_FUNC: n = VAR_TYPE_FUNC; break; - case VAR_LIST: n = VAR_TYPE_LIST; break; - case VAR_DICT: n = VAR_TYPE_DICT; break; - case VAR_FLOAT: n = VAR_TYPE_FLOAT; break; - case VAR_BOOL: n = VAR_TYPE_BOOL; break; - case VAR_SPECIAL:n = VAR_TYPE_SPECIAL; break; + case VAR_FUNC: n = VAR_TYPE_FUNC; break; + case VAR_LIST: n = VAR_TYPE_LIST; break; + case VAR_DICT: n = VAR_TYPE_DICT; break; + case VAR_FLOAT: n = VAR_TYPE_FLOAT; break; + case VAR_BOOL: n = VAR_TYPE_BOOL; break; + case VAR_SPECIAL: n = VAR_TYPE_SPECIAL; break; + case VAR_BLOB: n = VAR_TYPE_BLOB; break; case VAR_UNKNOWN: { internal_error("f_type(UNKNOWN)"); break; @@ -11678,16 +11834,16 @@ static void f_writefile(typval_T *argvars, typval_T *rettv, FunPtr fptr) return; } - if (argvars[0].v_type != VAR_LIST) { - EMSG2(_(e_listarg), "writefile()"); + if (argvars[0].v_type == VAR_LIST) { + TV_LIST_ITER_CONST(argvars[0].vval.v_list, li, { + if (!tv_check_str_or_nr(TV_LIST_ITEM_TV(li))) { + return; + } + }); + } else if (argvars[0].v_type != VAR_BLOB) { + EMSG2(_(e_invarg2), "writefile()"); return; } - const list_T *const list = argvars[0].vval.v_list; - TV_LIST_ITER_CONST(list, li, { - if (!tv_check_str_or_nr(TV_LIST_ITEM_TV(li))) { - return; - } - }); bool binary = false; bool append = false; @@ -11727,7 +11883,13 @@ static void f_writefile(typval_T *argvars, typval_T *rettv, FunPtr fptr) emsgf(_("E482: Can't open file %s for writing: %s"), fname, os_strerror(error)); } else { - if (write_list(&fp, list, binary)) { + bool write_ok; + if (argvars[0].v_type == VAR_BLOB) { + write_ok = write_blob(&fp, argvars[0].vval.v_blob); + } else { + write_ok = write_list(&fp, argvars[0].vval.v_list, binary); + } + if (write_ok) { rettv->vval.v_number = 0; } if ((error = file_close(&fp, do_fsync)) != 0) { diff --git a/src/nvim/eval/typval.c b/src/nvim/eval/typval.c index 22b3bf026b..86e43e0819 100644 --- a/src/nvim/eval/typval.c +++ b/src/nvim/eval/typval.c @@ -2125,6 +2125,73 @@ void tv_dict_set_keys_readonly(dict_T *const dict) }); } +//{{{1 Blobs +//{{{2 Alloc/free + +/// Allocate an empty blob. +/// +/// Caller should take care of the reference count. +/// +/// @return [allocated] new blob. +blob_T *tv_blob_alloc(void) + FUNC_ATTR_NONNULL_RET +{ + blob_T *const blob = xcalloc(1, sizeof(blob_T)); + ga_init(&blob->bv_ga, 1, 100); + return blob; +} + +/// Free a blob. Ignores the reference count. +/// +/// @param[in,out] b Blob to free. +void tv_blob_free(blob_T *const b) + FUNC_ATTR_NONNULL_ALL +{ + ga_clear(&b->bv_ga); + xfree(b); +} + +/// Unreference a blob. +/// +/// Decrements the reference count and frees blob when it becomes zero. +/// +/// @param[in,out] b Blob to operate on. +void tv_blob_unref(blob_T *const b) +{ + if (b != NULL && --b->bv_refcount <= 0) { + tv_blob_free(b); + } +} + +//{{{2 Operations on the whole blob + +/// Check whether two blobs are equal. +/// +/// @param[in] b1 First blob. +/// @param[in] b2 Second blob. +/// +/// @return true if blobs are equal, false otherwise. +bool tv_blob_equal(const blob_T *const b1, const blob_T *const b2) + FUNC_ATTR_WARN_UNUSED_RESULT +{ + if (b1 == NULL || b2 == NULL) { + return false; + } + if (b1 == b2) { + return true; + } + if (tv_blob_len(b1) != tv_blob_len(b2)) { + return false; + } + + for (int i = 0; i < b1->bv_ga.ga_len; i++) { + if (tv_blob_get(b1, i) != tv_blob_get(b2, i)) { + return false; + } + } + return true; +} + //{{{1 Generic typval operations //{{{2 Init/alloc/clear //{{{3 Alloc @@ -2169,6 +2236,18 @@ void tv_dict_alloc_ret(typval_T *const ret_tv) tv_dict_set_ret(ret_tv, d); } +/// Allocate an empty blob for a return value. +/// +/// Also sets reference count. +/// +/// @param[out] ret_tv Structure where blob is saved. +void tv_blob_alloc_ret(typval_T *const ret_tv) + FUNC_ATTR_NONNULL_ALL +{ + blob_T *const b = tv_blob_alloc(); + tv_blob_set_ret(ret_tv, b); +} + //{{{3 Clear #define TYPVAL_ENCODE_ALLOW_SPECIALS false @@ -2210,6 +2289,13 @@ void tv_dict_alloc_ret(typval_T *const ret_tv) #define TYPVAL_ENCODE_CONV_EXT_STRING(tv, buf, len, type) +#define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ + do { \ + tv_blob_unref(tv->vval.v_blob); \ + tv->vval.v_blob = NULL; \ + tv->v_lock = VAR_UNLOCKED; \ + } while (0) + static inline int _nothing_conv_func_start(typval_T *const tv, char_u *const fun) FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ARG(1) @@ -2392,6 +2478,7 @@ static inline void _nothing_conv_dict_end(typval_T *const tv, #undef TYPVAL_ENCODE_CONV_STRING #undef TYPVAL_ENCODE_CONV_STR_STRING #undef TYPVAL_ENCODE_CONV_EXT_STRING +#undef TYPVAL_ENCODE_CONV_BLOB #undef TYPVAL_ENCODE_CONV_FUNC_START #undef TYPVAL_ENCODE_CONV_FUNC_BEFORE_ARGS #undef TYPVAL_ENCODE_CONV_FUNC_BEFORE_SELF @@ -2449,6 +2536,10 @@ void tv_free(typval_T *tv) xfree(tv->vval.v_string); break; } + case VAR_BLOB: { + tv_blob_unref(tv->vval.v_blob); + break; + } case VAR_LIST: { tv_list_unref(tv->vval.v_list); break; @@ -2509,6 +2600,12 @@ void tv_copy(const typval_T *const from, typval_T *const to) } break; } + case VAR_BLOB: { + if (from->vval.v_blob != NULL) { + to->vval.v_blob->bv_refcount++; + } + break; + } case VAR_LIST: { tv_list_ref(to->vval.v_list); break; @@ -2560,6 +2657,13 @@ void tv_item_lock(typval_T *const tv, const int deep, const bool lock) CHANGE_LOCK(lock, tv->v_lock); switch (tv->v_type) { + case VAR_BLOB: { + blob_T *const b = tv->vval.v_blob; + if (b != NULL) { + CHANGE_LOCK(lock, b->bv_lock); + } + break; + } case VAR_LIST: { list_T *const l = tv->vval.v_list; if (l != NULL) { @@ -2646,10 +2750,11 @@ bool tv_check_lock(const typval_T *tv, const char *name, VarLockStatus lock = VAR_UNLOCKED; switch (tv->v_type) { - // case VAR_BLOB: - // if (tv->vval.v_blob != NULL) - // lock = tv->vval.v_blob->bv_lock; - // break; + case VAR_BLOB: + if (tv->vval.v_blob != NULL) { + lock = tv->vval.v_blob->bv_lock; + } + break; case VAR_LIST: if (tv->vval.v_list != NULL) { lock = tv->vval.v_list->lv_lock; @@ -2769,6 +2874,9 @@ bool tv_equal(typval_T *const tv1, typval_T *const tv2, const bool ic, recursive_cnt--; return r; } + case VAR_BLOB: { + return tv_blob_equal(tv1->vval.v_blob, tv2->vval.v_blob); + } case VAR_NUMBER: { return tv1->vval.v_number == tv2->vval.v_number; } @@ -2835,6 +2943,10 @@ bool tv_check_str_or_nr(const typval_T *const tv) EMSG(_("E728: Expected a Number or a String, Dictionary found")); return false; } + case VAR_BLOB: { + EMSG(_("E974: Expected a Number or a String, Blob found")); + return false; + } case VAR_BOOL: { EMSG(_("E5299: Expected a Number or a String, Boolean found")); return false; @@ -2860,6 +2972,7 @@ static const char *const num_errors[] = { [VAR_LIST]=N_("E745: Using a List as a Number"), [VAR_DICT]=N_("E728: Using a Dictionary as a Number"), [VAR_FLOAT]=N_("E805: Using a Float as a Number"), + [VAR_BLOB]=N_("E974: Using a Blob as a Number"), [VAR_UNKNOWN]=N_("E685: using an invalid value as a Number"), }; @@ -2888,6 +3001,7 @@ bool tv_check_num(const typval_T *const tv) case VAR_LIST: case VAR_DICT: case VAR_FLOAT: + case VAR_BLOB: case VAR_UNKNOWN: { EMSG(_(num_errors[tv->v_type])); return false; @@ -2905,6 +3019,7 @@ static const char *const str_errors[] = { [VAR_LIST]=N_("E730: using List as a String"), [VAR_DICT]=N_("E731: using Dictionary as a String"), [VAR_FLOAT]=((const char *)e_float_as_string), + [VAR_BLOB]=N_("E976: using Blob as a String"), [VAR_UNKNOWN]=N_("E908: using an invalid value as a String"), }; @@ -2933,6 +3048,7 @@ bool tv_check_str(const typval_T *const tv) case VAR_LIST: case VAR_DICT: case VAR_FLOAT: + case VAR_BLOB: case VAR_UNKNOWN: { EMSG(_(str_errors[tv->v_type])); return false; @@ -2980,6 +3096,7 @@ varnumber_T tv_get_number_chk(const typval_T *const tv, bool *const ret_error) case VAR_PARTIAL: case VAR_LIST: case VAR_DICT: + case VAR_BLOB: case VAR_FLOAT: { EMSG(_(num_errors[tv->v_type])); break; @@ -3075,6 +3192,10 @@ float_T tv_get_float(const typval_T *const tv) EMSG(_("E907: Using a special value as a Float")); break; } + case VAR_BLOB: { + EMSG(_("E975: Using a Blob as a Float")); + break; + } case VAR_UNKNOWN: { emsgf(_(e_intern2), "tv_get_float(UNKNOWN)"); break; @@ -3134,6 +3255,7 @@ const char *tv_get_string_buf_chk(const typval_T *const tv, char *const buf) case VAR_LIST: case VAR_DICT: case VAR_FLOAT: + case VAR_BLOB: case VAR_UNKNOWN: { EMSG(_(str_errors[tv->v_type])); return false; diff --git a/src/nvim/eval/typval.h b/src/nvim/eval/typval.h index ef49fa1de6..5aecaccee9 100644 --- a/src/nvim/eval/typval.h +++ b/src/nvim/eval/typval.h @@ -64,6 +64,7 @@ enum ListLenSpecials { typedef struct listvar_S list_T; typedef struct dictvar_S dict_T; typedef struct partial_S partial_T; +typedef struct blobvar_S blob_T; typedef struct ufunc ufunc_T; @@ -123,6 +124,7 @@ typedef enum { VAR_SPECIAL, ///< Special value (null), .v_special ///< is used. VAR_PARTIAL, ///< Partial, .v_partial is used. + VAR_BLOB, ///< Blob, .v_blob is used. } VarType; /// Structure that holds an internal variable value @@ -138,6 +140,7 @@ typedef struct { list_T *v_list; ///< List for VAR_LIST, can be NULL. dict_T *v_dict; ///< Dictionary for VAR_DICT, can be NULL. partial_T *v_partial; ///< Closure: function with args. + blob_T *v_blob; ///< Blob for VAR_BLOB, can be NULL. } vval; ///< Actual value. } typval_T; @@ -252,6 +255,13 @@ struct dictvar_S { LuaRef lua_table_ref; }; +/// Structure to hold info about a Blob +struct blobvar_S { + garray_T bv_ga; ///< Growarray with the data. + int bv_refcount; ///< Reference count. + VarLockStatus bv_lock; ///< VAR_UNLOCKED, VAR_LOCKED, VAR_FIXED. +}; + /// Type used for script ID typedef int scid_T; /// Format argument for scid_T @@ -711,6 +721,65 @@ static inline bool tv_dict_is_watched(const dict_T *const d) return d && !QUEUE_EMPTY(&d->watchers); } +static inline void tv_blob_set_ret(typval_T *const tv, blob_T *const b) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ARG(1); + +/// Set a blob as the return value. +/// +/// Increments the reference count. +/// +/// @param[out] tv Object to receive the blob. +/// @param[in,out] b Blob to pass to the object. +static inline void tv_blob_set_ret(typval_T *const tv, blob_T *const b) +{ + tv->v_type = VAR_BLOB; + tv->vval.v_blob = b; + if (b != NULL) { + b->bv_refcount++; + } +} + +static inline int tv_blob_len(const blob_T *const b) + REAL_FATTR_PURE REAL_FATTR_WARN_UNUSED_RESULT; + +/// Get the length of the data in the blob, in bytes. +/// +/// @param[in] b Blob to check. +static inline int tv_blob_len(const blob_T *const b) +{ + if (b == NULL) { + return 0; + } + return b->bv_ga.ga_len; +} + +static inline char_u tv_blob_get(const blob_T *const b, int idx) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ALL REAL_FATTR_WARN_UNUSED_RESULT; + +/// Get the byte at index `idx` in the blob. +/// +/// @param[in] b Blob to index. Cannot be NULL. +/// @param[in] idx Index in a blob. Must be valid. +/// +/// @return Byte value at the given index. +static inline char_u tv_blob_get(const blob_T *const b, int idx) +{ + return ((char_u *)b->bv_ga.ga_data)[idx]; +} + +static inline void tv_blob_set(blob_T *const b, int idx, char_u c) + REAL_FATTR_ALWAYS_INLINE REAL_FATTR_NONNULL_ALL; + +/// Store the byte `c` at index `idx` in the blob. +/// +/// @param[in] b Blob to index. Cannot be NULL. +/// @param[in] idx Index in a blob. Must be valid. +/// @param[in] c Value to store. +static inline void tv_blob_set(blob_T *const b, int idx, char_u c) +{ + ((char_u *)b->bv_ga.ga_data)[idx] = c; +} + /// Initialize VimL object /// /// Initializes to unlocked VAR_UNKNOWN object. diff --git a/src/nvim/eval/typval_encode.c.h b/src/nvim/eval/typval_encode.c.h index 91c948ce7e..cd1be1eecc 100644 --- a/src/nvim/eval/typval_encode.c.h +++ b/src/nvim/eval/typval_encode.c.h @@ -83,6 +83,13 @@ /// @param len String length. /// @param type EXT type. +/// @def TYPVAL_ENCODE_CONV_BLOB +/// @brief Macros used to convert a blob +/// +/// @param tv Pointer to typval where value is stored. May not be NULL. +/// @param blob Pointer to the blob to convert. +/// @param len Blob length. + /// @def TYPVAL_ENCODE_CONV_FUNC_START /// @brief Macros used when starting to convert a funcref or a partial /// @@ -330,6 +337,11 @@ static int _TYPVAL_ENCODE_CONVERT_ONE_VALUE( TYPVAL_ENCODE_CONV_FLOAT(tv, tv->vval.v_float); break; } + case VAR_BLOB: { + TYPVAL_ENCODE_CONV_BLOB(tv, tv->vval.v_blob, + tv_blob_len(tv->vval.v_blob)); + break; + } case VAR_FUNC: { TYPVAL_ENCODE_CONV_FUNC_START(tv, tv->vval.v_string); TYPVAL_ENCODE_CONV_FUNC_BEFORE_ARGS(tv, 0); diff --git a/src/nvim/globals.h b/src/nvim/globals.h index 2a72dbcd09..42989ea508 100644 --- a/src/nvim/globals.h +++ b/src/nvim/globals.h @@ -939,6 +939,8 @@ EXTERN char_u e_readonlyvar[] INIT(= N_( "E46: Cannot change read-only variable \"%.*s\"")); EXTERN char_u e_stringreq[] INIT(= N_("E928: String required")); EXTERN char_u e_dictreq[] INIT(= N_("E715: Dictionary required")); +EXTERN char_u e_blobidx[] INIT(= N_("E979: Blob index out of range: %" PRId64)); +EXTERN char_u e_invalblob[] INIT(= N_("E978: Invalid operation for Blob")); EXTERN char_u e_toomanyarg[] INIT(= N_( "E118: Too many arguments for function: %s")); EXTERN char_u e_dictkey[] INIT(= N_( diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c index 1a59cd94ae..007ef336cd 100644 --- a/src/nvim/lua/converter.c +++ b/src/nvim/lua/converter.c @@ -481,6 +481,9 @@ static bool typval_conv_special = false; #define TYPVAL_ENCODE_CONV_EXT_STRING(tv, str, len, type) \ TYPVAL_ENCODE_CONV_NIL(tv) +#define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ + abort() /* TODO(seandewar) */ \ + #define TYPVAL_ENCODE_CONV_FUNC_START(tv, fun) \ do { \ TYPVAL_ENCODE_CONV_NIL(tv); \ @@ -579,6 +582,7 @@ static bool typval_conv_special = false; #undef TYPVAL_ENCODE_CONV_STRING #undef TYPVAL_ENCODE_CONV_STR_STRING #undef TYPVAL_ENCODE_CONV_EXT_STRING +#undef TYPVAL_ENCODE_CONV_BLOB #undef TYPVAL_ENCODE_CONV_NUMBER #undef TYPVAL_ENCODE_CONV_FLOAT #undef TYPVAL_ENCODE_CONV_FUNC_START diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim new file mode 100644 index 0000000000..412bb69054 --- /dev/null +++ b/src/nvim/testdir/test_blob.vim @@ -0,0 +1,176 @@ +" Tests for the Blob types + +func TearDown() + " Run garbage collection after every test + call test_garbagecollect_now() +endfunc + +" Tests for Blob type + +" Blob creation from constant +func Test_blob_create() + let b = 0zDEADBEEF + call assert_equal(v:t_blob, type(b)) + call assert_equal(4, len(b)) + call assert_equal(0xDE, b[0]) + call assert_equal(0xAD, b[1]) + call assert_equal(0xBE, b[2]) + call assert_equal(0xEF, b[3]) + call assert_fails('let x = b[4]') + + call assert_equal(0xDE, get(b, 0)) + call assert_equal(0xEF, get(b, 3)) + call assert_fails('let x = get(b, 4)') +endfunc + +" assignment to a blob +func Test_blob_assign() + let b = 0zDEADBEEF + let b2 = b[1:2] + call assert_equal(0zADBE, b2) + + let bcopy = b[:] + call assert_equal(b, bcopy) + call assert_false(b is bcopy) +endfunc + +func Test_blob_to_string() + let b = 0zDEADBEEF + call assert_equal('[0xDE,0xAD,0xBE,0xEF]', string(b)) + call remove(b, 0, 3) + call assert_equal('[]', string(b)) +endfunc + +func Test_blob_compare() + let b1 = 0z0011 + let b2 = 0z1100 + call assert_false(b1 == b2) + call assert_true(b1 != b2) + call assert_true(b1 == 0z0011) + + call assert_false(b1 is b2) + let b2 = b1 + call assert_true(b1 is b2) + + call assert_fails('let x = b1 > b2') + call assert_fails('let x = b1 < b2') + call assert_fails('let x = b1 - b2') + call assert_fails('let x = b1 / b2') + call assert_fails('let x = b1 * b2') +endfunc + +" test for range assign +func Test_blob_range_assign() + let b = 0z00 + let b[1] = 0x11 + let b[2] = 0x22 + call assert_equal(0z001122, b) + call assert_fails('let b[4] = 0x33') +endfunc + +func Test_blob_for_loop() + let blob = 0z00010203 + let i = 0 + for byte in blob + call assert_equal(i, byte) + let i += 1 + endfor + + let blob = 0z00 + call remove(blob, 0) + call assert_equal(0, len(blob)) + for byte in blob + call assert_error('loop over empty blob') + endfor +endfunc + +func Test_blob_concatenate() + let b = 0z0011 + let b += 0z2233 + call assert_equal(0z00112233, b) + + call assert_fails('let b += "a"') + call assert_fails('let b += 88') + + let b = 0zDEAD + 0zBEEF + call assert_equal(0zDEADBEEF, b) +endfunc + +" Test removing items in blob +func Test_blob_func_remove() + " Test removing 1 element + let b = 0zDEADBEEF + call assert_equal(0xDE, remove(b, 0)) + call assert_equal(0zADBEEF, b) + + let b = 0zDEADBEEF + call assert_equal(0xEF, remove(b, -1)) + call assert_equal(0zDEADBE, b) + + let b = 0zDEADBEEF + call assert_equal(0xAD, remove(b, 1)) + call assert_equal(0zDEBEEF, b) + + " Test removing range of element(s) + let b = 0zDEADBEEF + call assert_equal(0zBE, remove(b, 2, 2)) + call assert_equal(0zDEADEF, b) + + let b = 0zDEADBEEF + call assert_equal(0zADBE, remove(b, 1, 2)) + call assert_equal(0zDEEF, b) + + " Test invalid cases + let b = 0zDEADBEEF + call assert_fails("call remove(b, 5)", 'E979:') + call assert_fails("call remove(b, 1, 5)", 'E979:') + call assert_fails("call remove(b, 3, 2)", 'E979:') + call assert_fails("call remove(1, 0)", 'E712:') + call assert_fails("call remove(b, b)", 'E974:') +endfunc + +func Test_blob_read_write() + let b = 0zDEADBEEF + call writefile(b, 'Xblob') + let br = readfile('Xblob', 'B') + call assert_equal(b, br) + call delete('Xblob') +endfunc + +" filter() item in blob +func Test_blob_filter() + let b = 0zDEADBEEF + call filter(b, 'v:val != 0xEF') + call assert_equal(0zDEADBE, b) +endfunc + +" map() item in blob +func Test_blob_map() + let b = 0zDEADBEEF + call map(b, 'v:val + 1') + call assert_equal(0zDFAEBFF0, b) +endfunc + +func Test_blob_index() + call assert_equal(2, index(0zDEADBEEF, 0xBE)) + call assert_equal(-1, index(0zDEADBEEF, 0)) +endfunc + +func Test_blob_insert() + let b = 0zDEADBEEF + call insert(b, 0x33) + call assert_equal(0z33DEADBEEF, b) + + let b = 0zDEADBEEF + call insert(b, 0x33, 2) + call assert_equal(0zDEAD33BEEF, b) +endfunc + +func Test_blob_reverse() + call assert_equal(0zEFBEADDE, reverse(0zDEADBEEF)) + call assert_equal(0zBEADDE, reverse(0zDEADBE)) + call assert_equal(0zADDE, reverse(0zDEAD)) + call assert_equal(0zDE, reverse(0zDE)) +endfunc + +" vim: shiftwidth=2 sts=2 expandtab diff --git a/src/nvim/testdir/test_method.vim b/src/nvim/testdir/test_method.vim index 7a6e6aa19d..d34448e09e 100644 --- a/src/nvim/testdir/test_method.vim +++ b/src/nvim/testdir/test_method.vim @@ -46,11 +46,8 @@ func Test_dict_method() call assert_equal(#{one: 1, two: 2, three: 3, four: 4}, d->extend(#{four: 4})) call assert_equal(#{one: 1, two: 2, three: 3}, d->filter('v:val != 4')) call assert_equal(2, d->get('two')) - " Nvim doesn't support Blobs yet; expect a different emsg - " call assert_fails("let x = d->index(2)", 'E897:') - " call assert_fails("let x = d->insert(0)", 'E899:') - call assert_fails("let x = d->index(2)", 'E714:') - call assert_fails("let x = d->insert(0)", 'E686:') + call assert_fails("let x = d->index(2)", 'E897:') + call assert_fails("let x = d->insert(0)", 'E899:') call assert_true(d->has_key('two')) call assert_equal([['one', 1], ['two', 2], ['three', 3]], d->items()) call assert_fails("let x = d->join()", 'E714:') @@ -63,9 +60,7 @@ func Test_dict_method() call assert_equal(2, d->remove("two")) let d.two = 2 call assert_fails('let x = d->repeat(2)', 'E731:') - " Nvim doesn't support Blobs yet; expect a different emsg - " call assert_fails('let x = d->reverse()', 'E899:') - call assert_fails('let x = d->reverse()', 'E686:') + call assert_fails('let x = d->reverse()', 'E899:') call assert_fails('let x = d->sort()', 'E686:') call assert_equal("{'one': 1, 'two': 2, 'three': 3}", d->string()) call assert_equal(v:t_dict, d->type()) diff --git a/src/nvim/vim.h b/src/nvim/vim.h index c719c064e2..d84979f6fe 100644 --- a/src/nvim/vim.h +++ b/src/nvim/vim.h @@ -102,6 +102,7 @@ typedef enum { #define VAR_TYPE_FLOAT 5 #define VAR_TYPE_BOOL 6 #define VAR_TYPE_SPECIAL 7 +#define VAR_TYPE_BLOB 10 // values for xp_context when doing command line completion diff --git a/test/functional/eval/null_spec.lua b/test/functional/eval/null_spec.lua index f866aca3ed..bc88e6c8b3 100644 --- a/test/functional/eval/null_spec.lua +++ b/test/functional/eval/null_spec.lua @@ -44,7 +44,7 @@ describe('NULL', function() -- Incorrect behaviour -- FIXME Should error out with different message null_test('makes :unlet act as if it is not a list', ':unlet L[0]', - 'Vim(unlet):E689: Can only index a List or Dictionary') + 'Vim(unlet):E689: Can only index a List, Dictionary or Blob') -- Subjectable behaviour diff --git a/test/functional/eval/writefile_spec.lua b/test/functional/eval/writefile_spec.lua index 356680ba7c..b2a601a767 100644 --- a/test/functional/eval/writefile_spec.lua +++ b/test/functional/eval/writefile_spec.lua @@ -119,7 +119,7 @@ describe('writefile()', function() eq('\nE118: Too many arguments for function: writefile', redir_exec(('call writefile([], "%s", "b", 1)'):format(fname))) for _, arg in ipairs({'0', '0.0', 'function("tr")', '{}', '"test"'}) do - eq('\nE686: Argument of writefile() must be a List', + eq('\nE475: Invalid argument: writefile()', redir_exec(('call writefile(%s, "%s", "b")'):format(arg, fname))) end for _, args in ipairs({'[], %s, "b"', '[], "' .. fname .. '", %s'}) do From bfeecd0b418cf03a974c57082f4fc647b90aef06 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Tue, 24 Nov 2020 21:46:43 +0000 Subject: [PATCH 02/37] vim-patch:8.1.0736: code for Blob not sufficiently tested Problem: Code for Blob not sufficiently tested. Solution: Add more tests. Fix uncovered crash. Add test_null_blob(). https://github.com/vim/vim/commit/c0f5a78c15b194f23bedb82e6825e34f481e6532 eval0 and ex_echo's emsg-specific changes have already been ported. These tests uncover another crash that was fixed in v8.1.0738. --- src/nvim/eval.c | 40 ++++++++++++++------- src/nvim/eval.h | 1 + src/nvim/eval/typval.c | 10 ++++-- src/nvim/testdir/test_blob.vim | 63 +++++++++++++++++++++++++++++++++- 4 files changed, 98 insertions(+), 16 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index c3a9c0b9ca..29df1a168f 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -242,6 +242,7 @@ static struct vimvar { VV(VV__NULL_STRING, "_null_string", VAR_STRING, VV_RO), VV(VV__NULL_LIST, "_null_list", VAR_LIST, VV_RO), VV(VV__NULL_DICT, "_null_dict", VAR_DICT, VV_RO), + VV(VV__NULL_BLOB, "_null_blob", VAR_BLOB, VV_RO), VV(VV_LUA, "lua", VAR_PARTIAL, VV_RO), }; #undef VV @@ -2125,8 +2126,8 @@ char_u *get_lval(char_u *const name, typval_T *const rettv, return NULL; } if (rettv != NULL - && !(rettv->v_type == VAR_LIST || rettv->vval.v_list != NULL) - && !(rettv->v_type == VAR_BLOB || rettv->vval.v_blob != NULL)) { + && !(rettv->v_type == VAR_LIST && rettv->vval.v_list != NULL) + && !(rettv->v_type == VAR_BLOB && rettv->vval.v_blob != NULL)) { if (!quiet) { EMSG(_("E709: [:] requires a List or Blob value")); } @@ -2252,15 +2253,24 @@ char_u *get_lval(char_u *const name, typval_T *const rettv, } tv_clear(&var1); - if (lp->ll_n1 < 0 || lp->ll_n1 > tv_blob_len(lp->ll_tv->vval.v_blob)) { + const int bloblen = tv_blob_len(lp->ll_tv->vval.v_blob); + if (lp->ll_n1 < 0 || lp->ll_n1 > bloblen + || (lp->ll_range && lp->ll_n1 == bloblen)) { if (!quiet) { - EMSGN(_(e_listidx), lp->ll_n1); + EMSGN(_(e_blobidx), lp->ll_n1); } + tv_clear(&var2); return NULL; } if (lp->ll_range && !lp->ll_empty2) { lp->ll_n2 = (long)tv_get_number(&var2); tv_clear(&var2); + if (lp->ll_n2 < 0 || lp->ll_n2 >= bloblen || lp->ll_n2 < lp->ll_n1) { + if (!quiet) { + EMSGN(_(e_blobidx), lp->ll_n2); + } + return NULL; + } } lp->ll_blob = lp->ll_tv->vval.v_blob; lp->ll_tv = NULL; @@ -2364,13 +2374,20 @@ static void set_var_lval(lval_T *lp, char_u *endp, typval_T *rettv, } if (lp->ll_range && rettv->v_type == VAR_BLOB) { - if (tv_blob_len(rettv->vval.v_blob) != tv_blob_len(lp->ll_blob)) { - EMSG(_("E972: Blob value has more items than target")); - return; + if (lp->ll_empty2) { + lp->ll_n2 = tv_blob_len(lp->ll_blob) - 1; } - for (int i = lp->ll_n1; i <= lp->ll_n2; i++) { - tv_blob_set(lp->ll_blob, i, tv_blob_get(rettv->vval.v_blob, i)); + if (lp->ll_n2 - lp->ll_n1 + 1 != tv_blob_len(rettv->vval.v_blob)) { + EMSG(_("E972: Blob value does not have the right number of bytes")); + return; + } + if (lp->ll_empty2) { + lp->ll_n2 = tv_blob_len(lp->ll_blob); + } + + for (int il = lp->ll_n1, ir = 0; il <= lp->ll_n2; il++) { + tv_blob_set(lp->ll_blob, il, tv_blob_get(rettv->vval.v_blob, ir++)); } } else { bool error = false; @@ -2386,9 +2403,8 @@ static void set_var_lval(lval_T *lp, char_u *endp, typval_T *rettv, if (lp->ll_n1 == gap->ga_len) { gap->ga_len++; } - } else { - EMSG(_(e_invrange)); } + // error for invalid range was already given in get_lval() } } } else if (op != NULL && *op != '=') { @@ -4591,7 +4607,7 @@ eval_index( case VAR_BLOB: { len = tv_blob_len(rettv->vval.v_blob); if (range) { - // The resulting variable is a substring. If the indexes + // The resulting variable is a sub-blob. If the indexes // are out of range the result is empty. if (n1 < 0) { n1 = len + n1; diff --git a/src/nvim/eval.h b/src/nvim/eval.h index e5a65122f1..2452a0a8c8 100644 --- a/src/nvim/eval.h +++ b/src/nvim/eval.h @@ -167,6 +167,7 @@ typedef enum { VV__NULL_STRING, // String with NULL value. For test purposes only. VV__NULL_LIST, // List with NULL value. For test purposes only. VV__NULL_DICT, // Dictionary with NULL value. For test purposes only. + VV__NULL_BLOB, // Blob with NULL value. For test purposes only. VV_LUA, } VimVarIndex; diff --git a/src/nvim/eval/typval.c b/src/nvim/eval/typval.c index 86e43e0819..e689571788 100644 --- a/src/nvim/eval/typval.c +++ b/src/nvim/eval/typval.c @@ -2174,13 +2174,17 @@ void tv_blob_unref(blob_T *const b) bool tv_blob_equal(const blob_T *const b1, const blob_T *const b2) FUNC_ATTR_WARN_UNUSED_RESULT { - if (b1 == NULL || b2 == NULL) { - return false; + const int len1 = tv_blob_len(b1); + const int len2 = tv_blob_len(b2); + + // empty and NULL are considered the same + if (len1 == 0 && len2 == 0) { + return true; } if (b1 == b2) { return true; } - if (tv_blob_len(b1) != tv_blob_len(b2)) { + if (len1 != len2) { return false; } diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index 412bb69054..df8efbb149 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -21,6 +21,12 @@ func Test_blob_create() call assert_equal(0xDE, get(b, 0)) call assert_equal(0xEF, get(b, 3)) call assert_fails('let x = get(b, 4)') + + call assert_fails('let b = 0z1', 'E973:') + call assert_fails('let b = 0z1x', 'E973:') + call assert_fails('let b = 0z12345', 'E973:') + + call assert_equal(0z, v:_null_blob) endfunc " assignment to a blob @@ -32,6 +38,45 @@ func Test_blob_assign() let bcopy = b[:] call assert_equal(b, bcopy) call assert_false(b is bcopy) + + let b = 0zDEADBEEF + let b2 = b + call assert_true(b is b2) + let b[:] = 0z11223344 + call assert_equal(0z11223344, b) + call assert_equal(0z11223344, b2) + call assert_true(b is b2) + + let b = 0zDEADBEEF + let b[3:] = 0z66 + call assert_equal(0zDEADBE66, b) + let b[:1] = 0z8899 + call assert_equal(0z8899BE66, b) + + call assert_fails('let b[2:3] = 0z112233', 'E972:') + call assert_fails('let b[2:3] = 0z11', 'E972:') + call assert_fails('let b[3:2] = 0z', 'E979:') + + let b = 0zDEADBEEF + let b += 0z99 + call assert_equal(0zDEADBEEF99, b) + + call assert_fails('let b .= 0z33', 'E734:') + call assert_fails('let b .= "xx"', 'E734:') + call assert_fails('let b += "xx"', 'E734:') + call assert_fails('let b[1:1] .= 0z55', 'E734:') +endfunc + +func Test_blob_get_range() + let b = 0z0011223344 + call assert_equal(0z2233, b[2:3]) + call assert_equal(0z223344, b[2:-1]) + call assert_equal(0z00, b[0:-5]) + call assert_equal(0z, b[0:-11]) + call assert_equal(0z44, b[-1:]) + call assert_equal(0z0011223344, b[:]) + call assert_equal(0z0011223344, b[:-1]) + call assert_equal(0z, b[5:6]) endfunc func Test_blob_to_string() @@ -44,8 +89,12 @@ endfunc func Test_blob_compare() let b1 = 0z0011 let b2 = 0z1100 + let b3 = 0z001122 + call assert_true(b1 == b1) call assert_false(b1 == b2) + call assert_false(b1 == b3) call assert_true(b1 != b2) + call assert_true(b1 != b3) call assert_true(b1 == 0z0011) call assert_false(b1 is b2) @@ -65,7 +114,7 @@ func Test_blob_range_assign() let b[1] = 0x11 let b[2] = 0x22 call assert_equal(0z001122, b) - call assert_fails('let b[4] = 0x33') + call assert_fails('let b[4] = 0x33', 'E979:') endfunc func Test_blob_for_loop() @@ -173,4 +222,16 @@ func Test_blob_reverse() call assert_equal(0zDE, reverse(0zDE)) endfunc +func Test_blob_lock() + let b = 0z112233 + lockvar b + call assert_fails('let b = 0z44', 'E741:') + unlockvar b + let b = 0z44 +endfunc + +func Test_blob_sort() + call assert_fails('call sort([1.0, 0z11], "f")', 'E975:') +endfunc + " vim: shiftwidth=2 sts=2 expandtab From 10aa60e806c300cc7bd1f84e52b4b043d6fac537 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 2 Aug 2021 19:35:06 +0100 Subject: [PATCH 03/37] feat(nlua): convert Blobs to strings --- src/nvim/lua/converter.c | 7 ++++++- test/functional/lua/luaeval_spec.lua | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/nvim/lua/converter.c b/src/nvim/lua/converter.c index 007ef336cd..0adbbdb953 100644 --- a/src/nvim/lua/converter.c +++ b/src/nvim/lua/converter.c @@ -482,7 +482,12 @@ static bool typval_conv_special = false; TYPVAL_ENCODE_CONV_NIL(tv) #define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ - abort() /* TODO(seandewar) */ \ + do { \ + const blob_T *const blob_ = (blob); \ + lua_pushlstring(lstate, \ + blob_ != NULL ? (const char *)blob_->bv_ga.ga_data : "", \ + (size_t)(len)); \ + } while (0) #define TYPVAL_ENCODE_CONV_FUNC_START(tv, fun) \ do { \ diff --git a/test/functional/lua/luaeval_spec.lua b/test/functional/lua/luaeval_spec.lua index 8ef77faa0f..36f6d4025f 100644 --- a/test/functional/lua/luaeval_spec.lua +++ b/test/functional/lua/luaeval_spec.lua @@ -458,6 +458,9 @@ describe('v:lua', function() function mymod.crashy() nonexistent() end + function mymod.whatis(value) + return type(value) .. ": " .. tostring(value) + end function mymod.omni(findstart, base) if findstart == 1 then return 5 @@ -476,6 +479,8 @@ describe('v:lua', function() eq(true, exec_lua([[return _G.val == vim.NIL]])) eq(NIL, eval('v:lua.mymod.noisy("eval")')) eq("hey eval", meths.get_current_line()) + eq("string: abc", eval('v:lua.mymod.whatis(0z616263)')) + eq("string: ", eval('v:lua.mymod.whatis(v:_null_blob)')) eq("Vim:E5108: Error executing lua [string \"\"]:0: attempt to call global 'nonexistent' (a nil value)", pcall_err(eval, 'v:lua.mymod.crashy()')) From 312c783d81544df7ab3342f4b3316ced931e89b8 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 2 Aug 2021 19:13:48 +0100 Subject: [PATCH 04/37] feat(api): convert Blobs to API strings Note that these are not NUL-terminated; the API supports this. --- src/nvim/api/private/helpers.c | 9 ++++++++- test/functional/eval/api_functions_spec.lua | 9 +++++++++ test/functional/lua/api_spec.lua | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/nvim/api/private/helpers.c b/src/nvim/api/private/helpers.c index 6ece819afa..ecbd4e13a3 100644 --- a/src/nvim/api/private/helpers.c +++ b/src/nvim/api/private/helpers.c @@ -445,7 +445,14 @@ void set_option_to(uint64_t channel_id, void *to, int type, TYPVAL_ENCODE_CONV_NIL(tv) #define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ - abort() /* TODO(seandewar) */ \ + do { \ + const size_t len_ = (size_t)(len); \ + const blob_T *const blob_ = (blob); \ + kvi_push(edata->stack, STRING_OBJ(((String) { \ + .data = len_ != 0 ? xmemdup(blob_->bv_ga.ga_data, len_) : NULL, \ + .size = len_ \ + }))); \ + } while (0) #define TYPVAL_ENCODE_CONV_FUNC_START(tv, fun) \ do { \ diff --git a/test/functional/eval/api_functions_spec.lua b/test/functional/eval/api_functions_spec.lua index 7d09a652ba..d07e74d40e 100644 --- a/test/functional/eval/api_functions_spec.lua +++ b/test/functional/eval/api_functions_spec.lua @@ -155,4 +155,13 @@ describe('eval-API', function() pcall_err(command, "sandbox call nvim_input('ievil')")) eq({''}, meths.buf_get_lines(0, 0, -1, true)) end) + + it('converts blobs to API strings', function() + command('let g:v1 = nvim__id(0z68656c6c6f)') + command('let g:v2 = nvim__id(v:_null_blob)') + eq(1, eval('type(g:v1)')) + eq(1, eval('type(g:v2)')) + eq('hello', eval('g:v1')) + eq('', eval('g:v2')) + end) end) diff --git a/test/functional/lua/api_spec.lua b/test/functional/lua/api_spec.lua index 896554f7a3..b2c343096a 100644 --- a/test/functional/lua/api_spec.lua +++ b/test/functional/lua/api_spec.lua @@ -64,6 +64,7 @@ describe('luaeval(vim.api.…)', function() it('correctly converts from API objects', function() eq(1, funcs.luaeval('vim.api.nvim_eval("1")')) eq('1', funcs.luaeval([[vim.api.nvim_eval('"1"')]])) + eq('Blobby', funcs.luaeval('vim.api.nvim_eval("0z426c6f626279")')) eq({}, funcs.luaeval('vim.api.nvim_eval("[]")')) eq({}, funcs.luaeval('vim.api.nvim_eval("{}")')) eq(1, funcs.luaeval('vim.api.nvim_eval("1.0")')) @@ -73,6 +74,7 @@ describe('luaeval(vim.api.…)', function() eq(0, eval([[type(luaeval('vim.api.nvim_eval("1")'))]])) eq(1, eval([[type(luaeval('vim.api.nvim_eval("''1''")'))]])) + eq(1, eval([[type(luaeval('vim.api.nvim_eval("0zbeef")'))]])) eq(3, eval([[type(luaeval('vim.api.nvim_eval("[]")'))]])) eq(4, eval([[type(luaeval('vim.api.nvim_eval("{}")'))]])) eq(5, eval([[type(luaeval('vim.api.nvim_eval("1.0")'))]])) From ab82369c8eb1bf6a58f848e7cb3fb3275d13ed8b Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 2 Aug 2021 18:59:55 +0100 Subject: [PATCH 05/37] feat(json): convert Blobs to array of byte values Similiar to how Vim does it, but to be consistent with how Nvim encodes lists, add a space after every comma. --- runtime/doc/eval.txt | 1 + src/nvim/eval/encode.c | 20 +++++++++++++++++++- test/functional/eval/json_functions_spec.lua | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index b9eeec9a6b..165d48eb42 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -6002,6 +6002,7 @@ json_encode({expr}) *json_encode()* surrogate pairs (such strings are not valid UTF-8 strings). Non-printable characters are converted into "\u1234" escapes or special escapes like "\t", other are dumped as-is. + |Blob|s are converted to arrays of the individual bytes. keys({dict}) *keys()* Return a |List| with all the keys of {dict}. The |List| is in diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 45927db0ac..35a97d4f50 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -729,7 +729,25 @@ static inline int convert_to_json_string(garray_T *const gap, #undef TYPVAL_ENCODE_CONV_BLOB #define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ - abort() /* TODO(seandewar) */ \ + do { \ + const blob_T *const blob_ = (blob); \ + const int len_ = (len); \ + if (len_ == 0) { \ + ga_concat(gap, "[]"); \ + } else { \ + ga_append(gap, '['); \ + char numbuf[NUMBUFLEN]; \ + for (int i_ = 0; i_ < len_; i_++) { \ + if (i_ > 0) { \ + ga_concat(gap, ", "); \ + } \ + vim_snprintf((char *)numbuf, ARRAY_SIZE(numbuf), "%d", \ + (int)tv_blob_get(blob_, i_)); \ + ga_concat(gap, numbuf); \ + } \ + ga_append(gap, ']'); \ + } \ + } while (0) #undef TYPVAL_ENCODE_CONV_FUNC_START #define TYPVAL_ENCODE_CONV_FUNC_START(tv, fun) \ diff --git a/test/functional/eval/json_functions_spec.lua b/test/functional/eval/json_functions_spec.lua index 8dcaea806e..9b5e207c07 100644 --- a/test/functional/eval/json_functions_spec.lua +++ b/test/functional/eval/json_functions_spec.lua @@ -538,6 +538,11 @@ describe('json_encode() function', function() eq('"þÿþ"', funcs.json_encode('þÿþ')) end) + it('dumps blobs', function() + eq('[]', eval('json_encode(0z)')) + eq('[222, 173, 190, 239]', eval('json_encode(0zDEADBEEF)')) + end) + it('dumps numbers', function() eq('0', funcs.json_encode(0)) eq('10', funcs.json_encode(10)) @@ -769,6 +774,10 @@ describe('json_encode() function', function() eq('""', eval('json_encode($XXX_UNEXISTENT_VAR_XXX)')) end) + it('can dump NULL blob', function() + eq('[]', eval('json_encode(v:_null_blob)')) + end) + it('can dump NULL list', function() eq('[]', eval('json_encode(v:_null_list)')) end) From af6f454f5c3349821b20bd2f0d846f7ae7343e2e Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 2 Aug 2021 19:27:52 +0100 Subject: [PATCH 06/37] feat(msgpack): convert Blobs to BIN strings --- runtime/doc/eval.txt | 4 ++-- src/nvim/eval/encode.c | 8 +++++++- test/functional/eval/msgpack_functions_spec.lua | 8 ++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 165d48eb42..e7820f5313 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -6692,7 +6692,7 @@ msgpackdump({list}) *msgpackdump()* 1. |Funcref|s cannot be dumped. 2. Containers that reference themselves cannot be dumped. 3. Dictionary keys are always dumped as STR strings. - 4. Other strings are always dumped as BIN strings. + 4. Other strings and |Blob|s are always dumped as BIN strings. 5. Points 3. and 4. do not apply to |msgpack-special-dict|s. msgpackparse({list}) *msgpackparse()* @@ -6706,7 +6706,7 @@ msgpackparse({list}) *msgpackparse()* Limitations: 1. Mapping ordering is not preserved unless messagepack - mapping is dumped using generic mapping + mapping is dumped using generic mapping (|msgpack-special-map|). 2. Since the parser aims to preserve all data untouched (except for 1.) some strings are parsed to diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 35a97d4f50..b6b37fce84 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -950,7 +950,13 @@ char *encode_tv2json(typval_T *tv, size_t *len) } while (0) #define TYPVAL_ENCODE_CONV_BLOB(tv, blob, len) \ - abort() /* TODO(seandewar) */ \ + do { \ + const size_t len_ = (size_t)(len); \ + msgpack_pack_bin(packer, len_); \ + if (len_ > 0) { \ + msgpack_pack_bin_body(packer, (blob)->bv_ga.ga_data, len_); \ + } \ + } while (0) #define TYPVAL_ENCODE_CONV_NUMBER(tv, num) \ msgpack_pack_int64(packer, (int64_t)(num)) diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index a8a413f68b..90e3ccc0e4 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -517,6 +517,10 @@ describe('msgpackdump() function', function() eq({"\196\004Test"}, eval('msgpackdump(obj)')) end) + it('dumps blob as BIN 8', function() + eq({'\196\005Bl\nb!'}, eval('msgpackdump([0z426c006221])')) + end) + it('can dump generic mapping with generic mapping keys and values', function() command('let todump = {"_TYPE": v:msgpack_types.map, "_VAL": []}') command('let todumpv1 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') @@ -716,6 +720,10 @@ describe('msgpackdump() function', function() eq({'\160'}, eval('msgpackdump([{"_TYPE": v:msgpack_types.string, "_VAL": [$XXX_UNEXISTENT_VAR_XXX]}])')) end) + it('can dump NULL blob', function() + eq({'\196\n'}, eval('msgpackdump([v:_null_blob])')) + end) + it('can dump NULL list', function() eq({'\144'}, eval('msgpackdump([v:_null_list])')) end) From ef729fb15b34480048c44e76c29657727053101c Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Fri, 30 Jul 2021 15:23:37 +0100 Subject: [PATCH 07/37] feat(shada): restore Blob globals properly As Strings and Blobs are encoded as msgpack BINs, the current ShaDa implementation will restore global Blob variables as Strings (or msgpack special dicts if they contain NULs). Encode an additional element with Blob globals to differentiate them from Strings so that we can restore them with the correct type. Adjust variables_spec.lua's autotest() to also check for proper type. --- src/nvim/shada.c | 40 +++++++++++++++++++++--- test/functional/shada/errors_spec.lua | 5 +++ test/functional/shada/variables_spec.lua | 8 +++-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/nvim/shada.c b/src/nvim/shada.c index c0e787380f..7d277fe5c8 100644 --- a/src/nvim/shada.c +++ b/src/nvim/shada.c @@ -1653,6 +1653,13 @@ static ShaDaWriteResult shada_pack_entry(msgpack_packer *const packer, break; } case kSDItemVariable: { + if (entry.data.global_var.value.v_type == VAR_TYPE_BLOB) { + // Strings and Blobs both pack as msgpack BINs; differentiate them by + // storing an additional VAR_TYPE_BLOB element alongside Blobs + list_T *const list = tv_list_alloc(1); + tv_list_append_number(list, VAR_TYPE_BLOB); + entry.data.global_var.additional_elements = list; + } const size_t arr_size = 2 + (size_t)( tv_list_len(entry.data.global_var.additional_elements)); msgpack_pack_array(spacker, arr_size); @@ -3937,15 +3944,38 @@ shada_read_next_item_start: entry->data.global_var.name = xmemdupz(unpacked.data.via.array.ptr[0].via.bin.ptr, unpacked.data.via.array.ptr[0].via.bin.size); - if (msgpack_to_vim(unpacked.data.via.array.ptr[1], - &(entry->data.global_var.value)) == FAIL) { + SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, 2, + entry->data.global_var.additional_elements, + "variable"); + bool is_blob = false; + // A msgpack BIN could be a String or Blob; an additional VAR_TYPE_BLOB + // element is stored with Blobs which can be used to differentiate them + if (unpacked.data.via.array.ptr[1].type == MSGPACK_OBJECT_BIN) { + const listitem_T *type_item + = tv_list_first(entry->data.global_var.additional_elements); + if (type_item != NULL) { + const typval_T *type_tv = TV_LIST_ITEM_TV(type_item); + if (type_tv->v_type != VAR_NUMBER + || type_tv->vval.v_number != VAR_TYPE_BLOB) { + emsgf(_(READERR("variable", "has wrong variable type")), + initial_fpos); + goto shada_read_next_item_error; + } + is_blob = true; + } + } + if (is_blob) { + const msgpack_object_bin *const bin + = &unpacked.data.via.array.ptr[1].via.bin; + blob_T *const blob = tv_blob_alloc(); + ga_concat_len(&blob->bv_ga, bin->ptr, (size_t)bin->size); + tv_blob_set_ret(&entry->data.global_var.value, blob); + } else if (msgpack_to_vim(unpacked.data.via.array.ptr[1], + &(entry->data.global_var.value)) == FAIL) { emsgf(_(READERR("variable", "has value that cannot " "be converted to the VimL value")), initial_fpos); goto shada_read_next_item_error; } - SET_ADDITIONAL_ELEMENTS(unpacked.data.via.array, 2, - entry->data.global_var.additional_elements, - "variable"); break; } case kSDItemSubString: { diff --git a/test/functional/shada/errors_spec.lua b/test/functional/shada/errors_spec.lua index 77a41caec7..ebfd73cf85 100644 --- a/test/functional/shada/errors_spec.lua +++ b/test/functional/shada/errors_spec.lua @@ -342,6 +342,11 @@ describe('ShaDa error handling', function() eq('Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 has wrong variable name type', exc_exec(sdrcmd())) end) + it('fails on variable item with BIN value and type value != VAR_TYPE_BLOB', function() + wshada('\006\000\007\147\196\001\065\196\000\000') + eq('Vim(rshada):E575: Error while reading ShaDa file: variable entry at position 0 has wrong variable type', exc_exec(sdrcmd())) + end) + it('fails on replacement item with NIL value', function() wshada('\003\000\001\192') eq('Vim(rshada):E575: Error while reading ShaDa file: sub string entry at position 0 is not an array', exc_exec(sdrcmd())) diff --git a/test/functional/shada/variables_spec.lua b/test/functional/shada/variables_spec.lua index cc0e7fa537..854add1363 100644 --- a/test/functional/shada/variables_spec.lua +++ b/test/functional/shada/variables_spec.lua @@ -1,7 +1,7 @@ -- ShaDa variables saving/reading support local helpers = require('test.functional.helpers')(after_each) -local meths, funcs, nvim_command, eq = - helpers.meths, helpers.funcs, helpers.command, helpers.eq +local meths, funcs, nvim_command, eq, eval = + helpers.meths, helpers.funcs, helpers.command, helpers.eq, helpers.eval local shada_helpers = require('test.functional.shada.helpers') local reset, clear = shada_helpers.reset, shada_helpers.clear @@ -30,10 +30,12 @@ describe('ShaDa support code', function() else meths.set_var(varname, varval) end + local vartype = eval('type(g:' .. varname .. ')') -- Exit during `reset` is not a regular exit: it does not write shada -- automatically nvim_command('qall') reset('set shada+=!') + eq(vartype, eval('type(g:' .. varname .. ')')) eq(varval, meths.get_var(varname)) end) end @@ -47,6 +49,8 @@ describe('ShaDa support code', function() autotest('false', 'FALSEVAR', false) autotest('null', 'NULLVAR', 'v:null', true) autotest('ext', 'EXTVAR', '{"_TYPE": v:msgpack_types.ext, "_VAL": [2, ["", ""]]}', true) + autotest('blob', 'BLOBVAR', '0z12ab34cd', true) + autotest('blob (with NULs)', 'BLOBVARNULS', '0z004e554c7300', true) it('does not read back variables without `!` in &shada', function() meths.set_var('STRVAR', 'foo') From de9df825d5e38e6a9e5ba254d1d27cb3cfc9f557 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Sat, 31 Jul 2021 21:45:58 +0100 Subject: [PATCH 08/37] feat(decode_string): decode binary string with NULs to Blob Strings that previously decoded into a msgpack special for representing BINs with NULs now convert to Blobs. It shouldn't be possible to decode into this special anymore after this change? Notably, Lua strings with NULs now convert to Blobs when passed to VimL. --- runtime/doc/eval.txt | 6 +-- runtime/doc/lua.txt | 3 +- src/nvim/eval/decode.c | 42 +++++++++++-------- .../eval/msgpack_functions_spec.lua | 3 +- test/functional/lua/api_spec.lua | 5 ++- test/functional/lua/luaeval_spec.lua | 29 ++++++------- 6 files changed, 46 insertions(+), 42 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index e7820f5313..70f970ffef 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -6750,9 +6750,9 @@ msgpackparse({list}) *msgpackparse()* zero byte or if string is a mapping key and mapping is being represented as special dictionary for other reasons. - binary |readfile()|-style list of strings. This value will - appear in |msgpackparse()| output if binary string - contains zero byte. + binary |String|, or |Blob| if binary string contains zero + byte. This value cannot appear in |msgpackparse()| + output since blobs were introduced. array |List|. This value cannot appear in |msgpackparse()| output. *msgpack-special-map* diff --git a/runtime/doc/lua.txt b/runtime/doc/lua.txt index 032c28c659..53d68fa5e6 100644 --- a/runtime/doc/lua.txt +++ b/runtime/doc/lua.txt @@ -326,7 +326,8 @@ semantically equivalent in Lua to: end Lua nils, numbers, strings, tables and booleans are converted to their -respective Vimscript types. Conversion of other Lua types is an error. +respective Vimscript types. If a Lua string contains a NUL byte, it will be +converted to a |Blob|. Conversion of other Lua types is an error. The magic global "_A" contains the second argument to luaeval(). diff --git a/src/nvim/eval/decode.c b/src/nvim/eval/decode.c index 89e1f04bfd..5c03f55621 100644 --- a/src/nvim/eval/decode.c +++ b/src/nvim/eval/decode.c @@ -246,14 +246,15 @@ list_T *decode_create_map_special_dict(typval_T *const ret_tv, /// Convert char* string to typval_T /// /// Depending on whether string has (no) NUL bytes, it may use a special -/// dictionary or decode string to VAR_STRING. +/// dictionary, VAR_BLOB, or decode string to VAR_STRING. /// /// @param[in] s String to decode. /// @param[in] len String length. /// @param[in] hasnul Whether string has NUL byte, not or it was not yet /// determined. -/// @param[in] binary If true, save special string type as kMPBinary, -/// otherwise kMPString. +/// @param[in] binary Determines decode type if string has NUL bytes. +/// If true convert string to VAR_BLOB, otherwise to the +/// kMPString special type. /// @param[in] s_allocated If true, then `s` was allocated and can be saved in /// a returned structure. If it is not saved there, it /// will be freed. @@ -269,21 +270,28 @@ typval_T decode_string(const char *const s, const size_t len, ? ((s != NULL) && (memchr(s, NUL, len) != NULL)) : (bool)hasnul); if (really_hasnul) { - list_T *const list = tv_list_alloc(kListLenMayKnow); - tv_list_ref(list); typval_T tv; - create_special_dict(&tv, binary ? kMPBinary : kMPString, ((typval_T) { - .v_type = VAR_LIST, - .v_lock = VAR_UNLOCKED, - .vval = { .v_list = list }, - })); - const int elw_ret = encode_list_write((void *)list, s, len); - if (s_allocated) { - xfree((void *)s); - } - if (elw_ret == -1) { - tv_clear(&tv); - return (typval_T) { .v_type = VAR_UNKNOWN, .v_lock = VAR_UNLOCKED }; + tv.v_lock = VAR_UNLOCKED; + if (binary) { + tv_blob_alloc_ret(&tv); + ga_concat_len(&tv.vval.v_blob->bv_ga, s, len); + } else { + list_T *const list = tv_list_alloc(kListLenMayKnow); + tv_list_ref(list); + create_special_dict(&tv, kMPString, + ((typval_T){ + .v_type = VAR_LIST, + .v_lock = VAR_UNLOCKED, + .vval = { .v_list = list }, + })); + const int elw_ret = encode_list_write((void *)list, s, len); + if (s_allocated) { + xfree((void *)s); + } + if (elw_ret == -1) { + tv_clear(&tv); + return (typval_T) { .v_type = VAR_UNKNOWN, .v_lock = VAR_UNLOCKED }; + } } return tv; } else { diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index 90e3ccc0e4..d95cea57e4 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -364,8 +364,7 @@ describe('msgpack*() functions', function() command('let dumped = ["\\xC4\\x01\\n"]') command('let parsed = msgpackparse(dumped)') command('let dumped2 = msgpackdump(parsed)') - eq({{_TYPE={}, _VAL={'\n'}}}, eval('parsed')) - eq(1, eval('parsed[0]._TYPE is v:msgpack_types.binary')) + eq({'\000'}, eval('parsed')) eq(1, eval('dumped ==# dumped2')) end) diff --git a/test/functional/lua/api_spec.lua b/test/functional/lua/api_spec.lua index b2c343096a..fdf79d55b2 100644 --- a/test/functional/lua/api_spec.lua +++ b/test/functional/lua/api_spec.lua @@ -15,7 +15,7 @@ describe('luaeval(vim.api.…)', function() describe('nvim_buf_get_lines', function() it('works', function() funcs.setline(1, {"abc", "def", "a\nb", "ttt"}) - eq({{_TYPE={}, _VAL={'a\nb'}}}, + eq({'a\000b'}, funcs.luaeval('vim.api.nvim_buf_get_lines(1, 2, 3, false)')) end) end) @@ -23,7 +23,7 @@ describe('luaeval(vim.api.…)', function() it('works', function() funcs.setline(1, {"abc", "def", "a\nb", "ttt"}) eq(NIL, funcs.luaeval('vim.api.nvim_buf_set_lines(1, 1, 2, false, {"b\\0a"})')) - eq({'abc', {_TYPE={}, _VAL={'b\na'}}, {_TYPE={}, _VAL={'a\nb'}}, 'ttt'}, + eq({'abc', 'b\000a', 'a\000b', 'ttt'}, funcs.luaeval('vim.api.nvim_buf_get_lines(1, 0, 4, false)')) end) end) @@ -68,6 +68,7 @@ describe('luaeval(vim.api.…)', function() eq({}, funcs.luaeval('vim.api.nvim_eval("[]")')) eq({}, funcs.luaeval('vim.api.nvim_eval("{}")')) eq(1, funcs.luaeval('vim.api.nvim_eval("1.0")')) + eq('\000', funcs.luaeval('vim.api.nvim_eval("0z00")')) eq(true, funcs.luaeval('vim.api.nvim_eval("v:true")')) eq(false, funcs.luaeval('vim.api.nvim_eval("v:false")')) eq(NIL, funcs.luaeval('vim.api.nvim_eval("v:null")')) diff --git a/test/functional/lua/luaeval_spec.lua b/test/functional/lua/luaeval_spec.lua index 36f6d4025f..255e99032f 100644 --- a/test/functional/lua/luaeval_spec.lua +++ b/test/functional/lua/luaeval_spec.lua @@ -63,11 +63,10 @@ describe('luaeval()', function() eq('\n[[...@0]]', funcs.execute('echo luaeval("l")')) end) end) - describe('strings', function() - it('are successfully converted to special dictionaries', function() + describe('strings with NULs', function() + it('are successfully converted to blobs', function() command([[let s = luaeval('"\0"')]]) - eq({_TYPE={}, _VAL={'\n'}}, meths.get_var('s')) - eq(1, funcs.eval('s._TYPE is v:msgpack_types.binary')) + eq('\000', meths.get_var('s')) end) it('are successfully converted to special dictionaries in table keys', function() @@ -76,13 +75,10 @@ describe('luaeval()', function() eq(1, funcs.eval('d._TYPE is v:msgpack_types.map')) eq(1, funcs.eval('d._VAL[0][0]._TYPE is v:msgpack_types.string')) end) - it('are successfully converted to special dictionaries from a list', + it('are successfully converted to blobs from a list', function() command([[let l = luaeval('{"abc", "a\0b", "c\0d", "def"}')]]) - eq({'abc', {_TYPE={}, _VAL={'a\nb'}}, {_TYPE={}, _VAL={'c\nd'}}, 'def'}, - meths.get_var('l')) - eq(1, funcs.eval('l[1]._TYPE is v:msgpack_types.binary')) - eq(1, funcs.eval('l[2]._TYPE is v:msgpack_types.binary')) + eq({'abc', 'a\000b', 'c\000d', 'def'}, meths.get_var('l')) end) end) @@ -100,9 +96,9 @@ describe('luaeval()', function() eq(1, eval('type(luaeval("\'test\'"))')) eq('', funcs.luaeval('""')) - eq({_TYPE={}, _VAL={'\n'}}, funcs.luaeval([['\0']])) - eq({_TYPE={}, _VAL={'\n', '\n'}}, funcs.luaeval([['\0\n\0']])) - eq(1, eval([[luaeval('"\0\n\0"')._TYPE is v:msgpack_types.binary]])) + eq('\000', funcs.luaeval([['\0']])) + eq('\000\n\000', funcs.luaeval([['\0\n\0']])) + eq(10, eval([[type(luaeval("'\\0\\n\\0'"))]])) eq(true, funcs.luaeval('true')) eq(false, funcs.luaeval('false')) @@ -122,12 +118,11 @@ describe('luaeval()', function() local level = 30 eq(nested_by_level[level].o, funcs.luaeval(nested_by_level[level].s)) - eq({_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, {_TYPE={}, _VAL={'\n', '\n\n'}}}}}, + eq({_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, '\000\n\000\000'}}}, funcs.luaeval([[{['\0\n\0']='\0\n\0\0'}]])) eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._TYPE is v:msgpack_types.map]])) eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._VAL[0][0]._TYPE is v:msgpack_types.string]])) - eq(1, eval([[luaeval('{["\0\n\0"]="\0\n\0\0"}')._VAL[0][1]._TYPE is v:msgpack_types.binary]])) - eq({nested={{_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, {_TYPE={}, _VAL={'\n', '\n\n'}}}}}}}, + eq({nested={{_TYPE={}, _VAL={{{_TYPE={}, _VAL={'\n', '\n'}}, '\000\n\000\000'}}}}}, funcs.luaeval([[{nested={{['\0\n\0']='\0\n\0\0'}}}]])) end) @@ -175,8 +170,8 @@ describe('luaeval()', function() end it('correctly passes special dictionaries', function() - eq({'binary', {'\n', '\n'}}, luaevalarg(sp('binary', '["\\n", "\\n"]'))) - eq({'binary', {'\n', '\n'}}, luaevalarg(sp('string', '["\\n", "\\n"]'))) + eq({0, '\000\n\000'}, luaevalarg(sp('binary', '["\\n", "\\n"]'))) + eq({0, '\000\n\000'}, luaevalarg(sp('string', '["\\n", "\\n"]'))) eq({0, true}, luaevalarg(sp('boolean', 1))) eq({0, false}, luaevalarg(sp('boolean', 0))) eq({0, NIL}, luaevalarg(sp('nil', 0))) From 7200454ee6f500bb851bea992707c019d9982cb9 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Tue, 24 Nov 2020 20:55:04 +0000 Subject: [PATCH 09/37] vim-patch:8.1.0738: using freed memory, for loop over blob leaks memory Problem: Using freed memory, for loop over blob leaks memory. Solution: Clear pointer after freeing memory. Decrement reference count after for loop over blob. https://github.com/vim/vim/commit/ecc8bc482ba601b9301a6c129c92a0d1f8527f72 --- src/nvim/eval.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 29df1a168f..eb8f0406c4 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -2603,6 +2603,8 @@ void *eval_for_line(const char_u *arg, bool *errp, char_u **nextcmdp, int skip) if (b == NULL) { tv_clear(&tv); } else { + // No need to increment the refcount, it's already set for + // the blob being used in "tv". fi->fi_blob = b; fi->fi_bi = 0; } @@ -2666,6 +2668,9 @@ void free_for_info(void *fi_void) tv_list_watch_remove(fi->fi_list, &fi->fi_lw); tv_list_unref(fi->fi_list); } + if (fi != NULL && fi->fi_blob != NULL) { + tv_blob_unref(fi->fi_blob); + } xfree(fi); } @@ -4072,9 +4077,12 @@ static int eval7( char_u *bp; for (bp = *arg + 2; ascii_isxdigit(bp[0]); bp += 2) { if (!ascii_isxdigit(bp[1])) { - EMSG(_("E973: Blob literal should have an even number of hex " - "characters")); - xfree(blob); + if (blob != NULL) { + EMSG(_("E973: Blob literal should have an even number of hex " + "characters")); + ga_clear(&blob->bv_ga); + XFREE_CLEAR(blob); + } ret = FAIL; break; } From d346ac536fbb7716e6a0d0f5ca7af9c39ec95de0 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Tue, 24 Nov 2020 21:37:52 +0000 Subject: [PATCH 10/37] vim-patch:8.1.0742: not all Blob operations are tested Problem: Not all Blob operations are tested. Solution: Add more testing for Blob. https://github.com/vim/vim/commit/05500ece6282407f9f7227aaf564e24147326863 Test_readfile_binary is already ported. --- src/nvim/eval/funcs.c | 15 +++++++++++++-- src/nvim/testdir/test_blob.vim | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index dbb00da911..3c25d76866 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -325,8 +325,13 @@ static void f_add(typval_T *argvars, typval_T *rettv, FunPtr fptr) blob_T *const b = argvars[0].vval.v_blob; if (b != NULL && !var_check_lock(b->bv_lock, N_("add() argument"), TV_TRANSLATE)) { - ga_append(&b->bv_ga, (char_u)tv_get_number(&argvars[1])); - tv_copy(&argvars[0], rettv); + bool error = false; + const varnumber_T n = tv_get_number_chk(&argvars[1], &error); + + if (!error) { + ga_append(&b->bv_ga, (int)n); + tv_copy(&argvars[0], rettv); + } } } else { EMSG(_(e_listreq)); @@ -4827,6 +4832,12 @@ static void f_index(typval_T *argvars, typval_T *rettv, FunPtr fptr) if (b == NULL) { return; } + if (start < 0) { + start = tv_blob_len(b) + start; + if (start < 0) { + start = 0; + } + } for (idx = start; idx < tv_blob_len(b); idx++) { typval_T tv; tv.v_type = VAR_NUMBER; diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index df8efbb149..87448e4fba 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -96,6 +96,8 @@ func Test_blob_compare() call assert_true(b1 != b2) call assert_true(b1 != b3) call assert_true(b1 == 0z0011) + call assert_fails('echo b1 == 9', 'E977:') + call assert_fails('echo b1 != 9', 'E977:') call assert_false(b1 is b2) let b2 = b1 @@ -145,6 +147,22 @@ func Test_blob_concatenate() call assert_equal(0zDEADBEEF, b) endfunc +func Test_blob_add() + let b = 0z0011 + call add(b, 0x22) + call assert_equal(0z001122, b) + call add(b, '51') + call assert_equal(0z00112233, b) + + call assert_fails('call add(b, [9])', 'E745:') +endfunc + +func Test_blob_empty() + call assert_false(empty(0z001122)) + call assert_true(empty(0z)) + call assert_true(empty(v:_null_blob)) +endfunc + " Test removing items in blob func Test_blob_func_remove() " Test removing 1 element @@ -198,11 +216,19 @@ func Test_blob_map() let b = 0zDEADBEEF call map(b, 'v:val + 1') call assert_equal(0zDFAEBFF0, b) + + call assert_fails("call map(b, '[9]')", 'E978:') endfunc func Test_blob_index() call assert_equal(2, index(0zDEADBEEF, 0xBE)) call assert_equal(-1, index(0zDEADBEEF, 0)) + call assert_equal(2, index(0z11111111, 0x11, 2)) + call assert_equal(3, index(0z11110111, 0x11, 2)) + call assert_equal(2, index(0z11111111, 0x11, -2)) + call assert_equal(3, index(0z11110111, 0x11, -2)) + + call assert_fails('call index("asdf", 0)', 'E714:') endfunc func Test_blob_insert() @@ -213,6 +239,10 @@ func Test_blob_insert() let b = 0zDEADBEEF call insert(b, 0x33, 2) call assert_equal(0zDEAD33BEEF, b) + + call assert_fails('call insert(b, -1)', 'E475:') + call assert_fails('call insert(b, 257)', 'E475:') + call assert_fails('call insert(b, 0, [9])', 'E745:') endfunc func Test_blob_reverse() From c1b8731ece81cdbc43fbb6050eaf12e84abdc4b0 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Wed, 25 Nov 2020 17:27:39 +0000 Subject: [PATCH 11/37] vim-patch:8.1.0755: error message for get() on a Blob with invalid index Problem: Error message for get() on a Blob with invalid index. Solution: Return an empty Blob, like get() on a List does. https://github.com/vim/vim/commit/2ea773b468a1143214c2f12b91ab5e1e7abb4a14 --- src/nvim/eval/funcs.c | 10 +++++++--- src/nvim/testdir/test_blob.vim | 13 ++++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 3c25d76866..e804029187 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -2809,14 +2809,18 @@ static void f_get(typval_T *argvars, typval_T *rettv, FunPtr fptr) if (argvars[0].v_type == VAR_BLOB) { bool error = false; - const int idx = tv_get_number_chk(&argvars[1], &error); + int idx = tv_get_number_chk(&argvars[1], &error); if (!error) { rettv->v_type = VAR_NUMBER; - if (idx >= tv_blob_len(argvars[0].vval.v_blob)) { - EMSGN(_(e_blobidx), idx); + if (idx < 0) { + idx = tv_blob_len(argvars[0].vval.v_blob) + idx; + } + if (idx < 0 || idx >= tv_blob_len(argvars[0].vval.v_blob)) { + rettv->vval.v_number = -1; } else { rettv->vval.v_number = tv_blob_get(argvars[0].vval.v_blob, idx); + tv = rettv; } } } else if (argvars[0].v_type == VAR_LIST) { diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index 87448e4fba..5bee25a0a2 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -20,7 +20,6 @@ func Test_blob_create() call assert_equal(0xDE, get(b, 0)) call assert_equal(0xEF, get(b, 3)) - call assert_fails('let x = get(b, 4)') call assert_fails('let b = 0z1', 'E973:') call assert_fails('let b = 0z1x', 'E973:') @@ -79,6 +78,18 @@ func Test_blob_get_range() call assert_equal(0z, b[5:6]) endfunc +func Test_blob_get() + let b = 0z0011223344 + call assert_equal(0x00, get(b, 0)) + call assert_equal(0x22, get(b, 2, 999)) + call assert_equal(0x44, get(b, 4)) + call assert_equal(0x44, get(b, -1)) + call assert_equal(-1, get(b, 5)) + call assert_equal(999, get(b, 5, 999)) + call assert_equal(-1, get(b, -8)) + call assert_equal(999, get(b, -8, 999)) +endfunc + func Test_blob_to_string() let b = 0zDEADBEEF call assert_equal('[0xDE,0xAD,0xBE,0xEF]', string(b)) From 2b98bdd75b7ffc8c876d1e03c96a4edc4c1e1b12 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Wed, 25 Nov 2020 16:21:00 +0000 Subject: [PATCH 12/37] vim-patch:8.1.0756: copy() does not make a copy of a Blob Problem: copy() does not make a copy of a Blob. Solution: Make a copy. https://github.com/vim/vim/commit/3d28b58c519c9fc3427587201423c74746cc219e Replace vim_memsave() with xmemdup(). --- src/nvim/eval.c | 14 +++++++++++++- src/nvim/testdir/test_blob.vim | 7 +++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index eb8f0406c4..35c3b1b876 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -9700,7 +9700,6 @@ int var_item_copy(const vimconv_T *const conv, case VAR_PARTIAL: case VAR_BOOL: case VAR_SPECIAL: - case VAR_BLOB: tv_copy(from, to); break; case VAR_STRING: @@ -9734,6 +9733,19 @@ int var_item_copy(const vimconv_T *const conv, ret = FAIL; } break; + case VAR_BLOB: + to->v_type = VAR_BLOB; + if (from->vval.v_blob == NULL) { + to->vval.v_blob = NULL; + } else { + tv_blob_alloc_ret(to); + const int len = from->vval.v_blob->bv_ga.ga_len; + + to->vval.v_blob->bv_ga.ga_data + = xmemdup(from->vval.v_blob->bv_ga.ga_data, (size_t)len); + to->vval.v_blob->bv_ga.ga_len = len; + } + break; case VAR_DICT: to->v_type = VAR_DICT; to->v_lock = VAR_UNLOCKED; diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index 5bee25a0a2..b2b5ddcfd6 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -112,7 +112,14 @@ func Test_blob_compare() call assert_false(b1 is b2) let b2 = b1 + call assert_true(b1 == b2) call assert_true(b1 is b2) + let b2 = copy(b1) + call assert_true(b1 == b2) + call assert_false(b1 is b2) + let b2 = b1[:] + call assert_true(b1 == b2) + call assert_false(b1 is b2) call assert_fails('let x = b1 > b2') call assert_fails('let x = b1 < b2') From 0eadd7e5fd4603fef313fd5fb4cced1b7b5f1679 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 01:13:55 +0100 Subject: [PATCH 13/37] vim-patch:8.1.0757: not enough documentation for Blobs Problem: Not enough documentation for Blobs. Solution: Add a section about Blobs. https://github.com/vim/vim/commit/d89682477cd01ec60edd6d88093b95b12e180127 Include doc changes for empty() from v7.4.1274. Include some minor typo fixes. --- runtime/doc/eval.txt | 149 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 136 insertions(+), 13 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 70f970ffef..2efd7077d9 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -34,7 +34,7 @@ Funcref A reference to a function |Funcref|. like a Partial. Example: function("Callback", [arg], myDict) -List An ordered sequence of items |List|. +List An ordered sequence of items, see |List| for details. Example: [1, 2, ['a', 'b']] Dictionary An associative, unordered array: Each entry has a key and a @@ -43,7 +43,8 @@ Dictionary An associative, unordered array: Each entry has a key and a {'blue': "#0000ff", 'red': "#ff0000"} #{blue: "#0000ff", red: "#ff0000"} -Blob Binary Large Object. Stores any sequence of bytes. *Blob* +Blob Binary Large Object. Stores any sequence of bytes. See |Blob| + for details. Example: 0zFF00ED015DAF 0z is an empty Blob. @@ -599,7 +600,120 @@ Functions that can be used with a Dictionary: > :call map(dict, '">> " . v:val') " prepend ">> " to each item -1.5 More about variables ~ +1.5 Blobs ~ + *blob* *Blob* *Blobs* *E978* +A Blob mostly behaves like a |List| of numbers, where the numbers have an +8-bit value, from 0 to 255. + + +Blob creation ~ + +A Blob can be created with a |blob-literal|: > + :let b = 0zFF00ED015DAF + +A blob can be read from a file with |readfile()| passing the {type} argument +set to "B", for example: > + :let b = readfile('image.png', 'B') + + +Blob index ~ + *blob-index* *E979* +A byte in the Blob can be accessed by putting the index in square brackets +after the Blob. Indexes are zero-based, thus the first byte has index zero. > + :let myblob = 0z00112233 + :let byte = myblob[0] " get the first byte: 0x00 + :let byte = myblob[2] " get the third byte: 0x22 + +A negative index is counted from the end. Index -1 refers to the last byte in +the Blob, -2 to the last but one byte, etc. > + :let last = myblob[-1] " get the last byte: 0x33 + +To avoid an error for an invalid index use the |get()| function. When an item +is not available it returns -1 or the default value you specify: > + :echo get(myblob, idx) + :echo get(myblob, idx, 999) + + +Blob concatenation ~ + +Two blobs can be concatenated with the "+" operator: > + :let longblob = myblob + 0z4455 + :let myblob += 0z6677 + +To change a blob in-place see |blob-modification| below. + + +Part of a blob ~ + +A part of the Blob can be obtained by specifying the first and last index, +separated by a colon in square brackets: > + :let myblob = 0z00112233 + :let shortblob = myblob[2:-1] " get 0z2233 + +Omitting the first index is similar to zero. Omitting the last index is +similar to -1. > + :let endblob = myblob[2:] " from item 2 to the end: 0z2233 + :let shortblob = myblob[2:2] " Blob with one byte: 0z22 + :let otherblob = myblob[:] " make a copy of the Blob + +If the first index is beyond the last byte of the Blob or the second byte is +before the first byte, the result is an empty Blob. There is no error +message. + +If the second index is equal to or greater than the length of the Blob the +length minus one is used: > + :echo myblob[2:8] " result: 0z2233 + + +Blob modification ~ + *blob-modification* +To change a specific byte of a blob use |:let| this way: > + :let blob[4] = 0x44 + +When the index is just one beyond the end of the Blob, it is appended. Any +higher index is an error. + +To change a sequence of bytes the [:] notation can be used: > + let blob[1:3] = 0z445566 +The length of the replaced bytes must be exactly the same as the value +provided. *E972* + +To change part of a blob you can specify the first and last byte to be +modified. The value must at least have the number of bytes in the range: > + :let blob[3:5] = [3, 4, 5] + +You can also use the functions |add()|, |remove()| and |insert()|. + + +Blob identity ~ + +Blobs can be compared for equality: > + if blob == 0z001122 +And for equal identity: > + if blob is otherblob +< *blob-identity* *E977* +When variable "aa" is a Blob and you assign it to another variable "bb", both +variables refer to the same Blob. Then the "is" operator returns true. + +When making a copy using [:] or |copy()| the values are the same, but the +identity is different: > + :let blob = 0z112233 + :let blob2 = blob + :echo blob == blob2 +< 1 > + :echo blob is blob2 +< 1 > + :let blob3 = blob[:] + :echo blob == blob3 +< 1 > + :echo blob is blob3 +< 0 + +Making a copy of a Blob is done with the |copy()| function. Using [:] also +works, as explained above. + + +1.6 More about variables ~ *more-variables* If you need to know the type of a variable or expression, use the |type()| function. @@ -1191,7 +1305,7 @@ encodings. Use "\u00ff" to store character 255 correctly as UTF-8. Note that "\000" and "\x00" force the end of the string. -blob-literal *blob-literal* *E973* *E977* *E978* +blob-literal *blob-literal* *E973* ------------ Hexadecimal starting with 0z or 0Z, with an arbitrary number of bytes. @@ -2124,7 +2238,7 @@ USAGE RESULT DESCRIPTION ~ abs({expr}) Float or Number absolute value of {expr} acos({expr}) Float arc cosine of {expr} -add({list}, {item}) List append {item} to |List| {list} +add({object}, {item}) List/Blob append {item} to {object} and({expr}, {expr}) Number bitwise AND api_info() Dict api metadata append({lnum}, {string}) Number append {string} below line {lnum} @@ -2668,13 +2782,14 @@ acos({expr}) *acos()* Can also be used as a |method|: > Compute()->acos() -add({list}, {expr}) *add()* - Append the item {expr} to |List| {list}. Returns the - resulting |List|. Examples: > +add({object}, {expr}) *add()* + Append the item {expr} to |List| or |Blob| {object}. Returns + the resulting |List| or |Blob|. Examples: > :let alist = add([1, 2, 3], item) :call add(mylist, "woodstock") < Note that when {expr} is a |List| it is appended as a single item. Use |extend()| to concatenate |Lists|. + When {object} is a |Blob| then {expr} must be a number. Use |insert()| to add an item at another position. Can also be used as a |method|: > @@ -3661,9 +3776,13 @@ diff_hlID({lnum}, {col}) *diff_hlID()* empty({expr}) *empty()* Return the Number 1 if {expr} is empty, zero otherwise. - A |List| or |Dictionary| is empty when it does not have any - items. A Number is empty when its value is zero. Special - variable is empty when it is |v:false| or |v:null|. + - A |List| or |Dictionary| is empty when it does not have any + items. + - A |String| is empty when its length is zero. + - A |Number| and |Float| are empty when their value is zero. + - |v:false| and |v:null| are empty, |v:true| is not. + - A |Blob| is empty when its length is zero. + Can also be used as a |method|: > mylist->empty() @@ -4383,6 +4502,10 @@ get({list}, {idx} [, {default}]) *get()* omitted. Can also be used as a |method|: > mylist->get(idx) +get({blob}, {idx} [, {default}]) + Get byte {idx} from |Blob| {blob}. When this byte is not + available return {default}. Return -1 when {default} is + omitted. get({dict}, {key} [, {default}]) Get item with key {key} from |Dictionary| {dict}. When this item is not available return {default}. Return zero when @@ -5573,10 +5696,10 @@ index({object}, {expr} [, {start} [, {ic}]]) *index()* If {object} is a |List| return the lowest index where the item has a value equal to {expr}. There is no automatic conversion, so the String "4" is different from the Number 4. - And the number 4 is different from the Float 4.0. The value + And the Number 4 is different from the Float 4.0. The value of 'ignorecase' is not used here, case always matters. - If {object} is |Blob| return the lowest index where the byte + If {object} is a |Blob| return the lowest index where the byte value is equal to {expr}. If {start} is given then start looking at the item with index From 6a02ccc2226ab427d7e6243a5b6d3e424557b0fd Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Wed, 2 Dec 2020 17:24:15 +0000 Subject: [PATCH 14/37] vim-patch:8.1.0765: string format of a Blob can't be parsed back Problem: String format of a Blob can't be parsed back. Solution: Use 0z format. https://github.com/vim/vim/commit/4131fd5509b283e978e8c6161f09643b64719787 --- src/nvim/eval.c | 3 +++ src/nvim/eval/encode.c | 16 +++++++++------- src/nvim/testdir/test_blob.vim | 15 ++++++++++++--- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 35c3b1b876..596ed19ff9 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -4089,6 +4089,9 @@ static int eval7( if (blob != NULL) { ga_append(&blob->bv_ga, (hex2nr(*bp) << 4) + hex2nr(*(bp + 1))); } + if (bp[2] == '.' && ascii_isxdigit(bp[3])) { + bp++; + } } if (blob != NULL) { tv_blob_set_ret(rettv, blob); diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index b6b37fce84..1c2274c410 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -324,20 +324,22 @@ int encode_read_from_list(ListReaderState *const state, char *const buf, const blob_T *const blob_ = (blob); \ const int len_ = (len); \ if (len_ == 0) { \ - ga_concat(gap, "[]"); \ + ga_concat(gap, "0z"); \ } else { \ - ga_grow(gap, 1 + len_ * 5); \ - ga_append(gap, '['); \ + /* Allocate space for "0z", the two hex chars per byte, and a */ \ + /* "." separator after every eight hex chars. */ \ + /* Example: "0z00112233.44556677.8899" */ \ + ga_grow(gap, 2 + 2 * len_ + (len_ - 1) / 4); \ + ga_concat(gap, "0z"); \ char numbuf[NUMBUFLEN]; \ for (int i_ = 0; i_ < len_; i_++) { \ - if (i_ > 0) { \ - ga_append(gap, ','); \ + if (i_ > 0 && (i_ & 3) == 0) { \ + ga_append(gap, '.'); \ } \ - vim_snprintf((char *)numbuf, ARRAY_SIZE(numbuf), "0x%02X", \ + vim_snprintf((char *)numbuf, ARRAY_SIZE(numbuf), "%02X", \ (int)tv_blob_get(blob_, i_)); \ ga_concat(gap, numbuf); \ } \ - ga_append(gap, ']'); \ } \ } while (0) diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index b2b5ddcfd6..dec37f0f10 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -26,6 +26,12 @@ func Test_blob_create() call assert_fails('let b = 0z12345', 'E973:') call assert_equal(0z, v:_null_blob) + + let b = 0z001122.33445566.778899.aabbcc.dd + call assert_equal(0z00112233445566778899aabbccdd, b) + call assert_fails('let b = 0z1.1') + call assert_fails('let b = 0z.') + call assert_fails('let b = 0z001122.') endfunc " assignment to a blob @@ -91,10 +97,13 @@ func Test_blob_get() endfunc func Test_blob_to_string() - let b = 0zDEADBEEF - call assert_equal('[0xDE,0xAD,0xBE,0xEF]', string(b)) + let b = 0z00112233445566778899aabbccdd + call assert_equal('0z00112233.44556677.8899AABB.CCDD', string(b)) + call assert_equal(b, eval(string(b))) + call remove(b, 4, -1) + call assert_equal('0z00112233', string(b)) call remove(b, 0, 3) - call assert_equal('[]', string(b)) + call assert_equal('0z', string(b)) endfunc func Test_blob_compare() From c57132ec2a9c16facff7858c8610c3206398fe7e Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Wed, 2 Dec 2020 18:37:18 +0000 Subject: [PATCH 15/37] vim-patch:8.1.0793: incorrect error messages for functions that take a Blob Problem: Incorrect error messages for functions that now take a Blob argument. Solution: Adjust the error messages. (Dominique Pelle, closes vim/vim#3846) https://github.com/vim/vim/commit/0d17f0d1c09fa6db306336695ba646c21ea24909 --- runtime/doc/eval.txt | 28 ++++++++++++++++++--------- src/nvim/eval/funcs.c | 13 +++++++------ src/nvim/globals.h | 3 +++ src/nvim/testdir/test_blob.vim | 6 ++++-- src/nvim/testdir/test_listdict.vim | 4 +++- test/functional/eval/execute_spec.lua | 8 ++++---- test/functional/lua/vim_spec.lua | 2 +- 7 files changed, 41 insertions(+), 23 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 2efd7077d9..1a9fc02a9b 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -14,7 +14,7 @@ Using expressions is introduced in chapter 41 of the user manual |usr_41.txt|. 1. Variables *variables* 1.1 Variable types ~ - *E712* + *E712* *E896* *E897* *E898* There are seven types of variables: *Number* *Integer* @@ -610,6 +610,9 @@ Blob creation ~ A Blob can be created with a |blob-literal|: > :let b = 0zFF00ED015DAF +Dots can be inserted between bytes (pair of hex characters) for readability, +they don't change the value: > + :let b = 0zFF00.ED01.5DAF A blob can be read from a file with |readfile()| passing the {type} argument set to "B", for example: > @@ -3805,8 +3808,8 @@ escape({string}, {chars}) *escape()* *eval()* eval({string}) Evaluate {string} and return the result. Especially useful to turn the result of |string()| back into the original value. - This works for Numbers, Floats, Strings and composites of - them. Also works for |Funcref|s that refer to existing + This works for Numbers, Floats, Strings, Blobs and composites + of them. Also works for |Funcref|s that refer to existing functions. Can also be used as a |method|: > @@ -5957,8 +5960,13 @@ items({dict}) *items()* Return a |List| with all the key-value pairs of {dict}. Each |List| item is a list with two items: the key of a {dict} entry and the value of this entry. The |List| is in arbitrary - order. - Can also be used as a |method|: > + order. Also see |keys()| and |values()|. + Example: > + for [key, value] in items(mydict) + echo key . ': ' . value + endfor + +< Can also be used as a |method|: > mydict->items() isnan({expr}) *isnan()* @@ -6129,7 +6137,8 @@ json_encode({expr}) *json_encode()* keys({dict}) *keys()* Return a |List| with all the keys of {dict}. The |List| is in - arbitrary order. + arbitrary order. Also see |items()| and |values()|. + Can also be used as a |method|: > mydict->keys() @@ -8970,14 +8979,15 @@ stridx({haystack}, {needle} [, {start}]) *stridx()* *string()* string({expr}) Return {expr} converted to a String. If {expr} is a Number, - Float, String or a composition of them, then the result can be - parsed back with |eval()|. + Float, String, Blob or a composition of them, then the result + can be parsed back with |eval()|. {expr} type result ~ String 'string' Number 123 Float 123.123456 or 1.123456e8 or `str2float('inf')` Funcref `function('name')` + Blob 0z00112233.44556677.8899 List [item, item] Dictionary {key: value, key: value} Note that in String values the ' character is doubled. @@ -9735,7 +9745,7 @@ uniq({list} [, {func} [, {dict}]]) *uniq()* *E882* values({dict}) *values()* Return a |List| with all the values of {dict}. The |List| is - in arbitrary order. + in arbitrary order. Also see |items()| and |keys()|. Can also be used as a |method|: > mydict->values() diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index e804029187..20affdf529 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -96,6 +96,7 @@ PRAGMA_DIAG_POP static char *e_listarg = N_("E686: Argument of %s must be a List"); +static char *e_listblobarg = N_("E898: Argument of %s must be a List or Blob"); static char *e_invalwindow = N_("E957: Invalid window number"); /// Dummy va_list for passing to vim_snprintf @@ -334,7 +335,7 @@ static void f_add(typval_T *argvars, typval_T *rettv, FunPtr fptr) } } } else { - EMSG(_(e_listreq)); + EMSG(_(e_listblobreq)); } } @@ -2884,7 +2885,7 @@ static void f_get(typval_T *argvars, typval_T *rettv, FunPtr fptr) } } } else { - EMSG2(_(e_listdictarg), "get()"); + EMSG2(_(e_listdictblobarg), "get()"); } if (tv == NULL) { @@ -4853,7 +4854,7 @@ static void f_index(typval_T *argvars, typval_T *rettv, FunPtr fptr) } return; } else if (argvars[0].v_type != VAR_LIST) { - EMSG(_(e_listreq)); + EMSG(_(e_listblobreq)); return; } list_T *const l = argvars[0].vval.v_list; @@ -5013,7 +5014,7 @@ static void f_insert(typval_T *argvars, typval_T *rettv, FunPtr fptr) tv_copy(&argvars[0], rettv); } else if (argvars[0].v_type != VAR_LIST) { - EMSG2(_(e_listarg), "insert()"); + EMSG2(_(e_listblobarg), "insert()"); } else if (!var_check_lock(tv_list_locked((l = argvars[0].vval.v_list)), N_("insert() argument"), TV_TRANSLATE)) { long before = 0; @@ -7349,7 +7350,7 @@ static void f_remove(typval_T *argvars, typval_T *rettv, FunPtr fptr) } } } else if (argvars[0].v_type != VAR_LIST) { - EMSG2(_(e_listdictarg), "remove()"); + EMSG2(_(e_listdictblobarg), "remove()"); } else if (!var_check_lock(tv_list_locked((l = argvars[0].vval.v_list)), arg_errmsg, TV_TRANSLATE)) { bool error = false; @@ -7634,7 +7635,7 @@ static void f_reverse(typval_T *argvars, typval_T *rettv, FunPtr fptr) } tv_blob_set_ret(rettv, b); } else if (argvars[0].v_type != VAR_LIST) { - EMSG2(_(e_listarg), "reverse()"); + EMSG2(_(e_listblobarg), "reverse()"); } else { list_T *const l = argvars[0].vval.v_list; if (!var_check_lock(tv_list_locked(l), N_("reverse() argument"), diff --git a/src/nvim/globals.h b/src/nvim/globals.h index 42989ea508..4d54907a75 100644 --- a/src/nvim/globals.h +++ b/src/nvim/globals.h @@ -946,8 +946,11 @@ EXTERN char_u e_toomanyarg[] INIT(= N_( EXTERN char_u e_dictkey[] INIT(= N_( "E716: Key not present in Dictionary: \"%s\"")); EXTERN char_u e_listreq[] INIT(= N_("E714: List required")); +EXTERN char_u e_listblobreq[] INIT(= N_("E897: List or Blob required")); EXTERN char_u e_listdictarg[] INIT(= N_( "E712: Argument of %s must be a List or Dictionary")); +EXTERN char_u e_listdictblobarg[] INIT(= N_( + "E896: Argument of %s must be a List, Dictionary or Blob")); EXTERN char_u e_readerrf[] INIT(= N_("E47: Error while reading errorfile")); EXTERN char_u e_sandbox[] INIT(= N_("E48: Not allowed in sandbox")); EXTERN char_u e_secure[] INIT(= N_("E523: Not allowed here")); diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index dec37f0f10..08342e35d2 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -32,6 +32,7 @@ func Test_blob_create() call assert_fails('let b = 0z1.1') call assert_fails('let b = 0z.') call assert_fails('let b = 0z001122.') + call assert_fails('call get("", 1)', 'E896:') endfunc " assignment to a blob @@ -182,6 +183,7 @@ func Test_blob_add() call assert_equal(0z00112233, b) call assert_fails('call add(b, [9])', 'E745:') + call assert_fails('call add("", 0x01)', 'E897:') endfunc func Test_blob_empty() @@ -219,7 +221,7 @@ func Test_blob_func_remove() call assert_fails("call remove(b, 5)", 'E979:') call assert_fails("call remove(b, 1, 5)", 'E979:') call assert_fails("call remove(b, 3, 2)", 'E979:') - call assert_fails("call remove(1, 0)", 'E712:') + call assert_fails("call remove(1, 0)", 'E896:') call assert_fails("call remove(b, b)", 'E974:') endfunc @@ -255,7 +257,7 @@ func Test_blob_index() call assert_equal(2, index(0z11111111, 0x11, -2)) call assert_equal(3, index(0z11110111, 0x11, -2)) - call assert_fails('call index("asdf", 0)', 'E714:') + call assert_fails('call index("asdf", 0)', 'E897:') endfunc func Test_blob_insert() diff --git a/src/nvim/testdir/test_listdict.vim b/src/nvim/testdir/test_listdict.vim index 5152af8f58..fa711dde2d 100644 --- a/src/nvim/testdir/test_listdict.vim +++ b/src/nvim/testdir/test_listdict.vim @@ -139,7 +139,7 @@ func Test_list_func_remove() call assert_fails("call remove(l, 5)", 'E684:') call assert_fails("call remove(l, 1, 5)", 'E684:') call assert_fails("call remove(l, 3, 2)", 'E16:') - call assert_fails("call remove(1, 0)", 'E712:') + call assert_fails("call remove(1, 0)", 'E896:') call assert_fails("call remove(l, l)", 'E745:') endfunc @@ -616,6 +616,8 @@ func Test_reverse_sort_uniq() call assert_equal(['bar', 'BAR', 'Bar', 'Foo', 'FOO', 'foo', 'FOOBAR', -1, 0, 0, 0.22, 1.0e-15, 12, 18, 22, 255, 7, 9, [], {}], sort(copy(l), 1)) call assert_equal(['bar', 'BAR', 'Bar', 'Foo', 'FOO', 'foo', 'FOOBAR', -1, 0, 0, 0.22, 1.0e-15, 12, 18, 22, 255, 7, 9, [], {}], sort(copy(l), 'i')) call assert_equal(['BAR', 'Bar', 'FOO', 'FOOBAR', 'Foo', 'bar', 'foo', -1, 0, 0, 0.22, 1.0e-15, 12, 18, 22, 255, 7, 9, [], {}], sort(copy(l))) + + call assert_fails('call reverse("")', 'E898:') endfunc " splitting a string to a List diff --git a/test/functional/eval/execute_spec.lua b/test/functional/eval/execute_spec.lua index f52ac4e59b..fccf52935b 100644 --- a/test/functional/eval/execute_spec.lua +++ b/test/functional/eval/execute_spec.lua @@ -322,16 +322,16 @@ describe('execute()', function() eq('Vim(call):E731: using Dictionary as a String', ret) ret = exc_exec('call execute("echo add(1, 1)", "")') - eq('Vim(echo):E714: List required', ret) + eq('Vim(echo):E897: List or Blob required', ret) ret = exc_exec('call execute(["echon 42", "echo add(1, 1)"], "")') - eq('Vim(echo):E714: List required', ret) + eq('Vim(echo):E897: List or Blob required', ret) ret = exc_exec('call execute("echo add(1, 1)", "silent")') - eq('Vim(echo):E714: List required', ret) + eq('Vim(echo):E897: List or Blob required', ret) ret = exc_exec('call execute(["echon 42", "echo add(1, 1)"], "silent")') - eq('Vim(echo):E714: List required', ret) + eq('Vim(echo):E897: List or Blob required', ret) end) end) end) diff --git a/test/functional/lua/vim_spec.lua b/test/functional/lua/vim_spec.lua index 2bedbd1453..a066cfbc10 100644 --- a/test/functional/lua/vim_spec.lua +++ b/test/functional/lua/vim_spec.lua @@ -739,7 +739,7 @@ describe('lua stdlib', function() eq({NIL, NIL}, exec_lua([[return vim.fn.Nilly()]])) -- error handling - eq({false, 'Vim:E714: List required'}, exec_lua([[return {pcall(vim.fn.add, "aa", "bb")}]])) + eq({false, 'Vim:E897: List or Blob required'}, exec_lua([[return {pcall(vim.fn.add, "aa", "bb")}]])) end) it('vim.fn should error when calling API function', function() From e140eec441006d0a05abcba49c6d9a0482609216 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Wed, 2 Dec 2020 19:01:48 +0000 Subject: [PATCH 16/37] vim-patch:8.1.0797: error E898 is used twice Problem: Error E898 is used twice. Solution: Rename the Blob error to E899. (closes vim/vim#3853) https://github.com/vim/vim/commit/bf821bccf18453b01d25bee53e4954b02a5dd0e6 --- runtime/doc/eval.txt | 2 +- src/nvim/eval/funcs.c | 2 +- src/nvim/testdir/test_listdict.vim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 1a9fc02a9b..7532bcfba7 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -14,7 +14,7 @@ Using expressions is introduced in chapter 41 of the user manual |usr_41.txt|. 1. Variables *variables* 1.1 Variable types ~ - *E712* *E896* *E897* *E898* + *E712* *E896* *E897* *E899* There are seven types of variables: *Number* *Integer* diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 20affdf529..2d58ce1d04 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -96,7 +96,7 @@ PRAGMA_DIAG_POP static char *e_listarg = N_("E686: Argument of %s must be a List"); -static char *e_listblobarg = N_("E898: Argument of %s must be a List or Blob"); +static char *e_listblobarg = N_("E899: Argument of %s must be a List or Blob"); static char *e_invalwindow = N_("E957: Invalid window number"); /// Dummy va_list for passing to vim_snprintf diff --git a/src/nvim/testdir/test_listdict.vim b/src/nvim/testdir/test_listdict.vim index fa711dde2d..ae035fa519 100644 --- a/src/nvim/testdir/test_listdict.vim +++ b/src/nvim/testdir/test_listdict.vim @@ -617,7 +617,7 @@ func Test_reverse_sort_uniq() call assert_equal(['bar', 'BAR', 'Bar', 'Foo', 'FOO', 'foo', 'FOOBAR', -1, 0, 0, 0.22, 1.0e-15, 12, 18, 22, 255, 7, 9, [], {}], sort(copy(l), 'i')) call assert_equal(['BAR', 'Bar', 'FOO', 'FOOBAR', 'Foo', 'bar', 'foo', -1, 0, 0, 0.22, 1.0e-15, 12, 18, 22, 255, 7, 9, [], {}], sort(copy(l))) - call assert_fails('call reverse("")', 'E898:') + call assert_fails('call reverse("")', 'E899:') endfunc " splitting a string to a List From 23f5999d28cdee049661ed38e22812a063e78fad Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 20:25:46 +0100 Subject: [PATCH 17/37] vim-patch:8.1.0798: changing a blob while iterating over it works strangely Problem: Changing a blob while iterating over it works strangely. Solution: Make a copy of the Blob before iterating. https://github.com/vim/vim/commit/dd29ea18050284526174b5685781469240f5bc4a --- src/nvim/eval.c | 29 ++++++++++------------------- src/nvim/eval/typval.c | 24 ++++++++++++++++++++++++ src/nvim/testdir/test_blob.vim | 14 ++++++++++++++ 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 596ed19ff9..b7dc00dc08 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -2599,15 +2599,16 @@ void *eval_for_line(const char_u *arg, bool *errp, char_u **nextcmdp, int skip) fi->fi_lw.lw_item = tv_list_first(l); } } else if (tv.v_type == VAR_BLOB) { - blob_T *const b = tv.vval.v_blob; - if (b == NULL) { - tv_clear(&tv); - } else { - // No need to increment the refcount, it's already set for - // the blob being used in "tv". - fi->fi_blob = b; - fi->fi_bi = 0; + fi->fi_bi = 0; + if (tv.vval.v_blob != NULL) { + typval_T btv; + + // Make a copy, so that the iteration still works when the + // blob is changed. + tv_blob_copy(&tv, &btv); + fi->fi_blob = btv.vval.v_blob; } + tv_clear(&tv); } else { EMSG(_(e_listreq)); tv_clear(&tv); @@ -9737,17 +9738,7 @@ int var_item_copy(const vimconv_T *const conv, } break; case VAR_BLOB: - to->v_type = VAR_BLOB; - if (from->vval.v_blob == NULL) { - to->vval.v_blob = NULL; - } else { - tv_blob_alloc_ret(to); - const int len = from->vval.v_blob->bv_ga.ga_len; - - to->vval.v_blob->bv_ga.ga_data - = xmemdup(from->vval.v_blob->bv_ga.ga_data, (size_t)len); - to->vval.v_blob->bv_ga.ga_len = len; - } + tv_blob_copy(from, to); break; case VAR_DICT: to->v_type = VAR_DICT; diff --git a/src/nvim/eval/typval.c b/src/nvim/eval/typval.c index e689571788..68fab1dacd 100644 --- a/src/nvim/eval/typval.c +++ b/src/nvim/eval/typval.c @@ -2252,6 +2252,30 @@ void tv_blob_alloc_ret(typval_T *const ret_tv) tv_blob_set_ret(ret_tv, b); } +/// Copy a blob typval to a different typval. +/// +/// @param[in] from Blob object to copy from. +/// @param[out] to Blob object to copy to. +void tv_blob_copy(typval_T *const from, typval_T *const to) + FUNC_ATTR_NONNULL_ALL +{ + assert(from->v_type == VAR_BLOB); + + to->v_type = VAR_BLOB; + if (from->vval.v_blob == NULL) { + to->vval.v_blob = NULL; + } else { + tv_blob_alloc_ret(to); + const int len = from->vval.v_blob->bv_ga.ga_len; + + if (len > 0) { + to->vval.v_blob->bv_ga.ga_data + = xmemdup(from->vval.v_blob->bv_ga.ga_data, (size_t)len); + } + to->vval.v_blob->bv_ga.ga_len = len; + } +} + //{{{3 Clear #define TYPVAL_ENCODE_ALLOW_SPECIALS false diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index 08342e35d2..568821f564 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -154,6 +154,7 @@ func Test_blob_for_loop() call assert_equal(i, byte) let i += 1 endfor + call assert_equal(4, i) let blob = 0z00 call remove(blob, 0) @@ -161,6 +162,19 @@ func Test_blob_for_loop() for byte in blob call assert_error('loop over empty blob') endfor + + let blob = 0z0001020304 + let i = 0 + for byte in blob + call assert_equal(i, byte) + if i == 1 + call remove(blob, 0) + elseif i == 3 + call remove(blob, 3) + endif + let i += 1 + endfor + call assert_equal(5, i) endfunc func Test_blob_concatenate() From bd9c787b4f3ec6d539246c24499bdf80c85248cf Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 22:54:47 +0100 Subject: [PATCH 18/37] vim-patch:8.1.0802: negative index doesn't work for Blob Problem: Negative index doesn't work for Blob. Solution: Make it work, add a test. (closes vim/vim#3856) https://github.com/vim/vim/commit/a5be9b62480a6f338a72c01e57c9edd0bca8048b Leave tv_blob_get()'s return type untouched. --- src/nvim/eval.c | 8 +++++--- src/nvim/eval/typval.c | 2 +- src/nvim/testdir/test_blob.vim | 7 +++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index b7dc00dc08..b16dfad48b 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -4647,9 +4647,11 @@ eval_index( tv_blob_set_ret(rettv, blob); } } else { - // The resulting variable is a string of a single - // character. If the index is too big or negative the - // result is empty. + // The resulting variable is a byte value. + // If the index is too big or negative that is an error. + if (n1 < 0) { + n1 = len + n1; + } if (n1 < len && n1 >= 0) { const int v = (int)tv_blob_get(rettv->vval.v_blob, n1); tv_clear(rettv); diff --git a/src/nvim/eval/typval.c b/src/nvim/eval/typval.c index 68fab1dacd..5b416dfa7d 100644 --- a/src/nvim/eval/typval.c +++ b/src/nvim/eval/typval.c @@ -2266,7 +2266,7 @@ void tv_blob_copy(typval_T *const from, typval_T *const to) to->vval.v_blob = NULL; } else { tv_blob_alloc_ret(to); - const int len = from->vval.v_blob->bv_ga.ga_len; + int len = from->vval.v_blob->bv_ga.ga_len; if (len > 0) { to->vval.v_blob->bv_ga.ga_data diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index 568821f564..1faa9af32c 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -95,6 +95,13 @@ func Test_blob_get() call assert_equal(999, get(b, 5, 999)) call assert_equal(-1, get(b, -8)) call assert_equal(999, get(b, -8, 999)) + + call assert_equal(0x00, b[0]) + call assert_equal(0x22, b[2]) + call assert_equal(0x44, b[4]) + call assert_equal(0x44, b[-1]) + call assert_fails('echo b[5]', 'E979:') + call assert_fails('echo b[-8]', 'E979:') endfunc func Test_blob_to_string() From e7772c051fde36aa856d413baf6bef6ef6cd82d4 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Tue, 24 Nov 2020 21:16:13 +0000 Subject: [PATCH 19/37] vim-patch:8.1.1023: may use NULL pointer when indexing a blob Problem: May use NULL pointer when indexing a blob. (Coverity) Solution: Break out of loop after using index on blob https://github.com/vim/vim/commit/61be376337b0374d55a4b1d8206b2ec87ca54252 --- src/nvim/eval.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index b16dfad48b..9d03a94d71 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -2274,6 +2274,7 @@ char_u *get_lval(char_u *const name, typval_T *const rettv, } lp->ll_blob = lp->ll_tv->vval.v_blob; lp->ll_tv = NULL; + break; } else { // Get the number and item for the only or first index of the List. if (empty1) { From 726b25528863ecd25f62f488b41bc984b842458c Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 21:06:12 +0100 Subject: [PATCH 20/37] vim-patch:8.1.1671: copying a blob may result in it being locked Problem: Copying a blob may result in it being locked. Solution: Reset v_lock. (Ken Takata, closes vim/vim#4648) https://github.com/vim/vim/commit/b7b9efbccfe17ee3f7b1bb877b7745d5bfbf0804 --- src/nvim/eval/typval.c | 1 + src/nvim/testdir/test_blob.vim | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/nvim/eval/typval.c b/src/nvim/eval/typval.c index 5b416dfa7d..d983ce14b5 100644 --- a/src/nvim/eval/typval.c +++ b/src/nvim/eval/typval.c @@ -2262,6 +2262,7 @@ void tv_blob_copy(typval_T *const from, typval_T *const to) assert(from->v_type == VAR_BLOB); to->v_type = VAR_BLOB; + to->v_lock = VAR_UNLOCKED; if (from->vval.v_blob == NULL) { to->vval.v_blob = NULL; } else { diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index 1faa9af32c..575c757837 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -71,6 +71,10 @@ func Test_blob_assign() call assert_fails('let b .= "xx"', 'E734:') call assert_fails('let b += "xx"', 'E734:') call assert_fails('let b[1:1] .= 0z55', 'E734:') + + let l = [0z12] + let m = deepcopy(l) + let m[0] = 0z34 " E742 or E741 should not occur. endfunc func Test_blob_get_range() From ef76238548ba758cb15b5316710b356b59334a41 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Wed, 25 Nov 2020 18:56:30 +0000 Subject: [PATCH 21/37] vim-patch:8.2.0121: filter() and map() on blob don't work Problem: filter() and map() on blob don't work. Solution: Correct the code. (closes vim/vim#5483) https://github.com/vim/vim/commit/49c57ce50019b667e5005ce1cfb8cdc2e48bf868 --- src/nvim/eval.c | 14 +++++++++----- src/nvim/testdir/test_blob.vim | 18 +++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 9d03a94d71..ba9cb11ed5 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -6449,7 +6449,8 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map) for (int i = 0; i < b->bv_ga.ga_len; i++) { typval_T tv; tv.v_type = VAR_NUMBER; - tv.vval.v_number = tv_blob_get(b, i); + const varnumber_T val = tv_blob_get(b, i); + tv.vval.v_number = val; vimvars[VV_KEY].vv_nr = idx; if (filter_map_one(&tv, expr, map, &rem) == FAIL || did_emsg) { break; @@ -6458,14 +6459,17 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map) EMSG(_(e_invalblob)); return; } - tv.v_type = VAR_NUMBER; - tv_blob_set(b, i, tv.vval.v_number); - if (!map && rem) { + if (map) { + if (tv.vval.v_number != val) { + tv_blob_set(b, i, tv.vval.v_number); + } + } else if (rem) { char_u *const p = (char_u *)argvars[0].vval.v_blob->bv_ga.ga_data; - memmove(p + idx, p + i + 1, (size_t)b->bv_ga.ga_len - i - 1); + memmove(p + i, p + i + 1, (size_t)b->bv_ga.ga_len - i - 1); b->bv_ga.ga_len--; i--; } + idx++; } } else { assert(argvars[0].v_type == VAR_LIST); diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index 575c757837..a74dd538e5 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -260,18 +260,22 @@ endfunc " filter() item in blob func Test_blob_filter() - let b = 0zDEADBEEF - call filter(b, 'v:val != 0xEF') - call assert_equal(0zDEADBE, b) + call assert_equal(0z, filter(0zDEADBEEF, '0')) + call assert_equal(0zADBEEF, filter(0zDEADBEEF, 'v:val != 0xDE')) + call assert_equal(0zDEADEF, filter(0zDEADBEEF, 'v:val != 0xBE')) + call assert_equal(0zDEADBE, filter(0zDEADBEEF, 'v:val != 0xEF')) + call assert_equal(0zDEADBEEF, filter(0zDEADBEEF, '1')) + call assert_equal(0z01030103, filter(0z010203010203, 'v:val != 0x02')) + call assert_equal(0zADEF, filter(0zDEADBEEF, 'v:key % 2')) endfunc " map() item in blob func Test_blob_map() - let b = 0zDEADBEEF - call map(b, 'v:val + 1') - call assert_equal(0zDFAEBFF0, b) + call assert_equal(0zDFAEBFF0, map(0zDEADBEEF, 'v:val + 1')) + call assert_equal(0z00010203, map(0zDEADBEEF, 'v:key')) + call assert_equal(0zDEAEC0F2, map(0zDEADBEEF, 'v:key + v:val')) - call assert_fails("call map(b, '[9]')", 'E978:') + call assert_fails("call map(0z00, '[9]')", 'E978:') endfunc func Test_blob_index() From 53f28f024c90399106aac5c006e4fd0dd605b564 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 21:30:01 +0100 Subject: [PATCH 22/37] vim-patch:8.2.0184: blob test fails Problem: Blob test fails. Solution: Check for different error when float feature is missing. https://github.com/vim/vim/commit/92be6e3f46120bb8e6c8fca0a7868a08df8b3345 Nvim always has the float feature, but include the changes to the test anyway. --- src/nvim/testdir/test_blob.vim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index a74dd538e5..19f1f6bd42 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -319,7 +319,11 @@ func Test_blob_lock() endfunc func Test_blob_sort() - call assert_fails('call sort([1.0, 0z11], "f")', 'E975:') + if has('float') + call assert_fails('call sort([1.0, 0z11], "f")', 'E975:') + else + call assert_fails('call sort(["abc", 0z11], "f")', 'E702:') + endif endfunc " vim: shiftwidth=2 sts=2 expandtab From ba34afb37849a8bdf5d601ced7f3c139a80cd396 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Mon, 2 Aug 2021 15:27:06 +0100 Subject: [PATCH 23/37] vim-patch:8.2.0404: writefile() error does not give a hint Problem: Writefile() error does not give a hint. Solution: Add remark about first argument. https://github.com/vim/vim/commit/18a2b87ca27c378a555b20f14a284d2ce3511427 --- src/nvim/eval/funcs.c | 3 ++- src/nvim/testdir/test_writefile.vim | 2 ++ test/functional/eval/writefile_spec.lua | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 2d58ce1d04..6dd7a92b61 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -11857,7 +11857,8 @@ static void f_writefile(typval_T *argvars, typval_T *rettv, FunPtr fptr) } }); } else if (argvars[0].v_type != VAR_BLOB) { - EMSG2(_(e_invarg2), "writefile()"); + EMSG2(_(e_invarg2), + _("writefile() first argument must be a List or a Blob")); return; } diff --git a/src/nvim/testdir/test_writefile.vim b/src/nvim/testdir/test_writefile.vim index 6922e2185d..2504fcb14e 100644 --- a/src/nvim/testdir/test_writefile.vim +++ b/src/nvim/testdir/test_writefile.vim @@ -17,6 +17,8 @@ func Test_writefile() call assert_equal("morning", l[3]) call assert_equal("vimmers", l[4]) call delete(f) + + call assert_fails('call writefile("text", "Xfile")', 'E475: Invalid argument: writefile() first argument must be a List or a Blob') endfunc func Test_writefile_ignore_regexp_error() diff --git a/test/functional/eval/writefile_spec.lua b/test/functional/eval/writefile_spec.lua index b2a601a767..14be8c377c 100644 --- a/test/functional/eval/writefile_spec.lua +++ b/test/functional/eval/writefile_spec.lua @@ -119,7 +119,7 @@ describe('writefile()', function() eq('\nE118: Too many arguments for function: writefile', redir_exec(('call writefile([], "%s", "b", 1)'):format(fname))) for _, arg in ipairs({'0', '0.0', 'function("tr")', '{}', '"test"'}) do - eq('\nE475: Invalid argument: writefile()', + eq('\nE475: Invalid argument: writefile() first argument must be a List or a Blob', redir_exec(('call writefile(%s, "%s", "b")'):format(arg, fname))) end for _, args in ipairs({'[], %s, "b"', '[], "' .. fname .. '", %s'}) do From 9b5c9dbfa7074789dd77201ff7893690073e0f0a Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Wed, 25 Nov 2020 16:02:10 +0000 Subject: [PATCH 24/37] vim-patch:8.2.0521: crash when reading a blob fails Problem: Crash when reading a blob fails. Solution: Avoid keeping a pointer to a freed blob object. (Dominique Pelle, closes vim/vim#5890) Adjust error messages. https://github.com/vim/vim/commit/15352dc6ec43fd50cc3be4f4fd1ad74d5619da20 --- src/nvim/eval/funcs.c | 9 ++++++++- src/nvim/testdir/test_blob.vim | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 6dd7a92b61..2b844af03f 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -7015,6 +7015,11 @@ static void f_readfile(typval_T *argvars, typval_T *rettv, FunPtr fptr) // Always open the file in binary mode, library functions have a mind of // their own about CR-LF conversion. const char *const fname = tv_get_string(&argvars[0]); + + if (os_isdir((const char_u *)fname)) { + EMSG2(_(e_isadir2), fname); + return; + } if (*fname == NUL || (fd = os_fopen(fname, READBIN)) == NULL) { EMSG2(_(e_notopen), *fname == NUL ? _("") : fname); return; @@ -7023,8 +7028,10 @@ static void f_readfile(typval_T *argvars, typval_T *rettv, FunPtr fptr) if (blob) { tv_blob_alloc_ret(rettv); if (!read_blob(fd, rettv->vval.v_blob)) { - EMSG("cannot read file"); + EMSG2(_(e_notread), fname); + // An empty blob is returned on error. tv_blob_free(rettv->vval.v_blob); + rettv->vval.v_blob = NULL; } fclose(fd); return; diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index 19f1f6bd42..a102a4d6bd 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -256,6 +256,9 @@ func Test_blob_read_write() let br = readfile('Xblob', 'B') call assert_equal(b, br) call delete('Xblob') + + " This was crashing when calling readfile() with a directory. + call assert_fails("call readfile('.', 'B')", 'E17: "." is a directory') endfunc " filter() item in blob From 19232593ba495aa7ac6347b45a9a78d218e6e871 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 22:32:22 +0100 Subject: [PATCH 25/37] fix(f_insert): partially port v8.2.0634 Fixes a crash in f_insert() when inserting into a NULL blob. Include blob-related test changes and some other simple changes. --- src/nvim/eval/funcs.c | 4 ++++ src/nvim/testdir/test_blob.vim | 5 +++++ src/nvim/testdir/test_fnamemodify.vim | 4 ++++ src/nvim/testdir/test_quickfix.vim | 4 ++++ src/nvim/testdir/test_vimscript.vim | 4 ++++ 5 files changed, 21 insertions(+) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 2b844af03f..b3afa332f8 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -4984,6 +4984,10 @@ static void f_insert(typval_T *argvars, typval_T *rettv, FunPtr fptr) bool error = false; if (argvars[0].v_type == VAR_BLOB) { + if (argvars[0].vval.v_blob == NULL) { + return; + } + long before = 0; const int len = tv_blob_len(argvars[0].vval.v_blob); diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index a102a4d6bd..b21fd936ad 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -33,6 +33,7 @@ func Test_blob_create() call assert_fails('let b = 0z.') call assert_fails('let b = 0z001122.') call assert_fails('call get("", 1)', 'E896:') + call assert_equal(0, len(v:_null_blob)) endfunc " assignment to a blob @@ -99,6 +100,7 @@ func Test_blob_get() call assert_equal(999, get(b, 5, 999)) call assert_equal(-1, get(b, -8)) call assert_equal(999, get(b, -8, 999)) + call assert_equal(10, get(v:_null_blob, 2, 10)) call assert_equal(0x00, b[0]) call assert_equal(0x22, b[2]) @@ -248,6 +250,7 @@ func Test_blob_func_remove() call assert_fails("call remove(b, 3, 2)", 'E979:') call assert_fails("call remove(1, 0)", 'E896:') call assert_fails("call remove(b, b)", 'E974:') + call assert_fails("call remove(v:_null_blob, 1, 2)", 'E979:') endfunc func Test_blob_read_write() @@ -304,6 +307,7 @@ func Test_blob_insert() call assert_fails('call insert(b, -1)', 'E475:') call assert_fails('call insert(b, 257)', 'E475:') call assert_fails('call insert(b, 0, [9])', 'E745:') + call assert_equal(0, insert(v:_null_blob, 0x33)) endfunc func Test_blob_reverse() @@ -311,6 +315,7 @@ func Test_blob_reverse() call assert_equal(0zBEADDE, reverse(0zDEADBE)) call assert_equal(0zADDE, reverse(0zDEAD)) call assert_equal(0zDE, reverse(0zDE)) + call assert_equal(0z, reverse(v:_null_blob)) endfunc func Test_blob_lock() diff --git a/src/nvim/testdir/test_fnamemodify.vim b/src/nvim/testdir/test_fnamemodify.vim index 116d23ba88..fe1df8fd4a 100644 --- a/src/nvim/testdir/test_fnamemodify.vim +++ b/src/nvim/testdir/test_fnamemodify.vim @@ -72,4 +72,8 @@ func Test_fnamemodify_er() " :e never includes the whole filename, so "a.b":e:e:e --> "b" call assert_equal('b.c', fnamemodify('a.b.c.d.e', ':r:r:e:e:e')) call assert_equal('b.c', fnamemodify('a.b.c.d.e', ':r:r:e:e:e:e')) + + call assert_equal('', fnamemodify(v:_null_string, v:_null_string)) endfunc + +" vim: shiftwidth=2 sts=2 expandtab diff --git a/src/nvim/testdir/test_quickfix.vim b/src/nvim/testdir/test_quickfix.vim index 283e7bbafe..18587b9b2c 100644 --- a/src/nvim/testdir/test_quickfix.vim +++ b/src/nvim/testdir/test_quickfix.vim @@ -2221,6 +2221,10 @@ func Xproperty_tests(cchar) call g:Xsetlist([], 'a', {'context':246}) let d = g:Xgetlist({'context':1}) call assert_equal(246, d.context) + " set other Vim data types as context + call g:Xsetlist([], 'a', {'context' : v:_null_blob}) + call g:Xsetlist([], 'a', {'context' : ''}) + call test_garbagecollect_now() if a:cchar == 'l' " Test for copying context across two different location lists new | only diff --git a/src/nvim/testdir/test_vimscript.vim b/src/nvim/testdir/test_vimscript.vim index d5837e88c9..b18ce563d3 100644 --- a/src/nvim/testdir/test_vimscript.vim +++ b/src/nvim/testdir/test_vimscript.vim @@ -1152,6 +1152,10 @@ func Test_type() call assert_equal(v:t_float, type(0.0)) call assert_equal(v:t_bool, type(v:false)) call assert_equal(v:t_bool, type(v:true)) + call assert_equal(v:t_string, type(v:_null_string)) + call assert_equal(v:t_list, type(v:_null_list)) + call assert_equal(v:t_dict, type(v:_null_dict)) + call assert_equal(v:t_blob, type(v:_null_blob)) endfunc "------------------------------------------------------------------------------- From ffaf881b42fd22b8bf423c5754d48ce93f59bf4b Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 21:40:30 +0100 Subject: [PATCH 26/37] vim-patch:8.2.0829: filter() may give misleading error message Problem: filter() may give misleading error message. Solution: Also mention Blob as an allowed argument. https://github.com/vim/vim/commit/fcb0b61d15f66f0e9116a6bc56d9d8105bb913cf Rename Test_map_fails() to Test_map_filter_fails() from v8.2.0610 and include the modeline. --- src/nvim/eval.c | 2 +- src/nvim/testdir/test_filter_map.vim | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index ba9cb11ed5..26fc02b7c9 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -6393,7 +6393,7 @@ void filter_map(typval_T *argvars, typval_T *rettv, int map) return; } } else { - EMSG2(_(e_listdictarg), ermsg); + EMSG2(_(e_listdictblobarg), ermsg); return; } diff --git a/src/nvim/testdir/test_filter_map.vim b/src/nvim/testdir/test_filter_map.vim index a15567bcf2..a52a66ac2f 100644 --- a/src/nvim/testdir/test_filter_map.vim +++ b/src/nvim/testdir/test_filter_map.vim @@ -81,7 +81,11 @@ func Test_filter_map_dict_expr_funcref() call assert_equal({"foo": "f", "bar": "b", "baz": "b"}, map(copy(dict), function('s:filter4'))) endfunc -func Test_map_fails() +func Test_map_filter_fails() call assert_fails('call map([1], "42 +")', 'E15:') call assert_fails('call filter([1], "42 +")', 'E15:') + call assert_fails("let l = map('abc', '\"> \" . v:val')", 'E896:') + call assert_fails("let l = filter('abc', '\"> \" . v:val')", 'E896:') endfunc + +" vim: shiftwidth=2 sts=2 expandtab From 9e38c4a79fe0351e9e18bc57cf48b960e8d31e88 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Sat, 31 Jul 2021 00:20:20 +0100 Subject: [PATCH 27/37] vim-patch:8.2.1473: items in a list given to :const can still be modified Problem: Items in a list given to :const can still be modified. Solution: Work like ":lockvar! name" but don't lock referenced items. Make locking a blob work. https://github.com/vim/vim/commit/021bda56710d98c09a6b35610a476ab2dd8c58ad --- runtime/doc/eval.txt | 14 +++++++++++--- src/nvim/eval.c | 16 +++++++++++----- src/nvim/eval/typval.c | 15 +++++++++------ src/nvim/testdir/test_const.vim | 33 ++++++++++++++++++++++++--------- test/unit/eval/typval_spec.lua | 16 ++++++++-------- 5 files changed, 63 insertions(+), 31 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 7532bcfba7..7f22bab7f2 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -10871,10 +10871,18 @@ text... :const x = 1 < is equivalent to: > :let x = 1 - :lockvar 1 x + :lockvar! x < This is useful if you want to make sure the variable - is not modified. - *E995* + is not modified. If the value is a List or Dictionary + literal then the items also cannot be changed: > + const ll = [1, 2, 3] + let ll[1] = 5 " Error! +< Nested references are not locked: > + let lvar = ['a'] + const lconst = [0, lvar] + let lconst[0] = 2 " Error! + let lconst[1][0] = 'b' " OK +< *E995* |:const| does not allow to for changing a variable. > :let x = 1 :const x = 2 " Error! diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 26fc02b7c9..903673123c 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -2373,6 +2373,9 @@ static void set_var_lval(lval_T *lp, char_u *endp, typval_T *rettv, EMSG2(_(e_letwrong), op); return; } + if (var_check_lock(lp->ll_blob->bv_lock, lp->ll_name, TV_CSTRING)) { + return; + } if (lp->ll_range && rettv->v_type == VAR_BLOB) { if (lp->ll_empty2) { @@ -3082,7 +3085,7 @@ static int do_lock_var(lval_T *lp, char_u *name_end FUNC_ATTR_UNUSED, } else { di->di_flags &= ~DI_FLAGS_LOCK; } - tv_item_lock(&di->di_tv, deep, lock); + tv_item_lock(&di->di_tv, deep, lock, false); } } } else if (lp->ll_range) { @@ -3090,16 +3093,16 @@ static int do_lock_var(lval_T *lp, char_u *name_end FUNC_ATTR_UNUSED, // (un)lock a range of List items. while (li != NULL && (lp->ll_empty2 || lp->ll_n2 >= lp->ll_n1)) { - tv_item_lock(TV_LIST_ITEM_TV(li), deep, lock); + tv_item_lock(TV_LIST_ITEM_TV(li), deep, lock, false); li = TV_LIST_ITEM_NEXT(lp->ll_list, li); lp->ll_n1++; } } else if (lp->ll_list != NULL) { // (un)lock a List item. - tv_item_lock(TV_LIST_ITEM_TV(lp->ll_li), deep, lock); + tv_item_lock(TV_LIST_ITEM_TV(lp->ll_li), deep, lock, false); } else { // (un)lock a Dictionary item. - tv_item_lock(&lp->ll_di->di_tv, deep, lock); + tv_item_lock(&lp->ll_di->di_tv, deep, lock, false); } return ret; @@ -9536,7 +9539,10 @@ static void set_var_const(const char *name, const size_t name_len, } if (is_const) { - tv_item_lock(&v->di_tv, 1, true); + // Like :lockvar! name: lock the value and what it contains, but only + // if the reference count is up to one. That locks only literal + // values. + tv_item_lock(&v->di_tv, DICT_MAXNEST, true, true); } } diff --git a/src/nvim/eval/typval.c b/src/nvim/eval/typval.c index d983ce14b5..b08fe274f0 100644 --- a/src/nvim/eval/typval.c +++ b/src/nvim/eval/typval.c @@ -2659,7 +2659,10 @@ void tv_copy(const typval_T *const from, typval_T *const to) /// @param[out] tv Item to (un)lock. /// @param[in] deep Levels to (un)lock, -1 to (un)lock everything. /// @param[in] lock True if it is needed to lock an item, false to unlock. -void tv_item_lock(typval_T *const tv, const int deep, const bool lock) +/// @param[in] check_refcount If true, do not lock a list or dict with a +/// reference count larger than 1. +void tv_item_lock(typval_T *const tv, const int deep, const bool lock, + const bool check_refcount) FUNC_ATTR_NONNULL_ALL { // TODO(ZyX-I): Make this not recursive @@ -2688,19 +2691,19 @@ void tv_item_lock(typval_T *const tv, const int deep, const bool lock) switch (tv->v_type) { case VAR_BLOB: { blob_T *const b = tv->vval.v_blob; - if (b != NULL) { + if (b != NULL && !(check_refcount && b->bv_refcount > 1)) { CHANGE_LOCK(lock, b->bv_lock); } break; } case VAR_LIST: { list_T *const l = tv->vval.v_list; - if (l != NULL) { + if (l != NULL && !(check_refcount && l->lv_refcount > 1)) { CHANGE_LOCK(lock, l->lv_lock); if (deep < 0 || deep > 1) { // Recursive: lock/unlock the items the List contains. TV_LIST_ITER(l, li, { - tv_item_lock(TV_LIST_ITEM_TV(li), deep - 1, lock); + tv_item_lock(TV_LIST_ITEM_TV(li), deep - 1, lock, check_refcount); }); } } @@ -2708,12 +2711,12 @@ void tv_item_lock(typval_T *const tv, const int deep, const bool lock) } case VAR_DICT: { dict_T *const d = tv->vval.v_dict; - if (d != NULL) { + if (d != NULL && !(check_refcount && d->dv_refcount > 1)) { CHANGE_LOCK(lock, d->dv_lock); if (deep < 0 || deep > 1) { // recursive: lock/unlock the items the List contains TV_DICT_ITER(d, di, { - tv_item_lock(&di->di_tv, deep - 1, lock); + tv_item_lock(&di->di_tv, deep - 1, lock, check_refcount); }); } } diff --git a/src/nvim/testdir/test_const.vim b/src/nvim/testdir/test_const.vim index ea69c8cba4..0d064617a5 100644 --- a/src/nvim/testdir/test_const.vim +++ b/src/nvim/testdir/test_const.vim @@ -244,18 +244,33 @@ func Test_const_with_eval_name() call assert_fails('const {s2} = "bar"', 'E995:') endfunc -func Test_lock_depth_is_1() - const l = [1, 2, 3] - const d = {'foo': 10} - - " Modify list - setting item is OK, adding/removing items not - let l[0] = 42 +func Test_lock_depth_is_2() + " Modify list - error when changing item or adding/removing items + const l = [1, 2, [3, 4]] + call assert_fails('let l[0] = 42', 'E741:') + call assert_fails('let l[2][0] = 42', 'E741:') call assert_fails('call add(l, 4)', 'E741:') call assert_fails('unlet l[1]', 'E741:') - " Modify dict - changing item is OK, adding/removing items not - let d['foo'] = 'hello' - let d.foo = 44 + " Modify blob - error when changing + const b = 0z001122 + call assert_fails('let b[0] = 42', 'E741:') + + " Modify dict - error when changing item or adding/removing items + const d = {'foo': 10} + call assert_fails("let d['foo'] = 'hello'", 'E741:') + call assert_fails("let d.foo = 'hello'", 'E741:') call assert_fails("let d['bar'] = 'hello'", 'E741:') call assert_fails("unlet d['foo']", 'E741:') + + " Modifying list or dict item contents is OK. + let lvar = ['a', 'b'] + let bvar = 0z1122 + const l2 = [0, lvar, bvar] + let l2[1][0] = 'c' + let l2[2][1] = 0x33 + call assert_equal([0, ['c', 'b'], 0z1133], l2) + + const d2 = #{a: 0, b: lvar, c: 4} + let d2.b[1] = 'd' endfunc diff --git a/test/unit/eval/typval_spec.lua b/test/unit/eval/typval_spec.lua index d81e272877..e61b568f3a 100644 --- a/test/unit/eval/typval_spec.lua +++ b/test/unit/eval/typval_spec.lua @@ -2531,7 +2531,7 @@ describe('typval.c', function() value='tr', dict={}, }) - lib.tv_item_lock(p_tv, -1, true) + lib.tv_item_lock(p_tv, -1, true, false) eq(lib.VAR_UNLOCKED, p_tv.vval.v_partial.pt_dict.dv_lock) end) itp('does not change VAR_FIXED values', function() @@ -2542,14 +2542,14 @@ describe('typval.c', function() d_tv.vval.v_dict.dv_lock = lib.VAR_FIXED l_tv.v_lock = lib.VAR_FIXED l_tv.vval.v_list.lv_lock = lib.VAR_FIXED - lib.tv_item_lock(d_tv, 1, true) - lib.tv_item_lock(l_tv, 1, true) + lib.tv_item_lock(d_tv, 1, true, false) + lib.tv_item_lock(l_tv, 1, true, false) eq(lib.VAR_FIXED, d_tv.v_lock) eq(lib.VAR_FIXED, l_tv.v_lock) eq(lib.VAR_FIXED, d_tv.vval.v_dict.dv_lock) eq(lib.VAR_FIXED, l_tv.vval.v_list.lv_lock) - lib.tv_item_lock(d_tv, 1, false) - lib.tv_item_lock(l_tv, 1, false) + lib.tv_item_lock(d_tv, 1, false, false) + lib.tv_item_lock(l_tv, 1, false, false) eq(lib.VAR_FIXED, d_tv.v_lock) eq(lib.VAR_FIXED, l_tv.v_lock) eq(lib.VAR_FIXED, d_tv.vval.v_dict.dv_lock) @@ -2561,9 +2561,9 @@ describe('typval.c', function() local d_tv = lua2typvalt(null_dict) local s_tv = lua2typvalt(null_string) alloc_log:clear() - lib.tv_item_lock(l_tv, 1, true) - lib.tv_item_lock(d_tv, 1, true) - lib.tv_item_lock(s_tv, 1, true) + lib.tv_item_lock(l_tv, 1, true, false) + lib.tv_item_lock(d_tv, 1, true, false) + lib.tv_item_lock(s_tv, 1, true, false) eq(null_list, typvalt2lua(l_tv)) eq(null_dict, typvalt2lua(d_tv)) eq(null_string, typvalt2lua(s_tv)) From ecb54238e00f66f46070a0a2d859531258377272 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 22:44:51 +0100 Subject: [PATCH 28/37] vim-patch:8.2.2712: memory leak when adding to a blob fails Problem: Memory leak when adding to a blob fails. Solution: Clear the second typval before returning. https://github.com/vim/vim/commit/f2dd9cb9958962302d33fab8fe6439645a2d1e1b --- src/nvim/eval.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 903673123c..8777ae1281 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -3799,10 +3799,12 @@ static int eval5(char_u **arg, typval_T *rettv, int evaluate) } else { n1 = tv_get_number_chk(rettv, &error); if (error) { - /* This can only happen for "list + non-list". For - * "non-list + ..." or "something - ...", we returned - * before evaluating the 2nd operand. */ + // This can only happen for "list + non-list" or + // "blob + non-blob". For "non-list + ..." or + // "something - ...", we returned before evaluating the + // 2nd operand. tv_clear(rettv); + tv_clear(&var2); return FAIL; } if (var2.v_type == VAR_FLOAT) From 3d6bb8b3fbe21d8a7a4ba975682eb9a4e5170859 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 22:47:33 +0100 Subject: [PATCH 29/37] fix(f_remove): partially port v8.2.2779 Fixes remove() copying one extra byte after the end of a Blob's buffer. Can't be fully ported as the change is from blob_remove(), which hasn't been ported yet. --- src/nvim/eval/funcs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index b3afa332f8..660c069e7f 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -7334,7 +7334,7 @@ static void f_remove(typval_T *argvars, typval_T *rettv, FunPtr fptr) memmove(p + idx, p + idx + 1, (size_t)len - idx - 1); b->bv_ga.ga_len--; } else { - // Remove range of items, return list with values. + // Remove range of items, return blob with values. end = (long)tv_get_number_chk(&argvars[2], &error); if (error) { return; @@ -7356,7 +7356,9 @@ static void f_remove(typval_T *argvars, typval_T *rettv, FunPtr fptr) (size_t)(end - idx + 1)); tv_blob_set_ret(rettv, blob); - memmove(p + idx, p + end + 1, (size_t)(len - end)); + if (len - end - 1 > 0) { + memmove(p + idx, p + end + 1, (size_t)(len - end - 1)); + } b->bv_ga.ga_len -= end - idx + 1; } } From e88961943b85434ceebff6a31089c635ac39cd66 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Sun, 1 Aug 2021 18:27:41 +0100 Subject: [PATCH 30/37] fix(eval): partially port v8.2.3284 These were issues that I found while porting that I fixed upstream. :^) Very little of the patch can be exactly ported as we're a bit behind on dependant patches (we also can't use the exact :for emsg, as we don't support iterating over Strings yet), so just translate the fixes as best as we can for now. Include latest relevant doc changes from: - v8.1.0815 - v8.2.2658 --- runtime/doc/eval.txt | 50 +++++++++++++++++++--------- src/nvim/eval.c | 2 +- src/nvim/eval/funcs.c | 21 ++++++++---- src/nvim/testdir/test_blob.vim | 12 +++++++ src/nvim/testdir/test_eval_stuff.vim | 8 +++-- 5 files changed, 67 insertions(+), 26 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 7f22bab7f2..a628b9d030 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -372,8 +372,8 @@ Changing the order of items in a list: > For loop ~ -The |:for| loop executes commands for each item in a list. A variable is set -to each item in the list in sequence. Example: > +The |:for| loop executes commands for each item in a |List| or |Blob|. +A variable is set to each item in the sequence. Example with a List: > :for item in mylist : call Doit(item) :endfor @@ -406,6 +406,8 @@ It is also possible to put remaining items in a List variable: > : endif :endfor +For a Blob one byte at a time is used. + List functions ~ *E714* @@ -637,6 +639,16 @@ is not available it returns -1 or the default value you specify: > :echo get(myblob, idx, 999) +Blob iteration ~ + +The |:for| loop executes commands for each byte of a Blob. The loop variable is +set to each byte in the Blob. Example: > + :for byte in 0z112233 + : call Doit(byte) + :endfor +This calls Doit() with 0x11, 0x22 and 0x33. + + Blob concatenation ~ Two blobs can be concatenated with the "+" operator: > @@ -10999,28 +11011,34 @@ text... NOTE: The ":append" and ":insert" commands don't work properly inside a ":while" and ":for" loop. -:for {var} in {list} *:for* *E690* *E732* +:for {var} in {object} *:for* *E690* *E732* :endfo[r] *:endfo* *:endfor* Repeat the commands between ":for" and ":endfor" for - each item in {list}. Variable {var} is set to the - value of each item. - When an error is detected for a command inside the - loop, execution continues after the "endfor". - Changing {list} inside the loop affects what items are - used. Make a copy if this is unwanted: > + each item in {object}. {object} can be a |List| or + a |Blob|. Variable {var} is set to the value of each + item. When an error is detected for a command inside + the loop, execution continues after the "endfor". + Changing {object} inside the loop affects what items + are used. Make a copy if this is unwanted: > :for item in copy(mylist) -< When not making a copy, Vim stores a reference to the - next item in the list, before executing the commands - with the current item. Thus the current item can be - removed without effect. Removing any later item means - it will not be found. Thus the following example - works (an inefficient way to make a list empty): > +< + When {object} is a |List| and not making a copy, Vim + stores a reference to the next item in the |List| + before executing the commands with the current item. + Thus the current item can be removed without effect. + Removing any later item means it will not be found. + Thus the following example works (an inefficient way + to make a |List| empty): > for item in mylist call remove(mylist, 0) endfor -< Note that reordering the list (e.g., with sort() or +< Note that reordering the |List| (e.g., with sort() or reverse()) may have unexpected effects. + When {object} is a |Blob|, Vim always makes a copy to + iterate over. Unlike with |List|, modifying the + |Blob| does not affect the iteration. + :for [{var1}, {var2}, ...] in {listlist} :endfo[r] Like ":for" above, but each item in {listlist} must be diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 8777ae1281..5607de3cc1 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -2614,7 +2614,7 @@ void *eval_for_line(const char_u *arg, bool *errp, char_u **nextcmdp, int skip) } tv_clear(&tv); } else { - EMSG(_(e_listreq)); + EMSG(_(e_listblobreq)); tv_clear(&tv); } } diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index 660c069e7f..ec13aa7fbd 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -4984,12 +4984,16 @@ static void f_insert(typval_T *argvars, typval_T *rettv, FunPtr fptr) bool error = false; if (argvars[0].v_type == VAR_BLOB) { - if (argvars[0].vval.v_blob == NULL) { + blob_T *const b = argvars[0].vval.v_blob; + + if (b == NULL + || var_check_lock(b->bv_lock, N_("insert() argument"), + TV_TRANSLATE)) { return; } long before = 0; - const int len = tv_blob_len(argvars[0].vval.v_blob); + const int len = tv_blob_len(b); if (argvars[2].v_type != VAR_UNKNOWN) { before = (long)tv_get_number_chk(&argvars[2], &error); @@ -5010,11 +5014,11 @@ static void f_insert(typval_T *argvars, typval_T *rettv, FunPtr fptr) return; } - ga_grow(&argvars[0].vval.v_blob->bv_ga, 1); - char_u *const p = (char_u *)argvars[0].vval.v_blob->bv_ga.ga_data; + ga_grow(&b->bv_ga, 1); + char_u *const p = (char_u *)b->bv_ga.ga_data; memmove(p + before + 1, p + before, (size_t)len - before); *(p + before) = val; - argvars[0].vval.v_blob->bv_ga.ga_len++; + b->bv_ga.ga_len++; tv_copy(&argvars[0], rettv); } else if (argvars[0].v_type != VAR_LIST) { @@ -7312,11 +7316,16 @@ static void f_remove(typval_T *argvars, typval_T *rettv, FunPtr fptr) } } } else if (argvars[0].v_type == VAR_BLOB) { + blob_T *const b = argvars[0].vval.v_blob; + + if (b != NULL && var_check_lock(b->bv_lock, arg_errmsg, TV_TRANSLATE)) { + return; + } + bool error = false; idx = (long)tv_get_number_chk(&argvars[1], &error); if (!error) { - blob_T *const b = argvars[0].vval.v_blob; const int len = tv_blob_len(b); if (idx < 0) { diff --git a/src/nvim/testdir/test_blob.vim b/src/nvim/testdir/test_blob.vim index b21fd936ad..20758b0c0a 100644 --- a/src/nvim/testdir/test_blob.vim +++ b/src/nvim/testdir/test_blob.vim @@ -251,6 +251,12 @@ func Test_blob_func_remove() call assert_fails("call remove(1, 0)", 'E896:') call assert_fails("call remove(b, b)", 'E974:') call assert_fails("call remove(v:_null_blob, 1, 2)", 'E979:') + + " Translated from v8.2.3284 + let b = 0zDEADBEEF + lockvar b + call assert_fails('call remove(b, 0)', 'E741:') + unlockvar b endfunc func Test_blob_read_write() @@ -308,6 +314,12 @@ func Test_blob_insert() call assert_fails('call insert(b, 257)', 'E475:') call assert_fails('call insert(b, 0, [9])', 'E745:') call assert_equal(0, insert(v:_null_blob, 0x33)) + + " Translated from v8.2.3284 + let b = 0zDEADBEEF + lockvar b + call assert_fails('call insert(b, 3)', 'E741:') + unlockvar b endfunc func Test_blob_reverse() diff --git a/src/nvim/testdir/test_eval_stuff.vim b/src/nvim/testdir/test_eval_stuff.vim index 084c856ba0..98e098387e 100644 --- a/src/nvim/testdir/test_eval_stuff.vim +++ b/src/nvim/testdir/test_eval_stuff.vim @@ -23,9 +23,11 @@ func Test_E963() endfunc func Test_for_invalid() - call assert_fails("for x in 99", 'E714:') - call assert_fails("for x in function('winnr')", 'E714:') - call assert_fails("for x in {'a': 9}", 'E714:') + " Vim gives incorrect emsg here until v8.2.3284, but the exact emsg from that + " patch cannot be used until v8.2.2658 is ported (for loop over Strings) + call assert_fails("for x in 99", 'E897:') + call assert_fails("for x in function('winnr')", 'E897:') + call assert_fails("for x in {'a': 9}", 'E897:') if 0 /1/5/2/s/\n From 7e9ea083213c3eb195fbf206f12a3aac1fa29033 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 12:26:13 +0100 Subject: [PATCH 31/37] feat(f_chansend): support Blob data argument --- runtime/doc/eval.txt | 4 ++-- src/nvim/channel.c | 1 + src/nvim/eval/funcs.c | 12 +++++++++++- test/functional/core/channels_spec.lua | 5 +++++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index a628b9d030..94be7b595e 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -3217,8 +3217,8 @@ chansend({id}, {data}) *chansend()* written if the write succeeded, 0 otherwise. See |channel-bytes| for more information. - {data} may be a string, string convertible, or a list. If - {data} is a list, the items will be joined by newlines; any + {data} may be a string, string convertible, |Blob|, or a list. + If {data} is a list, the items will be joined by newlines; any newlines in an item will be sent as NUL. To send a final newline, include a final empty string. Example: > :call chansend(id, ["abc", "123\n456", ""]) diff --git a/src/nvim/channel.c b/src/nvim/channel.c index 94db7fb3b9..54a59f6cc1 100644 --- a/src/nvim/channel.c +++ b/src/nvim/channel.c @@ -516,6 +516,7 @@ uint64_t channel_from_stdio(bool rpc, CallbackReader on_output, /// @param data will be consumed size_t channel_send(uint64_t id, char *data, size_t len, bool data_owned, const char **error) + FUNC_ATTR_NONNULL_ALL { Channel *chan = find_channel(id); size_t written = 0; diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index ec13aa7fbd..e83e0ac0cc 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -972,7 +972,17 @@ static void f_chansend(typval_T *argvars, typval_T *rettv, FunPtr fptr) } ptrdiff_t input_len = 0; - char *input = save_tv_as_string(&argvars[1], &input_len, false); + char *input = NULL; + if (argvars[1].v_type == VAR_BLOB) { + const blob_T *const b = argvars[1].vval.v_blob; + input_len = tv_blob_len(b); + if (input_len > 0) { + input = xmemdup(b->bv_ga.ga_data, input_len); + } + } else { + input = save_tv_as_string(&argvars[1], &input_len, false); + } + if (!input) { // Either the error has been handled by save_tv_as_string(), // or there is no input to send. diff --git a/test/functional/core/channels_spec.lua b/test/functional/core/channels_spec.lua index 1ef34c7318..6efa4f9b80 100644 --- a/test/functional/core/channels_spec.lua +++ b/test/functional/core/channels_spec.lua @@ -89,6 +89,9 @@ describe('channels', function() command("call chansend(id, 'howdy')") eq({"notification", "stdout", {id, {"[1, ['howdy'], 'stdin']"}}}, next_msg()) + command("call chansend(id, 0z686f6c61)") + eq({"notification", "stdout", {id, {"[1, ['hola'], 'stdin']"}}}, next_msg()) + command("call chanclose(id, 'stdin')") expect_twostreams({{"notification", "stdout", {id, {"[1, [''], 'stdin']"}}}, {'notification', 'stdout', {id, {''}}}}, @@ -131,6 +134,8 @@ describe('channels', function() command("call chansend(id, 'TEXT\n')") expect_twoline(id, "stdout", "TEXT\r", "[1, ['TEXT', ''], 'stdin']") + command("call chansend(id, 0z426c6f6273210a)") + expect_twoline(id, "stdout", "Blobs!\r", "[1, ['Blobs!', ''], 'stdin']") command("call chansend(id, 'neovan')") eq({"notification", "stdout", {id, {"neovan"}}}, next_msg()) From 5fdf741f77c3b6ebed9b5cdc0e9d1043080beb3c Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 14:05:55 +0100 Subject: [PATCH 32/37] feat(f_msgpackdump): support dumping to Blob --- runtime/doc/eval.txt | 14 +++-- src/nvim/eval.lua | 2 +- src/nvim/eval/encode.c | 8 +++ src/nvim/eval/funcs.c | 15 ++++-- .../eval/msgpack_functions_spec.lua | 54 +++++++++++-------- 5 files changed, 60 insertions(+), 33 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 94be7b595e..013bff3d20 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -2534,7 +2534,7 @@ min({expr}) Number minimum value of items in {expr} mkdir({name} [, {path} [, {prot}]]) Number create directory {name} mode([expr]) String current editing mode -msgpackdump({list}) List dump a list of objects to msgpack +msgpackdump({list} [, {type}]) List/Blob dump objects to msgpack msgpackparse({list}) List parse msgpack to a list of objects nextnonblank({lnum}) Number line nr of non-blank line >= {lnum} nr2char({expr}[, {utf8}]) String single char with ASCII/UTF8 value {expr} @@ -6824,11 +6824,15 @@ mode([expr]) Return a string that indicates the current mode. the leading character(s). Also see |visualmode()|. -msgpackdump({list}) *msgpackdump()* - Convert a list of VimL objects to msgpack. Returned value is - |readfile()|-style list. Example: > +msgpackdump({list} [, {type}]) *msgpackdump()* + Convert a list of VimL objects to msgpack. Returned value is a + |readfile()|-style list. When {type} contains "B", a |Blob| is + returned instead. Example: > call writefile(msgpackdump([{}]), 'fname.mpack', 'b') -< This will write the single 0x80 byte to `fname.mpack` file +< or, using a |Blob|: > + call writefile(msgpackdump([{}], 'B'), 'fname.mpack') +< + This will write the single 0x80 byte to a `fname.mpack` file (dictionary with zero items is represented by 0x80 byte in messagepack). diff --git a/src/nvim/eval.lua b/src/nvim/eval.lua index a7242ba73a..5f355abe3c 100644 --- a/src/nvim/eval.lua +++ b/src/nvim/eval.lua @@ -250,7 +250,7 @@ return { min={args=1, base=1}, mkdir={args={1, 3}}, mode={args={0, 1}}, - msgpackdump={args=1}, + msgpackdump={args={1, 2}}, msgpackparse={args=1}, nextnonblank={args=1}, nr2char={args={1, 2}}, diff --git a/src/nvim/eval/encode.c b/src/nvim/eval/encode.c index 1c2274c410..b5e50e7ef5 100644 --- a/src/nvim/eval/encode.c +++ b/src/nvim/eval/encode.c @@ -47,6 +47,14 @@ const char *const encode_special_var_names[] = { # include "eval/encode.c.generated.h" #endif +/// Msgpack callback for writing to a Blob +int encode_blob_write(void *const data, const char *const buf, const size_t len) + FUNC_ATTR_NONNULL_ARG(1) +{ + ga_concat_len(&((blob_T *)data)->bv_ga, buf, len); + return (int)len; +} + /// Msgpack callback for writing to readfile()-style list int encode_list_write(void *const data, const char *const buf, const size_t len) FUNC_ATTR_NONNULL_ARG(1) diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index e83e0ac0cc..a2ea74b19b 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -6506,9 +6506,16 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv, FunPtr fptr) EMSG2(_(e_listarg), "msgpackdump()"); return; } - list_T *const ret_list = tv_list_alloc_ret(rettv, kListLenMayKnow); list_T *const list = argvars[0].vval.v_list; - msgpack_packer *lpacker = msgpack_packer_new(ret_list, &encode_list_write); + msgpack_packer *packer; + if (argvars[1].v_type != VAR_UNKNOWN + && strequal(tv_get_string(&argvars[1]), "B")) { + tv_blob_alloc_ret(rettv); + packer = msgpack_packer_new(rettv->vval.v_blob, &encode_blob_write); + } else { + packer = msgpack_packer_new(tv_list_alloc_ret(rettv, kListLenMayKnow), + &encode_list_write); + } const char *const msg = _("msgpackdump() argument, index %i"); // Assume that translation will not take more then 4 times more space char msgbuf[sizeof("msgpackdump() argument, index ") * 4 + NUMBUFLEN]; @@ -6516,11 +6523,11 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv, FunPtr fptr) TV_LIST_ITER(list, li, { vim_snprintf(msgbuf, sizeof(msgbuf), (char *)msg, idx); idx++; - if (encode_vim_to_msgpack(lpacker, TV_LIST_ITEM_TV(li), msgbuf) == FAIL) { + if (encode_vim_to_msgpack(packer, TV_LIST_ITEM_TV(li), msgbuf) == FAIL) { break; } }); - msgpack_packer_free(lpacker); + msgpack_packer_free(packer); } /// "msgpackparse" function diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index d95cea57e4..3db977257c 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -1,6 +1,5 @@ local helpers = require('test.functional.helpers')(after_each) local clear = helpers.clear -local funcs = helpers.funcs local eval, eq = helpers.eval, helpers.eq local command = helpers.command local nvim = helpers.nvim @@ -511,13 +510,22 @@ end) describe('msgpackdump() function', function() before_each(clear) + local dump_eq = function(exp_list, arg_expr) + local l = {} + for i,v in ipairs(exp_list) do + l[i] = v:gsub('\n', '\000') + end + local exp_blobstr = table.concat(l, '\n') + eq(exp_list, eval('msgpackdump(' .. arg_expr .. ')')) + eq(exp_blobstr, eval('msgpackdump(' .. arg_expr .. ', "B")')) + end + it('dumps string as BIN 8', function() - nvim('set_var', 'obj', {'Test'}) - eq({"\196\004Test"}, eval('msgpackdump(obj)')) + dump_eq({'\196\004Test'}, '["Test"]') end) it('dumps blob as BIN 8', function() - eq({'\196\005Bl\nb!'}, eval('msgpackdump([0z426c006221])')) + dump_eq({'\196\005Bl\nb!'}, '[0z426c006221]') end) it('can dump generic mapping with generic mapping keys and values', function() @@ -525,56 +533,56 @@ describe('msgpackdump() function', function() command('let todumpv1 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') command('let todumpv2 = {"_TYPE": v:msgpack_types.map, "_VAL": []}') command('call add(todump._VAL, [todumpv1, todumpv2])') - eq({'\129\128\128'}, eval('msgpackdump([todump])')) + dump_eq({'\129\128\128'}, '[todump]') end) it('can dump v:true', function() - eq({'\195'}, funcs.msgpackdump({true})) + dump_eq({'\195'}, '[v:true]') end) it('can dump v:false', function() - eq({'\194'}, funcs.msgpackdump({false})) + dump_eq({'\194'}, '[v:false]') end) - it('can v:null', function() - command('let todump = v:null') + it('can dump v:null', function() + dump_eq({'\192'}, '[v:null]') end) it('can dump special bool mapping (true)', function() command('let todump = {"_TYPE": v:msgpack_types.boolean, "_VAL": 1}') - eq({'\195'}, eval('msgpackdump([todump])')) + dump_eq({'\195'}, '[todump]') end) it('can dump special bool mapping (false)', function() command('let todump = {"_TYPE": v:msgpack_types.boolean, "_VAL": 0}') - eq({'\194'}, eval('msgpackdump([todump])')) + dump_eq({'\194'}, '[todump]') end) it('can dump special nil mapping', function() command('let todump = {"_TYPE": v:msgpack_types.nil, "_VAL": 0}') - eq({'\192'}, eval('msgpackdump([todump])')) + dump_eq({'\192'}, '[todump]') end) it('can dump special ext mapping', function() command('let todump = {"_TYPE": v:msgpack_types.ext, "_VAL": [5, ["",""]]}') - eq({'\212\005', ''}, eval('msgpackdump([todump])')) + dump_eq({'\212\005', ''}, '[todump]') end) it('can dump special array mapping', function() command('let todump = {"_TYPE": v:msgpack_types.array, "_VAL": [5, [""]]}') - eq({'\146\005\145\196\n'}, eval('msgpackdump([todump])')) + dump_eq({'\146\005\145\196\n'}, '[todump]') end) it('can dump special UINT64_MAX mapping', function() command('let todump = {"_TYPE": v:msgpack_types.integer}') command('let todump._VAL = [1, 3, 0x7FFFFFFF, 0x7FFFFFFF]') - eq({'\207\255\255\255\255\255\255\255\255'}, eval('msgpackdump([todump])')) + dump_eq({'\207\255\255\255\255\255\255\255\255'}, '[todump]') end) it('can dump special INT64_MIN mapping', function() command('let todump = {"_TYPE": v:msgpack_types.integer}') command('let todump._VAL = [-1, 2, 0, 0]') - eq({'\211\128\n\n\n\n\n\n\n'}, eval('msgpackdump([todump])')) + dump_eq({'\211\128\n\n\n\n\n\n\n'}, '[todump]') end) it('fails to dump a function reference', function() @@ -613,13 +621,13 @@ describe('msgpackdump() function', function() it('can dump dict with two same dicts inside', function() command('let inter = {}') command('let todump = {"a": inter, "b": inter}') - eq({"\130\161a\128\161b\128"}, eval('msgpackdump([todump])')) + dump_eq({"\130\161a\128\161b\128"}, '[todump]') end) it('can dump list with two same lists inside', function() command('let inter = []') command('let todump = [inter, inter]') - eq({"\146\144\144"}, eval('msgpackdump([todump])')) + dump_eq({"\146\144\144"}, '[todump]') end) it('fails to dump a recursive list in a special dict', function() @@ -670,9 +678,9 @@ describe('msgpackdump() function', function() exc_exec('call msgpackdump()')) end) - it('fails when called with two arguments', function() + it('fails when called with three arguments', function() eq('Vim(call):E118: Too many arguments for function: msgpackdump', - exc_exec('call msgpackdump(["", ""], 1)')) + exc_exec('call msgpackdump(["", ""], 1, 2)')) end) it('fails to dump a string', function() @@ -714,9 +722,9 @@ describe('msgpackdump() function', function() end) it('can dump NULL string', function() - eq({'\196\n'}, eval('msgpackdump([$XXX_UNEXISTENT_VAR_XXX])')) - eq({'\196\n'}, eval('msgpackdump([{"_TYPE": v:msgpack_types.binary, "_VAL": [$XXX_UNEXISTENT_VAR_XXX]}])')) - eq({'\160'}, eval('msgpackdump([{"_TYPE": v:msgpack_types.string, "_VAL": [$XXX_UNEXISTENT_VAR_XXX]}])')) + dump_eq({'\196\n'}, '[$XXX_UNEXISTENT_VAR_XXX]') + dump_eq({'\196\n'}, '[{"_TYPE": v:msgpack_types.binary, "_VAL": [$XXX_UNEXISTENT_VAR_XXX]}]') + dump_eq({'\160'}, '[{"_TYPE": v:msgpack_types.string, "_VAL": [$XXX_UNEXISTENT_VAR_XXX]}]') end) it('can dump NULL blob', function() From ddaa0cc9bebd8e094a7169f43947f298a3436ba9 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Sat, 31 Jul 2021 00:47:24 +0100 Subject: [PATCH 33/37] fix(test/dumplog): tostring(rv) before formatting as string For example, implicitly converting a table to a string works in LuaJIT, but needs to be done explicitly with tostring() in Lua 5.1. This can cause issues when testing a non-JIT build if eq(), for example, fails with a table argument. E.g: eq({}, {1}) will not print the details of the assertion failure, but will instead print a less helpful "string expected, got table" error. --- test/helpers.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helpers.lua b/test/helpers.lua index 469aee53f0..499d68b0d6 100644 --- a/test/helpers.lua +++ b/test/helpers.lua @@ -70,7 +70,7 @@ local function dumplog(logfile, fn, ...) if status == false then logfile = logfile or os.getenv('NVIM_LOG_FILE') or '.nvimlog' local logtail = module.read_nvim_log(logfile) - error(string.format('%s\n%s', rv, logtail)) + error(string.format('%s\n%s', tostring(rv), logtail)) end end function module.eq(expected, actual, context, logfile) From e53b71627fb84025fb658a44ee5f90adf276cde7 Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 29 Jul 2021 17:46:45 +0100 Subject: [PATCH 34/37] feat(f_msgpackparse): support parsing from Blob Note that it is not possible for msgpack_unpack_next() and msgpack_unpacker_next() to return MSGPACK_UNPACK_EXTRA_BYTES, so it should be fine to abort() on that. Lua 5.1 doesn't support string hex escapes (\xXX) like VimL does (though LuaJIT does), so convert them to decimal escapes (\DDD) in tests. --- runtime/doc/eval.txt | 7 +- src/nvim/eval/funcs.c | 117 +++++++++++++----- .../eval/msgpack_functions_spec.lua | 86 +++++++------ 3 files changed, 137 insertions(+), 73 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 013bff3d20..93826660b1 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -2535,7 +2535,7 @@ mkdir({name} [, {path} [, {prot}]]) Number create directory {name} mode([expr]) String current editing mode msgpackdump({list} [, {type}]) List/Blob dump objects to msgpack -msgpackparse({list}) List parse msgpack to a list of objects +msgpackparse({data}) List parse msgpack to a list of objects nextnonblank({lnum}) Number line nr of non-blank line >= {lnum} nr2char({expr}[, {utf8}]) String single char with ASCII/UTF8 value {expr} nvim_...({args}...) any call nvim |api| functions @@ -6843,8 +6843,9 @@ msgpackdump({list} [, {type}]) *msgpackdump()* 4. Other strings and |Blob|s are always dumped as BIN strings. 5. Points 3. and 4. do not apply to |msgpack-special-dict|s. -msgpackparse({list}) *msgpackparse()* - Convert a |readfile()|-style list to a list of VimL objects. +msgpackparse({data}) *msgpackparse()* + Convert a |readfile()|-style list or a |Blob| to a list of + VimL objects. Example: > let fname = expand('~/.config/nvim/shada/main.shada') let mpack = readfile(fname, 'b') diff --git a/src/nvim/eval/funcs.c b/src/nvim/eval/funcs.c index a2ea74b19b..fda33fdfdd 100644 --- a/src/nvim/eval/funcs.c +++ b/src/nvim/eval/funcs.c @@ -6530,16 +6530,43 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv, FunPtr fptr) msgpack_packer_free(packer); } -/// "msgpackparse" function -static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr) +static int msgpackparse_convert_item(const msgpack_object data, + const msgpack_unpack_return result, + list_T *const ret_list, + const bool fail_if_incomplete) FUNC_ATTR_NONNULL_ALL { - if (argvars[0].v_type != VAR_LIST) { - EMSG2(_(e_listarg), "msgpackparse()"); - return; + switch (result) { + case MSGPACK_UNPACK_PARSE_ERROR: + EMSG2(_(e_invarg2), "Failed to parse msgpack string"); + return FAIL; + case MSGPACK_UNPACK_NOMEM_ERROR: + EMSG(_(e_outofmem)); + return FAIL; + case MSGPACK_UNPACK_CONTINUE: + if (fail_if_incomplete) { + EMSG2(_(e_invarg2), "Incomplete msgpack string"); + return FAIL; + } + return NOTDONE; + case MSGPACK_UNPACK_SUCCESS: { + typval_T tv = { .v_type = VAR_UNKNOWN }; + if (msgpack_to_vim(data, &tv) == FAIL) { + EMSG2(_(e_invarg2), "Failed to convert msgpack string"); + return FAIL; + } + tv_list_append_owned_tv(ret_list, tv); + return OK; + } + default: + abort(); } - list_T *const ret_list = tv_list_alloc_ret(rettv, kListLenMayKnow); - const list_T *const list = argvars[0].vval.v_list; +} + +static void msgpackparse_unpack_list(const list_T *const list, + list_T *const ret_list) + FUNC_ATTR_NONNULL_ARG(2) +{ if (tv_list_len(list) == 0) { return; } @@ -6558,43 +6585,28 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr) do { if (!msgpack_unpacker_reserve_buffer(unpacker, IOSIZE)) { EMSG(_(e_outofmem)); - goto f_msgpackparse_exit; + goto end; } size_t read_bytes; const int rlret = encode_read_from_list( &lrstate, msgpack_unpacker_buffer(unpacker), IOSIZE, &read_bytes); if (rlret == FAIL) { EMSG2(_(e_invarg2), "List item is not a string"); - goto f_msgpackparse_exit; + goto end; } msgpack_unpacker_buffer_consumed(unpacker, read_bytes); if (read_bytes == 0) { break; } while (unpacker->off < unpacker->used) { - const msgpack_unpack_return result = msgpack_unpacker_next(unpacker, - &unpacked); - if (result == MSGPACK_UNPACK_PARSE_ERROR) { - EMSG2(_(e_invarg2), "Failed to parse msgpack string"); - goto f_msgpackparse_exit; - } - if (result == MSGPACK_UNPACK_NOMEM_ERROR) { - EMSG(_(e_outofmem)); - goto f_msgpackparse_exit; - } - if (result == MSGPACK_UNPACK_SUCCESS) { - typval_T tv = { .v_type = VAR_UNKNOWN }; - if (msgpack_to_vim(unpacked.data, &tv) == FAIL) { - EMSG2(_(e_invarg2), "Failed to convert msgpack string"); - goto f_msgpackparse_exit; - } - tv_list_append_owned_tv(ret_list, tv); - } - if (result == MSGPACK_UNPACK_CONTINUE) { - if (rlret == OK) { - EMSG2(_(e_invarg2), "Incomplete msgpack string"); - } + const msgpack_unpack_return result + = msgpack_unpacker_next(unpacker, &unpacked); + const int conv_result = msgpackparse_convert_item(unpacked.data, result, + ret_list, rlret == OK); + if (conv_result == NOTDONE) { break; + } else if (conv_result == FAIL) { + goto end; } } if (rlret == OK) { @@ -6602,10 +6614,47 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr) } } while (true); -f_msgpackparse_exit: - msgpack_unpacked_destroy(&unpacked); +end: msgpack_unpacker_free(unpacker); - return; + msgpack_unpacked_destroy(&unpacked); +} + +static void msgpackparse_unpack_blob(const blob_T *const blob, + list_T *const ret_list) + FUNC_ATTR_NONNULL_ARG(2) +{ + const int len = tv_blob_len(blob); + if (len == 0) { + return; + } + msgpack_unpacked unpacked; + msgpack_unpacked_init(&unpacked); + for (size_t offset = 0; offset < (size_t)len;) { + const msgpack_unpack_return result + = msgpack_unpack_next(&unpacked, blob->bv_ga.ga_data, len, &offset); + if (msgpackparse_convert_item(unpacked.data, result, ret_list, true) + != OK) { + break; + } + } + + msgpack_unpacked_destroy(&unpacked); +} + +/// "msgpackparse" function +static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr) + FUNC_ATTR_NONNULL_ALL +{ + if (argvars[0].v_type != VAR_LIST && argvars[0].v_type != VAR_BLOB) { + EMSG2(_(e_listblobarg), "msgpackparse()"); + return; + } + list_T *const ret_list = tv_list_alloc_ret(rettv, kListLenMayKnow); + if (argvars[0].v_type == VAR_LIST) { + msgpackparse_unpack_list(argvars[0].vval.v_list, ret_list); + } else { + msgpackparse_unpack_blob(argvars[0].vval.v_blob, ret_list); + } } /* diff --git a/test/functional/eval/msgpack_functions_spec.lua b/test/functional/eval/msgpack_functions_spec.lua index 3db977257c..837b629858 100644 --- a/test/functional/eval/msgpack_functions_spec.lua +++ b/test/functional/eval/msgpack_functions_spec.lua @@ -1,5 +1,6 @@ local helpers = require('test.functional.helpers')(after_each) local clear = helpers.clear +local funcs = helpers.funcs local eval, eq = helpers.eval, helpers.eq local command = helpers.command local nvim = helpers.nvim @@ -12,6 +13,7 @@ describe('msgpack*() functions', function() it(msg, function() nvim('set_var', 'obj', obj) eq(obj, eval('msgpackparse(msgpackdump(g:obj))')) + eq(obj, eval('msgpackparse(msgpackdump(g:obj, "B"))')) end) end @@ -390,56 +392,61 @@ describe('msgpack*() functions', function() end) end) +local blobstr = function(list) + local l = {} + for i,v in ipairs(list) do + l[i] = v:gsub('\n', '\000') + end + return table.concat(l, '\n') +end + +-- Test msgpackparse() with a readfile()-style list and a blob argument +local parse_eq = function(expect, list_arg) + local blob_expr = '0z' .. blobstr(list_arg):gsub('(.)', function(c) + return ('%.2x'):format(c:byte()) + end) + eq(expect, funcs.msgpackparse(list_arg)) + command('let g:parsed = msgpackparse(' .. blob_expr .. ')') + eq(expect, eval('g:parsed')) +end + describe('msgpackparse() function', function() before_each(clear) it('restores nil as v:null', function() - command('let dumped = ["\\xC0"]') - command('let parsed = msgpackparse(dumped)') - eq('[v:null]', eval('string(parsed)')) + parse_eq(eval('[v:null]'), {'\192'}) end) it('restores boolean false as v:false', function() - command('let dumped = ["\\xC2"]') - command('let parsed = msgpackparse(dumped)') - eq({false}, eval('parsed')) + parse_eq({false}, {'\194'}) end) it('restores boolean true as v:true', function() - command('let dumped = ["\\xC3"]') - command('let parsed = msgpackparse(dumped)') - eq({true}, eval('parsed')) + parse_eq({true}, {'\195'}) end) it('restores FIXSTR as special dict', function() - command('let dumped = ["\\xa2ab"]') - command('let parsed = msgpackparse(dumped)') - eq({{_TYPE={}, _VAL={'ab'}}}, eval('parsed')) + parse_eq({{_TYPE={}, _VAL={'ab'}}}, {'\162ab'}) eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.string')) end) it('restores BIN 8 as string', function() - command('let dumped = ["\\xC4\\x02ab"]') - eq({'ab'}, eval('msgpackparse(dumped)')) + parse_eq({'ab'}, {'\196\002ab'}) end) it('restores FIXEXT1 as special dictionary', function() - command('let dumped = ["\\xD4\\x10", ""]') - command('let parsed = msgpackparse(dumped)') - eq({{_TYPE={}, _VAL={0x10, {"", ""}}}}, eval('parsed')) + parse_eq({{_TYPE={}, _VAL={0x10, {"", ""}}}}, {'\212\016', ''}) eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.ext')) end) it('restores MAP with BIN key as special dictionary', function() - command('let dumped = ["\\x81\\xC4\\x01a\\xC4\\n"]') - command('let parsed = msgpackparse(dumped)') - eq({{_TYPE={}, _VAL={{'a', ''}}}}, eval('parsed')) + parse_eq({{_TYPE={}, _VAL={{'a', ''}}}}, {'\129\196\001a\196\n'}) eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map')) end) it('restores MAP with duplicate STR keys as special dictionary', function() command('let dumped = ["\\x82\\xA1a\\xC4\\n\\xA1a\\xC4\\n"]') - -- FIXME Internal error bug + -- FIXME Internal error bug, can't use parse_eq() here command('silent! let parsed = msgpackparse(dumped)') eq({{_TYPE={}, _VAL={ {{_TYPE={}, _VAL={'a'}}, ''}, {{_TYPE={}, _VAL={'a'}}, ''}}} }, eval('parsed')) @@ -449,9 +456,7 @@ describe('msgpackparse() function', function() end) it('restores MAP with MAP key as special dictionary', function() - command('let dumped = ["\\x81\\x80\\xC4\\n"]') - command('let parsed = msgpackparse(dumped)') - eq({{_TYPE={}, _VAL={{{}, ''}}}}, eval('parsed')) + parse_eq({{_TYPE={}, _VAL={{{}, ''}}}}, {'\129\128\196\n'}) eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map')) end) @@ -476,48 +481,57 @@ describe('msgpackparse() function', function() end) it('fails to parse a string', function() - eq('Vim(call):E686: Argument of msgpackparse() must be a List', + eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob', exc_exec('call msgpackparse("abcdefghijklmnopqrstuvwxyz")')) end) it('fails to parse a number', function() - eq('Vim(call):E686: Argument of msgpackparse() must be a List', + eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob', exc_exec('call msgpackparse(127)')) end) it('fails to parse a dictionary', function() - eq('Vim(call):E686: Argument of msgpackparse() must be a List', + eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob', exc_exec('call msgpackparse({})')) end) it('fails to parse a funcref', function() - eq('Vim(call):E686: Argument of msgpackparse() must be a List', + eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob', exc_exec('call msgpackparse(function("tr"))')) end) it('fails to parse a partial', function() command('function T() dict\nendfunction') - eq('Vim(call):E686: Argument of msgpackparse() must be a List', + eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob', exc_exec('call msgpackparse(function("T", [1, 2], {}))')) end) it('fails to parse a float', function() - eq('Vim(call):E686: Argument of msgpackparse() must be a List', + eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob', exc_exec('call msgpackparse(0.0)')) end) + + it('fails on incomplete msgpack string', function() + local expected = 'Vim(call):E475: Invalid argument: Incomplete msgpack string' + eq(expected, exc_exec([[call msgpackparse(["\xc4"])]])) + eq(expected, exc_exec([[call msgpackparse(["\xca", "\x02\x03"])]])) + eq(expected, exc_exec('call msgpackparse(0zc4)')) + eq(expected, exc_exec('call msgpackparse(0zca0a0203)')) + end) + + it('fails when unable to parse msgpack string', function() + local expected = 'Vim(call):E475: Invalid argument: Failed to parse msgpack string' + eq(expected, exc_exec([[call msgpackparse(["\xc1"])]])) + eq(expected, exc_exec('call msgpackparse(0zc1)')) + end) end) describe('msgpackdump() function', function() before_each(clear) local dump_eq = function(exp_list, arg_expr) - local l = {} - for i,v in ipairs(exp_list) do - l[i] = v:gsub('\n', '\000') - end - local exp_blobstr = table.concat(l, '\n') eq(exp_list, eval('msgpackdump(' .. arg_expr .. ')')) - eq(exp_blobstr, eval('msgpackdump(' .. arg_expr .. ', "B")')) + eq(blobstr(exp_list), eval('msgpackdump(' .. arg_expr .. ', "B")')) end it('dumps string as BIN 8', function() From 6eb41e5c8d935e2308770d5c3a03744c3a59079c Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Fri, 30 Jul 2021 17:36:00 +0100 Subject: [PATCH 35/37] test(oldtest): unskip existing tests that use Blobs --- src/nvim/testdir/test_rename.vim | 1 - src/nvim/testdir/test_swap.vim | 1 - src/nvim/testdir/test_undo.vim | 1 - 3 files changed, 3 deletions(-) diff --git a/src/nvim/testdir/test_rename.vim b/src/nvim/testdir/test_rename.vim index e4228188bd..2311caf790 100644 --- a/src/nvim/testdir/test_rename.vim +++ b/src/nvim/testdir/test_rename.vim @@ -95,7 +95,6 @@ func Test_rename_copy() endfunc func Test_rename_fails() - throw 'skipped: TODO: ' call writefile(['foo'], 'Xrenamefile') " Can't rename into a non-existing directory. diff --git a/src/nvim/testdir/test_swap.vim b/src/nvim/testdir/test_swap.vim index 02bc297de1..e3101d4e44 100644 --- a/src/nvim/testdir/test_swap.vim +++ b/src/nvim/testdir/test_swap.vim @@ -168,7 +168,6 @@ func Test_swapname() endfunc func Test_swapfile_delete() - throw 'skipped: need the "blob" feature for this test' autocmd! SwapExists function s:swap_exists() let v:swapchoice = s:swap_choice diff --git a/src/nvim/testdir/test_undo.vim b/src/nvim/testdir/test_undo.vim index 54caed3983..c7dcaa0f36 100644 --- a/src/nvim/testdir/test_undo.vim +++ b/src/nvim/testdir/test_undo.vim @@ -368,7 +368,6 @@ endfunc " Check that reading a truncted undo file doesn't hang. func Test_undofile_truncated() - throw 'skipped: TODO: ' new call setline(1, 'hello') set ul=100 From 10283915d6fe463314a4dd44a88c4f6435996a1c Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Sun, 1 Aug 2021 19:14:34 +0100 Subject: [PATCH 36/37] doc(eval): include latest relevant Blob changes Also includes some small relevant nearby non-Blob changes and typo fixes. Changes are included from: - v8.1.0815 - v8.1.0846 - v8.1.1084 - v8.1.2326 - v8.2.1969 - d89682477c - d09091d495 - 53f7fccc94 --- runtime/doc/eval.txt | 66 +++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt index 93826660b1..99e48e602b 100644 --- a/runtime/doc/eval.txt +++ b/runtime/doc/eval.txt @@ -604,8 +604,11 @@ Functions that can be used with a Dictionary: > 1.5 Blobs ~ *blob* *Blob* *Blobs* *E978* -A Blob mostly behaves like a |List| of numbers, where the numbers have an -8-bit value, from 0 to 255. +A Blob is a binary object. It can be used to read an image from a file and +send it over a channel, for example. + +A Blob mostly behaves like a |List| of numbers, where each number has the +value of an 8-bit byte, from 0 to 255. Blob creation ~ @@ -663,6 +666,7 @@ Part of a blob ~ A part of the Blob can be obtained by specifying the first and last index, separated by a colon in square brackets: > :let myblob = 0z00112233 + :let shortblob = myblob[1:2] " get 0z1122 :let shortblob = myblob[2:-1] " get 0z2233 Omitting the first index is similar to zero. Omitting the last index is @@ -779,8 +783,9 @@ Expression syntax summary, from least to most significant: etc. As above, append ? for ignoring case, # for matching case - expr5 is expr5 same |List| instance - expr5 isnot expr5 different |List| instance + expr5 is expr5 same |List|, |Dictionary| or |Blob| instance + expr5 isnot expr5 different |List|, |Dictionary| or |Blob| + instance |expr5| expr6 expr6 + expr6 ... number addition, list or blob concatenation @@ -951,12 +956,12 @@ Dictionary and arguments, use |get()| to get the function name: > if get(Part1, 'name') == get(Part2, 'name') " Part1 and Part2 refer to the same function -When using "is" or "isnot" with a |List| or a |Dictionary| this checks if the -expressions are referring to the same |List| or |Dictionary| instance. A copy -of a |List| is different from the original |List|. When using "is" without -a |List| or a |Dictionary| it is equivalent to using "equal", using "isnot" -equivalent to using "not equal". Except that a different type means the -values are different: > +Using "is" or "isnot" with a |List|, |Dictionary| or |Blob| checks whether +the expressions are referring to the same |List|, |Dictionary| or |Blob| +instance. A copy of a |List| is different from the original |List|. When +using "is" without a |List|, |Dictionary| or |Blob|, it is equivalent to +using "equal", using "isnot" is equivalent to using "not equal". Except that +a different type means the values are different: > echo 4 == '4' 1 echo 4 is '4' @@ -2155,21 +2160,21 @@ v:swapcommand Normal mode command to be executed after a file has been For ":edit +cmd file" the value is ":cmd\r". *v:t_TYPE* *v:t_bool* *t_bool-variable* -v:t_bool Value of Boolean type. Read-only. See: |type()| +v:t_bool Value of |Boolean| type. Read-only. See: |type()| *v:t_dict* *t_dict-variable* -v:t_dict Value of Dictionary type. Read-only. See: |type()| +v:t_dict Value of |Dictionary| type. Read-only. See: |type()| *v:t_float* *t_float-variable* -v:t_float Value of Float type. Read-only. See: |type()| +v:t_float Value of |Float| type. Read-only. See: |type()| *v:t_func* *t_func-variable* -v:t_func Value of Funcref type. Read-only. See: |type()| +v:t_func Value of |Funcref| type. Read-only. See: |type()| *v:t_list* *t_list-variable* -v:t_list Value of List type. Read-only. See: |type()| +v:t_list Value of |List| type. Read-only. See: |type()| *v:t_number* *t_number-variable* -v:t_number Value of Number type. Read-only. See: |type()| +v:t_number Value of |Number| type. Read-only. See: |type()| *v:t_string* *t_string-variable* -v:t_string Value of String type. Read-only. See: |type()| +v:t_string Value of |String| type. Read-only. See: |type()| *v:t_blob* *t_blob-variable* -v:t_blob Value of Blob type. Read-only. See: |type()| +v:t_blob Value of |Blob| type. Read-only. See: |type()| *v:termresponse* *termresponse-variable* v:termresponse The escape sequence returned by the terminal for the DA @@ -2574,7 +2579,10 @@ remote_read({serverid} [, {timeout}]) remote_send({server}, {string} [, {idvar}]) String send key sequence remote_startserver({name}) none become server {name} -remove({list}, {idx} [, {end}]) any remove items {idx}-{end} from {list} +remove({list}, {idx} [, {end}]) any/List + remove items {idx}-{end} from {list} +remove({blob}, {idx} [, {end}]) Number/Blob + remove bytes {idx}-{end} from {blob} remove({dict}, {key}) any remove entry {key} from {dict} rename({from}, {to}) Number rename (move) file from {from} to {to} repeat({expr}, {count}) String repeat {expr} {count} times @@ -4236,7 +4244,7 @@ filter({expr1}, {expr2}) *filter()* |Dictionary| to remain unmodified make a copy first: > :let l = filter(copy(mylist), 'v:val =~ "KEEP"') -< Returns {expr1}, the |List| , |Blob| or |Dictionary| that was +< Returns {expr1}, the |List|, |Blob| or |Dictionary| that was filtered. When an error is encountered while evaluating {expr2} no further items in {expr1} are processed. When {expr2} is a Funcref errors inside a function are ignored, @@ -6314,12 +6322,14 @@ luaeval({expr}[, {expr}]) to Vim data structures. See |lua-eval| for more details. map({expr1}, {expr2}) *map()* - {expr1} must be a |List| or a |Dictionary|. + {expr1} must be a |List|, |Blob| or |Dictionary|. Replace each item in {expr1} with the result of evaluating - {expr2}. {expr2} must be a |string| or |Funcref|. + {expr2}. For a |Blob| each byte is replaced. + + {expr2} must be a |string| or |Funcref|. If {expr2} is a |string|, inside {expr2} |v:val| has the value - of the current item. For a |Dictionary| |v:key| has the key + of the current item. For a |Dictionary| |v:key| has the key of the current item and for a |List| |v:key| has the index of the current item. For a |Blob| |v:key| has the index of the current byte. @@ -6352,11 +6362,11 @@ map({expr1}, {expr2}) *map()* |Dictionary| to remain unmodified make a copy first: > :let tlist = map(copy(mylist), ' v:val . "\t"') -< Returns {expr1}, the |List| or |Dictionary| that was filtered. - When an error is encountered while evaluating {expr2} no - further items in {expr1} are processed. When {expr2} is a - Funcref errors inside a function are ignored, unless it was - defined with the "abort" flag. +< Returns {expr1}, the |List|, |Blob| or |Dictionary| that was + filtered. When an error is encountered while evaluating + {expr2} no further items in {expr1} are processed. When + {expr2} is a Funcref errors inside a function are ignored, + unless it was defined with the "abort" flag. Can also be used as a |method|: > mylist->map(expr2) From 89f7f7a991de6135f9009b832cd7736dabf2050c Mon Sep 17 00:00:00 2001 From: Sean Dewar Date: Thu, 16 Sep 2021 20:16:48 +0100 Subject: [PATCH 37/37] vim-patch:8.2.1866: Vim9: appending to pushed blob gives wrong result Problem: Vim9: appending to pushed blob gives wrong result. Solution: Set ga_maxlen when copying a blob. https://github.com/vim/vim/commit/66fa5fd54f550c0790d36c20124c49493b323bfa Vim9script is N/A. --- src/nvim/eval/typval.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nvim/eval/typval.c b/src/nvim/eval/typval.c index b08fe274f0..381d70ea1b 100644 --- a/src/nvim/eval/typval.c +++ b/src/nvim/eval/typval.c @@ -2274,6 +2274,7 @@ void tv_blob_copy(typval_T *const from, typval_T *const to) = xmemdup(from->vval.v_blob->bv_ga.ga_data, (size_t)len); } to->vval.v_blob->bv_ga.ga_len = len; + to->vval.v_blob->bv_ga.ga_maxlen = len; } }