feat(f_msgpackparse): support parsing from Blob

Note that it is not possible for msgpack_unpack_next() and
msgpack_unpacker_next() to return MSGPACK_UNPACK_EXTRA_BYTES, so it
should be fine to abort() on that.

Lua 5.1 doesn't support string hex escapes (\xXX) like VimL does (though
LuaJIT does), so convert them to decimal escapes (\DDD) in tests.

runtime/doc/eval.txt

@@ -2535,7 +2535,7 @@ mkdir({name} [, {path} [, {prot}]])
 				Number	create directory {name}
 mode([expr])			String	current editing mode
 msgpackdump({list} [, {type}])	List/Blob  dump objects to msgpack
-msgpackparse({list})		List	parse msgpack to a list of objects
+msgpackparse({data})		List	parse msgpack to a list of objects
 nextnonblank({lnum})		Number	line nr of non-blank line >= {lnum}
 nr2char({expr}[, {utf8}])	String	single char with ASCII/UTF8 value {expr}
 nvim_...({args}...)		any	call nvim |api| functions
@@ -6843,8 +6843,9 @@ msgpackdump({list} [, {type}])				*msgpackdump()*
 		4. Other strings and |Blob|s are always dumped as BIN strings.
 		5. Points 3. and 4. do not apply to |msgpack-special-dict|s.
 
-msgpackparse({list})					*msgpackparse()*
+msgpackparse({data})					*msgpackparse()*
-		Convert a |readfile()|-style list to a list of VimL objects.
+		Convert a |readfile()|-style list or a |Blob| to a list of
+		VimL objects.
 		Example: >
 			let fname = expand('~/.config/nvim/shada/main.shada')
 			let mpack = readfile(fname, 'b')

src/nvim/eval/funcs.c

@@ -6530,16 +6530,43 @@ static void f_msgpackdump(typval_T *argvars, typval_T *rettv, FunPtr fptr)
   msgpack_packer_free(packer);
 }
 
-/// "msgpackparse" function
+static int msgpackparse_convert_item(const msgpack_object data,
-static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr)
+                                     const msgpack_unpack_return result,
+                                     list_T *const ret_list,
+                                     const bool fail_if_incomplete)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (argvars[0].v_type != VAR_LIST) {
+  switch (result) {
-    EMSG2(_(e_listarg), "msgpackparse()");
+    case MSGPACK_UNPACK_PARSE_ERROR:
-    return;
+      EMSG2(_(e_invarg2), "Failed to parse msgpack string");
+      return FAIL;
+    case MSGPACK_UNPACK_NOMEM_ERROR:
+      EMSG(_(e_outofmem));
+      return FAIL;
+    case MSGPACK_UNPACK_CONTINUE:
+      if (fail_if_incomplete) {
+        EMSG2(_(e_invarg2), "Incomplete msgpack string");
+        return FAIL;
+      }
+      return NOTDONE;
+    case MSGPACK_UNPACK_SUCCESS: {
+      typval_T tv = { .v_type = VAR_UNKNOWN };
+      if (msgpack_to_vim(data, &tv) == FAIL) {
+        EMSG2(_(e_invarg2), "Failed to convert msgpack string");
+        return FAIL;
+      }
+      tv_list_append_owned_tv(ret_list, tv);
+      return OK;
+    }
+    default:
+      abort();
   }
-  list_T *const ret_list = tv_list_alloc_ret(rettv, kListLenMayKnow);
+}
-  const list_T *const list = argvars[0].vval.v_list;
+
+static void msgpackparse_unpack_list(const list_T *const list,
+                                     list_T *const ret_list)
+  FUNC_ATTR_NONNULL_ARG(2)
+{
   if (tv_list_len(list) == 0) {
     return;
   }
@@ -6558,43 +6585,28 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr)
   do {
     if (!msgpack_unpacker_reserve_buffer(unpacker, IOSIZE)) {
       EMSG(_(e_outofmem));
-      goto f_msgpackparse_exit;
+      goto end;
     }
     size_t read_bytes;
     const int rlret = encode_read_from_list(
         &lrstate, msgpack_unpacker_buffer(unpacker), IOSIZE, &read_bytes);
     if (rlret == FAIL) {
       EMSG2(_(e_invarg2), "List item is not a string");
-      goto f_msgpackparse_exit;
+      goto end;
     }
     msgpack_unpacker_buffer_consumed(unpacker, read_bytes);
     if (read_bytes == 0) {
       break;
     }
     while (unpacker->off < unpacker->used) {
-      const msgpack_unpack_return result = msgpack_unpacker_next(unpacker,
+      const msgpack_unpack_return result
-                                                                 &unpacked);
+          = msgpack_unpacker_next(unpacker, &unpacked);
-      if (result == MSGPACK_UNPACK_PARSE_ERROR) {
+      const int conv_result = msgpackparse_convert_item(unpacked.data, result,
-        EMSG2(_(e_invarg2), "Failed to parse msgpack string");
+                                                        ret_list, rlret == OK);
-        goto f_msgpackparse_exit;
+      if (conv_result == NOTDONE) {
-      }
-      if (result == MSGPACK_UNPACK_NOMEM_ERROR) {
-        EMSG(_(e_outofmem));
-        goto f_msgpackparse_exit;
-      }
-      if (result == MSGPACK_UNPACK_SUCCESS) {
-        typval_T tv = { .v_type = VAR_UNKNOWN };
-        if (msgpack_to_vim(unpacked.data, &tv) == FAIL) {
-          EMSG2(_(e_invarg2), "Failed to convert msgpack string");
-          goto f_msgpackparse_exit;
-        }
-        tv_list_append_owned_tv(ret_list, tv);
-      }
-      if (result == MSGPACK_UNPACK_CONTINUE) {
-        if (rlret == OK) {
-          EMSG2(_(e_invarg2), "Incomplete msgpack string");
-        }
         break;
+      } else if (conv_result == FAIL) {
+        goto end;
       }
     }
     if (rlret == OK) {
@@ -6602,10 +6614,47 @@ static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr)
     }
   } while (true);
 
-f_msgpackparse_exit:
+end:
-  msgpack_unpacked_destroy(&unpacked);
   msgpack_unpacker_free(unpacker);
-  return;
+  msgpack_unpacked_destroy(&unpacked);
+}
+
+static void msgpackparse_unpack_blob(const blob_T *const blob,
+                                     list_T *const ret_list)
+  FUNC_ATTR_NONNULL_ARG(2)
+{
+  const int len = tv_blob_len(blob);
+  if (len == 0) {
+    return;
+  }
+  msgpack_unpacked unpacked;
+  msgpack_unpacked_init(&unpacked);
+  for (size_t offset = 0; offset < (size_t)len;) {
+    const msgpack_unpack_return result
+        = msgpack_unpack_next(&unpacked, blob->bv_ga.ga_data, len, &offset);
+    if (msgpackparse_convert_item(unpacked.data, result, ret_list, true)
+        != OK) {
+      break;
+    }
+  }
+
+  msgpack_unpacked_destroy(&unpacked);
+}
+
+/// "msgpackparse" function
+static void f_msgpackparse(typval_T *argvars, typval_T *rettv, FunPtr fptr)
+  FUNC_ATTR_NONNULL_ALL
+{
+  if (argvars[0].v_type != VAR_LIST && argvars[0].v_type != VAR_BLOB) {
+    EMSG2(_(e_listblobarg), "msgpackparse()");
+    return;
+  }
+  list_T *const ret_list = tv_list_alloc_ret(rettv, kListLenMayKnow);
+  if (argvars[0].v_type == VAR_LIST) {
+    msgpackparse_unpack_list(argvars[0].vval.v_list, ret_list);
+  } else {
+    msgpackparse_unpack_blob(argvars[0].vval.v_blob, ret_list);
+  }
 }
 
 /*

test/functional/eval/msgpack_functions_spec.lua

@@ -1,5 +1,6 @@
 local helpers = require('test.functional.helpers')(after_each)
 local clear = helpers.clear
+local funcs = helpers.funcs
 local eval, eq = helpers.eval, helpers.eq
 local command = helpers.command
 local nvim = helpers.nvim
@@ -12,6 +13,7 @@ describe('msgpack*() functions', function()
     it(msg, function()
       nvim('set_var', 'obj', obj)
       eq(obj, eval('msgpackparse(msgpackdump(g:obj))'))
+      eq(obj, eval('msgpackparse(msgpackdump(g:obj, "B"))'))
     end)
   end
 
@@ -390,56 +392,61 @@ describe('msgpack*() functions', function()
   end)
 end)
 
+local blobstr = function(list)
+  local l = {}
+  for i,v in ipairs(list) do
+    l[i] = v:gsub('\n', '\000')
+  end
+  return table.concat(l, '\n')
+end
+
+-- Test msgpackparse() with a readfile()-style list and a blob argument
+local parse_eq = function(expect, list_arg)
+  local blob_expr = '0z' .. blobstr(list_arg):gsub('(.)', function(c)
+    return ('%.2x'):format(c:byte())
+  end)
+  eq(expect, funcs.msgpackparse(list_arg))
+  command('let g:parsed = msgpackparse(' .. blob_expr .. ')')
+  eq(expect, eval('g:parsed'))
+end
+
 describe('msgpackparse() function', function()
   before_each(clear)
 
   it('restores nil as v:null', function()
-    command('let dumped = ["\\xC0"]')
+    parse_eq(eval('[v:null]'), {'\192'})
-    command('let parsed = msgpackparse(dumped)')
-    eq('[v:null]', eval('string(parsed)'))
   end)
 
   it('restores boolean false as v:false', function()
-    command('let dumped = ["\\xC2"]')
+    parse_eq({false}, {'\194'})
-    command('let parsed = msgpackparse(dumped)')
-    eq({false}, eval('parsed'))
   end)
 
   it('restores boolean true as v:true', function()
-    command('let dumped = ["\\xC3"]')
+    parse_eq({true}, {'\195'})
-    command('let parsed = msgpackparse(dumped)')
-    eq({true}, eval('parsed'))
   end)
 
   it('restores FIXSTR as special dict', function()
-    command('let dumped = ["\\xa2ab"]')
+    parse_eq({{_TYPE={}, _VAL={'ab'}}}, {'\162ab'})
-    command('let parsed = msgpackparse(dumped)')
-    eq({{_TYPE={}, _VAL={'ab'}}}, eval('parsed'))
     eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.string'))
   end)
 
   it('restores BIN 8 as string', function()
-    command('let dumped = ["\\xC4\\x02ab"]')
+    parse_eq({'ab'}, {'\196\002ab'})
-    eq({'ab'}, eval('msgpackparse(dumped)'))
   end)
 
   it('restores FIXEXT1 as special dictionary', function()
-    command('let dumped = ["\\xD4\\x10", ""]')
+    parse_eq({{_TYPE={}, _VAL={0x10, {"", ""}}}}, {'\212\016', ''})
-    command('let parsed = msgpackparse(dumped)')
-    eq({{_TYPE={}, _VAL={0x10, {"", ""}}}}, eval('parsed'))
     eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.ext'))
   end)
 
   it('restores MAP with BIN key as special dictionary', function()
-    command('let dumped = ["\\x81\\xC4\\x01a\\xC4\\n"]')
+    parse_eq({{_TYPE={}, _VAL={{'a', ''}}}}, {'\129\196\001a\196\n'})
-    command('let parsed = msgpackparse(dumped)')
-    eq({{_TYPE={}, _VAL={{'a', ''}}}}, eval('parsed'))
     eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map'))
   end)
 
   it('restores MAP with duplicate STR keys as special dictionary', function()
     command('let dumped = ["\\x82\\xA1a\\xC4\\n\\xA1a\\xC4\\n"]')
-    -- FIXME Internal error bug
+    -- FIXME Internal error bug, can't use parse_eq() here
     command('silent! let parsed = msgpackparse(dumped)')
     eq({{_TYPE={}, _VAL={ {{_TYPE={}, _VAL={'a'}}, ''},
                           {{_TYPE={}, _VAL={'a'}}, ''}}} }, eval('parsed'))
@@ -449,9 +456,7 @@ describe('msgpackparse() function', function()
   end)
 
   it('restores MAP with MAP key as special dictionary', function()
-    command('let dumped = ["\\x81\\x80\\xC4\\n"]')
+    parse_eq({{_TYPE={}, _VAL={{{}, ''}}}}, {'\129\128\196\n'})
-    command('let parsed = msgpackparse(dumped)')
-    eq({{_TYPE={}, _VAL={{{}, ''}}}}, eval('parsed'))
     eq(1, eval('g:parsed[0]._TYPE is v:msgpack_types.map'))
   end)
 
@@ -476,48 +481,57 @@ describe('msgpackparse() function', function()
   end)
 
   it('fails to parse a string', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse("abcdefghijklmnopqrstuvwxyz")'))
   end)
 
   it('fails to parse a number', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse(127)'))
   end)
 
   it('fails to parse a dictionary', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse({})'))
   end)
 
   it('fails to parse a funcref', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse(function("tr"))'))
   end)
 
   it('fails to parse a partial', function()
     command('function T() dict\nendfunction')
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse(function("T", [1, 2], {}))'))
   end)
 
   it('fails to parse a float', function()
-    eq('Vim(call):E686: Argument of msgpackparse() must be a List',
+    eq('Vim(call):E899: Argument of msgpackparse() must be a List or Blob',
        exc_exec('call msgpackparse(0.0)'))
   end)
+
+  it('fails on incomplete msgpack string', function()
+    local expected = 'Vim(call):E475: Invalid argument: Incomplete msgpack string'
+    eq(expected, exc_exec([[call msgpackparse(["\xc4"])]]))
+    eq(expected, exc_exec([[call msgpackparse(["\xca", "\x02\x03"])]]))
+    eq(expected, exc_exec('call msgpackparse(0zc4)'))
+    eq(expected, exc_exec('call msgpackparse(0zca0a0203)'))
+  end)
+
+  it('fails when unable to parse msgpack string', function()
+    local expected = 'Vim(call):E475: Invalid argument: Failed to parse msgpack string'
+    eq(expected, exc_exec([[call msgpackparse(["\xc1"])]]))
+    eq(expected, exc_exec('call msgpackparse(0zc1)'))
+  end)
 end)
 
 describe('msgpackdump() function', function()
   before_each(clear)
 
   local dump_eq = function(exp_list, arg_expr)
-    local l = {}
-    for i,v in ipairs(exp_list) do
-      l[i] = v:gsub('\n', '\000')
-    end
-    local exp_blobstr = table.concat(l, '\n')
     eq(exp_list, eval('msgpackdump(' .. arg_expr .. ')'))
-    eq(exp_blobstr, eval('msgpackdump(' .. arg_expr .. ', "B")'))
+    eq(blobstr(exp_list), eval('msgpackdump(' .. arg_expr .. ', "B")'))
   end
 
   it('dumps string as BIN 8', function()