object_to_vim: Fix buffer/window/tabpage conversion on BE systems

Since data.integer is a different (larger) integer type than
data.{buffer,window,tabpage}, we cannot abuse the union by using
data.integer to access the value for all 4 types.  Instead, remove the
{buffer,window,tabpage} members and always use the integer member.

In order to accomodate this, perform distinct validation and coercion
between the Integer type and Buffer/Window/Tabpage types in
object_to_vim, msgpack_rpc helpers, and gendispatch.lua.

scripts/gendispatch.lua

@@ -232,8 +232,14 @@ for i = 1, #functions do
       converted = 'arg_'..j
       local rt = real_type(param[1])
       if rt ~= 'Object' then
-        output:write('\n  if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {')
-        output:write('\n    '..converted..' = args.items['..(j - 1)..'].data.'..rt:lower()..';')
+        if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') then
+          -- Buffer, Window, and Tabpage have a specific type, but are stored in integer
+          output:write('\n  if (args.items['..(j - 1)..'].type == kObjectType'..rt..' && args.items['..(j - 1)..'].data.integer >= 0) {')
+          output:write('\n    '..converted..' = (handle_T)args.items['..(j - 1)..'].data.integer;')
+        else
+          output:write('\n  if (args.items['..(j - 1)..'].type == kObjectType'..rt..') {')
+          output:write('\n    '..converted..' = args.items['..(j - 1)..'].data.'..rt:lower()..';')
+        end
         if rt:match('^Buffer$') or rt:match('^Window$') or rt:match('^Tabpage$') or rt:match('^Boolean$') then
           -- accept nonnegative integers for Booleans, Buffers, Windows and Tabpages
           output:write('\n  } else if (args.items['..(j - 1)..'].type == kObjectTypeInteger && args.items['..(j - 1)..'].data.integer >= 0) {')

src/nvim/api/private/defs.h

@@ -91,9 +91,6 @@ typedef enum {
 struct object {
   ObjectType type;
   union {
-    Buffer buffer;
-    Window window;
-    Tabpage tabpage;
     Boolean boolean;
     Integer integer;
     Float floating;

src/nvim/api/private/helpers.c

@@ -616,13 +616,13 @@ bool object_to_vim(Object obj, typval_T *tv, Error *err)
     case kObjectTypeWindow:
     case kObjectTypeTabpage:
     case kObjectTypeInteger:
-      if (obj.data.integer > INT_MAX || obj.data.integer < INT_MIN) {
+      if (obj.data.integer > VARNUMBER_MAX || obj.data.integer < VARNUMBER_MIN) {
         api_set_error(err, Validation, _("Integer value outside range"));
         return false;
       }
 
       tv->v_type = VAR_NUMBER;
-      tv->vval.v_number = (int)obj.data.integer;
+      tv->vval.v_number = (varnumber_T)obj.data.integer;
       break;
 
     case kObjectTypeFloat:

src/nvim/api/private/helpers.h

@@ -37,15 +37,15 @@
 
 #define BUFFER_OBJ(s) ((Object) { \
     .type = kObjectTypeBuffer, \
-    .data.buffer = s })
+    .data.integer = s })
 
 #define WINDOW_OBJ(s) ((Object) { \
     .type = kObjectTypeWindow, \
-    .data.window = s })
+    .data.integer = s })
 
 #define TABPAGE_OBJ(s) ((Object) { \
     .type = kObjectTypeTabpage, \
-    .data.tabpage = s })
+    .data.integer = s })
 
 #define ARRAY_OBJ(a) ((Object) { \
     .type = kObjectTypeArray, \

src/nvim/msgpack_rpc/helpers.c

@@ -21,7 +21,7 @@ static msgpack_zone zone;
 static msgpack_sbuffer sbuffer;
 
 #define HANDLE_TYPE_CONVERSION_IMPL(t, lt) \
-  bool msgpack_rpc_to_##lt(const msgpack_object *const obj, t *const arg) \
+  bool msgpack_rpc_to_##lt(const msgpack_object *const obj, Integer *const arg) \
     FUNC_ATTR_NONNULL_ALL \
   { \
     if (obj->type != MSGPACK_OBJECT_EXT \
@@ -44,12 +44,12 @@ static msgpack_sbuffer sbuffer;
     return true; \
   } \
   \
-  void msgpack_rpc_from_##lt(t o, msgpack_packer *res) \
+  void msgpack_rpc_from_##lt(Integer o, msgpack_packer *res) \
     FUNC_ATTR_NONNULL_ARG(2) \
   { \
     msgpack_packer pac; \
     msgpack_packer_init(&pac, &sbuffer, msgpack_sbuffer_write); \
-    msgpack_pack_int64(&pac, o); \
+    msgpack_pack_int64(&pac, (handle_T)o); \
     msgpack_pack_ext(res, sbuffer.size, kObjectType##t); \
     msgpack_pack_ext_body(res, sbuffer.data, sbuffer.size); \
     msgpack_sbuffer_clear(&sbuffer); \
@@ -213,17 +213,17 @@ bool msgpack_rpc_to_object(const msgpack_object *const obj, Object *const arg)
         switch (cur.mobj->via.ext.type) {
           case kObjectTypeBuffer: {
             cur.aobj->type = kObjectTypeBuffer;
-            ret = msgpack_rpc_to_buffer(cur.mobj, &cur.aobj->data.buffer);
+            ret = msgpack_rpc_to_buffer(cur.mobj, &cur.aobj->data.integer);
             break;
           }
           case kObjectTypeWindow: {
             cur.aobj->type = kObjectTypeWindow;
-            ret = msgpack_rpc_to_window(cur.mobj, &cur.aobj->data.window);
+            ret = msgpack_rpc_to_window(cur.mobj, &cur.aobj->data.integer);
             break;
           }
           case kObjectTypeTabpage: {
             cur.aobj->type = kObjectTypeTabpage;
-            ret = msgpack_rpc_to_tabpage(cur.mobj, &cur.aobj->data.tabpage);
+            ret = msgpack_rpc_to_tabpage(cur.mobj, &cur.aobj->data.integer);
             break;
           }
         }
@@ -369,15 +369,15 @@ void msgpack_rpc_from_object(const Object result, msgpack_packer *const res)
         break;
       }
       case kObjectTypeBuffer: {
-        msgpack_rpc_from_buffer(cur.aobj->data.buffer, res);
+        msgpack_rpc_from_buffer(cur.aobj->data.integer, res);
         break;
       }
       case kObjectTypeWindow: {
-        msgpack_rpc_from_window(cur.aobj->data.window, res);
+        msgpack_rpc_from_window(cur.aobj->data.integer, res);
         break;
       }
       case kObjectTypeTabpage: {
-        msgpack_rpc_from_tabpage(cur.aobj->data.tabpage, res);
+        msgpack_rpc_from_tabpage(cur.aobj->data.integer, res);
         break;
       }
       case kObjectTypeArray: {