Merge #5903 from ZyX-I/fix-5901

Reset copyID also when dictionary is referenced

src/nvim/api/private/helpers.c

@@ -387,6 +387,8 @@ static inline void typval_encode_list_start(EncodedData *const edata,
 #define TYPVAL_ENCODE_CONV_LIST_START(tv, len) \
     typval_encode_list_start(edata, (size_t)(len))
 
+#define TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START(tv, mpsv)
+
 static inline void typval_encode_between_list_items(EncodedData *const edata)
   FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_NONNULL_ALL
 {
@@ -427,6 +429,8 @@ static inline void typval_encode_dict_start(EncodedData *const edata,
 #define TYPVAL_ENCODE_CONV_DICT_START(tv, dict, len) \
     typval_encode_dict_start(edata, (size_t)(len))
 
+#define TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START(tv, dict, mpsv)
+
 #define TYPVAL_ENCODE_SPECIAL_DICT_KEY_CHECK(label, kv_pair)
 
 static inline void typval_encode_after_key(EncodedData *const edata)
@@ -499,11 +503,13 @@ static inline void typval_encode_dict_end(EncodedData *const edata)
 #undef TYPVAL_ENCODE_CONV_FUNC_END
 #undef TYPVAL_ENCODE_CONV_EMPTY_LIST
 #undef TYPVAL_ENCODE_CONV_LIST_START
+#undef TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START
 #undef TYPVAL_ENCODE_CONV_EMPTY_DICT
 #undef TYPVAL_ENCODE_CONV_NIL
 #undef TYPVAL_ENCODE_CONV_BOOL
 #undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER
 #undef TYPVAL_ENCODE_CONV_DICT_START
+#undef TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START
 #undef TYPVAL_ENCODE_CONV_DICT_END
 #undef TYPVAL_ENCODE_CONV_DICT_AFTER_KEY
 #undef TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS

src/nvim/eval.c

@@ -6445,7 +6445,7 @@ void dict_free(dict_T *d) {
  */
 dictitem_T *dictitem_alloc(char_u *key) FUNC_ATTR_NONNULL_RET
 {
-  dictitem_T *di = xmalloc(sizeof(dictitem_T) + STRLEN(key));
+  dictitem_T *di = xmalloc(offsetof(dictitem_T, di_key) + STRLEN(key) + 1);
 #ifndef __clang_analyzer__
   STRCPY(di->di_key, key);
 #endif
@@ -19147,23 +19147,25 @@ static inline void _nothing_conv_func_end(typval_T *const tv, const int copyID)
       } \
     } while (0)
 
-static inline int _nothing_conv_list_start(typval_T *const tv)
+static inline int _nothing_conv_real_list_after_start(
+    typval_T *const tv, MPConvStackVal *const mpsv)
   FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT
 {
-  if (tv == NULL) {
+  assert(tv != NULL);
-    return NOTDONE;
-  }
   tv->v_lock = VAR_UNLOCKED;
   if (tv->vval.v_list->lv_refcount > 1) {
     tv->vval.v_list->lv_refcount--;
     tv->vval.v_list = NULL;
+    mpsv->data.l.li = NULL;
     return OK;
   }
   return NOTDONE;
 }
-#define TYPVAL_ENCODE_CONV_LIST_START(tv, len) \
+#define TYPVAL_ENCODE_CONV_LIST_START(tv, len)
+
+#define TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START(tv, mpsv) \
     do { \
-      if (_nothing_conv_list_start(tv) != NOTDONE) { \
+      if (_nothing_conv_real_list_after_start(tv, &mpsv) != NOTDONE) { \
         goto typval_encode_stop_converting_one_item; \
       } \
     } while (0)
@@ -19183,9 +19185,9 @@ static inline void _nothing_conv_list_end(typval_T *const tv)
 }
 #define TYPVAL_ENCODE_CONV_LIST_END(tv) _nothing_conv_list_end(tv)
 
-static inline int _nothing_conv_dict_start(typval_T *const tv,
+static inline int _nothing_conv_real_dict_after_start(
-                                           dict_T **const dictp,
+    typval_T *const tv, dict_T **const dictp, const void *const nodictvar,
-                                           const void *const nodictvar)
+    MPConvStackVal *const mpsv)
   FUNC_ATTR_ALWAYS_INLINE FUNC_ATTR_WARN_UNUSED_RESULT
 {
   if (tv != NULL) {
@@ -19194,15 +19196,18 @@ static inline int _nothing_conv_dict_start(typval_T *const tv,
   if ((const void *)dictp != nodictvar && (*dictp)->dv_refcount > 1) {
     (*dictp)->dv_refcount--;
     *dictp = NULL;
+    mpsv->data.d.todo = 0;
     return OK;
   }
   return NOTDONE;
 }
-#define TYPVAL_ENCODE_CONV_DICT_START(tv, dict, len) \
+#define TYPVAL_ENCODE_CONV_DICT_START(tv, dict, len)
+
+#define TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START(tv, dict, mpsv) \
     do { \
-      if (_nothing_conv_dict_start(tv, (dict_T **)&dict, \
+      if (_nothing_conv_real_dict_after_start( \
-                                   (void *)&TYPVAL_ENCODE_NODICT_VAR) \
+          tv, (dict_T **)&dict, (void *)&TYPVAL_ENCODE_NODICT_VAR, \
-          != NOTDONE) { \
+          &mpsv) != NOTDONE) { \
         goto typval_encode_stop_converting_one_item; \
       } \
     } while (0)
@@ -19253,9 +19258,11 @@ static inline void _nothing_conv_dict_end(typval_T *const tv,
 #undef TYPVAL_ENCODE_CONV_EMPTY_LIST
 #undef TYPVAL_ENCODE_CONV_EMPTY_DICT
 #undef TYPVAL_ENCODE_CONV_LIST_START
+#undef TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START
 #undef TYPVAL_ENCODE_CONV_LIST_BETWEEN_ITEMS
 #undef TYPVAL_ENCODE_CONV_LIST_END
 #undef TYPVAL_ENCODE_CONV_DICT_START
+#undef TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START
 #undef TYPVAL_ENCODE_SPECIAL_DICT_KEY_CHECK
 #undef TYPVAL_ENCODE_CONV_DICT_AFTER_KEY
 #undef TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS
@@ -21658,9 +21665,12 @@ void func_unref(char_u *name)
     fp = find_func(name);
     if (fp == NULL) {
 #ifdef EXITFREE
-      if (!entered_free_all_mem)  // NOLINT(readability/braces)
+      if (!entered_free_all_mem) {
-#endif
         EMSG2(_(e_intern2), "func_unref()");
+      }
+#else
+      EMSG2(_(e_intern2), "func_unref()");
+#endif
     } else {
       user_func_unref(fp);
     }

src/nvim/eval/encode.c

@@ -369,6 +369,8 @@ int encode_read_from_list(ListReaderState *const state, char *const buf,
 #define TYPVAL_ENCODE_CONV_LIST_START(tv, len) \
     ga_append(gap, '[')
 
+#define TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START(tv, mpsv)
+
 #define TYPVAL_ENCODE_CONV_EMPTY_DICT(tv, dict) \
     ga_concat(gap, "{}")
 
@@ -383,6 +385,8 @@ int encode_read_from_list(ListReaderState *const state, char *const buf,
 #define TYPVAL_ENCODE_CONV_DICT_START(tv, dict, len) \
     ga_append(gap, '{')
 
+#define TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START(tv, dict, mpsv)
+
 #define TYPVAL_ENCODE_CONV_DICT_END(tv, dict) \
     ga_append(gap, '}')
 
@@ -789,11 +793,13 @@ bool encode_check_json_key(const typval_T *const tv)
 #undef TYPVAL_ENCODE_CONV_FUNC_END
 #undef TYPVAL_ENCODE_CONV_EMPTY_LIST
 #undef TYPVAL_ENCODE_CONV_LIST_START
+#undef TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START
 #undef TYPVAL_ENCODE_CONV_EMPTY_DICT
 #undef TYPVAL_ENCODE_CONV_NIL
 #undef TYPVAL_ENCODE_CONV_BOOL
 #undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER
 #undef TYPVAL_ENCODE_CONV_DICT_START
+#undef TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START
 #undef TYPVAL_ENCODE_CONV_DICT_END
 #undef TYPVAL_ENCODE_CONV_DICT_AFTER_KEY
 #undef TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS
@@ -933,6 +939,8 @@ char *encode_tv2json(typval_T *tv, size_t *len)
 #define TYPVAL_ENCODE_CONV_LIST_START(tv, len) \
     msgpack_pack_array(packer, (size_t)(len))
 
+#define TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START(tv, mpsv)
+
 #define TYPVAL_ENCODE_CONV_EMPTY_DICT(tv, dict) \
     msgpack_pack_map(packer, 0)
 
@@ -954,6 +962,8 @@ char *encode_tv2json(typval_T *tv, size_t *len)
 #define TYPVAL_ENCODE_CONV_DICT_START(tv, dict, len) \
     msgpack_pack_map(packer, (size_t)(len))
 
+#define TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START(tv, dict, mpsv)
+
 #define TYPVAL_ENCODE_CONV_DICT_END(tv, dict)
 
 #define TYPVAL_ENCODE_CONV_DICT_AFTER_KEY(tv, dict)
@@ -994,11 +1004,13 @@ char *encode_tv2json(typval_T *tv, size_t *len)
 #undef TYPVAL_ENCODE_CONV_FUNC_END
 #undef TYPVAL_ENCODE_CONV_EMPTY_LIST
 #undef TYPVAL_ENCODE_CONV_LIST_START
+#undef TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START
 #undef TYPVAL_ENCODE_CONV_EMPTY_DICT
 #undef TYPVAL_ENCODE_CONV_NIL
 #undef TYPVAL_ENCODE_CONV_BOOL
 #undef TYPVAL_ENCODE_CONV_UNSIGNED_NUMBER
 #undef TYPVAL_ENCODE_CONV_DICT_START
+#undef TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START
 #undef TYPVAL_ENCODE_CONV_DICT_END
 #undef TYPVAL_ENCODE_CONV_DICT_AFTER_KEY
 #undef TYPVAL_ENCODE_CONV_DICT_BETWEEN_ITEMS

src/nvim/eval/typval_encode.c.h

@@ -129,6 +129,16 @@
 ///             point to a special dictionary.
 /// @param  len  List length. Is an expression which evaluates to an integer.
 
+/// @def TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START
+/// @brief Macros used after pushing list onto the stack
+///
+/// Only used for real list_T* lists, not for special dictionaries or partial
+/// arguments.
+///
+/// @param  tv  Pointer to typval where value is stored. May be NULL. May
+///             point to a special dictionary.
+/// @param  mpsv  Pushed MPConvStackVal value.
+
 /// @def TYPVAL_ENCODE_CONV_LIST_BETWEEN_ITEMS
 /// @brief Macros used after finishing converting non-last list item
 ///
@@ -142,6 +152,9 @@
 /// @def TYPVAL_ENCODE_CONV_DICT_START
 /// @brief Macros used before starting to convert non-empty dictionary
 ///
+/// Only used for real dict_T* dictionaries, not for special dictionaries. Also
+/// used for partial self dictionary.
+///
 /// @param  tv  Pointer to typval where dictionary is stored. May be NULL. May
 ///             point to a special dictionary.
 /// @param  dict  Converted dictionary, lvalue or #TYPVAL_ENCODE_NODICT_VAR
@@ -149,6 +162,14 @@
 /// @param  len  Dictionary length. Is an expression which evaluates to an
 ///              integer.
 
+/// @def TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START
+/// @brief Macros used after pushing dictionary onto the stack
+///
+/// @param  tv  Pointer to typval where dictionary is stored. May be NULL.
+///             May not point to a special dictionary.
+/// @param  dict  Converted dictionary, lvalue.
+/// @param  mpsv  Pushed MPConvStackVal value.
+
 /// @def TYPVAL_ENCODE_SPECIAL_DICT_KEY_CHECK
 /// @brief Macros used to check special dictionary key
 ///
@@ -354,6 +375,7 @@ static int _TYPVAL_ENCODE_CONVERT_ONE_VALUE(
           },
         },
       }));
+      TYPVAL_ENCODE_CONV_REAL_LIST_AFTER_START(tv, _mp_last(*mpstack));
       break;
     }
     case VAR_SPECIAL: {
@@ -564,6 +586,8 @@ _convert_one_value_regular_dict:
           },
         },
       }));
+      TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START(tv, tv->vval.v_dict,
+                                               _mp_last(*mpstack));
       break;
     }
     case VAR_UNKNOWN: {
@@ -732,6 +756,8 @@ typval_encode_stop_converting_one_item:
                   },
                 },
               }));
+              TYPVAL_ENCODE_CONV_REAL_DICT_AFTER_START(NULL, pt->pt_dict,
+                                                       _mp_last(mpstack));
             } else {
               TYPVAL_ENCODE_CONV_FUNC_BEFORE_SELF(tv, -1);
             }

src/nvim/eval/typval_encode.h

@@ -108,37 +108,38 @@ static inline size_t tv_strlen(const typval_T *const tv)
       } \
     } while (0)
 
-#define _TYPVAL_ENCODE_CHECK_SELF_REFERENCE_INNER_2(name) \
+#define _TYPVAL_ENCODE_FUNC_NAME_INNER_2(pref, name, suf) \
-    _typval_encode_##name##_check_self_reference
+    pref##name##suf
-#define _TYPVAL_ENCODE_CHECK_SELF_REFERENCE_INNER(name) \
+#define _TYPVAL_ENCODE_FUNC_NAME_INNER(pref, name, suf) \
-    _TYPVAL_ENCODE_CHECK_SELF_REFERENCE_INNER_2(name)
+    _TYPVAL_ENCODE_FUNC_NAME_INNER_2(pref, name, suf)
+
+/// Construct function name, possibly using macros
+///
+/// Is used to expand macros that may appear in arguments.
+///
+/// @note Expands all arguments, even if only one is needed.
+///
+/// @param[in]  pref  Prefix.
+/// @param[in]  suf  Suffix.
+///
+/// @return Concat: pref + #TYPVAL_ENCODE_NAME + suf.
+#define _TYPVAL_ENCODE_FUNC_NAME(pref, suf) \
+    _TYPVAL_ENCODE_FUNC_NAME_INNER(pref, TYPVAL_ENCODE_NAME, suf)
 
 /// Self reference checker function name
 #define _TYPVAL_ENCODE_CHECK_SELF_REFERENCE \
-    _TYPVAL_ENCODE_CHECK_SELF_REFERENCE_INNER(TYPVAL_ENCODE_NAME)
+    _TYPVAL_ENCODE_FUNC_NAME(_typval_encode_, _check_self_reference)
-
-#define _TYPVAL_ENCODE_ENCODE_INNER_2(name) encode_vim_to_##name
-#define _TYPVAL_ENCODE_ENCODE_INNER(name) _TYPVAL_ENCODE_ENCODE_INNER_2(name)
 
 /// Entry point function name
-#define _TYPVAL_ENCODE_ENCODE _TYPVAL_ENCODE_ENCODE_INNER(TYPVAL_ENCODE_NAME)
+#define _TYPVAL_ENCODE_ENCODE \
-
+    _TYPVAL_ENCODE_FUNC_NAME(encode_vim_to_, )
-#define _TYPVAL_ENCODE_CONVERT_ONE_VALUE_INNER_2(name) \
-    _typval_encode_##name##_convert_one_value
-#define _TYPVAL_ENCODE_CONVERT_ONE_VALUE_INNER(name) \
-    _TYPVAL_ENCODE_CONVERT_ONE_VALUE_INNER_2(name)
 
 /// Name of the …convert_one_value function
 #define _TYPVAL_ENCODE_CONVERT_ONE_VALUE \
-    _TYPVAL_ENCODE_CONVERT_ONE_VALUE_INNER(TYPVAL_ENCODE_NAME)
+    _TYPVAL_ENCODE_FUNC_NAME(_typval_encode_, _convert_one_value)
-
-#define _TYPVAL_ENCODE_NODICT_VAR_INNER_2(name) \
-    _typval_encode_##name##_nodict_var
-#define _TYPVAL_ENCODE_NODICT_VAR_INNER(name) \
-    _TYPVAL_ENCODE_NODICT_VAR_INNER_2(name)
 
 /// Name of the dummy const dict_T *const variable
 #define TYPVAL_ENCODE_NODICT_VAR \
-    _TYPVAL_ENCODE_NODICT_VAR_INNER(TYPVAL_ENCODE_NAME)
+    _TYPVAL_ENCODE_FUNC_NAME(_typval_encode_, _nodict_var)
 
 #endif  // NVIM_EVAL_TYPVAL_ENCODE_H

src/nvim/globals.h

@@ -409,10 +409,6 @@ EXTERN struct caller_scope {
 } provider_caller_scope;
 EXTERN int provider_call_nesting INIT(= 0);
 
-/* Magic number used for hashitem "hi_key" value indicating a deleted item.
- * Only the address is used. */
-EXTERN char_u hash_removed;
-
 
 EXTERN int t_colors INIT(= 256);                // int value of T_CCO
 
@@ -636,10 +632,6 @@ EXTERN int exiting INIT(= FALSE);
 /* TRUE when planning to exit Vim.  Might
  * still keep on running if there is a changed
  * buffer. */
-#if defined(EXITFREE)
-// true when in or after free_all_mem()
-EXTERN bool entered_free_all_mem INIT(= false);
-#endif
 // volatile because it is used in signal handler deathtrap().
 EXTERN volatile int full_screen INIT(= false);
 // TRUE when doing full-screen output

src/nvim/hashtab.c

@@ -36,6 +36,8 @@
 # include "hashtab.c.generated.h"
 #endif
 
+char hash_removed;
+
 /// Initialize an empty hash table.
 void hash_init(hashtab_T *ht)
 {
@@ -380,3 +382,13 @@ hash_T hash_hash(char_u *key)
 
   return hash;
 }
+
+/// Function to get HI_KEY_REMOVED value
+///
+/// Used for testing because luajit ffi does not allow getting addresses of
+/// globals.
+const char_u *_hash_key_removed(void)
+  FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  return HI_KEY_REMOVED;
+}

src/nvim/hashtab.h

@@ -5,14 +5,19 @@
 
 #include "nvim/types.h"
 
+/// Magic number used for hashitem "hi_key" value indicating a deleted item
+///
+/// Only the address is used.
+extern char hash_removed;
+
 /// Type for hash number (hash calculation result).
 typedef size_t hash_T;
 
 /// The address of "hash_removed" is used as a magic number
 /// for hi_key to indicate a removed item.
-#define HI_KEY_REMOVED &hash_removed
+#define HI_KEY_REMOVED ((char_u *)&hash_removed)
 #define HASHITEM_EMPTY(hi) ((hi)->hi_key == NULL \
-                            || (hi)->hi_key == &hash_removed)
+                            || (hi)->hi_key == (char_u *)&hash_removed)
 
 /// A hastable item.
 ///

src/nvim/memory.c

@@ -17,16 +17,41 @@
 // Force je_ prefix on jemalloc functions.
 # define JEMALLOC_NO_DEMANGLE
 # include <jemalloc/jemalloc.h>
-# define malloc(size) je_malloc(size)
+#endif
-# define calloc(count, size) je_calloc(count, size)
+
-# define realloc(ptr, size) je_realloc(ptr, size)
+#ifdef UNIT_TESTING
-# define free(ptr) je_free(ptr)
+# define malloc(size) mem_malloc(size)
+# define calloc(count, size) mem_calloc(count, size)
+# define realloc(ptr, size) mem_realloc(ptr, size)
+# define free(ptr) mem_free(ptr)
+# ifdef HAVE_JEMALLOC
+MemMalloc mem_malloc = &je_malloc;
+MemFree mem_free = &je_free;
+MemCalloc mem_calloc = &je_calloc;
+MemRealloc mem_realloc = &je_realloc;
+# else
+MemMalloc mem_malloc = &malloc;
+MemFree mem_free = &free;
+MemCalloc mem_calloc = &calloc;
+MemRealloc mem_realloc = &realloc;
+# endif
+#else
+# ifdef HAVE_JEMALLOC
+#  define malloc(size) je_malloc(size)
+#  define calloc(count, size) je_calloc(count, size)
+#  define realloc(ptr, size) je_realloc(ptr, size)
+#  define free(ptr) je_free(ptr)
+# endif
 #endif
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "memory.c.generated.h"
 #endif
 
+#ifdef EXITFREE
+bool entered_free_all_mem = false;
+#endif
+
 /// Try to free memory. Used when trying to recover from out of memory errors.
 /// @see {xmalloc}
 void try_to_free_memory(void)
@@ -353,15 +378,15 @@ char *xstpncpy(char *restrict dst, const char *restrict src, size_t maxlen)
 size_t xstrlcpy(char *restrict dst, const char *restrict src, size_t size)
   FUNC_ATTR_NONNULL_ALL
 {
-    size_t ret = strlen(src);
+  size_t ret = strlen(src);
 
-    if (size) {
+  if (size) {
-        size_t len = (ret >= size) ? size - 1 : ret;
+    size_t len = (ret >= size) ? size - 1 : ret;
-        memcpy(dst, src, len);
+    memcpy(dst, src, len);
-        dst[len] = '\0';
+    dst[len] = '\0';
-    }
+  }
 
-    return ret;
+  return ret;
 }
 
 /// strdup() wrapper
@@ -371,6 +396,7 @@ size_t xstrlcpy(char *restrict dst, const char *restrict src, size_t size)
 /// @return pointer to a copy of the string
 char *xstrdup(const char *str)
   FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_RET
+  FUNC_ATTR_NONNULL_ALL
 {
   return xmemdupz(str, strlen(str));
 }
@@ -401,6 +427,7 @@ void *xmemrchr(const void *src, uint8_t c, size_t len)
 /// @return pointer to a copy of the string
 char *xstrndup(const char *str, size_t len)
   FUNC_ATTR_MALLOC FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_RET
+  FUNC_ATTR_NONNULL_ALL
 {
   char *p = memchr(str, '\0', len);
   return xmemdupz(str, p ? (size_t)(p - str) : len);

src/nvim/memory.h

@@ -1,9 +1,41 @@
 #ifndef NVIM_MEMORY_H
 #define NVIM_MEMORY_H
 
+#include <stdbool.h>  // for bool
 #include <stdint.h>  // for uint8_t
 #include <stddef.h>  // for size_t
-#include <time.h>    // for time_t
+#include <time.h>  // for time_t
+
+/// `malloc()` function signature
+typedef void *(*MemMalloc)(size_t);
+
+/// `free()` function signature
+typedef void (*MemFree)(void *);
+
+/// `calloc()` function signature
+typedef void *(*MemCalloc)(size_t, size_t);
+
+/// `realloc()` function signature
+typedef void *(*MemRealloc)(void *, size_t);
+
+#ifdef UNIT_TESTING
+/// When unit testing: pointer to the `malloc()` function, may be altered
+extern MemMalloc mem_malloc;
+
+/// When unit testing: pointer to the `free()` function, may be altered
+extern MemFree mem_free;
+
+/// When unit testing: pointer to the `calloc()` function, may be altered
+extern MemCalloc mem_calloc;
+
+/// When unit testing: pointer to the `realloc()` function, may be altered
+extern MemRealloc mem_realloc;
+#endif
+
+#ifdef EXITFREE
+/// Indicates that free_all_mem function was or is running
+extern bool entered_free_all_mem;
+#endif
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "memory.h.generated.h"

test/unit/eval/helpers.lua

@@ -5,7 +5,8 @@ local to_cstr = helpers.to_cstr
 local ffi = helpers.ffi
 local eq = helpers.eq
 
-local eval = cimport('./src/nvim/eval.h', './src/nvim/eval_defs.h')
+local eval = cimport('./src/nvim/eval.h', './src/nvim/eval_defs.h',
+                     './src/nvim/hashtab.h')
 
 local null_string = {[true]='NULL string'}
 local null_list = {[true]='NULL list'}
@@ -122,6 +123,32 @@ typvalt2lua = function(t, processed)
   end)(t, processed or {}))
 end
 
+local function list_iter(l)
+  local init_s = {
+    idx=0,
+    li=l.lv_first,
+  }
+  local function f(s, _)
+    -- (listitem_T *) NULL is equal to nil, but yet it is not false.
+    if s.li == nil then
+      return nil
+    end
+    local ret_li = s.li
+    s.li = s.li.li_next
+    s.idx = s.idx + 1
+    return s.idx, ret_li
+  end
+  return f, init_s, nil
+end
+
+local function list_items(l)
+  local ret = {}
+  for i, li in list_iter(l) do
+    ret[i] = li
+  end
+  return ret
+end
+
 lst2tbl = function(l, processed)
   if l == nil then
     return null_list
@@ -133,11 +160,8 @@ lst2tbl = function(l, processed)
   end
   local ret = {[type_key]=list_type}
   processed[p_key] = ret
-  local li = l.lv_first
+  for i, li in list_iter(l) do
-  -- (listitem_T *) NULL is equal to nil, but yet it is not false.
+    ret[i] = typvalt2lua(li.li_tv, processed)
-  while li ~= nil do
-    ret[#ret + 1] = typvalt2lua(li.li_tv, processed)
-    li = li.li_next
   end
   if ret[1] then
     ret[type_key] = nil
@@ -145,7 +169,9 @@ lst2tbl = function(l, processed)
   return ret
 end
 
-local function dict_iter(d)
+local hi_key_removed = eval._hash_key_removed()
+
+local function dict_iter(d, return_hi)
   local init_s = {
     todo=d.dv_hashtab.ht_used,
     hi=d.dv_hashtab.ht_array,
@@ -153,13 +179,18 @@ local function dict_iter(d)
   local function f(s, _)
     if s.todo == 0 then return nil end
     while s.todo > 0 do
-      if s.hi.hi_key ~= nil and s.hi ~= eval.hash_removed then
+      if s.hi.hi_key ~= nil and s.hi.hi_key ~= hi_key_removed then
         local key = ffi.string(s.hi.hi_key)
-        local di = ffi.cast('dictitem_T*',
+        local ret
-                            s.hi.hi_key - ffi.offsetof('dictitem_T', 'di_key'))
+        if return_hi then
+          ret = s.hi
+        else
+          ret = ffi.cast('dictitem_T*',
+                         s.hi.hi_key - ffi.offsetof('dictitem_T', 'di_key'))
+        end
         s.todo = s.todo - 1
         s.hi = s.hi + 1
-        return key, di
+        return key, ret
       end
       s.hi = s.hi + 1
     end
@@ -172,6 +203,16 @@ local function first_di(d)
   return select(2, f(init_s, v))
 end
 
+local function dict_items(d)
+  local ret = {[0]=0}
+  for k, hi in dict_iter(d) do
+    ret[k] = hi
+    ret[0] = ret[0] + 1
+    ret[ret[0]] = hi
+  end
+  return ret
+end
+
 dct2tbl = function(d, processed)
   if d == nil then
     return null_dict
@@ -324,6 +365,22 @@ lua2typvalt = function(l, processed)
   end
 end
 
+local function void(ptr)
+  return ffi.cast('void*', ptr)
+end
+
+local alloc_logging_helpers = {
+  list = function(l) return {func='calloc', args={1, ffi.sizeof('list_T')}, ret=void(l)} end,
+  li = function(li) return {func='malloc', args={ffi.sizeof('listitem_T')}, ret=void(li)} end,
+  dict = function(d) return {func='malloc', args={ffi.sizeof('dict_T')}, ret=void(d)} end,
+  di = function(di, size)
+    return {func='malloc', args={ffi.offsetof('dictitem_T', 'di_key') + size + 1}, ret=void(di)}
+  end,
+  str = function(s, size) return {func='malloc', args={size + 1}, ret=void(s)} end,
+
+  freed = function(p) return {func='free', args={p and void(p)}} end,
+}
+
 return {
   null_string=null_string,
   null_list=null_list,
@@ -350,5 +407,11 @@ return {
   li_alloc=li_alloc,
 
   dict_iter=dict_iter,
+  list_iter=list_iter,
   first_di=first_di,
+
+  alloc_logging_helpers=alloc_logging_helpers,
+
+  list_items=list_items,
+  dict_items=dict_items,
 }

test/unit/eval/tv_clear_spec.lua

@@ -0,0 +1,127 @@
+local helpers = require('test.unit.helpers')
+local eval_helpers = require('test.unit.eval.helpers')
+
+local alloc_log_new = helpers.alloc_log_new
+local cimport = helpers.cimport
+local ffi = helpers.ffi
+local eq = helpers.eq
+
+local a = eval_helpers.alloc_logging_helpers
+local type_key = eval_helpers.type_key
+local list_type = eval_helpers.list_type
+local list_items = eval_helpers.list_items
+local dict_items = eval_helpers.dict_items
+local lua2typvalt = eval_helpers.lua2typvalt
+
+local lib = cimport('./src/nvim/eval_defs.h', './src/nvim/eval.h')
+
+local alloc_log = alloc_log_new()
+
+before_each(function()
+  alloc_log:before_each()
+end)
+
+after_each(function()
+  alloc_log:after_each()
+end)
+
+describe('clear_tv()', function()
+  it('successfully frees all lists in [&l [1], *l, *l]', function()
+    local l_inner = {1}
+    local list = {l_inner, l_inner, l_inner}
+    local list_tv = ffi.gc(lua2typvalt(list), nil)
+    local list_p = list_tv.vval.v_list
+    local lis = list_items(list_p)
+    local list_inner_p = lis[1].li_tv.vval.v_list
+    local lis_inner = list_items(list_inner_p)
+    alloc_log:check({
+      a.list(list_p),
+      a.list(list_inner_p),
+      a.li(lis_inner[1]),
+      a.li(lis[1]),
+      a.li(lis[2]),
+      a.li(lis[3]),
+    })
+    eq(3, list_inner_p.lv_refcount)
+    lib.clear_tv(list_tv)
+    alloc_log:check({
+      a.freed(lis_inner[1]),
+      a.freed(list_inner_p),
+      a.freed(lis[1]),
+      a.freed(lis[2]),
+      a.freed(lis[3]),
+      a.freed(list_p),
+    })
+  end)
+  it('successfully frees all lists in [&l [], *l, *l]', function()
+    local l_inner = {[type_key]=list_type}
+    local list = {l_inner, l_inner, l_inner}
+    local list_tv = ffi.gc(lua2typvalt(list), nil)
+    local list_p = list_tv.vval.v_list
+    local lis = list_items(list_p)
+    local list_inner_p = lis[1].li_tv.vval.v_list
+    alloc_log:check({
+      a.list(list_p),
+      a.list(list_inner_p),
+      a.li(lis[1]),
+      a.li(lis[2]),
+      a.li(lis[3]),
+    })
+    eq(3, list_inner_p.lv_refcount)
+    lib.clear_tv(list_tv)
+    alloc_log:check({
+      a.freed(list_inner_p),
+      a.freed(lis[1]),
+      a.freed(lis[2]),
+      a.freed(lis[3]),
+      a.freed(list_p),
+    })
+  end)
+  it('successfully frees all dictionaries in [&d {}, *d]', function()
+    local d_inner = {}
+    local list = {d_inner, d_inner}
+    local list_tv = ffi.gc(lua2typvalt(list), nil)
+    local list_p = list_tv.vval.v_list
+    local lis = list_items(list_p)
+    local dict_inner_p = lis[1].li_tv.vval.v_dict
+    alloc_log:check({
+      a.list(list_p),
+      a.dict(dict_inner_p),
+      a.li(lis[1]),
+      a.li(lis[2]),
+    })
+    eq(2, dict_inner_p.dv_refcount)
+    lib.clear_tv(list_tv)
+    alloc_log:check({
+      a.freed(dict_inner_p),
+      a.freed(lis[1]),
+      a.freed(lis[2]),
+      a.freed(list_p),
+    })
+  end)
+  it('successfully frees all dictionaries in [&d {a: 1}, *d]', function()
+    local d_inner = {a=1}
+    local list = {d_inner, d_inner}
+    local list_tv = ffi.gc(lua2typvalt(list), nil)
+    local list_p = list_tv.vval.v_list
+    local lis = list_items(list_p)
+    local dict_inner_p = lis[1].li_tv.vval.v_dict
+    local dis = dict_items(dict_inner_p)
+    alloc_log:check({
+      a.list(list_p),
+      a.dict(dict_inner_p),
+      a.di(dis.a, 1),
+      a.li(lis[1]),
+      a.li(lis[2]),
+    })
+    eq(2, dict_inner_p.dv_refcount)
+    lib.clear_tv(list_tv)
+    alloc_log:check({
+      a.freed(dis.a),
+      a.freed(dict_inner_p),
+      a.freed(lis[1]),
+      a.freed(lis[2]),
+      a.freed(list_p),
+    })
+  end)
+end)

test/unit/helpers.lua

@@ -9,6 +9,12 @@ local neq = global_helpers.neq
 local eq = global_helpers.eq
 local ok = global_helpers.ok
 
+-- C constants.
+local NULL = ffi.cast('void*', 0)
+
+local OK = 1
+local FAIL = 0
+
 -- add some standard header locations
 for _, p in ipairs(Paths.include_paths) do
   Preprocess.add_to_include_path(p)
@@ -118,6 +124,67 @@ local function cppimport(path)
   return cimport(Paths.test_include_path .. '/' .. path)
 end
 
+local function alloc_log_new()
+  local log = {
+    log={},
+    lib=cimport('./src/nvim/memory.h'),
+    original_functions={},
+    null={['\0:is_null']=true},
+  }
+  local allocator_functions = {'malloc', 'free', 'calloc', 'realloc'}
+  function log:save_original_functions()
+    for _, funcname in ipairs(allocator_functions) do
+      self.original_functions[funcname] = self.lib['mem_' .. funcname]
+    end
+  end
+  function log:set_mocks()
+    for _, k in ipairs(allocator_functions) do
+      do
+        local kk = k
+        self.lib['mem_' .. k] = function(...)
+          local log_entry = {func=kk, args={...}}
+          self.log[#self.log + 1] = log_entry
+          if kk == 'free' then
+            self.original_functions[kk](...)
+          else
+            log_entry.ret = self.original_functions[kk](...)
+          end
+          for i, v in ipairs(log_entry.args) do
+            if v == nil then
+              -- XXX This thing thinks that {NULL} ~= {NULL}.
+              log_entry.args[i] = self.null
+            end
+          end
+          if self.hook then self:hook(log_entry) end
+          if log_entry.ret then
+            return log_entry.ret
+          end
+        end
+      end
+    end
+  end
+  function log:clear()
+    self.log = {}
+  end
+  function log:check(exp)
+    eq(exp, self.log)
+    self:clear()
+  end
+  function log:restore_original_functions()
+    for k, v in pairs(self.original_functions) do
+      self.lib['mem_' .. k] = v
+    end
+  end
+  function log:before_each()
+    log:save_original_functions()
+    log:set_mocks()
+  end
+  function log:after_each()
+    log:restore_original_functions()
+  end
+  return log
+end
+
 cimport('./src/nvim/types.h')
 
 -- take a pointer to a C-allocated string and return an interned
@@ -142,12 +209,6 @@ do
   main.event_init()
 end
 
--- C constants.
-local NULL = ffi.cast('void*', 0)
-
-local OK = 1
-local FAIL = 0
-
 return {
   cimport = cimport,
   cppimport = cppimport,
@@ -161,5 +222,6 @@ return {
   to_cstr = to_cstr,
   NULL = NULL,
   OK = OK,
-  FAIL = FAIL
+  FAIL = FAIL,
+  alloc_log_new = alloc_log_new,
 }

test/unit/preprocess.lua

@@ -123,6 +123,7 @@ function Gcc:init_defines()
   self:define('INIT', {'...'}, '')
   self:define('_GNU_SOURCE')
   self:define('INCLUDE_GENERATED_DECLARATIONS')
+  self:define('UNIT_TESTING')
   -- Needed for FreeBSD
   self:define('_Thread_local', nil, '')
   -- Needed for macOS Sierra