decor: sketch new decorations API

return decorations back

lol no nvim_buf_get_virtual_text

share decorations that are hl only to avoid alloc avalanche

src/nvim/api/buffer.c

@@ -1108,15 +1108,65 @@ ArrayOf(Integer, 2) nvim_buf_get_mark(Buffer buffer, String name, Error *err)
   return rv;
 }
 
+static Array extmark_to_array(ExtmarkInfo extmark, bool id, bool add_dict)
+{
+  Array rv = ARRAY_DICT_INIT;
+  if (id) {
+    ADD(rv, INTEGER_OBJ((Integer)extmark.mark_id));
+  }
+  ADD(rv, INTEGER_OBJ(extmark.row));
+  ADD(rv, INTEGER_OBJ(extmark.col));
+
+  if (add_dict) {
+    Dictionary dict = ARRAY_DICT_INIT;
+
+    if (extmark.end_row >= 0) {
+      PUT(dict, "end_row", INTEGER_OBJ(extmark.end_row));
+      PUT(dict, "end_col", INTEGER_OBJ(extmark.end_col));
+    }
+
+    if (extmark.decor) {
+      Decoration *decor = extmark.decor;
+      if (decor->hl_id) {
+        String name = cstr_to_string((const char *)syn_id2name(decor->hl_id));
+        PUT(dict, "hl_group", STRING_OBJ(name));
+      }
+      if (kv_size(decor->virt_text)) {
+        Array chunks = ARRAY_DICT_INIT;
+        for (size_t i = 0; i < decor->virt_text.size; i++) {
+          Array chunk = ARRAY_DICT_INIT;
+          VirtTextChunk *vtc = &decor->virt_text.items[i];
+          ADD(chunk, STRING_OBJ(cstr_to_string(vtc->text)));
+          if (vtc->hl_id > 0) {
+            ADD(chunk,
+                STRING_OBJ(cstr_to_string(
+                    (const char *)syn_id2name(vtc->hl_id))));
+          }
+          ADD(chunks, ARRAY_OBJ(chunk));
+        }
+        PUT(dict, "virt_text", ARRAY_OBJ(chunks));
+      }
+    }
+
+    if (dict.size) {
+      ADD(rv, DICTIONARY_OBJ(dict));
+    }
+  }
+
+  return rv;
+}
+
 /// Returns position for a given extmark id
 ///
 /// @param buffer  Buffer handle, or 0 for current buffer
 /// @param ns_id  Namespace id from |nvim_create_namespace()|
 /// @param id  Extmark id
+/// @param details Wether to include the details dict
 /// @param[out] err   Error details, if any
 /// @return (row, col) tuple or empty list () if extmark id was absent
 ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
-                                            Integer id, Error *err)
+                                            Integer id, Boolean details,
+                                            Error *err)
   FUNC_API_SINCE(7)
 {
   Array rv = ARRAY_DICT_INIT;
@@ -1136,9 +1186,7 @@ ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
   if (extmark.row < 0) {
     return rv;
   }
-  ADD(rv, INTEGER_OBJ((Integer)extmark.row));
+  return extmark_to_array(extmark, false, (bool)details);
-  ADD(rv, INTEGER_OBJ((Integer)extmark.col));
-  return rv;
 }
 
 /// Gets extmarks in "traversal order" from a |charwise| region defined by
@@ -1181,10 +1229,13 @@ ArrayOf(Integer) nvim_buf_get_extmark_by_id(Buffer buffer, Integer ns_id,
 ///             (whose position defines the bound)
 /// @param opts  Optional parameters. Keys:
 ///          - limit:  Maximum number of marks to return
+/// @param details Wether to include the details dict
 /// @param[out] err   Error details, if any
 /// @return List of [extmark_id, row, col] tuples in "traversal order".
-Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start,
+Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id,
-                            Object end, Dictionary opts, Error *err)
+                            Object start, Object end,
+                            Dictionary opts, Boolean details,
+                            Error *err)
   FUNC_API_SINCE(7)
 {
   Array rv = ARRAY_DICT_INIT;
@@ -1241,16 +1292,11 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start,
   }
 
 
-  ExtmarkArray marks = extmark_get(buf, (uint64_t)ns_id, l_row, l_col, u_row,
+  ExtmarkInfoArray marks = extmark_get(buf, (uint64_t)ns_id, l_row, l_col,
-                                   u_col, (int64_t)limit, reverse);
+                                       u_row, u_col, (int64_t)limit, reverse);
 
   for (size_t i = 0; i < kv_size(marks); i++) {
-    Array mark = ARRAY_DICT_INIT;
+    ADD(rv, ARRAY_OBJ(extmark_to_array(kv_A(marks, i), true, (bool)details)));
-    ExtmarkInfo extmark = kv_A(marks, i);
-    ADD(mark, INTEGER_OBJ((Integer)extmark.mark_id));
-    ADD(mark, INTEGER_OBJ(extmark.row));
-    ADD(mark, INTEGER_OBJ(extmark.col));
-    ADD(rv, ARRAY_OBJ(mark));
   }
 
   kv_destroy(marks);
@@ -1260,18 +1306,26 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start,
 /// Creates or updates an extmark.
 ///
 /// To create a new extmark, pass id=0. The extmark id will be returned.
-//  To move an existing mark, pass its id.
+/// To move an existing mark, pass its id.
 ///
 /// It is also allowed to create a new mark by passing in a previously unused
 /// id, but the caller must then keep track of existing and unused ids itself.
 /// (Useful over RPC, to avoid waiting for the return value.)
 ///
+/// Using the optional arguments, it is possible to use this to highlight
+/// a range of text, and also to associate virtual text to the mark.
+///
 /// @param buffer  Buffer handle, or 0 for current buffer
 /// @param ns_id  Namespace id from |nvim_create_namespace()|
-/// @param id  Extmark id, or 0 to create new
 /// @param line  Line number where to place the mark
 /// @param col  Column where to place the mark
-/// @param opts  Optional parameters. Currently not used.
+/// @param opts  Optional parameters.
+///               - id : id of the extmark to edit.
+///               - end_line : ending line of the mark, 0-based inclusive.
+///               - end_col : ending col of the mark, 0-based inclusive.
+///               - hl_group : name of the highlight group used to highlight
+///                   this mark.
+///               - virt_text : virtual text to link to this mark.
 /// @param[out]  err   Error details, if any
 /// @return Id of the created/updated extmark
 Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
@@ -1281,6 +1335,7 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
 {
   buf_T *buf = find_buffer_by_handle(buffer, err);
   if (!buf) {
+    api_set_error(err, kErrorTypeValidation, "Invalid buffer id");
     return 0;
   }
 
@@ -1305,6 +1360,9 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
   }
 
   uint64_t id = 0;
+  int line2 = -1, hl_id = 0;
+  colnr_T col2 = 0;
+  VirtText virt_text = KV_INITIAL_VALUE;
   for (size_t i = 0; i < opts.size; i++) {
     String k = opts.items[i].key;
     Object *v = &opts.items[i].value;
@@ -1316,19 +1374,97 @@ Integer nvim_buf_set_extmark(Buffer buffer, Integer ns_id,
       }
 
       id = (uint64_t)v->data.integer;
+    } else if (strequal("end_line", k.data)) {
+      if (v->type != kObjectTypeInteger) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_line is not an integer");
+        goto error;
+      }
+      if (v->data.integer < 0 || v->data.integer > buf->b_ml.ml_line_count) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_line value outside range");
+        goto error;
+      }
+
+      line2 = (int)v->data.integer;
+    } else if (strequal("end_col", k.data)) {
+      if (v->type != kObjectTypeInteger) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_col is not an integer");
+        goto error;
+      }
+      if (v->data.integer < 0 || v->data.integer > MAXCOL) {
+        api_set_error(err, kErrorTypeValidation,
+                      "end_col value outside range");
+        goto error;
+      }
+
+      col2 = (colnr_T)v->data.integer;
+    } else if (strequal("hl_group", k.data)) {
+      String hl_group;
+      switch (v->type) {
+        case kObjectTypeString:
+          hl_group = v->data.string;
+          hl_id = syn_check_group(
+              (char_u *)(hl_group.data),
+              (int)hl_group.size);
+          break;
+        case kObjectTypeInteger:
+          hl_id = (int)v->data.integer;
+          break;
+        default:
+          api_set_error(err, kErrorTypeValidation,
+                        "hl_group is not valid.");
+          goto error;
+      }
+    } else if (strequal("virt_text", k.data)) {
+      if (v->type != kObjectTypeArray) {
+        api_set_error(err, kErrorTypeValidation,
+                      "virt_text is not an Array");
+        goto error;
+      }
+      virt_text = parse_virt_text(v->data.array, err);
+      if (ERROR_SET(err)) {
+        goto error;
+      }
     } else {
       api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
       goto error;
     }
   }
 
+  if (col2 >= 0) {
+    if (line2 >= 0) {
+      len = STRLEN(ml_get_buf(buf, (linenr_T)line2+1, false));
+    } else {
+      // reuse len from before
+      line2 = (int)line;
+    }
+    if (col2 > (Integer)len) {
+      api_set_error(err, kErrorTypeValidation,
+                    "end_col value outside range");
+      goto error;
+    }
+  } else if (line2 >= 0) {
+    col2 = 0;
+  }
+
+  Decoration *decor = NULL;
+  if (kv_size(virt_text)) {
+    decor = xcalloc(1, sizeof(*decor));
+    decor->hl_id = hl_id;
+    decor->virt_text = virt_text;
+  } else if (hl_id) {
+    decor = decoration_hl(hl_id);
+  }
 
   id = extmark_set(buf, (uint64_t)ns_id, id,
-                   (int)line, (colnr_T)col, -1, -1, NULL, kExtmarkUndo);
+                   (int)line, (colnr_T)col, line2, col2, decor, kExtmarkUndo);
 
   return (Integer)id;
 
 error:
+  clear_virttext(&virt_text);
   return 0;
 }
 
@@ -1421,9 +1557,9 @@ Integer nvim_buf_add_highlight(Buffer buffer,
     return src_id;
   }
 
-  int hlg_id = 0;
+  int hl_id = 0;
   if (hl_group.size > 0) {
-    hlg_id = syn_check_group((char_u *)hl_group.data, (int)hl_group.size);
+    hl_id = syn_check_group((char_u *)hl_group.data, (int)hl_group.size);
   } else {
     return src_id;
   }
@@ -1434,13 +1570,10 @@ Integer nvim_buf_add_highlight(Buffer buffer,
     end_line++;
   }
 
-  Decoration *decor = xcalloc(1, sizeof(*decor));
-  decor->hl_id = hlg_id;
-
   ns_id = extmark_set(buf, ns_id, 0,
                       (int)line, (colnr_T)col_start,
                       end_line, (colnr_T)col_end,
-                      decor, kExtmarkUndo);
+                      decoration_hl(hl_id), kExtmarkUndo);
   return src_id;
 }
 
@@ -1611,114 +1744,6 @@ Integer nvim_buf_set_virtual_text(Buffer buffer,
   return src_id;
 }
 
-/// Get the virtual text (annotation) for a buffer line.
-///
-/// The virtual text is returned as list of lists, whereas the inner lists have
-/// either one or two elements. The first element is the actual text, the
-/// optional second element is the highlight group.
-///
-/// The format is exactly the same as given to nvim_buf_set_virtual_text().
-///
-/// If there is no virtual text associated with the given line, an empty list
-/// is returned.
-///
-/// @param buffer   Buffer handle, or 0 for current buffer
-/// @param line     Line to get the virtual text from (zero-indexed)
-/// @param[out] err Error details, if any
-/// @return         List of virtual text chunks
-Array nvim_buf_get_virtual_text(Buffer buffer, Integer line, Error *err)
-  FUNC_API_SINCE(7)
-{
-  Array chunks = ARRAY_DICT_INIT;
-
-  buf_T *buf = find_buffer_by_handle(buffer, err);
-  if (!buf) {
-    return chunks;
-  }
-
-  if (line < 0 || line >= MAXLNUM) {
-    api_set_error(err, kErrorTypeValidation, "Line number outside range");
-    return chunks;
-  }
-
-  VirtText *virt_text = extmark_find_virttext(buf, (int)line, 0);
-
-  if (!virt_text) {
-    return chunks;
-  }
-
-  for (size_t i = 0; i < virt_text->size; i++) {
-    Array chunk = ARRAY_DICT_INIT;
-    VirtTextChunk *vtc = &virt_text->items[i];
-    ADD(chunk, STRING_OBJ(cstr_to_string(vtc->text)));
-    if (vtc->hl_id > 0) {
-      ADD(chunk, STRING_OBJ(cstr_to_string(
-          (const char *)syn_id2name(vtc->hl_id))));
-    }
-    ADD(chunks, ARRAY_OBJ(chunk));
-  }
-
-  return chunks;
-}
-
-Integer nvim__buf_add_decoration(Buffer buffer, Integer ns_id, String hl_group,
-                                 Integer start_row, Integer start_col,
-                                 Integer end_row, Integer end_col,
-                                 Array virt_text,
-                                 Error *err)
-{
-  buf_T *buf = find_buffer_by_handle(buffer, err);
-  if (!buf) {
-    return 0;
-  }
-
-  if (!ns_initialized((uint64_t)ns_id)) {
-    api_set_error(err, kErrorTypeValidation, "Invalid ns_id");
-    return 0;
-  }
-
-
-  if (start_row < 0 || start_row >= MAXLNUM || end_row > MAXCOL) {
-    api_set_error(err, kErrorTypeValidation, "Line number outside range");
-    return 0;
-  }
-
-  if (start_col < 0 || start_col > MAXCOL || end_col > MAXCOL) {
-    api_set_error(err, kErrorTypeValidation, "Column value outside range");
-    return 0;
-  }
-  if (end_row < 0 || end_col < 0) {
-    end_row = -1;
-    end_col = -1;
-  }
-
-  if (start_row >= buf->b_ml.ml_line_count
-      || end_row >= buf->b_ml.ml_line_count) {
-    // safety check, we can't add marks outside the range
-    return 0;
-  }
-
-  int hlg_id = 0;
-  if (hl_group.size > 0) {
-    hlg_id = syn_check_group((char_u *)hl_group.data, (int)hl_group.size);
-  }
-
-  VirtText vt = parse_virt_text(virt_text, err);
-  if (ERROR_SET(err)) {
-    return 0;
-  }
-
-  Decoration *decor = xcalloc(1, sizeof(*decor));
-  decor->hl_id = hlg_id;
-  decor->virt_text = vt;
-
-  uint64_t mark_id = extmark_set(buf, (uint64_t)ns_id, 0,
-                                 (int)start_row, (colnr_T)start_col,
-                                 (int)end_row, (colnr_T)end_col, decor,
-                                 kExtmarkUndo);
-  return (Integer)mark_id;
-}
-
 Dictionary nvim__buf_stats(Buffer buffer, Error *err)
 {
   Dictionary rv = ARRAY_DICT_INIT;

src/nvim/extmark.c

@@ -43,6 +43,13 @@
 # include "extmark.c.generated.h"
 #endif
 
+static PMap(uint64_t) *hl_decors;
+
+void extmark_init(void)
+{
+  hl_decors = pmap_new(uint64_t)();
+}
+
 static ExtmarkNs *buf_ns_ref(buf_T *buf, uint64_t ns_id, bool put) {
   if (!buf->b_extmark_ns) {
     if (!put) {
@@ -291,31 +298,44 @@ bool extmark_clear(buf_T *buf, uint64_t ns_id,
 // will be searched to the start, or end
 // dir can be set to control the order of the array
 // amount = amount of marks to find or -1 for all
-ExtmarkArray extmark_get(buf_T *buf, uint64_t ns_id,
+ExtmarkInfoArray extmark_get(buf_T *buf, uint64_t ns_id,
-                         int l_row, colnr_T l_col,
+                             int l_row, colnr_T l_col,
-                         int u_row, colnr_T u_col,
+                             int u_row, colnr_T u_col,
-                         int64_t amount, bool reverse)
+                             int64_t amount, bool reverse)
 {
-  ExtmarkArray array = KV_INITIAL_VALUE;
+  ExtmarkInfoArray array = KV_INITIAL_VALUE;
-  MarkTreeIter itr[1] = { 0 };
+  MarkTreeIter itr[1];
   // Find all the marks
   marktree_itr_get_ext(buf->b_marktree, (mtpos_t){ l_row, l_col },
                        itr, reverse, false, NULL);
   int order = reverse ? -1 : 1;
   while ((int64_t)kv_size(array) < amount) {
     mtmark_t mark = marktree_itr_current(itr);
+    mtpos_t endpos = { -1, -1 };
     if (mark.row < 0
         || (mark.row - u_row) * order > 0
         || (mark.row == u_row && (mark.col - u_col) * order > 0)) {
       break;
     }
+    if (mark.id & MARKTREE_END_FLAG) {
+      goto next_mark;
+    } else if (mark.id & MARKTREE_PAIRED_FLAG) {
+      endpos = marktree_lookup(buf->b_marktree, mark.id | MARKTREE_END_FLAG,
+                               NULL);
+    }
+
+
     ExtmarkItem item = map_get(uint64_t, ExtmarkItem)(buf->b_extmark_index,
                                                       mark.id);
     if (item.ns_id == ns_id) {
       kv_push(array, ((ExtmarkInfo) { .ns_id = item.ns_id,
                                       .mark_id = item.mark_id,
-                                      .row = mark.row, .col = mark.col }));
+                                      .row = mark.row, .col = mark.col,
+                                      .end_row = endpos.row,
+                                      .end_col = endpos.col,
+                                      .decor = item.decor }));
     }
+next_mark:
     if (reverse) {
       marktree_itr_prev(buf->b_marktree, itr);
     } else {
@@ -329,7 +349,7 @@ ExtmarkArray extmark_get(buf_T *buf, uint64_t ns_id,
 ExtmarkInfo extmark_from_id(buf_T *buf, uint64_t ns_id, uint64_t id)
 {
   ExtmarkNs *ns = buf_ns_ref(buf, ns_id, false);
-  ExtmarkInfo ret = { 0, 0, -1, -1 };
+  ExtmarkInfo ret = { 0, 0, -1, -1, -1, -1, NULL };
   if (!ns) {
     return ret;
   }
@@ -340,12 +360,22 @@ ExtmarkInfo extmark_from_id(buf_T *buf, uint64_t ns_id, uint64_t id)
   }
 
   mtpos_t pos = marktree_lookup(buf->b_marktree, mark, NULL);
+  mtpos_t endpos = { -1, -1 };
+  if (mark & MARKTREE_PAIRED_FLAG) {
+    endpos = marktree_lookup(buf->b_marktree, mark | MARKTREE_END_FLAG, NULL);
+  }
   assert(pos.row >= 0);
 
+  ExtmarkItem item = map_get(uint64_t, ExtmarkItem)(buf->b_extmark_index,
+                                                    mark);
+
   ret.ns_id = ns_id;
   ret.mark_id = id;
   ret.row = pos.row;
   ret.col = pos.col;
+  ret.end_row = endpos.row;
+  ret.end_col = endpos.col;
+  ret.decor = item.decor;
 
   return ret;
 }
@@ -682,6 +712,7 @@ void bufhl_add_hl_pos_offset(buf_T *buf,
 {
   colnr_T hl_start = 0;
   colnr_T hl_end = 0;
+  Decoration *decor = decoration_hl(hl_id);
 
   // TODO(bfredl): if decoration had blocky mode, we could avoid this loop
   for (linenr_T lnum = pos_start.lnum; lnum <= pos_end.lnum; lnum ++) {
@@ -706,14 +737,28 @@ void bufhl_add_hl_pos_offset(buf_T *buf,
       hl_start = pos_start.col + offset;
       hl_end = pos_end.col + offset;
     }
-    Decoration *decor = xcalloc(1, sizeof(*decor));
-    decor->hl_id = hl_id;
     (void)extmark_set(buf, (uint64_t)src_id, 0,
                       (int)lnum-1, hl_start, (int)lnum-1+end_off, hl_end,
                       decor, kExtmarkUndo);
   }
 }
 
+Decoration *decoration_hl(int hl_id)
+{
+  assert(hl_id > 0);
+  Decoration **dp = (Decoration **)pmap_ref(uint64_t)(hl_decors,
+                                                      (uint64_t)hl_id, true);
+  if (*dp) {
+    return *dp;
+  }
+
+  Decoration *decor = xcalloc(1, sizeof(*decor));
+  decor->hl_id = hl_id;
+  decor->shared = true;
+  *dp = decor;
+  return decor;
+}
+
 void decoration_redraw(buf_T *buf, int row1, int row2, Decoration *decor)
 {
   if (decor->hl_id && row2 >= row1) {
@@ -727,7 +772,7 @@ void decoration_redraw(buf_T *buf, int row1, int row2, Decoration *decor)
 
 void free_decoration(Decoration *decor)
 {
-  if (decor) {
+  if (decor && !decor->shared) {
     clear_virttext(&decor->virt_text);
     xfree(decor);
   }

src/nvim/extmark.h

@@ -13,9 +13,12 @@ typedef struct
   uint64_t mark_id;
   int row;
   colnr_T col;
+  int end_row;
+  colnr_T end_col;
+  Decoration *decor;
 } ExtmarkInfo;
 
-typedef kvec_t(ExtmarkInfo) ExtmarkArray;
+typedef kvec_t(ExtmarkInfo) ExtmarkInfoArray;
 
 
 // delete the columns between mincol and endcol

src/nvim/extmark_defs.h

@@ -17,6 +17,7 @@ typedef struct
   int hl_id;  // highlight group
   VirtText virt_text;
   // TODO(bfredl): style, signs, etc
+  bool shared;  // shared decoration, don't free
 } Decoration;
 
 typedef struct

src/nvim/main.c

@@ -21,6 +21,7 @@
 #include "nvim/ex_cmds.h"
 #include "nvim/ex_cmds2.h"
 #include "nvim/ex_docmd.h"
+#include "nvim/extmark.h"
 #include "nvim/fileio.h"
 #include "nvim/fold.h"
 #include "nvim/getchar.h"
@@ -160,6 +161,7 @@ void early_init(mparm_T *paramp)
   env_init();
   fs_init();
   handle_init();
+  extmark_init();
   eval_init();          // init global variables
   init_path(argv0 ? argv0 : "nvim");
   init_normal_cmds();   // Init the table of Normal mode commands.

src/nvim/map.h

@@ -73,6 +73,7 @@ MAP_DECLS(String, handle_T)
 #define pmap_has(T) map_has(T, ptr_t)
 #define pmap_key(T) map_key(T, ptr_t)
 #define pmap_put(T) map_put(T, ptr_t)
+#define pmap_ref(T) map_ref(T, ptr_t)
 /// @see pmap_del2
 #define pmap_del(T) map_del(T, ptr_t)
 #define pmap_clear(T) map_clear(T, ptr_t)

test/functional/api/extmark_spec.lua

@@ -18,13 +18,13 @@ local function expect(contents)
 end
 
 local function check_undo_redo(ns, mark, sr, sc, er, ec) --s = start, e = end
-  local rv = curbufmeths.get_extmark_by_id(ns, mark)
+  local rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({er, ec}, rv)
   feed("u")
-  rv = curbufmeths.get_extmark_by_id(ns, mark)
+  rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({sr, sc}, rv)
   feed("<c-r>")
-  rv = curbufmeths.get_extmark_by_id(ns, mark)
+  rv = curbufmeths.get_extmark_by_id(ns, mark, false)
   eq({er, ec}, rv)
 end
 
@@ -42,7 +42,7 @@ local function get_extmarks(ns_id, start, end_, opts)
   if opts == nil then
     opts = {}
   end
-  return curbufmeths.get_extmarks(ns_id, start, end_, opts)
+  return curbufmeths.get_extmarks(ns_id, start, end_, opts, false)
 end
 
 local function batch_set(ns_id, positions)
@@ -96,7 +96,7 @@ describe('API/extmarks', function()
   it('adds, updates  and deletes marks', function()
     local rv = set_extmark(ns, marks[1], positions[1][1], positions[1][2])
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({positions[1][1], positions[1][2]}, rv)
     -- Test adding a second mark on same row works
     rv = set_extmark(ns, marks[2], positions[2][1], positions[2][2])
@@ -105,14 +105,14 @@ describe('API/extmarks', function()
     -- Test an update, (same pos)
     rv = set_extmark(ns, marks[1], positions[1][1], positions[1][2])
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[2])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[2], false)
     eq({positions[2][1], positions[2][2]}, rv)
     -- Test an update, (new pos)
     row = positions[1][1]
     col = positions[1][2] + 1
     rv = set_extmark(ns, marks[1], row, col)
     eq(marks[1], rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({row, col}, rv)
 
     -- remove the test marks
@@ -435,7 +435,7 @@ describe('API/extmarks', function()
       ~              |
                      |
     ]])
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 6}, rv)
     check_undo_redo(ns, marks[1], 0, 3, 0, 6)
   end)
@@ -909,9 +909,9 @@ describe('API/extmarks', function()
     -- Set the mark before the cursor, should stay there
     set_extmark(ns, marks[2], 0, 10)
     feed("i<cr><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[2])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[2], false)
     eq({0, 10}, rv)
     check_undo_redo(ns, marks[1], 0, 12, 1, 3)
   end)
@@ -924,12 +924,12 @@ describe('API/extmarks', function()
     feed("0iint <esc>A {<cr><esc>0i1M1<esc>")
     set_extmark(ns, marks[1], 1, 1)
     feed("0i<c-f><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
     check_undo_redo(ns, marks[1], 1, 1, 1, 3)
     -- now check when cursor at eol
     feed("uA<c-f><esc>")
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({1, 3}, rv)
   end)
 
@@ -940,12 +940,12 @@ describe('API/extmarks', function()
     feed("0i<tab><esc>")
     set_extmark(ns, marks[1], 0, 3)
     feed("bi<c-d><esc>")
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 1}, rv)
     check_undo_redo(ns, marks[1], 0, 3, 0, 1)
     -- check when cursor at eol
     feed("uA<c-d><esc>")
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, 1}, rv)
   end)
 
@@ -1075,7 +1075,7 @@ describe('API/extmarks', function()
     check_undo_redo(ns, marks[5], 2, 0, 3, 0)
     feed('u')
     feed([[:1,2s:3:\rxx<cr>]])
-    eq({1, 3}, curbufmeths.get_extmark_by_id(ns, marks[3]))
+    eq({1, 3}, curbufmeths.get_extmark_by_id(ns, marks[3], false))
   end)
 
   it('substitions over multiple lines with replace in substition', function()
@@ -1314,16 +1314,16 @@ describe('API/extmarks', function()
     eq("Invalid ns_id", pcall_err(set_extmark, ns_invalid, marks[1], positions[1][1], positions[1][2]))
     eq("Invalid ns_id", pcall_err(curbufmeths.del_extmark, ns_invalid, marks[1]))
     eq("Invalid ns_id", pcall_err(get_extmarks, ns_invalid, positions[1], positions[2]))
-    eq("Invalid ns_id", pcall_err(curbufmeths.get_extmark_by_id, ns_invalid, marks[1]))
+    eq("Invalid ns_id", pcall_err(curbufmeths.get_extmark_by_id, ns_invalid, marks[1], false))
   end)
 
   it('when col = line-length, set the mark on eol', function()
     set_extmark(ns, marks[1], 0, -1)
-    local rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    local rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, init_text:len()}, rv)
     -- Test another
     set_extmark(ns, marks[1], 0, -1)
-    rv = curbufmeths.get_extmark_by_id(ns, marks[1])
+    rv = curbufmeths.get_extmark_by_id(ns, marks[1], false)
     eq({0, init_text:len()}, rv)
   end)
 
@@ -1336,7 +1336,7 @@ describe('API/extmarks', function()
     local invalid_col = init_text:len() + 1
     local invalid_lnum = 3
     eq('line value outside range', pcall_err(set_extmark, ns, marks[1], invalid_lnum, invalid_col))
-    eq({}, curbufmeths.get_extmark_by_id(ns, marks[1]))
+    eq({}, curbufmeths.get_extmark_by_id(ns, marks[1], false))
   end)
 
   it('bug from check_col in extmark_set', function()
@@ -1361,7 +1361,7 @@ describe('API/extmarks', function()
     local buf = request('nvim_create_buf', 0, 1)
     request('nvim_buf_set_lines', buf, 0, -1, 1, {"", ""})
     local id = bufmeths.set_extmark(buf, ns, 1, 0, {})
-    eq({{id, 1, 0}}, bufmeths.get_extmarks(buf, ns, 0, -1, {}))
+    eq({{id, 1, 0}}, bufmeths.get_extmarks(buf, ns, 0, -1, {}, false))
   end)
 
   it('does not crash with append/delete/undo seqence', function()

test/functional/ui/bufhl_spec.lua

@@ -690,7 +690,7 @@ describe('Buffer highlighting', function()
     end)
 
     it('can be retrieved', function()
-      local get_virtual_text = curbufmeths.get_virtual_text
+      local get_extmarks = curbufmeths.get_extmarks
       local line_count = curbufmeths.line_count
 
       local s1 = {{'Köttbullar', 'Comment'}, {'Kräuterbutter'}}
@@ -699,12 +699,14 @@ describe('Buffer highlighting', function()
       -- TODO: only a virtual text from the same ns curretly overrides
       -- an existing virtual text. We might add a prioritation system.
       set_virtual_text(id1, 0, s1, {})
-      eq(s1, get_virtual_text(0))
+      eq({{1, 0, 0, {virt_text = s1}}}, get_extmarks(id1, {0,0}, {0, -1}, {}, true))
 
-      set_virtual_text(-1, line_count(), s2, {})
+      -- TODO: is this really valid? shouldn't the max be line_count()-1?
-      eq(s2, get_virtual_text(line_count()))
+      local lastline = line_count()
+      set_virtual_text(id1, line_count(), s2, {})
+      eq({{3, lastline, 0, {virt_text = s2}}}, get_extmarks(id1, {lastline,0}, {lastline, -1}, {}, true))
 
-      eq({}, get_virtual_text(line_count() + 9000))
+      eq({}, get_extmarks(id1, {lastline+9000,0}, {lastline+9000, -1}, {}, false))
     end)
 
     it('is not highlighted by visual selection', function()