api/buffer: add "on_bytes" callback to nvim_buf_attach

This implements byte-resolution updates of buffer changes.
Note: there is no promise that the buffer state is valid inside
the callback!

src/nvim/api/buffer.c

@@ -175,8 +175,7 @@ Boolean nvim_buf_attach(uint64_t channel_id,
       }
       cb.on_lines = v->data.luaref;
       v->data.luaref = LUA_NOREF;
-    } else if (is_lua && strequal("_on_bytes", k.data)) {
+    } else if (is_lua && strequal("on_bytes", k.data)) {
-      // NB: undocumented, untested and incomplete interface!
       if (v->type != kObjectTypeLuaRef) {
         api_set_error(err, kErrorTypeValidation, "callback is not a function");
         goto error;
@@ -1796,6 +1795,7 @@ Dictionary nvim__buf_stats(Buffer buffer, Error *err)
   // NB: this should be zero at any time API functions are called,
   // this exists to debug issues
   PUT(rv, "dirty_bytes", INTEGER_OBJ((Integer)buf->deleted_bytes));
+  PUT(rv, "dirty_bytes2", INTEGER_OBJ((Integer)buf->deleted_bytes2));
 
   u_header_T *uhp = NULL;
   if (buf->b_u_curhead != NULL) {

src/nvim/buffer_defs.h

@@ -835,6 +835,7 @@ struct file_buffer {
   // tree-sitter) or the corresponding UTF-32/UTF-16 size (like LSP) of the
   // deleted text.
   size_t deleted_bytes;
+  size_t deleted_bytes2;
   size_t deleted_codepoints;
   size_t deleted_codeunits;
 

src/nvim/buffer_updates.c

@@ -2,6 +2,7 @@
 // it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com
 
 #include "nvim/buffer_updates.h"
+#include "nvim/extmark.h"
 #include "nvim/memline.h"
 #include "nvim/api/private/helpers.h"
 #include "nvim/msgpack_rpc/channel.h"
@@ -282,9 +283,9 @@ void buf_updates_send_changes(buf_T *buf,
 }
 
 void buf_updates_send_splice(buf_T *buf,
-                             linenr_T start_line, colnr_T start_col,
+                             int start_row, colnr_T start_col, bcount_t start_byte,
-                             linenr_T oldextent_line, colnr_T oldextent_col,
+                             int old_row, colnr_T old_col, bcount_t old_byte,
-                             linenr_T newextent_line, colnr_T newextent_col)
+                             int new_row, colnr_T new_col, bcount_t new_byte)
 {
   if (!buf_updates_active(buf)) {
     return;
@@ -296,7 +297,7 @@ void buf_updates_send_splice(buf_T *buf,
     BufUpdateCallbacks cb = kv_A(buf->update_callbacks, i);
     bool keep = true;
     if (cb.on_bytes != LUA_NOREF) {
-      FIXED_TEMP_ARRAY(args, 8);
+      FIXED_TEMP_ARRAY(args, 11);
 
       // the first argument is always the buffer handle
       args.items[0] = BUFFER_OBJ(buf->handle);
@@ -304,12 +305,15 @@ void buf_updates_send_splice(buf_T *buf,
       // next argument is b:changedtick
       args.items[1] = INTEGER_OBJ(buf_get_changedtick(buf));
 
-      args.items[2] = INTEGER_OBJ(start_line);
+      args.items[2] = INTEGER_OBJ(start_row);
       args.items[3] = INTEGER_OBJ(start_col);
-      args.items[4] = INTEGER_OBJ(oldextent_line);
+      args.items[4] = INTEGER_OBJ(start_byte);
-      args.items[5] = INTEGER_OBJ(oldextent_col);
+      args.items[5] = INTEGER_OBJ(old_row);
-      args.items[6] = INTEGER_OBJ(newextent_line);
+      args.items[6] = INTEGER_OBJ(old_col);
-      args.items[7] = INTEGER_OBJ(newextent_col);
+      args.items[7] = INTEGER_OBJ(old_byte);
+      args.items[8] = INTEGER_OBJ(new_row);
+      args.items[9] = INTEGER_OBJ(new_col);
+      args.items[10] = INTEGER_OBJ(new_byte);
 
       textlock++;
       Object res = executor_exec_lua_cb(cb.on_bytes, "bytes", args, true, NULL);

src/nvim/buffer_updates.h

@@ -2,6 +2,7 @@
 #define NVIM_BUFFER_UPDATES_H
 
 #include "nvim/buffer_defs.h"
+#include "nvim/extmark.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "buffer_updates.h.generated.h"

src/nvim/change.c

@@ -1597,7 +1597,7 @@ int open_line(
     if (curwin->w_cursor.lnum + 1 < curbuf->b_ml.ml_line_count
         || curwin->w_p_diff) {
       mark_adjust(curwin->w_cursor.lnum + 1, (linenr_T)MAXLNUM, 1L, 0L,
-                  kExtmarkUndo);
+                  kExtmarkNOOP);
     }
     did_append = true;
   } else {
@@ -1611,6 +1611,7 @@ int open_line(
     }
     ml_replace(curwin->w_cursor.lnum, p_extra, true);
     changed_bytes(curwin->w_cursor.lnum, 0);
+    // TODO: extmark_splice_cols here??
     curwin->w_cursor.lnum--;
     did_append = false;
   }
@@ -1691,8 +1692,9 @@ int open_line(
         // Always move extmarks - Here we move only the line where the
         // cursor is, the previous mark_adjust takes care of the lines after
         int cols_added = mincol-1+less_cols_off-less_cols;
-        extmark_splice(curbuf, (int)lnum-1, mincol-1, 0, less_cols_off,
+        extmark_splice(curbuf, (int)lnum-1, mincol-1,
-                       1, cols_added, kExtmarkUndo);
+                       0, less_cols_off, less_cols_off,
+                       1, cols_added, 1 + cols_added, kExtmarkUndo);
       } else {
         changed_bytes(curwin->w_cursor.lnum, curwin->w_cursor.col);
       }
@@ -1704,8 +1706,10 @@ int open_line(
   }
   if (did_append) {
     changed_lines(curwin->w_cursor.lnum, 0, curwin->w_cursor.lnum, 1L, true);
-    extmark_splice(curbuf, (int)curwin->w_cursor.lnum-1,
+    // bail out and just get the final lenght of the line we just manipulated
-                   0, 0, 0, 1, 0, kExtmarkUndo);
+    bcount_t extra = (bcount_t)STRLEN(ml_get(curwin->w_cursor.lnum));
+    extmark_splice(curbuf, (int)curwin->w_cursor.lnum-1, 0,
+                   0, 0, 0, 1, 0, 1+extra, kExtmarkUndo);
   }
   curbuf_splice_pending--;
 

src/nvim/ex_cmds.c

@@ -851,6 +851,11 @@ int do_move(linenr_T line1, linenr_T line2, linenr_T dest)
     return OK;
   }
 
+  bcount_t start_byte = ml_find_line_or_offset(curbuf, line1, NULL, true);
+  bcount_t end_byte = ml_find_line_or_offset(curbuf, line2+1, NULL, true);
+  bcount_t extent_byte = end_byte-start_byte;
+  bcount_t dest_byte = ml_find_line_or_offset(curbuf, dest+1, NULL, true);
+
   num_lines = line2 - line1 + 1;
 
   /*
@@ -885,6 +890,8 @@ int do_move(linenr_T line1, linenr_T line2, linenr_T dest)
   last_line = curbuf->b_ml.ml_line_count;
   mark_adjust_nofold(line1, line2, last_line - line2, 0L, kExtmarkNOOP);
   changed_lines(last_line - num_lines + 1, 0, last_line + 1, num_lines, false);
+  int line_off = 0;
+  bcount_t byte_off = 0;
   if (dest >= line2) {
     mark_adjust_nofold(line2 + 1, dest, -num_lines, 0L, kExtmarkNOOP);
     FOR_ALL_TAB_WINDOWS(tab, win) {
@@ -894,6 +901,8 @@ int do_move(linenr_T line1, linenr_T line2, linenr_T dest)
     }
     curbuf->b_op_start.lnum = dest - num_lines + 1;
     curbuf->b_op_end.lnum = dest;
+    line_off = -num_lines;
+    byte_off = -extent_byte;
   } else {
     mark_adjust_nofold(dest + 1, line1 - 1, num_lines, 0L, kExtmarkNOOP);
     FOR_ALL_TAB_WINDOWS(tab, win) {
@@ -909,11 +918,10 @@ int do_move(linenr_T line1, linenr_T line2, linenr_T dest)
                      -(last_line - dest - extra), 0L, kExtmarkNOOP);
 
   // extmarks are handled separately
-  int size = line2-line1+1;
+  extmark_move_region(curbuf, line1-1, 0, start_byte,
-  int off = dest >= line2 ? -size : 0;
+                      line2-line1+1, 0, extent_byte,
-  extmark_move_region(curbuf, line1-1, 0,
+                      dest+line_off, 0, dest_byte+byte_off,
-                      line2-line1+1, 0,
+                      kExtmarkUndo);
-                      dest+off, 0, kExtmarkUndo);
 
   changed_lines(last_line - num_lines + 1, 0, last_line + 1, -extra, false);
 
@@ -3913,6 +3921,18 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout,
 
           ADJUST_SUB_FIRSTLNUM();
 
+          // TODO(bfredl): adjust also in preview, because decorations?
+          // this has some robustness issues, will look into later.
+          bool do_splice = !preview;
+          bcount_t replaced_bytes = 0;
+          lpos_T start = regmatch.startpos[0], end = regmatch.endpos[0];
+          if (do_splice) {
+            for (i = 0; i < nmatch-1; i++) {
+              replaced_bytes += STRLEN(ml_get(lnum_start+i)) + 1;
+            }
+            replaced_bytes += end.col - start.col;
+          }
+
 
           // Now the trick is to replace CTRL-M chars with a real line
           // break.  This would make it impossible to insert a CTRL-M in
@@ -3956,16 +3976,13 @@ static buf_T *do_sub(exarg_T *eap, proftime_T timeout,
           current_match.end.col = new_endcol;
           current_match.end.lnum = lnum;
 
-          // TODO(bfredl): adjust in preview, because decorations?
+          if (do_splice) {
-          // this has some robustness issues, will look into later.
-          if (!preview) {
-            lpos_T start = regmatch.startpos[0], end = regmatch.endpos[0];
             int matchcols = end.col - ((end.lnum == start.lnum)
                                        ? start.col : 0);
             int subcols = new_endcol - ((lnum == lnum_start) ? start_col : 0);
             extmark_splice(curbuf, lnum_start-1, start_col,
-                           end.lnum-start.lnum, matchcols,
+                           end.lnum-start.lnum, matchcols, replaced_bytes,
-                           lnum-lnum_start, subcols, kExtmarkUndo);
+                           lnum-lnum_start, subcols, sublen-1, kExtmarkUndo);
           }
         }
 

src/nvim/extmark.c

@@ -479,17 +479,17 @@ void extmark_apply_undo(ExtmarkUndoObject undo_info, bool undo)
     // Undo
     ExtmarkSplice splice = undo_info.data.splice;
     if (undo) {
-      extmark_splice(curbuf,
+      extmark_splice_impl(curbuf,
-                     splice.start_row, splice.start_col,
+                          splice.start_row, splice.start_col, splice.start_byte,
-                     splice.new_row, splice.new_col,
+                          splice.new_row, splice.new_col, splice.new_byte,
-                     splice.old_row, splice.old_col,
+                          splice.old_row, splice.old_col, splice.old_byte,
                           kExtmarkNoUndo);
 
     } else {
-      extmark_splice(curbuf,
+      extmark_splice_impl(curbuf,
-                     splice.start_row, splice.start_col,
+                          splice.start_row, splice.start_col, splice.start_byte,
-                     splice.old_row, splice.old_col,
+                          splice.old_row, splice.old_col, splice.old_byte,
-                     splice.new_row, splice.new_col,
+                          splice.new_row, splice.new_col, splice.new_byte,
                           kExtmarkNoUndo);
     }
   // kExtmarkSavePos
@@ -509,15 +509,15 @@ void extmark_apply_undo(ExtmarkUndoObject undo_info, bool undo)
     ExtmarkMove move = undo_info.data.move;
     if (undo) {
       extmark_move_region(curbuf,
-                          move.new_row, move.new_col,
+                          move.new_row, move.new_col, move.new_byte,
-                          move.extent_row, move.extent_col,
+                          move.extent_row, move.extent_col, move.extent_byte,
-                          move.start_row, move.start_col,
+                          move.start_row, move.start_col, move.start_byte,
                           kExtmarkNoUndo);
     } else {
       extmark_move_region(curbuf,
-                          move.start_row, move.start_col,
+                          move.start_row, move.start_col, move.start_byte,
-                          move.extent_row, move.extent_col,
+                          move.extent_row, move.extent_col, move.extent_byte,
-                          move.new_row, move.new_col,
+                          move.new_row, move.new_col, move.new_byte,
                           kExtmarkNoUndo);
     }
   }
@@ -532,36 +532,57 @@ void extmark_adjust(buf_T *buf,
                     long amount_after,
                     ExtmarkOp undo)
 {
-  if (!curbuf_splice_pending) {
+  if (curbuf_splice_pending) {
-    int old_extent, new_extent;
+    return;
+  }
+  bcount_t start_byte = ml_find_line_or_offset(buf, line1, NULL, true);
+  bcount_t old_byte = 0, new_byte = 0;
+  int old_row, new_row;
   if (amount == MAXLNUM) {
-      old_extent = (int)(line2 - line1+1);
+    old_row = (int)(line2 - line1+1);
-      new_extent = (int)(amount_after + old_extent);
+    // TODO: ej kasta?
+    old_byte = (bcount_t)buf->deleted_bytes2;
+
+    new_row = (int)(amount_after + old_row);
   } else {
     // A region is either deleted (amount == MAXLNUM) or
     // added (line2 == MAXLNUM). The only other case is :move
     // which is handled by a separate entry point extmark_move_region.
     assert(line2 == MAXLNUM);
-      old_extent = 0;
+    old_row = 0;
-      new_extent = (int)amount;
+    new_row = (int)amount;
   }
-    extmark_splice(buf,
+  if (new_row > 0) {
-                   (int)line1-1, 0,
+    new_byte = ml_find_line_or_offset(buf, line1+new_row, NULL, true)-start_byte;
-                   old_extent, 0,
-                   new_extent, 0, undo);
   }
+  extmark_splice_impl(buf,
+                      (int)line1-1, 0, start_byte,
+                      old_row, 0, old_byte,
+                      new_row, 0, new_byte, undo);
 }
 
-
 void extmark_splice(buf_T *buf,
                          int start_row, colnr_T start_col,
-                    int old_row, colnr_T old_col,
+                         int old_row, colnr_T old_col, bcount_t old_byte,
-                    int new_row, colnr_T new_col,
+                         int new_row, colnr_T new_col, bcount_t new_byte,
                          ExtmarkOp undo)
 {
-  buf_updates_send_splice(buf, start_row, start_col,
+  long offset = ml_find_line_or_offset(buf, start_row+1, NULL, true);
-                          old_row, old_col,
+  extmark_splice_impl(buf, start_row, start_col, offset+start_col,
-                          new_row, new_col);
+                      old_row, old_col, old_byte, new_row, new_col, new_byte,
+                      undo);
+}
+
+void extmark_splice_impl(buf_T *buf,
+                         int start_row, colnr_T start_col, bcount_t start_byte,
+                         int old_row, colnr_T old_col, bcount_t old_byte,
+                         int new_row, colnr_T new_col, bcount_t new_byte,
+                         ExtmarkOp undo)
+{
+  curbuf->deleted_bytes2 = 0;
+  buf_updates_send_splice(buf, start_row, start_col, start_byte,
+                          old_row, old_col, old_byte,
+                          new_row, new_col, new_byte);
 
   if (undo == kExtmarkUndo && (old_row > 0 || old_col > 0)) {
     // Copy marks that would be effected by delete
@@ -599,15 +620,19 @@ void extmark_splice(buf_T *buf,
           if (old_col == 0 && start_col >= splice->start_col
               && start_col <= splice->start_col+splice->new_col) {
             splice->new_col += new_col;
+            splice->new_byte += new_byte;
             merged = true;
           } else if (new_col == 0
                      && start_col == splice->start_col+splice->new_col) {
             splice->old_col += old_col;
+            splice->old_byte += old_byte;
             merged = true;
           } else if (new_col == 0
                      && start_col + old_col == splice->start_col) {
             splice->start_col = start_col;
+            splice->start_byte = start_byte;
             splice->old_col += old_col;
+            splice->old_byte += old_byte;
             merged = true;
           }
         }
@@ -618,10 +643,13 @@ void extmark_splice(buf_T *buf,
       ExtmarkSplice splice;
       splice.start_row = start_row;
       splice.start_col = start_col;
+      splice.start_byte = start_byte;
       splice.old_row = old_row;
       splice.old_col = old_col;
+      splice.old_byte = old_byte;
       splice.new_row = new_row;
       splice.new_col = new_col;
+      splice.new_byte = new_byte;
 
       kv_push(uhp->uh_extmark,
               ((ExtmarkUndoObject){ .type = kExtmarkSplice,
@@ -635,29 +663,31 @@ void extmark_splice_cols(buf_T *buf,
                          colnr_T old_col, colnr_T new_col,
                          ExtmarkOp undo)
 {
-  extmark_splice(buf, start_row, start_col, 0, old_col, 0, new_col, undo);
+  extmark_splice(buf, start_row, start_col,
+                 0, old_col, old_col,
+                 0, new_col, new_col, undo);
 }
 
 void extmark_move_region(buf_T *buf,
-                         int start_row, colnr_T start_col,
+                         int start_row, colnr_T start_col, bcount_t start_byte,
-                         int extent_row, colnr_T extent_col,
+                         int extent_row, colnr_T extent_col, bcount_t extent_byte,
-                         int new_row, colnr_T new_col,
+                         int new_row, colnr_T new_col, bcount_t new_byte,
                          ExtmarkOp undo)
 {
   // TODO(bfredl): this is not synced to the buffer state inside the callback.
   // But unless we make the undo implementation smarter, this is not ensured
   // anyway.
-  buf_updates_send_splice(buf, start_row, start_col,
+  buf_updates_send_splice(buf, start_row, start_col, start_byte,
-                          extent_row, extent_col,
+                          extent_row, extent_col, extent_byte,
-                          0, 0);
+                          0, 0, 0);
 
   marktree_move_region(buf->b_marktree, start_row, start_col,
                        extent_row, extent_col,
                        new_row, new_col);
 
-  buf_updates_send_splice(buf, new_row, new_col,
+  buf_updates_send_splice(buf, new_row, new_col, new_byte,
-                          0, 0,
+                          0, 0, 0,
-                          extent_row, extent_col);
+                          extent_row, extent_col, extent_byte);
 
 
   if (undo == kExtmarkUndo) {
@@ -669,10 +699,13 @@ void extmark_move_region(buf_T *buf,
     ExtmarkMove move;
     move.start_row = start_row;
     move.start_col = start_col;
+    move.start_byte = start_byte;
     move.extent_row = extent_row;
     move.extent_col = extent_col;
+    move.extent_byte = extent_byte;
     move.new_row = new_row;
     move.new_col = new_col;
+    move.new_byte = new_byte;
 
     kv_push(uhp->uh_extmark,
             ((ExtmarkUndoObject){ .type = kExtmarkMove,

src/nvim/extmark.h

@@ -20,6 +20,9 @@ typedef struct
 
 typedef kvec_t(ExtmarkInfo) ExtmarkInfoArray;
 
+// TODO(bfredl): good enough name for now.
+typedef ptrdiff_t bcount_t;
+
 
 // delete the columns between mincol and endcol
 typedef struct {
@@ -29,9 +32,9 @@ typedef struct {
   colnr_T old_col;
   int new_row;
   colnr_T new_col;
-  size_t start_byte;
+  bcount_t start_byte;
-  size_t old_byte;
+  bcount_t old_byte;
-  size_t new_byte;
+  bcount_t new_byte;
 } ExtmarkSplice;
 
 // adjust marks after :move operation
@@ -42,6 +45,9 @@ typedef struct {
   int extent_col;
   int new_row;
   int new_col;
+  bcount_t start_byte;
+  bcount_t extent_byte;
+  bcount_t new_byte;
 } ExtmarkMove;
 
 // extmark was updated

src/nvim/fileio.c

@@ -1797,6 +1797,7 @@ failed:
       linecnt--;
     }
     curbuf->deleted_bytes = 0;
+    curbuf->deleted_bytes2 = 0;
     curbuf->deleted_codepoints = 0;
     curbuf->deleted_codeunits = 0;
     linecnt = curbuf->b_ml.ml_line_count - linecnt;

src/nvim/indent.c

@@ -297,10 +297,9 @@ int set_indent(int size, int flags)
   if (!(flags & SIN_UNDO) || (u_savesub(curwin->w_cursor.lnum) == OK)) {
     ml_replace(curwin->w_cursor.lnum, newline, false);
     if (!(flags & SIN_NOMARK)) {
-      extmark_splice(curbuf,
+      extmark_splice_cols(curbuf,
                      (int)curwin->w_cursor.lnum-1, skipcols,
-                     0, (int)(p-oldline) - skipcols,
+                     (int)(p-oldline) - skipcols, (int)(s-newline) - skipcols,
-                     0, (int)(s-newline) - skipcols,
                      kExtmarkUndo);
     }
 

src/nvim/memline.c

@@ -2405,12 +2405,13 @@ void ml_add_deleted_len_buf(buf_T *buf, char_u *ptr, ssize_t len)
   if (len == -1) {
     len = STRLEN(ptr);
   }
-  buf->deleted_bytes += len+1;
+  curbuf->deleted_bytes += len+1;
-  if (buf->update_need_codepoints) {
+  curbuf->deleted_bytes2 += len+1;
-    mb_utflen(ptr, len, &buf->deleted_codepoints,
+  if (curbuf->update_need_codepoints) {
-              &buf->deleted_codeunits);
+    mb_utflen(ptr, len, &curbuf->deleted_codepoints,
-    buf->deleted_codepoints++;  // NL char
+              &curbuf->deleted_codeunits);
-    buf->deleted_codeunits++;
+    curbuf->deleted_codepoints++;  // NL char
+    curbuf->deleted_codeunits++;
   }
 }
 

src/nvim/ops.c

@@ -1659,17 +1659,20 @@ int op_delete(oparg_T *oap)
       curpos = curwin->w_cursor;  // remember curwin->w_cursor
       curwin->w_cursor.lnum++;
       del_lines(oap->line_count - 2, false);
+      bcount_t deleted_bytes = (bcount_t)curbuf->deleted_bytes2 - startpos.col;
 
       // delete from start of line until op_end
       n = (oap->end.col + 1 - !oap->inclusive);
       curwin->w_cursor.col = 0;
       (void)del_bytes((colnr_T)n, !virtual_op,
                       oap->op_type == OP_DELETE && !oap->is_VIsual);
+      deleted_bytes += n;
       curwin->w_cursor = curpos;  // restore curwin->w_cursor
       (void)do_join(2, false, false, false, false);
       curbuf_splice_pending--;
       extmark_splice(curbuf, (int)startpos.lnum-1, startpos.col,
-                     (int)oap->line_count-1, n, 0, 0, kExtmarkUndo);
+                     (int)oap->line_count-1, n, deleted_bytes,
+                     0, 0, 0, kExtmarkUndo);
     }
   }
 
@@ -1854,6 +1857,7 @@ int op_replace(oparg_T *oap, int c)
       }
       // replace the line
       ml_replace(curwin->w_cursor.lnum, newp, false);
+      curbuf_splice_pending++;
       linenr_T baselnum = curwin->w_cursor.lnum;
       if (after_p != NULL) {
         ml_append(curwin->w_cursor.lnum++, after_p, (int)after_p_len, false);
@@ -1861,9 +1865,10 @@ int op_replace(oparg_T *oap, int c)
         oap->end.lnum++;
         xfree(after_p);
       }
+      curbuf_splice_pending--;
       extmark_splice(curbuf, (int)baselnum-1, bd.textcol,
-                     0, bd.textlen,
+                     0, bd.textlen, bd.textlen,
-                     newrows, newcols, kExtmarkUndo);
+                     newrows, newcols, newrows+newcols, kExtmarkUndo);
     }
   } else {
     // Characterwise or linewise motion replace.
@@ -3369,13 +3374,23 @@ void do_put(int regname, yankreg_T *reg, int dir, long count, int flags)
           }
         }
 
+        bcount_t totsize = 0;
+        int lastsize = 0;
+        if (y_type == kMTCharWise
+            || (y_type == kMTLineWise && flags & PUT_LINE_SPLIT)) {
+          for (i = 0; i < y_size-1; i++) {
+            totsize += (bcount_t)STRLEN(y_array[i]) + 1;
+          }
+          lastsize = (int)STRLEN(y_array[y_size-1]);
+          totsize += lastsize;
+        }
         if (y_type == kMTCharWise) {
-          extmark_splice(curbuf, (int)new_cursor.lnum-1, col, 0, 0,
+          extmark_splice(curbuf, (int)new_cursor.lnum-1, col, 0, 0, 0,
-                         (int)y_size-1, (int)STRLEN(y_array[y_size-1]),
+                         (int)y_size-1, lastsize, totsize,
                          kExtmarkUndo);
         } else if (y_type == kMTLineWise && flags & PUT_LINE_SPLIT) {
-          extmark_splice(curbuf, (int)new_cursor.lnum-1, col, 0, 0,
+          extmark_splice(curbuf, (int)new_cursor.lnum-1, col, 0, 0, 0,
-                         (int)y_size+1, 0, kExtmarkUndo);
+                         (int)y_size+1, 0, totsize+1, kExtmarkUndo);
         }
       }
 
@@ -3819,9 +3834,10 @@ int do_join(size_t count,
     }
 
     if (t > 0 && curbuf_splice_pending == 0) {
+      colnr_T removed = (int)(curr- curr_start);
       extmark_splice(curbuf, (int)curwin->w_cursor.lnum-1, sumsize,
-                     1, (int)(curr- curr_start),
+                     1, removed, removed + 1,
-                     0, spaces[t],
+                     0, spaces[t], spaces[t],
                      kExtmarkUndo);
     }
     currsize = (int)STRLEN(curr);

test/functional/lua/buffer_updates_spec.lua

@@ -7,6 +7,7 @@ local clear = helpers.clear
 local eq = helpers.eq
 local exec_lua = helpers.exec_lua
 local feed = helpers.feed
+local deepcopy = helpers.deepcopy
 
 local origlines = {"original line 1",
                    "original line 2",
@@ -16,10 +17,9 @@ local origlines = {"original line 1",
                    "original line 6",
                    "    indented line"}
 
-describe('lua: buffer event callbacks', function()
+local function attach_buffer(evname)
-  before_each(function()
-    clear()
   exec_lua([[
+    local evname = ...
     local events = {}
 
     function test_register(bufnr, id, changedtick, utf_sizes)
@@ -29,7 +29,7 @@ describe('lua: buffer event callbacks', function()
           return true
         end
       end
-        local opts = {on_lines=callback, on_detach=callback, utf_sizes=utf_sizes}
+      local opts = {[evname]=callback, on_detach=callback, utf_sizes=utf_sizes}
       if changedtick then
         opts.on_changedtick = callback
       end
@@ -41,7 +41,13 @@ describe('lua: buffer event callbacks', function()
       events = {}
       return ret_events
     end
-    ]])
+  ]], evname)
+end
+
+describe('lua buffer event callbacks: on_lines', function()
+  before_each(function()
+    clear()
+    attach_buffer('on_lines')
   end)
 
 
@@ -62,7 +68,7 @@ describe('lua: buffer event callbacks', function()
     local function check_events(expected)
       local events = exec_lua("return get_events(...)" )
       if utf_sizes then
-        -- this test case uses ASCII only, so sizes sshould be the same.
+        -- this test case uses ASCII only, so sizes should be the same.
         -- Unicode is tested below.
         for _, event in ipairs(expected) do
           event[9] = event[8]
@@ -236,3 +242,69 @@ describe('lua: buffer event callbacks', function()
   end)
 
 end)
+
+describe('lua buffer event callbacks: on_bytes', function()
+  before_each(function()
+    clear()
+    attach_buffer('on_bytes')
+  end)
+
+
+  -- verifying the sizes with nvim_buf_get_offset is nice (checks we cannot
+  -- assert the wrong thing), but masks errors with unflushed lines (as
+  -- nvim_buf_get_offset forces a flush of the memline). To be safe run the
+  -- test both ways.
+  local function check(verify)
+    local lastsize
+    meths.buf_set_lines(0, 0, -1, true, origlines)
+    local shadow = deepcopy(origlines)
+    local shadowbytes = table.concat(shadow, '\n') .. '\n'
+    if verify then
+      lastsize = meths.buf_get_offset(0, meths.buf_line_count(0))
+    end
+    exec_lua("return test_register(...)", 0, "test1",false,utf_sizes)
+    local tick = meths.buf_get_changedtick(0)
+
+    local verify_name = "test1"
+    local function check_events(expected)
+      local events = exec_lua("return get_events(...)" )
+      eq(expected, events)
+      if verify then
+        for _, event in ipairs(events) do
+          if event[1] == verify_name and event[2] == "bytes" then
+            local _, _, buf, tick, start_row, start_col, start_byte, old_row, old_col, old_byte, new_row, new_col, new_byte = unpack(event)
+            local before = string.sub(shadowbytes, 1, start_byte)
+            -- no text in the tests will contain 0xff bytes (invalid UTF-8)
+            -- so we can use it as marker for unknown bytes
+            local unknown = string.rep('\255', new_byte)
+            local after = string.sub(shadowbytes, start_byte + old_byte + 1)
+            shadowbytes = before .. unknown .. after
+          end
+        end
+        local text = meths.buf_get_lines(0, 0, -1, true)
+        local bytes = table.concat(text, '\n') .. '\n'
+        eq(string.len(bytes), string.len(shadowbytes), shadowbytes)
+        for i = 1, string.len(shadowbytes) do
+          local shadowbyte = string.sub(shadowbytes, i, i)
+          if shadowbyte ~= '\255' then
+            eq(string.sub(bytes, i, i), shadowbyte, i)
+          end
+        end
+      end
+    end
+
+    feed('ggJ')
+    check_events({{'test1', 'bytes', 1, 3, 0, 15, 15, 1, 0, 1, 0, 1, 1}})
+
+    feed('3J')
+    check_events({
+        {'test1', 'bytes', 1, 5, 0, 31, 31, 1, 0, 1, 0, 1, 1},
+        {'test1', 'bytes', 1, 5, 0, 47, 47, 1, 0, 1, 0, 1, 1},
+    })
+  end
+
+  it('works with verify', function()
+    check(true)
+  end)
+end)
+