Merge pull request #25503 from bfredl/unishape

refactor(grid): do arabic shaping in one place

src/nvim/arabic.c

@@ -297,13 +297,12 @@ static int A_is_valid(int c)
 }
 
 // Do Arabic shaping on character "c".  Returns the shaped character.
-// out:    "ccp" points to the first byte of the character to be shaped.
 // in/out: "c1p" points to the first composing char for "c".
 // in:     "prev_c"  is the previous character (not shaped)
 // in:     "prev_c1" is the first composing char for the previous char
 //          (not shaped)
 // in:     "next_c"  is the next character (not shaped).
-int arabic_shape(int c, int *ccp, int *c1p, int prev_c, int prev_c1, int next_c)
+int arabic_shape(int c, int *c1p, int prev_c, int prev_c1, int next_c)
 {
   // Deal only with Arabic character, pass back all others
   if (!A_is_ok(c)) {
@@ -347,14 +346,6 @@ int arabic_shape(int c, int *ccp, int *c1p, int prev_c, int prev_c1, int next_c)
     curr_c = c;
   }
 
-  if ((curr_c != c) && (ccp != NULL)) {
-    char buf[MB_MAXBYTES + 1];
-
-    // Update the first byte of the character
-    utf_char2bytes(curr_c, buf);
-    *ccp = (uint8_t)buf[0];
-  }
-
   // Return the shaped character
   return curr_c;
 }

src/nvim/drawline.c

@@ -12,7 +12,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "nvim/arabic.h"
 #include "nvim/ascii.h"
 #include "nvim/buffer.h"
 #include "nvim/charset.h"
@@ -141,15 +140,6 @@ typedef struct {
                              ///< to be added to wlv.vcol later
 } winlinevars_T;
 
-/// for line_putchar. Contains the state that needs to be remembered from
-/// putting one character to the next.
-typedef struct {
-  const char *p;
-  int prev_c;   ///< previous Arabic character
-  int prev_c1;  ///< first composing char for prev_c
-} LineState;
-#define LINE_STATE(p) { p, 0, 0 }
-
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "drawline.c.generated.h"
 #endif
@@ -220,10 +210,10 @@ static void line_check_overwrite(schar_T *dest, int cells, int maxcells, bool rl
 
 /// Put a single char from an UTF-8 buffer into a line buffer.
 ///
-/// Handles composing chars and arabic shaping state.
+/// Handles composing chars
-static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, bool rl, int vcol)
+static int line_putchar(buf_T *buf, const char **pp, schar_T *dest, int maxcells, bool rl, int vcol)
 {
-  const char *p = s->p;
+  const char *p = *pp;
   int cells = utf_ptr2cells(p);
   int c_len = utfc_ptr2len(p);
   int u8c, u8cc[MAX_MCO];
@@ -244,39 +234,14 @@ static int line_putchar(buf_T *buf, LineState *s, schar_T *dest, int maxcells, b
     goto done;
   } else if ((uint8_t)(*p) < 0x80 && u8cc[0] == 0) {
     dest[0] = schar_from_ascii(*p);
-    s->prev_c = u8c;
   } else {
-    if (p_arshape && !p_tbidi && ARABIC_CHAR(u8c)) {
-      // Do Arabic shaping.
-      int pc, pc1, nc;
-      int pcc[MAX_MCO];
-      int firstbyte = (uint8_t)(*p);
-
-      // The idea of what is the previous and next
-      // character depends on 'rightleft'.
-      if (rl) {
-        pc = s->prev_c;
-        pc1 = s->prev_c1;
-        nc = utf_ptr2char(p + c_len);
-        s->prev_c1 = u8cc[0];
-      } else {
-        pc = utfc_ptr2char(p + c_len, pcc);
-        nc = s->prev_c;
-        pc1 = pcc[0];
-      }
-      s->prev_c = u8c;
-
-      u8c = arabic_shape(u8c, &firstbyte, &u8cc[0], pc, pc1, nc);
-    } else {
-      s->prev_c = u8c;
-    }
     dest[0] = schar_from_cc(u8c, u8cc);
   }
   if (cells > 1) {
     dest[rl ? -1 : 1] = 0;
   }
 done:
-  s->p += c_len;
+  *pp += c_len;
   return cells;
 }
 
@@ -344,22 +309,22 @@ static void draw_virt_text(win_T *wp, buf_T *buf, int col_off, int *end_col, int
 static int draw_virt_text_item(buf_T *buf, int col, VirtText vt, HlMode hl_mode, int max_col,
                                int vcol, bool rl)
 {
-  LineState s = LINE_STATE("");
+  const char *p = "";
   int virt_attr = 0;
   size_t virt_pos = 0;
 
   while (rl ? col > max_col : col < max_col) {
-    if (*s.p == NUL) {
+    if (!*p) {
       if (virt_pos >= kv_size(vt)) {
         break;
       }
       virt_attr = 0;
-      s.p = next_virt_text_chunk(vt, &virt_pos, &virt_attr);
+      p = next_virt_text_chunk(vt, &virt_pos, &virt_attr);
-      if (s.p == NULL) {
+      if (p == NULL) {
         break;
       }
     }
-    if (*s.p == NUL) {
+    if (*p == NUL) {
       continue;
     }
     int attr;
@@ -367,14 +332,14 @@ static int draw_virt_text_item(buf_T *buf, int col, VirtText vt, HlMode hl_mode,
     if (hl_mode == kHlModeCombine) {
       attr = hl_combine_attr(linebuf_attr[col], virt_attr);
     } else if (hl_mode == kHlModeBlend) {
-      through = (*s.p == ' ');
+      through = (*p == ' ');
       attr = hl_blend_attrs(linebuf_attr[col], virt_attr, &through);
     } else {
       attr = virt_attr;
     }
     schar_T dummy[2];
     int maxcells = rl ? col - max_col : max_col - col;
-    int cells = line_putchar(buf, &s, through ? dummy : &linebuf_char[col],
+    int cells = line_putchar(buf, &p, through ? dummy : &linebuf_char[col],
                              maxcells, rl, vcol);
     // If we failed to emit a char, we still need to put a space and advance.
     if (cells < 1) {
@@ -1171,8 +1136,6 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
   int multispace_pos = 0;               // position in lcs-multispace string
   int line_attr_save;
   int line_attr_lowprio_save;
-  int prev_c = 0;                       // previous Arabic character
-  int prev_c1 = 0;                      // first composing char for prev_c
 
   bool search_attr_from_match = false;  // if search_attr is from :match
   bool has_decor = false;               // this buffer has decoration
@@ -2160,28 +2123,6 @@ int win_line(win_T *wp, linenr_T lnum, int startrow, int endrow, bool number_onl
         }
       } else if (mb_l == 0) {        // at the NUL at end-of-line
         mb_l = 1;
-      } else if (p_arshape && !p_tbidi && ARABIC_CHAR(mb_c)) {
-        // Do Arabic shaping.
-        int pc, pc1, nc;
-        int pcc[MAX_MCO];
-
-        // The idea of what is the previous and next
-        // character depends on 'rightleft'.
-        if (wp->w_p_rl) {
-          pc = prev_c;
-          pc1 = prev_c1;
-          nc = utf_ptr2char(ptr + mb_l);
-          prev_c1 = u8cc[0];
-        } else {
-          pc = utfc_ptr2char(ptr + mb_l, pcc);
-          nc = prev_c;
-          pc1 = pcc[0];
-        }
-        prev_c = mb_c;
-
-        mb_c = arabic_shape(mb_c, &c, &u8cc[0], pc, pc1, nc);
-      } else {
-        prev_c = mb_c;
       }
       // If a double-width char doesn't fit display a '>' in the
       // last column; the character is displayed at the start of the

src/nvim/drawscreen.c

@@ -2391,10 +2391,10 @@ static void win_update(win_T *wp, DecorProviders *providers)
       wp->w_botline = lnum;
     } else if (dy_flags & DY_LASTLINE) {      // 'display' has "lastline"
       // Last line isn't finished: Display "@@@" at the end.
-      // TODO(bfredl): this display ">@@@" when ">" was a left-halve
+      // If this would split a doublewidth char in two, we need to display "@@@@" instead
-      // maybe "@@@@" is preferred when this happens.
       grid_line_start(&wp->w_grid, wp->w_grid.rows - 1);
-      grid_line_fill(MAX(wp->w_grid.cols - 3, 0), wp->w_grid.cols,
+      int width = grid_line_getchar(MAX(wp->w_grid.cols - 3, 0), NULL) == NUL ? 4 : 3;
+      grid_line_fill(MAX(wp->w_grid.cols - width, 0), wp->w_grid.cols,
                      wp->w_p_fcs_chars.lastline, at_attr);
       grid_line_flush();
       set_empty_rows(wp, srow);

src/nvim/ex_getln.c

@@ -2998,16 +2998,6 @@ void realloc_cmdbuff(int len)
   }
 }
 
-static char *arshape_buf = NULL;
-
-#if defined(EXITFREE)
-void free_arshape_buf(void)
-{
-  xfree(arshape_buf);
-}
-
-#endif
-
 enum { MAX_CB_ERRORS = 1, };
 
 /// Color expression cmdline using built-in expressions parser
@@ -3317,98 +3307,7 @@ static void draw_cmdline(int start, int len)
       msg_putchar('*');
       i += utfc_ptr2len(ccline.cmdbuff + start + i) - 1;
     }
-  } else if (p_arshape && !p_tbidi && len > 0) {
-    bool do_arabicshape = false;
-    int mb_l;
-    for (int i = start; i < start + len; i += mb_l) {
-      char *p = ccline.cmdbuff + i;
-      int u8cc[MAX_MCO];
-      int u8c = utfc_ptr2char_len(p, u8cc, start + len - i);
-      mb_l = utfc_ptr2len_len(p, start + len - i);
-      if (ARABIC_CHAR(u8c)) {
-        do_arabicshape = true;
-        break;
-      }
-    }
-    if (!do_arabicshape) {
-      goto draw_cmdline_no_arabicshape;
-    }
-
-    static size_t buflen = 0;
-    assert(len >= 0);
-
-    // Do arabic shaping into a temporary buffer.  This is very
-    // inefficient!
-    if ((size_t)len * 2 + 2 > buflen) {
-      // Re-allocate the buffer.  We keep it around to avoid a lot of
-      // alloc()/free() calls.
-      xfree(arshape_buf);
-      buflen = (size_t)len * 2 + 2;
-      arshape_buf = xmalloc(buflen);
-    }
-
-    int newlen = 0;
-    if (utf_iscomposing(utf_ptr2char(ccline.cmdbuff + start))) {
-      // Prepend a space to draw the leading composing char on.
-      arshape_buf[0] = ' ';
-      newlen = 1;
-    }
-
-    int prev_c = 0;
-    int prev_c1 = 0;
-    for (int i = start; i < start + len; i += mb_l) {
-      char *p = ccline.cmdbuff + i;
-      int u8cc[MAX_MCO];
-      int u8c = utfc_ptr2char_len(p, u8cc, start + len - i);
-      mb_l = utfc_ptr2len_len(p, start + len - i);
-      if (ARABIC_CHAR(u8c)) {
-        int pc;
-        int pc1 = 0;
-        int nc = 0;
-        // Do Arabic shaping.
-        if (cmdmsg_rl) {
-          // Displaying from right to left.
-          pc = prev_c;
-          pc1 = prev_c1;
-          prev_c1 = u8cc[0];
-          if (i + mb_l >= start + len) {
-            nc = NUL;
-          } else {
-            nc = utf_ptr2char(p + mb_l);
-          }
-        } else {
-          // Displaying from left to right.
-          if (i + mb_l >= start + len) {
-            pc = NUL;
-          } else {
-            int pcc[MAX_MCO];
-
-            pc = utfc_ptr2char_len(p + mb_l, pcc, start + len - i - mb_l);
-            pc1 = pcc[0];
-          }
-          nc = prev_c;
-        }
-        prev_c = u8c;
-
-        u8c = arabic_shape(u8c, NULL, &u8cc[0], pc, pc1, nc);
-
-        newlen += utf_char2bytes(u8c, arshape_buf + newlen);
-        if (u8cc[0] != 0) {
-          newlen += utf_char2bytes(u8cc[0], arshape_buf + newlen);
-          if (u8cc[1] != 0) {
-            newlen += utf_char2bytes(u8cc[1], arshape_buf + newlen);
-          }
-        }
-      } else {
-        prev_c = u8c;
-        memmove(arshape_buf + newlen, p, (size_t)mb_l);
-        newlen += mb_l;
-      }
-    }
-
-    msg_outtrans_len(arshape_buf, newlen, 0);
   } else {
-draw_cmdline_no_arabicshape:
     if (kv_size(ccline.last_colors.colors)) {
       for (size_t i = 0; i < kv_size(ccline.last_colors.colors); i++) {
         CmdlineColorChunk chunk = kv_A(ccline.last_colors.colors, i);

src/nvim/grid.c

@@ -152,17 +152,17 @@ bool schar_high(schar_T sc)
 #endif
 }
 
+#ifdef ORDER_BIG_ENDIAN
+# define schar_idx(sc) (sc & (0x00FFFFFF))
+#else
+# define schar_idx(sc) (sc >> 8)
+#endif
+
 void schar_get(char *buf_out, schar_T sc)
 {
   if (schar_high(sc)) {
-#ifdef ORDER_BIG_ENDIAN
+    uint32_t idx = schar_idx(sc);
-    uint32_t idx = sc & (0x00FFFFFF);
+    assert(idx < glyph_cache.h.n_keys);
-#else
-    uint32_t idx = sc >> 8;
-#endif
-    if (idx >= glyph_cache.h.n_keys) {
-      abort();
-    }
     xstrlcpy(buf_out, &glyph_cache.keys[idx], 32);
   } else {
     memcpy(buf_out, (char *)&sc, 4);
@@ -170,6 +170,13 @@ void schar_get(char *buf_out, schar_T sc)
   }
 }
 
+/// gets first raw UTF-8 byte of an schar
+static char schar_get_first_byte(schar_T sc)
+{
+  assert(!(schar_high(sc) && schar_idx(sc) >= glyph_cache.h.n_keys));
+  return schar_high(sc) ? glyph_cache.keys[schar_idx(sc)] : *(char *)&sc;
+}
+
 /// @return ascii char or NUL if not ascii
 char schar_get_ascii(schar_T sc)
 {
@@ -179,6 +186,90 @@ char schar_get_ascii(schar_T sc)
   return (sc < 0x80) ? (char)sc : NUL;
 #endif
 }
+
+static bool schar_in_arabic_block(schar_T sc)
+{
+  char first_byte = schar_get_first_byte(sc);
+  return ((uint8_t)first_byte & 0xFE) == 0xD8;
+}
+
+/// Get the first two codepoints of an schar, or NUL when not available
+static void schar_get_first_two_codepoints(schar_T sc, int *c0, int *c1)
+{
+  char sc_buf[MAX_SCHAR_SIZE];
+  schar_get(sc_buf, sc);
+
+  *c0 = utf_ptr2char(sc_buf);
+  int len = utf_ptr2len(sc_buf);
+  if (*c0 == NUL) {
+    *c1 = NUL;
+  } else {
+    *c1 = utf_ptr2char(sc_buf + len);
+  }
+}
+
+void line_do_arabic_shape(schar_T *buf, int cols)
+{
+  int i = 0;
+
+  for (i = 0; i < cols; i++) {
+    // quickly skip over non-arabic text
+    if (schar_in_arabic_block(buf[i])) {
+      break;
+    }
+  }
+
+  if (i == cols) {
+    return;
+  }
+
+  int c0prev = 0;
+  int c0, c1;
+  schar_get_first_two_codepoints(buf[i], &c0, &c1);
+
+  for (; i < cols; i++) {
+    int c0next, c1next;
+    schar_get_first_two_codepoints(i + 1 < cols ? buf[i + 1] : 0, &c0next, &c1next);
+
+    if (!ARABIC_CHAR(c0)) {
+      goto next;
+    }
+
+    int c1new = c1;
+    int c0new = arabic_shape(c0, &c1new, c0next, c1next, c0prev);
+
+    if (c0new == c0 && c1new == c1) {
+      goto next;  // unchanged
+    }
+
+    char scbuf[MAX_SCHAR_SIZE];
+    schar_get(scbuf, buf[i]);
+
+    char scbuf_new[MAX_SCHAR_SIZE];
+    int len = utf_char2bytes(c0new, scbuf_new);
+    if (c1new) {
+      len += utf_char2bytes(c1new, scbuf_new + len);
+    }
+
+    int off = utf_char2len(c0) + (c1 ? utf_char2len(c1) : 0);
+    size_t rest = strlen(scbuf + off);
+    if (rest + (size_t)off + 1 > MAX_SCHAR_SIZE) {
+      // TODO(bfredl): this cannot happen just yet, as we only construct
+      // schar_T values with up to MAX_MCO+1 composing codepoints. When code
+      // is improved so that MAX_SCHAR_SIZE becomes the only/sharp limit,
+      // we need be able to peel off a composing char which doesn't fit anymore.
+      abort();
+    }
+    memcpy(scbuf_new + len, scbuf + off, rest);
+    buf[i] = schar_from_buf(scbuf_new, (size_t)len + rest);
+
+next:
+    c0prev = c0;
+    c0 = c0next;
+    c1 = c1next;
+  }
+}
+
 /// clear a line in the grid starting at "off" until "width" characters
 /// are cleared.
 void grid_clear_line(ScreenGrid *grid, size_t off, int width, bool valid)
@@ -242,6 +333,15 @@ void grid_line_start(ScreenGrid *grid, int row)
   grid_line_first = (int)linebuf_size;
   grid_line_maxcol = grid->cols - grid_line_coloff;
   grid_line_last = 0;
+
+  assert((size_t)grid_line_maxcol <= linebuf_size);
+
+  if (rdb_flags & RDB_INVALID) {
+    // Current batch must not depend on previous contents of linebuf_char.
+    // Set invalid values which will cause assertion failures later if they are used.
+    memset(linebuf_char, 0xFF, sizeof(schar_T) * linebuf_size);
+    memset(linebuf_attr, 0xFF, sizeof(sattr_T) * linebuf_size);
+  }
 }
 
 /// Get present char from current rendered screen line
@@ -287,11 +387,7 @@ int grid_line_puts(int col, const char *text, int textlen, int attr)
 {
   const char *ptr = text;
   int len = textlen;
-  int c;
   int u8cc[MAX_MCO];
-  int prev_c = 0;  // previous Arabic character
-  int pc, nc, nc1;
-  int pcc[MAX_MCO];
 
   assert(grid_line_grid);
 
@@ -301,7 +397,6 @@ int grid_line_puts(int col, const char *text, int textlen, int attr)
   while (col < max_col
          && (len < 0 || (int)(ptr - text) < len)
          && *ptr != NUL) {
-    c = (unsigned char)(*ptr);
     // check if this is the first byte of a multibyte
     int mbyte_blen = len > 0
       ? utfc_ptr2len_len(ptr, (int)((text + len) - ptr))
@@ -316,37 +411,16 @@ int grid_line_puts(int col, const char *text, int textlen, int attr)
       u8cc[0] = 0;
     }
 
-    if (p_arshape && !p_tbidi && ARABIC_CHAR(u8c)) {
-      // Do Arabic shaping.
-      if (len >= 0 && (int)(ptr - text) + mbyte_blen >= len) {
-        // Past end of string to be displayed.
-        nc = NUL;
-        nc1 = NUL;
-      } else {
-        nc = len >= 0
-          ? utfc_ptr2char_len(ptr + mbyte_blen, pcc,
-                              (int)((text + len) - ptr - mbyte_blen))
-          : utfc_ptr2char(ptr + mbyte_blen, pcc);
-        nc1 = pcc[0];
-      }
-      pc = prev_c;
-      prev_c = u8c;
-      u8c = arabic_shape(u8c, &c, &u8cc[0], nc, nc1, pc);
-    } else {
-      prev_c = u8c;
-    }
     if (col + mbyte_cells > max_col) {
       // Only 1 cell left, but character requires 2 cells:
       // display a '>' in the last column to avoid wrapping. */
-      c = '>';
       u8c = '>';
       u8cc[0] = 0;
       mbyte_cells = 1;
     }
 
     schar_T buf;
-    // TODO(bfredl): why not just keep the original byte sequence. arabshape is
+    // TODO(bfredl): why not just keep the original byte sequence.
-    // an edge case, treat it as such..
     buf = schar_from_cc(u8c, u8cc);
 
     // When at the start of the text and overwriting the right half of a
@@ -545,14 +619,12 @@ static int grid_char_needs_redraw(ScreenGrid *grid, int col, size_t off_to, int
 /// If "wrap" is true, then hint to the UI that "row" contains a line
 /// which has wrapped into the next row.
 void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int col, int endcol, int clear_width,
-                      int rl, int bg_attr, bool wrap, bool invalid_row)
+                      bool rl, int bg_attr, bool wrap, bool invalid_row)
 {
   bool redraw_next;                         // redraw_this for next character
   bool clear_next = false;
   int char_cells;                           // 1: normal char
                                             // 2: occupies two display cells
-  int start_dirty = -1, end_dirty = 0;
-
   assert(0 <= row && row < grid->rows);
   // TODO(bfredl): check all callsites and eliminate
   // Check for illegal col, just in case
@@ -591,6 +663,10 @@ void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int col, int endcol
     endcol = (clear_width > 0 ? clear_width : -clear_width);
   }
 
+  if (p_arshape && !p_tbidi) {
+    line_do_arabic_shape(linebuf_char + col, endcol - col);
+  }
+
   if (bg_attr) {
     for (int c = col; c < endcol; c++) {
       linebuf_attr[c] = hl_combine_attr(bg_attr, linebuf_attr[c]);
@@ -599,6 +675,8 @@ void grid_put_linebuf(ScreenGrid *grid, int row, int coloff, int col, int endcol
 
   redraw_next = grid_char_needs_redraw(grid, col, (size_t)col + off_to, endcol - col);
 
+  int start_dirty = -1, end_dirty = 0;
+
   while (col < endcol) {
     char_cells = 1;
     if (col + 1 < endcol && linebuf_char[col + 1] == 0) {

src/nvim/memory.c

@@ -767,8 +767,6 @@ void free_all_mem(void)
   // Free all option values.  Must come after closing windows.
   free_all_options();
 
-  free_arshape_buf();
-
   // Clear registers.
   clear_registers();
   ResetRedobuff();

test/functional/terminal/tui_spec.lua

@@ -1641,7 +1641,7 @@ describe('TUI', function()
       {13:℃}{12: ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ }|
       {12:℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ }|
       {12:℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ }{15:$}{12:                             }|
-      ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ >{4:@@@}|
+      ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ ℃ {4:@@@@}|
       {5:[No Name] [+]                                     }|
                                                         |
       {3:-- TERMINAL --}                                    |

test/functional/ui/fold_spec.lua

@@ -1156,7 +1156,7 @@ describe("folded lines", function()
           [2:---------------------------------------------]|
           [3:---------------------------------------------]|
         ## grid 2
-          {5:^+--  2 lines: å 语 x̎͂̀̂͛͛ العَرَبِيَّة·················}|
+          {5:^+--  2 lines: å 语 x̎͂̀̂͛͛ ﺎﻠﻋَﺮَﺒِﻳَّﺓ·················}|
           {1:~                                            }|
           {1:~                                            }|
           {1:~                                            }|
@@ -1168,7 +1168,7 @@ describe("folded lines", function()
         ]])
       else
         screen:expect([[
-          {5:^+--  2 lines: å 语 x̎͂̀̂͛͛ العَرَبِيَّة·················}|
+          {5:^+--  2 lines: å 语 x̎͂̀̂͛͛ ﺎﻠﻋَﺮَﺒِﻳَّﺓ·················}|
           {1:~                                            }|
           {1:~                                            }|
           {1:~                                            }|
@@ -1337,7 +1337,7 @@ describe("folded lines", function()
           [2:---------------------------------------------]|
           [3:---------------------------------------------]|
         ## grid 2
-          {5:·················ةيَّبِرَعَلا x̎͂̀̂͛͛ 语 å :senil 2  --^+}|
+          {5:·················ﺔﻴَّﺑِﺮَﻌَﻟﺍ x̎͂̀̂͛͛ 语 å :senil 2  --^+}|
           {1:                                            ~}|
           {1:                                            ~}|
           {1:                                            ~}|
@@ -1349,7 +1349,7 @@ describe("folded lines", function()
         ]])
       else
         screen:expect([[
-          {5:·················ةيَّبِرَعَلا x̎͂̀̂͛͛ 语 å :senil 2  --^+}|
+          {5:·················ﺔﻴَّﺑِﺮَﻌَﻟﺍ x̎͂̀̂͛͛ 语 å :senil 2  --^+}|
           {1:                                            ~}|
           {1:                                            ~}|
           {1:                                            ~}|

test/functional/ui/multibyte_spec.lua

@@ -177,6 +177,57 @@ describe("multibyte rendering", function()
                                                                        |
     ]], reset=true}
   end)
+
+  it('works with arabic input and arabicshape', function()
+    command('set arabic')
+
+    command('set noarabicshape')
+    feed('isghl!<esc>')
+    screen:expect{grid=[[
+                                                             ^!مالس|
+      {1:                                                           ~}|
+      {1:                                                           ~}|
+      {1:                                                           ~}|
+      {1:                                                           ~}|
+                                                                  |
+    ]]}
+
+    command('set arabicshape')
+    screen:expect{grid=[[
+                                                              ^!ﻡﻼﺳ|
+      {1:                                                           ~}|
+      {1:                                                           ~}|
+      {1:                                                           ~}|
+      {1:                                                           ~}|
+                                                                  |
+    ]]}
+  end)
+
+  it('works with arabic input and arabicshape and norightleft', function()
+    command('set arabic norightleft')
+
+    command('set noarabicshape')
+    feed('isghl!<esc>')
+    screen:expect{grid=[[
+      سلام^!                                                       |
+      {1:~                                                           }|
+      {1:~                                                           }|
+      {1:~                                                           }|
+      {1:~                                                           }|
+                                                                  |
+    ]]}
+
+    command('set arabicshape')
+    screen:expect{grid=[[
+      ﺱﻼﻣ^!                                                        |
+      {1:~                                                           }|
+      {1:~                                                           }|
+      {1:~                                                           }|
+      {1:~                                                           }|
+                                                                  |
+    ]]}
+
+  end)
 end)
 
 describe('multibyte rendering: statusline', function()