vim-patch:8.0.0645: no error for illegal back reference in NFA engine

Problem:    The new regexp engine does not give an error for using a back
            reference where it is not allowed. (Dominique Pelle)
Solution:   Check the back reference like the old engine. (closes vim/vim#1774)
1ef9bbe215

src/nvim/regexp.c

@@ -1210,6 +1210,31 @@ char_u *skip_regexp(char_u *startp, int dirc, int magic, char_u **newp)
   return p;
 }
 
+/*
+ * Return TRUE if the back reference is legal. We must have seen the close
+ * brace.
+ * TODO: Should also check that we don't refer to something that is repeated
+ * (+*=): what instance of the repetition should we match?
+ */
+static int seen_endbrace(int refnum)
+{
+  if (!had_endbrace[refnum]) {
+      char_u *p;
+
+      /* Trick: check if "@<=" or "@<!" follows, in which case
+       * the \1 can appear before the referenced match. */
+      for (p = regparse; *p != NUL; ++p)
+        if (p[0] == '@' && p[1] == '<' && (p[2] == '!' || p[2] == '='))
+          break;
+
+    if (*p == NUL) {
+      EMSG(_("E65: Illegal back reference"));
+      rc_did_emsg = TRUE;
+      return FALSE;
+    }
+  }
+  return TRUE;
+}
 
 /*
  * bt_regcomp() - compile a regular expression into internal code for the
@@ -1928,23 +1953,8 @@ static char_u *regatom(int *flagp)
     int refnum;
 
     refnum = c - Magic('0');
-    /*
-     * Check if the back reference is legal. We must have seen the
-     * close brace.
-     * TODO: Should also check that we don't refer to something
-     * that is repeated (+*=): what instance of the repetition
-     * should we match?
-     */
-    if (!had_endbrace[refnum]) {
-      /* Trick: check if "@<=" or "@<!" follows, in which case
-       * the \1 can appear before the referenced match. */
-      for (p = regparse; *p != NUL; ++p)
-        if (p[0] == '@' && p[1] == '<'
-            && (p[2] == '!' || p[2] == '='))
-          break;
-      if (*p == NUL)
-        EMSG_RET_NULL(_("E65: Illegal back reference"));
-    }
+    if (!seen_endbrace(refnum))
+      return NULL;
     ret = regnode(BACKREF + refnum);
   }
   break;

src/nvim/regexp_nfa.c

@@ -1338,8 +1338,14 @@ static int nfa_regatom(void)
   case Magic('7'):
   case Magic('8'):
   case Magic('9'):
-    EMIT(NFA_BACKREF1 + (no_Magic(c) - '1'));
-    nfa_has_backref = TRUE;
+    {
+      int refnum = no_Magic(c) - '1';
+
+      if (!seen_endbrace(refnum + 1))
+          return FAIL;
+      EMIT(NFA_BACKREF1 + refnum);
+      nfa_has_backref = TRUE;
+    }
     break;
 
   case Magic('z'):

src/nvim/testdir/test_hlsearch.vim

@@ -37,11 +37,11 @@ func Test_hlsearch_hangs()
     return
   endif
 
-  " This pattern takes forever to match, it should timeout.
+  " This pattern takes a long time to match, it should timeout.
   help
   let start = reltime()
   set hlsearch nolazyredraw redrawtime=101
-  let @/ = '\%#=2\v(a|\1)*'
+  let @/ = '\%#=1a*.*X\@<=b*'
   redraw
   let elapsed = reltimefloat(reltime(start))
   call assert_true(elapsed > 0.1)

src/nvim/testdir/test_regexp_latin.vim

@@ -63,3 +63,13 @@ func Test_rex_init()
   bwipe!
   set re=0
 endfunc
+
+func Test_backref()
+  new
+  call setline(1, ['one', 'two', 'three', 'four', 'five'])
+  call assert_equal(3, search('\%#=1\(e\)\1'))
+  call assert_equal(3, search('\%#=2\(e\)\1'))
+  call assert_fails('call search("\\%#=1\\(e\\1\\)")', 'E65:')
+  call assert_fails('call search("\\%#=2\\(e\\1\\)")', 'E65:')
+  bwipe!
+endfunc

src/nvim/testdir/test_statusline.vim

@@ -223,7 +223,7 @@ func Test_statusline()
   set statusline=ab%(cd%q%)de
   call assert_match('^abde\s*$', s:get_statusline())
   copen
-  call assert_match('^abcd\[Quickfix List\1]de\s*$', s:get_statusline())
+  call assert_match('^abcd\[Quickfix List]de\s*$', s:get_statusline())
   cclose
 
   " %#: Set highlight group. The name must follow and then a # again.