:stopinsert should leave terminal-mode #9856

Problem:  Calling :stopinsert from RPC while in terminal-mode does not
          go back to normal-mode.
Solution: Implement a check() handler for state_enter(), adapted from
          insert_check().

Fix #7807

ci/build.ps1

@@ -29,7 +29,7 @@ if (-Not (Test-Path -PathType container $nvimCmakeVars["DEPS_BUILD_DIR"])) {
   write-host "cache dir not found: $($nvimCmakeVars['DEPS_BUILD_DIR'])"
   mkdir $nvimCmakeVars["DEPS_BUILD_DIR"]
 } else {
-  write-host "cache dir ($nvimCmakeVars['DEPS_BUILD_DIR']) size: $(Get-ChildItem $nvimCmakeVars['DEPS_BUILD_DIR'] -recurse | Measure-Object -property length -sum | Select -expand sum)"
+  write-host "cache dir $($nvimCmakeVars['DEPS_BUILD_DIR']) size: $(Get-ChildItem $nvimCmakeVars['DEPS_BUILD_DIR'] -recurse | Measure-Object -property length -sum | Select -expand sum)"
 }
 
 if ($compiler -eq 'MINGW') {

runtime/doc/insert.txt

@@ -1844,7 +1844,8 @@ NOTE: These commands cannot be used with |:global| or |:vglobal|.
 ":endif", ":for" and ":endfor", ":while" and ":endwhile".
 
 							*:start* *:startinsert*
-:star[tinsert][!]	Start Insert mode just after executing this command.
+:star[tinsert][!]	Start Insert mode (or |Terminal-mode| in a |terminal|
+			buffer) just after executing this command.
 			Works like typing "i" in Normal mode.  When the ! is
 			included it works like "A", append to the line.
 			Otherwise insertion starts at the cursor position.
@@ -1854,8 +1855,8 @@ NOTE: These commands cannot be used with |:global| or |:vglobal|.
 			This command does not work from |:normal|.
 
 							*:stopi* *:stopinsert*
-:stopi[nsert]		Stop Insert mode as soon as possible.  Works like
+:stopi[nsert]		Stop Insert mode or |Terminal-mode| as soon as
-			typing <Esc> in Insert mode.
+			possible.  Works like typing <Esc> in Insert mode.
 			Can be used in an autocommand, example: >
 				:au BufEnter scratch stopinsert
 <

src/nvim/terminal.c

@@ -403,6 +403,7 @@ void terminal_enter(void)
   redraw(false);
 
   s->state.execute = terminal_execute;
+  s->state.check = terminal_check;
   state_enter(&s->state);
 
   restart_edit = 0;
@@ -427,6 +428,19 @@ void terminal_enter(void)
   }
 }
 
+// Function executed before each iteration of terminal mode.
+// Return:
+//   1 if the iteration should continue normally
+//   0 if the main loop must exit
+static int terminal_check(VimState *state)
+{
+  if (stop_insert_mode) {
+    stop_insert_mode = false;
+    return 0;
+  }
+  return 1;
+}
+
 static int terminal_execute(VimState *state, int key)
 {
   TerminalState *s = (TerminalState *)state;

test/functional/terminal/ex_terminal_spec.lua

@@ -99,6 +99,13 @@ describe(':terminal', function()
     eq(3, #jumps)
   end)
 
+  it(':stopinsert RPC request exits terminal-mode #7807', function()
+    command(':terminal')
+    feed('i[tui] insert-mode')
+    eq({ blocking=false, mode='t' }, nvim('get_mode'))
+    command('stopinsert')
+    eq({ blocking=false, mode='n' }, nvim('get_mode'))
+  end)
 end)
 
 describe(':terminal (with fake shell)', function()