diff --git a/src/nvim/window.c b/src/nvim/window.c
index d9a05cde17..66d090b0c8 100644
--- a/src/nvim/window.c
+++ b/src/nvim/window.c
@@ -2320,7 +2320,7 @@ void leaving_window(win_T *const win)
   // When leaving the window (or closing the window) was done from a
   // callback we need to break out of the Insert mode loop and restart Insert
   // mode when entering the window again.
-  if (State & MODE_INSERT) {
+  if ((State & MODE_INSERT) && !stop_insert_mode) {
     stop_insert_mode = true;
     if (win->w_buffer->b_prompt_insert == NUL) {
       win->w_buffer->b_prompt_insert = 'A';
diff --git a/test/functional/legacy/prompt_buffer_spec.lua b/test/functional/legacy/prompt_buffer_spec.lua
index 0c6898526e..59a9283868 100644
--- a/test/functional/legacy/prompt_buffer_spec.lua
+++ b/test/functional/legacy/prompt_buffer_spec.lua
@@ -4,6 +4,7 @@ local feed = helpers.feed
 local source = helpers.source
 local clear = helpers.clear
 local command = helpers.command
+local expect = helpers.expect
 local poke_eventloop = helpers.poke_eventloop
 local api = helpers.api
 local eq = helpers.eq
@@ -217,4 +218,24 @@ describe('prompt buffer', function()
     command('call DoAppend()')
     eq({ mode = 'i', blocking = false }, api.nvim_get_mode())
   end)
+
+  -- oldtest: Test_prompt_close_modify_hidden()
+  it('modifying hidden buffer does not prevent prompt buffer mode change', function()
+    source([[
+      file hidden
+      set bufhidden=hide
+      enew
+      new prompt
+      set buftype=prompt
+
+      inoremap <buffer> q <Cmd>bwipe!<CR>
+      autocmd BufWinLeave prompt call setbufline('hidden', 1, 'Test')
+    ]])
+    feed('a')
+    eq({ mode = 'i', blocking = false }, api.nvim_get_mode())
+    feed('q')
+    eq({ mode = 'n', blocking = false }, api.nvim_get_mode())
+    command('bwipe!')
+    expect('Test')
+  end)
 end)
diff --git a/test/old/testdir/test_prompt_buffer.vim b/test/old/testdir/test_prompt_buffer.vim
index 3dfbbcece6..20daa07a3d 100644
--- a/test/old/testdir/test_prompt_buffer.vim
+++ b/test/old/testdir/test_prompt_buffer.vim
@@ -298,4 +298,36 @@ func Test_prompt_appending_while_hidden()
   call StopVimInTerminal(buf)
 endfunc
 
+" Modifying a hidden buffer while closing a prompt buffer should not prevent
+" stopping of Insert mode.
+func Test_prompt_close_modify_hidden()
+  call CanTestPromptBuffer()
+
+  let script =<< trim END
+      file hidden
+      set bufhidden=hide
+      enew
+      new prompt
+      set buftype=prompt
+
+      inoremap <buffer> q <Cmd>bwipe!<CR>
+      autocmd BufWinLeave prompt call setbufline('hidden', 1, 'Test')
+  END
+  call writefile(script, 'XpromptCloseModifyHidden', 'D')
+
+  let buf = RunVimInTerminal('-S XpromptCloseModifyHidden', {'rows': 10})
+  call TermWait(buf)
+
+  call term_sendkeys(buf, "a")
+  call WaitForAssert({-> assert_match('-- INSERT --', term_getline(buf, 10))})
+
+  call term_sendkeys(buf, "q")
+  call WaitForAssert({-> assert_notmatch('-- INSERT --', term_getline(buf, 10))})
+
+  call term_sendkeys(buf, ":bwipe!\<CR>")
+  call WaitForAssert({-> assert_equal('Test', term_getline(buf, 1))})
+
+  call StopVimInTerminal(buf)
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab