diff --git a/src/nvim/move.c b/src/nvim/move.c
index 5d817fd7aa..5934ad90e9 100644
--- a/src/nvim/move.c
+++ b/src/nvim/move.c
@@ -1344,7 +1344,6 @@ bool scrollup(long line_count, int byfold)
     int width1 = curwin->w_width_inner - curwin_col_off();
     int width2 = width1 + curwin_col_off2();
     unsigned size = 0;
-    linenr_T prev_topline = curwin->w_topline;
     const colnr_T prev_skipcol = curwin->w_skipcol;
 
     if (do_sms) {
@@ -1397,9 +1396,9 @@ bool scrollup(long line_count, int byfold)
       }
     }
 
-    if (curwin->w_topline == prev_topline
-        || curwin->w_skipcol != prev_skipcol) {
-      // need to redraw because wl_size of the topline may now be invalid
+    if (prev_skipcol > 0 || curwin->w_skipcol > 0) {
+      // need to redraw more, because wl_size of the (new) topline may
+      // now be invalid
       redraw_later(curwin, UPD_NOT_VALID);
     }
   } else {
diff --git a/test/functional/legacy/scroll_opt_spec.lua b/test/functional/legacy/scroll_opt_spec.lua
index bb0ada15e9..a5a6aded79 100644
--- a/test/functional/legacy/scroll_opt_spec.lua
+++ b/test/functional/legacy/scroll_opt_spec.lua
@@ -874,10 +874,11 @@ describe('smoothscroll', function()
     })
     exec([[
       setlocal cursorline scrolloff=0 smoothscroll
-      call setline(1, repeat([''], 9))
+      call setline(1, repeat([''], 8))
       call setline(3, repeat('a', 50))
-      call setline(8, 'bbb')
-      call setline(9, 'ccc')
+      call setline(4, repeat('a', 50))
+      call setline(7, 'bbb')
+      call setline(8, 'ccc')
       redraw
     ]])
     screen:expect([[
@@ -885,8 +886,8 @@ describe('smoothscroll', function()
                                               |
       aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa|
       aaaaaaaaaa                              |
-                                              |
-                                              |
+      aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa|
+      aaaaaaaaaa                              |
                                               |
                                               |
       bbb                                     |
@@ -895,8 +896,8 @@ describe('smoothscroll', function()
     feed('3<C-E>')
     screen:expect([[
       {0:<<<}{1:aaaaaa^a                              }|
-                                              |
-                                              |
+      aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa|
+      aaaaaaaaaa                              |
                                               |
                                               |
       bbb                                     |
@@ -905,6 +906,19 @@ describe('smoothscroll', function()
       {0:~                                       }|
                                               |
     ]])
+    feed('2<C-E>')
+    screen:expect([[
+      {0:<<<}{1:aaaaaa^a                              }|
+                                              |
+                                              |
+      bbb                                     |
+      ccc                                     |
+      {0:~                                       }|
+      {0:~                                       }|
+      {0:~                                       }|
+      {0:~                                       }|
+                                              |
+    ]])
   end)
 
   it("works with virt_lines above and below", function()
diff --git a/test/old/testdir/test_scroll_opt.vim b/test/old/testdir/test_scroll_opt.vim
index 1cd8bbfc84..f9f33b20eb 100644
--- a/test/old/testdir/test_scroll_opt.vim
+++ b/test/old/testdir/test_scroll_opt.vim
@@ -801,10 +801,11 @@ func Test_smoothscroll_multi_skipcol()
 
   let lines =<< trim END
       setlocal cursorline scrolloff=0 smoothscroll
-      call setline(1, repeat([''], 9))
+      call setline(1, repeat([''], 8))
       call setline(3, repeat('a', 50))
-      call setline(8, 'bbb')
-      call setline(9, 'ccc')
+      call setline(4, repeat('a', 50))
+      call setline(7, 'bbb')
+      call setline(8, 'ccc')
       redraw
   END
   call writefile(lines, 'XSmoothMultiSkipcol', 'D')
@@ -814,6 +815,9 @@ func Test_smoothscroll_multi_skipcol()
   call term_sendkeys(buf, "3\<C-E>")
   call VerifyScreenDump(buf, 'Test_smooth_multi_skipcol_2', {})
 
+  call term_sendkeys(buf, "2\<C-E>")
+  call VerifyScreenDump(buf, 'Test_smooth_multi_skipcol_3', {})
+
   call StopVimInTerminal(buf)
 endfunc