input: Do not set high-bit; preserve ALT modifier.

Background: Vim internally prefers to represent ALT/META chords as
single-byte keys, by setting the high bit of the key byte.
extract_modifiers() _discards_ the meta/alt modifier, but we need it for
libvterm and libtermkey.

Closes #2440
Closes #3727
Closes #2017
References #2277
References #2254

https://github.com/neovim/neovim/issues/2017#issuecomment-140423557
> We [not libtermkey] are setting the high bit for some reason

https://github.com/neovim/neovim/issues/176#issuecomment-77834715
> libvtermkey requires the leading esc to parse alt/meta

https://github.com/neovim/neovim/pull/3246#issuecomment-136328450
> A program could do better than the current logic on some terminals, by
> asking for pure 8bit mode (S8C1T) and then immediately querying the
> mode again. If the result comes back as an 8bit single-byte CSI, then
> it can presume the mode setting was successful, and now the ESC prefix
> byte won't be seen in multibyte sequences; only as an Alt- prefix or
> a real Escape key. On such a terminal, it could therefore avoid
> needing to use that waiting timeout.

src/nvim/keymap.c

@@ -18,6 +18,9 @@
 #include "nvim/strings.h"
 #include "nvim/mouse.h"
 
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "keymap.c.generated.h"
+#endif
 
 /*
  * Some useful tables.
@@ -637,11 +640,11 @@ find_special_key (
             key = DEL;
         }
 
-        /*
-         * Normal Key with modifier: Try to make a single byte code.
-         */
-        if (!IS_SPECIAL(key))
+        // Normal Key with modifier:
+        // Try to make a single byte code (except for Alt/Meta modifiers).
+        if (!IS_SPECIAL(key)) {
           key = extract_modifiers(key, &modifiers);
+        }
 
         *modp = modifiers;
         *srcp = end_of_name;
@@ -652,11 +655,9 @@ find_special_key (
   return 0;
 }
 
-/*
- * Try to include modifiers in the key.
- * Changes "Shift-a" to 'A', "Alt-A" to 0xc0, etc.
- */
-int extract_modifiers(int key, int *modp)
+/// Try to include modifiers (except alt/meta) in the key.
+/// Changes "Shift-a" to 'A', "Ctrl-@" to <Nul>, etc.
+static int extract_modifiers(int key, int *modp)
 {
   int modifiers = *modp;
 
@@ -665,19 +666,12 @@ int extract_modifiers(int key, int *modp)
     modifiers &= ~MOD_MASK_SHIFT;
   }
   if ((modifiers & MOD_MASK_CTRL)
-      && ((key >= '?' && key <= '_') || ASCII_ISALPHA(key))
-      ) {
+      && ((key >= '?' && key <= '_') || ASCII_ISALPHA(key))) {
     key = Ctrl_chr(key);
     modifiers &= ~MOD_MASK_CTRL;
-    /* <C-@> is <Nul> */
-    if (key == 0)
+    if (key == 0) {  // <C-@> is <Nul>
       key = K_ZERO;
-  }
-  if ((modifiers & MOD_MASK_ALT) && key < 0x80
-      && !enc_dbcs                      /* avoid creating a lead byte */
-      ) {
-    key |= 0x80;
-    modifiers &= ~MOD_MASK_ALT;         /* remove the META modifier */
+    }
   }
 
   *modp = modifiers;

src/nvim/terminal.c

@@ -1,5 +1,5 @@
-// VT220/xterm-like terminal emulator implementation for Neovim. Powered by
-// libvterm(http://www.leonerd.org.uk/code/libvterm/).
+// VT220/xterm-like terminal emulator implementation for nvim. Powered by
+// libvterm (http://www.leonerd.org.uk/code/libvterm/).
 //
 // libvterm is a pure C99 terminal emulation library with abstract input and
 // display. This means that the library needs to read data from the master fd
@@ -10,31 +10,31 @@
 // vterm_keyboard_key/vterm_keyboard_unichar, which generates byte streams that
 // must be fed back to the master fd.
 //
-// This implementation uses Neovim buffers as the display mechanism for both
+// This implementation uses nvim buffers as the display mechanism for both
 // the visible screen and the scrollback buffer. When focused, the window
 // "pins" to the bottom of the buffer and mirrors libvterm screen state.
 //
 // When a line becomes invisible due to a decrease in screen height or because
 // a line was pushed up during normal terminal output, we store the line
-// information in the scrollback buffer, which is mirrored in the Neovim buffer
+// information in the scrollback buffer, which is mirrored in the nvim buffer
 // by appending lines just above the visible part of the buffer.
 //
 // When the screen height increases, libvterm will ask for a row in the
-// scrollback buffer, which is mirrored in the Neovim buffer displaying lines
+// scrollback buffer, which is mirrored in the nvim buffer displaying lines
 // that were previously invisible.
 //
-// The vterm->Neovim synchronization is performed in intervals of 10
+// The vterm->nvim synchronization is performed in intervals of 10
 // milliseconds. This is done to minimize screen updates when receiving large
 // bursts of data.
 //
 // This module is decoupled from the processes that normally feed it data, so
-// it's possible to use it as a general purpose console buffer(possibly as a
-// log/display mechanism for Neovim in the future)
+// it's possible to use it as a general purpose console buffer (possibly as a
+// log/display mechanism for nvim in the future)
 //
-// Inspired by vimshell(http://www.wana.at/vimshell/) and
-// Conque(https://code.google.com/p/conque/).  Libvterm usage instructions (plus
+// Inspired by vimshell (http://www.wana.at/vimshell/) and
+// Conque (https://code.google.com/p/conque/). Libvterm usage instructions (plus
 // some extra code) were taken from
-// pangoterm(http://www.leonerd.org.uk/code/pangoterm/)
+// pangoterm (http://www.leonerd.org.uk/code/pangoterm/)
 #include <assert.h>
 #include <stdio.h>
 #include <stdint.h>
@@ -402,11 +402,11 @@ static int terminal_execute(VimState *state, int key)
   TerminalState *s = (TerminalState *)state;
 
   switch (key) {
-    case K_FOCUSGAINED:  // Neovim has been given focus
+    case K_FOCUSGAINED:  // nvim has been given focus
       apply_autocmds(EVENT_FOCUSGAINED, NULL, NULL, false, curbuf);
       break;
 
-    case K_FOCUSLOST:   // Neovim has lost focus
+    case K_FOCUSLOST:   // nvim has lost focus
       apply_autocmds(EVENT_FOCUSLOST, NULL, NULL, false, curbuf);
       break;
 

test/functional/terminal/tui_spec.lua

@@ -53,7 +53,7 @@ describe('tui', function()
     ]])
   end)
 
-  it('interprets leading esc byte as the alt modifier', function()
+  it('interprets leading <Esc> byte as ALT modifier in normal-mode', function()
     local keys = 'dfghjkl'
     for c in keys:gmatch('.') do
       execute('nnoremap <a-'..c..'> ialt-'..c..'<cr><esc>')
@@ -80,6 +80,25 @@ describe('tui', function()
     ]])
   end)
 
+  it('does not mangle unmapped ALT-key chord', function()
+    -- Vim represents ALT/META by setting the "high bit" of the modified key;
+    -- we do _not_. #3982
+    --
+    -- Example: for input ALT+j:
+    --    * Vim (Nvim prior to #3982) sets high-bit, inserts "ê".
+    --    * Nvim (after #3982) inserts "j".
+    feed('i\x1bj')
+    screen:expect([[
+      j{1: }                                                |
+      ~                                                 |
+      ~                                                 |
+      ~                                                 |
+      [No Name] [+]                                     |
+      -- INSERT --                                      |
+      -- TERMINAL --                                    |
+    ]])
+  end)
+
   it('accepts ascii control sequences', function()
     feed('i')
     feed('\x16\x07') -- ctrl+g