Merge #7653 from justinmk/tui-termcap

runtime/doc/eval.txt

@@ -8889,11 +8889,6 @@ This does NOT work: >
 			value and the global value are changed.
 			Example: >
 				:let &path = &path . ',/usr/local/include'
-<			This also works for terminal codes in the form t_xx.
-			But only for alphanumerical names.  Example: >
-				:let &t_k1 = "\<Esc>[234;"
-<			When the code does not exist yet it will be created as
-			a terminal key code, there is no error.
 
 :let &{option-name} .= {expr1}
 			For a string option: Append {expr1} to the value.

runtime/doc/options.txt

@@ -6424,6 +6424,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 	Currently, these messages are given:
 	>= 1	When the shada file is read or written.
 	>= 2	When a file is ":source"'ed.
+	>= 3	UI info, terminal capabilities
 	>= 5	Every searched tags file and include file.
 	>= 8	Files for which a group of autocommands is executed.
 	>= 9	Every executed autocommand.

runtime/doc/starting.txt

@@ -249,14 +249,14 @@ argument.
 		for reading or writing a ShaDa file.  Can be used to find
 		out what is happening upon startup and exit.
 		Example: >
-			vim -V8 foobar
+			nvim -V8
 
 -V[N]{filename}
-		Like -V and set 'verbosefile' to {filename}.  The result is
+		Like -V and set 'verbosefile' to {filename}.  Messages are not
-		that messages are not displayed but written to the file
+		displayed; instead they are written to the file {filename}.
-		{filename}.  {filename} must not start with a digit.
+		{filename} must not start with a digit.
 		Example: >
-			vim -V20vimlog foobar
+			nvim -V20vimlog
 <
 							*-D*
 -D		Debugging.  Go to debugging mode when executing the first

runtime/doc/vim_diff.txt

@@ -327,22 +327,26 @@ Ed-compatible mode:
   ":set noedcompatible" is ignored
   ":set edcompatible" is an error
 
-	*t_xx* *:set-termcap* *termcap-options* *t_AB* *t_Sb* *t_vb* *t_SI*
+			*t_xx* *termcap-options* *t_AB* *t_Sb* *t_vb* *t_SI*
 Nvim does not have special `t_XX` options nor <t_XX> keycodes to configure
 terminal capabilities. Instead Nvim treats the terminal as any other UI. For
 example, 'guicursor' sets the terminal cursor style if possible.
 
-						  *'term'* *E529* *E530* *E531*
+			*:set-termcap*
+Start Nvim with 'verbose' level 3 to see the terminal capabilities. >
+	nvim -V3
+<
+			*'term'* *E529* *E530* *E531*
 'term' reflects the terminal type derived from |$TERM| and other environment
 checks.  For debugging only; not reliable during startup. >
 	:echo &term
 "builtin_x" means one of the |builtin-terms| was chosen, because the expected
 terminfo file was not found on the system.
 
-								*termcap*
+			*termcap*
 Nvim never uses the termcap database, only |terminfo| and |builtin-terms|.
 
-						*xterm-8bit* *xterm-8-bit*
+			*xterm-8bit* *xterm-8-bit*
 Xterm can be run in a mode where it uses true 8-bit CSI.  Supporting this
 requires autodetection of whether the terminal is in UTF-8 mode or non-UTF-8
 mode, as the 8-bit CSI character has to be written differently in each case.

src/nvim/README.md

@@ -32,6 +32,39 @@ The source files use extensions to hint about their purpose.
 - `*.h.generated.h` - exported functions’ declarations.
 - `*.c.generated.h` - static functions’ declarations.
 
+TUI debugging
+-------------
+
+### TUI troubleshoot
+
+Nvim logs its internal terminfo state at 'verbose' level 3.  This makes it
+possible to see exactly what terminfo values Nvim is using on any system.
+
+    nvim -V3log
+
+### TUI trace
+
+The ancient `script` command is still the "state of the art" for tracing
+terminal behavior. The libvterm `vterm-dump` utility formats the result for
+human-readability.
+
+Record a Nvim terminal session and format it with `vterm-dump`:
+
+    script foo
+    ./build/bin/nvim -u NONE
+    # Exit the script session with CTRL-d
+
+    # Use `vterm-dump` utility to format the result.
+    ./.deps/usr/bin/vterm-dump foo > bar
+
+Then you can compare `bar` with another session, to debug TUI behavior.
+
+### Terminal reference
+
+- `man terminfo`
+- http://bazaar.launchpad.net/~libvterm/libvterm/trunk/view/head:/doc/seqs.txt
+- http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
+
 Nvim lifecycle
 --------------
 
@@ -39,7 +72,7 @@ Following describes how Nvim processes input.
 
 Consider a typical Vim-like editing session:
 
-01. Vim dispays the welcome screen
+01. Vim displays the welcome screen
 02. User types: `:`
 03. Vim enters command-line mode
 04. User types: `edit README.txt<CR>`

src/nvim/message.c

@@ -1237,31 +1237,30 @@ void msg_make(char_u *arg)
   }
 }
 
-/*
+/// Output the string 'str' upto a NUL character.
- * Output the string 'str' upto a NUL character.
+/// Return the number of characters it takes on the screen.
- * Return the number of characters it takes on the screen.
+///
- *
+/// If K_SPECIAL is encountered, then it is taken in conjunction with the
- * If K_SPECIAL is encountered, then it is taken in conjunction with the
+/// following character and shown as <F1>, <S-Up> etc.  Any other character
- * following character and shown as <F1>, <S-Up> etc.  Any other character
+/// which is not printable shown in <> form.
- * which is not printable shown in <> form.
+/// If 'from' is TRUE (lhs of a mapping), a space is shown as <Space>.
- * If 'from' is TRUE (lhs of a mapping), a space is shown as <Space>.
+/// If a character is displayed in one of these special ways, is also
- * If a character is displayed in one of these special ways, is also
+/// highlighted (its highlight name is '8' in the p_hl variable).
- * highlighted (its highlight name is '8' in the p_hl variable).
+/// Otherwise characters are not highlighted.
- * Otherwise characters are not highlighted.
+/// This function is used to show mappings, where we want to see how to type
- * This function is used to show mappings, where we want to see how to type
+/// the character/string -- webb
- * the character/string -- webb
+int msg_outtrans_special(
- */
+    const char_u *strstart,
-int
+    int from               ///< true for LHS of a mapping
-msg_outtrans_special (
-    char_u *strstart,
-    int from               /* TRUE for lhs of a mapping */
 )
 {
-  char_u      *str = strstart;
+  if (strstart == NULL) {
+    return 0;  // Do nothing.
+  }
+  const char_u *str = strstart;
   int retval = 0;
-  int attr;
+  int attr = hl_attr(HLF_8);
 
-  attr = hl_attr(HLF_8);
   while (*str != NUL) {
     const char *string;
     // Leading and trailing spaces need to be displayed in <> form.
@@ -1307,7 +1306,7 @@ char *str2special_save(const char *const str, const bool replace_spaces,
   return (char *)ga.ga_data;
 }
 
-/// Convert character, replacing key one key code with printable representation
+/// Convert character, replacing key with printable representation.
 ///
 /// @param[in,out]  sp  String to convert. Is advanced to the next key code.
 /// @param[in]  replace_spaces  Convert spaces into <Space>, normally used for
@@ -1392,7 +1391,7 @@ void str2specialbuf(const char *sp, char *buf, size_t len)
   while (*sp) {
     const char *s = str2special(&sp, false, false);
     const size_t s_len = strlen(s);
-    if (s_len <= len) {
+    if (len <= s_len) {
       break;
     }
     memcpy(buf, s, s_len);

src/nvim/option.c

@@ -4906,15 +4906,14 @@ showoptions (
 
   vimoption_T **items = xmalloc(sizeof(vimoption_T *) * PARAM_COUNT);
 
-  /* Highlight title */
+  // Highlight title
-  if (all == 2)
+  if (opt_flags & OPT_GLOBAL) {
-    MSG_PUTS_TITLE(_("\n--- Terminal codes ---"));
-  else if (opt_flags & OPT_GLOBAL)
     MSG_PUTS_TITLE(_("\n--- Global option values ---"));
-  else if (opt_flags & OPT_LOCAL)
+  } else if (opt_flags & OPT_LOCAL) {
     MSG_PUTS_TITLE(_("\n--- Local option values ---"));
-  else
+  } else {
     MSG_PUTS_TITLE(_("\n--- Options ---"));
+  }
 
   /*
    * do the loop two times:

src/nvim/tui/terminfo.c

@@ -9,7 +9,10 @@
 #include <unibilium.h>
 
 #include "nvim/log.h"
+#include "nvim/globals.h"
 #include "nvim/memory.h"
+#include "nvim/message.h"
+#include "nvim/option.h"
 #include "nvim/tui/terminfo.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
@@ -166,3 +169,87 @@ unibi_term *terminfo_from_builtin(const char *term, char **termname)
   unibi_set_bool(ut, unibi_back_color_erase, false);
   return ut;
 }
+
+/// Dumps termcap info to the messages area.
+/// Serves a similar purpose as Vim `:set termcap` (removed in Nvim).
+///
+/// @note adapted from unibilium unibi-dump.c
+void terminfo_info_msg(const unibi_term *const ut)
+{
+  if (exiting) {
+    return;
+  }
+  msg_puts_title("\n\n--- Terminal info --- {{{\n");
+
+  char *term;
+  get_tty_option("term", &term);
+  msg_printf_attr(0, "&term: %s\n", term);
+  msg_printf_attr(0, "Description: %s\n", unibi_get_name(ut));
+  const char **a = unibi_get_aliases(ut);
+  if (*a) {
+    msg_puts("Aliases: ");
+    do {
+      msg_printf_attr(0, "%s%s\n", *a, a[1] ? " | " : "");
+      a++;
+    } while (*a);
+  }
+
+  msg_puts("Boolean capabilities:\n");
+  for (enum unibi_boolean i = unibi_boolean_begin_ + 1;
+       i < unibi_boolean_end_; i++) {
+    msg_printf_attr(0, "  %-25s %-10s = %s\n", unibi_name_bool(i),
+                    unibi_short_name_bool(i),
+                    unibi_get_bool(ut, i) ? "true" : "false");
+  }
+
+  msg_puts("Numeric capabilities:\n");
+  for (enum unibi_numeric i = unibi_numeric_begin_ + 1;
+       i < unibi_numeric_end_; i++) {
+    int n = unibi_get_num(ut, i);  // -1 means "empty"
+    msg_printf_attr(0, "  %-25s %-10s = %hd\n", unibi_name_num(i),
+                    unibi_short_name_num(i), n);
+  }
+
+  msg_puts("String capabilities:\n");
+  for (enum unibi_string i = unibi_string_begin_ + 1;
+       i < unibi_string_end_; i++) {
+    const char *s = unibi_get_str(ut, i);
+    if (s) {
+      msg_printf_attr(0, "  %-25s %-10s = ", unibi_name_str(i),
+                      unibi_short_name_str(i));
+      // Most of these strings will contain escape sequences.
+      msg_outtrans_special((char_u *)s, false);
+      msg_putchar('\n');
+    }
+  }
+
+  if (unibi_count_ext_bool(ut)) {
+    msg_puts("Extended boolean capabilities:\n");
+    for (size_t i = 0; i < unibi_count_ext_bool(ut); i++) {
+      msg_printf_attr(0, "  %-25s = %s\n",
+                      unibi_get_ext_bool_name(ut, i),
+                      unibi_get_ext_bool(ut, i) ? "true" : "false");
+    }
+  }
+
+  if (unibi_count_ext_num(ut)) {
+    msg_puts("Extended numeric capabilities:\n");
+    for (size_t i = 0; i < unibi_count_ext_num(ut); i++) {
+      msg_printf_attr(0, "  %-25s = %hd\n",
+                      unibi_get_ext_num_name(ut, i),
+                      unibi_get_ext_num(ut, i));
+    }
+  }
+
+  if (unibi_count_ext_str(ut)) {
+    msg_puts("Extended string capabilities:\n");
+    for (size_t i = 0; i < unibi_count_ext_str(ut); i++) {
+      msg_printf_attr(0, "  %-25s = ", unibi_get_ext_str_name(ut, i));
+      msg_outtrans_special((char_u *)unibi_get_ext_str(ut, i), false);
+      msg_putchar('\n');
+    }
+  }
+
+  msg_puts("}}}\n");
+  xfree(term);
+}

src/nvim/tui/tui.c

@@ -354,10 +354,12 @@ static void tui_main(UIBridgeData *bridge, UI *ui)
   tui_terminal_start(ui);
   data->stop = false;
 
-  // allow the main thread to continue, we are ready to start handling UI
+  // Allow main thread to continue, we are ready to handle UI callbacks.
-  // callbacks
   CONTINUE(bridge);
 
+  loop_schedule_deferred(&main_loop,
+                         event_create(show_termcap_event, 1, data->ut));
+
   while (!data->stop) {
     loop_poll_events(&tui_loop, -1);  // tui_loop.events is never processed
   }
@@ -1061,6 +1063,24 @@ static void tui_flush(UI *ui)
   flush_buf(ui, true);
 }
 
+/// Dumps termcap info to the messages area, if 'verbose' >= 3.
+static void show_termcap_event(void **argv)
+{
+  if (p_verbose < 3) {
+    return;
+  }
+  const unibi_term *const ut = argv[0];
+  if (!ut) {
+    abort();
+  }
+  verbose_enter();
+  // XXX: (future) if unibi_term is modified (e.g. after a terminal
+  // query-response) this is a race condition.
+  terminfo_info_msg(ut);
+  verbose_leave();
+  verbose_stop();  // flush now
+}
+
 #ifdef UNIX
 static void suspend_event(void **argv)
 {
@@ -1577,11 +1597,13 @@ static void augment_terminfo(TUIData *data, const char *term,
       || konsole     // per commentary in VT102Emulation.cpp
       || teraterm    // per TeraTerm "Supported Control Functions" doco
       || rxvt) {     // per command.C
-    data->unibi_ext.resize_screen = (int)unibi_add_ext_str(ut, NULL,
+    data->unibi_ext.resize_screen = (int)unibi_add_ext_str(ut,
+      "ext.resize_screen",
       "\x1b[8;%p1%d;%p2%dt");
   }
   if (putty || xterm || rxvt) {
-    data->unibi_ext.reset_scroll_region = (int)unibi_add_ext_str(ut, NULL,
+    data->unibi_ext.reset_scroll_region = (int)unibi_add_ext_str(ut,
+      "ext.reset_scroll_region",
       "\x1b[r");
   }
 
@@ -1639,21 +1661,29 @@ static void augment_terminfo(TUIData *data, const char *term,
 
   /// Terminals usually ignore unrecognized private modes, and there is no
   /// known ambiguity with these. So we just set them unconditionally.
-  data->unibi_ext.enable_lr_margin = (int)unibi_add_ext_str(ut, NULL,
+  data->unibi_ext.enable_lr_margin = (int)unibi_add_ext_str(ut,
+      "ext.enable_lr_margin",
       "\x1b[?69h");
-  data->unibi_ext.disable_lr_margin = (int)unibi_add_ext_str(ut, NULL,
+  data->unibi_ext.disable_lr_margin = (int)unibi_add_ext_str(ut,
+      "ext.disable_lr_margin",
       "\x1b[?69l");
-  data->unibi_ext.enable_bracketed_paste = (int)unibi_add_ext_str(ut, NULL,
+  data->unibi_ext.enable_bracketed_paste = (int)unibi_add_ext_str(ut,
+      "ext.enable_bpaste",
       "\x1b[?2004h");
-  data->unibi_ext.disable_bracketed_paste = (int)unibi_add_ext_str(ut, NULL,
+  data->unibi_ext.disable_bracketed_paste = (int)unibi_add_ext_str(ut,
+      "ext.disable_bpaste",
       "\x1b[?2004l");
-  data->unibi_ext.enable_focus_reporting = (int)unibi_add_ext_str(ut, NULL,
+  data->unibi_ext.enable_focus_reporting = (int)unibi_add_ext_str(ut,
+      "ext.enable_focus",
       rxvt ? "\x1b]777;focus;on\x7" : "\x1b[?1004h");
-  data->unibi_ext.disable_focus_reporting = (int)unibi_add_ext_str(ut, NULL,
+  data->unibi_ext.disable_focus_reporting = (int)unibi_add_ext_str(ut,
+      "ext.disable_focus",
       rxvt ? "\x1b]777;focus;off\x7" : "\x1b[?1004l");
-  data->unibi_ext.enable_mouse = (int)unibi_add_ext_str(ut, NULL,
+  data->unibi_ext.enable_mouse = (int)unibi_add_ext_str(ut,
+      "ext.enable_mouse",
       "\x1b[?1002h\x1b[?1006h");
-  data->unibi_ext.disable_mouse = (int)unibi_add_ext_str(ut, NULL,
+  data->unibi_ext.disable_mouse = (int)unibi_add_ext_str(ut,
+      "ext.disable_mouse",
       "\x1b[?1002l\x1b[?1006l");
 }
 

src/nvim/ui_bridge.c

@@ -82,6 +82,7 @@ UI *ui_bridge_attach(UI *ui, ui_main_fn ui_main, event_scheduler scheduler)
     abort();
   }
 
+  // Suspend the main thread until CONTINUE is called by the UI thread.
   while (!rv->ready) {
     uv_cond_wait(&rv->cond, &rv->mutex);
   }
@@ -149,7 +150,7 @@ static void ui_bridge_suspend(UI *b)
   uv_mutex_lock(&data->mutex);
   UI_BRIDGE_CALL(b, suspend, 1, b);
   data->ready = false;
-  // suspend the main thread until CONTINUE is called by the UI thread
+  // Suspend the main thread until CONTINUE is called by the UI thread.
   while (!data->ready) {
     uv_cond_wait(&data->cond, &data->mutex);
   }

test/functional/helpers.lua

@@ -261,6 +261,7 @@ local function retry(max, max_ms, fn)
     if status then
       return result
     end
+    luv.update_time()  -- Update cached value of luv.now() (libuv: uv_now()).
     if (max and tries >= max) or (luv.now() - start_time > timeout) then
       if type(result) == "string" then
         result = "\nretry() attempts: "..tostring(tries).."\n"..result
@@ -333,8 +334,8 @@ local function feed_command(...)
 end
 
 -- Dedent the given text and write it to the file name.
-local function write_file(name, text, dont_dedent)
+local function write_file(name, text, no_dedent, append)
-  local file = io.open(name, 'w')
+  local file = io.open(name, (append and 'a' or 'w'))
   if type(text) == 'table' then
     -- Byte blob
     local bytes = text
@@ -342,7 +343,7 @@ local function write_file(name, text, dont_dedent)
     for _, char in ipairs(bytes) do
       text = ('%s%c'):format(text, char)
     end
-  elseif not dont_dedent then
+  elseif not no_dedent then
     text = dedent(text)
   end
   file:write(text)

test/functional/terminal/tui_spec.lua

@@ -4,6 +4,7 @@ local global_helpers = require('test.helpers')
 local uname = global_helpers.uname
 local helpers = require('test.functional.helpers')(after_each)
 local thelpers = require('test.functional.terminal.helpers')
+local eq = helpers.eq
 local feed_data = thelpers.feed_data
 local feed_command = helpers.feed_command
 local clear = helpers.clear
@@ -11,6 +12,8 @@ local nvim_dir = helpers.nvim_dir
 local retry = helpers.retry
 local nvim_prog = helpers.nvim_prog
 local nvim_set = helpers.nvim_set
+local ok = helpers.ok
+local read_file = helpers.read_file
 
 if helpers.pending_win32(pending) then return end
 
@@ -21,9 +24,6 @@ describe('tui', function()
     clear()
     screen = thelpers.screen_setup(0, '["'..nvim_prog
       ..'", "-u", "NONE", "-i", "NONE", "--cmd", "set noswapfile noshowcmd noruler undodir=. directory=. viewdir=. backupdir=."]')
-    -- right now pasting can be really slow in the TUI, especially in ASAN.
-    -- this will be fixed later but for now we require a high timeout.
-    screen.timeout = 60000
     screen:expect([[
       {1: }                                                 |
       {4:~                                                 }|
@@ -125,6 +125,9 @@ describe('tui', function()
   end)
 
   it('automatically sends <Paste> for bracketed paste sequences', function()
+    -- Pasting can be really slow in the TUI, specially in ASAN.
+    -- This will be fixed later but for now we require a high timeout.
+    screen.timeout = 60000
     feed_data('i\027[200~')
     screen:expect([[
       {1: }                                                 |
@@ -158,6 +161,8 @@ describe('tui', function()
   end)
 
   it('can handle arbitrarily long bursts of input', function()
+    -- Need extra time for this test, specially in ASAN.
+    screen.timeout = 60000
     feed_command('set ruler')
     local t = {}
     for i = 1, 3000 do
@@ -639,6 +644,7 @@ end)
 describe("tui 'term' option", function()
   local screen
   local is_bsd = not not string.find(string.lower(uname()), 'bsd')
+  local is_macos = not not string.find(string.lower(uname()), 'darwin')
 
   local function assert_term(term_envvar, term_expected)
     clear()
@@ -664,11 +670,62 @@ describe("tui 'term' option", function()
   end)
 
   it('gets system-provided term if $TERM is valid', function()
-    if is_bsd then  -- BSD lacks terminfo, we always use builtin there.
+    if is_bsd then  -- BSD lacks terminfo, builtin is always used.
       assert_term("xterm", "builtin_xterm")
+    elseif is_macos then
+      local status, _ = pcall(assert_term, "xterm", "xterm")
+      if not status then
+        pending("macOS: unibilium could not find terminfo", function() end)
+      end
     else
       assert_term("xterm", "xterm")
     end
   end)
 
 end)
+
+-- These tests require `thelpers` because --headless/--embed
+-- does not initialize the TUI.
+describe("tui", function()
+  local screen
+  local logfile = 'Xtest_tui_verbose_log'
+  after_each(function()
+    os.remove(logfile)
+  end)
+
+  -- Runs (child) `nvim` in a TTY (:terminal), to start the builtin TUI.
+  local function nvim_tui(extra_args)
+    clear()
+    -- This is ugly because :term/termopen() forces TERM=xterm-256color.
+    -- TODO: Revisit this after jobstart/termopen accept `env` dict.
+    local cmd = string.format(
+      [=[['sh', '-c', 'LANG=C %s -u NONE -i NONE %s --cmd "%s"']]=],
+      nvim_prog,
+      extra_args or "",
+      nvim_set)
+    screen = thelpers.screen_setup(0, cmd)
+  end
+
+  it('-V3log logs terminfo values', function()
+    nvim_tui('-V3'..logfile)
+
+    -- Wait for TUI to start.
+    feed_data('Gitext')
+    screen:expect([[
+      text{1: }                                             |
+      {4:~                                                 }|
+      {4:~                                                 }|
+      {4:~                                                 }|
+      {4:~                                                 }|
+      {3:-- INSERT --}                                      |
+      {3:-- TERMINAL --}                                    |
+    ]])
+
+    retry(nil, 3000, function()  -- Wait for log file to be flushed.
+      local log = read_file('Xtest_tui_verbose_log') or ''
+      eq('--- Terminal info --- {{{\n', string.match(log, '--- Terminal.-\n'))
+      ok(#log > 50)
+    end)
+  end)
+
+end)