terminal: fix rgb rendering of palette colors

simplify handling of default colors

nvim is always true color internally, remove ui_rgb_attached() check.

Fix "runtime termguicolors" test. The test actually reflected broken behavior
in (parent) nvim: nvim_ui_set_option("rgb", true) was not respected by existing
:terminal instances, so all 16-palette colors became dark blue.

src/nvim/terminal.c

@@ -152,8 +152,6 @@ static VTermScreenCallbacks vterm_screen_callbacks = {
 };
 
 static PMap(ptr_t) *invalidated_terminals;
-static int default_vt_fg, default_vt_bg;
-static VTermColor default_vt_bg_rgb;
 
 void terminal_init(void)
 {
@@ -161,16 +159,6 @@ void terminal_init(void)
   time_watcher_init(&main_loop, &refresh_timer, NULL);
   // refresh_timer_cb will redraw the screen which can call vimscript
   refresh_timer.events = multiqueue_new_child(main_loop.events);
-
-  VTerm *vt = vterm_new(24, 80);
-  VTermState *state = vterm_obtain_state(vt);
-
-  VTermColor fg, bg;
-  vterm_state_get_default_colors(state, &fg, &bg);
-  default_vt_fg = RGB_(fg.rgb.red, fg.rgb.green, fg.rgb.blue);
-  default_vt_bg = RGB_(bg.rgb.red, bg.rgb.green, bg.rgb.blue);
-  default_vt_bg_rgb = bg;
-  vterm_free(vt);
 }
 
 void terminal_teardown(void)
@@ -185,7 +173,6 @@ void terminal_teardown(void)
 
 Terminal *terminal_open(TerminalOptions opts)
 {
-  bool true_color = ui_rgb_attached();
   // Create a new terminal instance and configure it
   Terminal *rv = xcalloc(1, sizeof(Terminal));
   rv->opts = opts;
@@ -233,10 +220,6 @@ Terminal *terminal_open(TerminalOptions opts)
   rv->sb_size = (size_t)curbuf->b_p_scbk;
   rv->sb_buffer = xmalloc(sizeof(ScrollbackLine *) * rv->sb_size);
 
-  if (!true_color) {
-    return rv;
-  }
-
   vterm_state_set_bold_highbright(state, true);
 
   // Configure the color palette. Try to get the color from:
@@ -583,11 +566,19 @@ void terminal_receive(Terminal *term, char *data, size_t len)
   vterm_screen_flush_damage(term->vts);
 }
 
+static int get_rgb(VTermState *state, VTermColor color)
+{
+  vterm_state_convert_color_to_rgb(state, &color);
+  return RGB_(color.rgb.red, color.rgb.green, color.rgb.blue);
+}
+
+
 void terminal_get_line_attributes(Terminal *term, win_T *wp, int linenr,
                                   int *term_attrs)
 {
   int height, width;
   vterm_get_size(term->vt, &height, &width);
+  VTermState *state = vterm_obtain_state(term->vt);
   assert(linenr);
   int row = linenr_to_row(term, linenr);
   if (row >= height) {
@@ -598,18 +589,18 @@ void terminal_get_line_attributes(Terminal *term, win_T *wp, int linenr,
 
   for (int col = 0; col < width; col++) {
     VTermScreenCell cell;
-    fetch_cell(term, row, col, &cell);
+    bool color_valid = fetch_cell(term, row, col, &cell);
+    bool fg_default = !color_valid || VTERM_COLOR_IS_DEFAULT_FG(&cell.fg);
+    bool bg_default = !color_valid || VTERM_COLOR_IS_DEFAULT_BG(&cell.bg);
+
     // Get the rgb value set by libvterm.
-    int vt_fg = RGB_(cell.fg.rgb.red, cell.fg.rgb.green, cell.fg.rgb.blue);
+    int vt_fg = fg_default ? -1 : get_rgb(state, cell.fg);
-    int vt_bg = RGB_(cell.bg.rgb.red, cell.bg.rgb.green, cell.bg.rgb.blue);
+    int vt_bg = bg_default ? -1 : get_rgb(state, cell.bg);
-    vt_fg = vt_fg != default_vt_fg ? vt_fg : - 1;
+
-    vt_bg = vt_bg != default_vt_bg ? vt_bg : - 1;
+    int vt_fg_idx = ((!fg_default && VTERM_COLOR_IS_INDEXED(&cell.fg))
-    int vt_fg_idx = VTERM_COLOR_IS_DEFAULT_FG(&cell.fg)
+                     ? cell.fg.indexed.idx + 1 : 0);
-        ? 0 : VTERM_COLOR_IS_INDEXED(&cell.fg)
+    int vt_bg_idx = ((!bg_default && VTERM_COLOR_IS_INDEXED(&cell.bg))
-            ? cell.fg.indexed.idx + 1 : 0;
+                     ? cell.bg.indexed.idx + 1 : 0);
-    int vt_bg_idx = VTERM_COLOR_IS_DEFAULT_BG(&cell.bg)
-        ? 0 : VTERM_COLOR_IS_INDEXED(&cell.bg)
-            ? cell.bg.indexed.idx + 1 : 0;
 
     int hl_attrs = (cell.attrs.bold ? HL_BOLD : 0)
                  | (cell.attrs.italic ? HL_ITALIC : 0)
@@ -618,7 +609,7 @@ void terminal_get_line_attributes(Terminal *term, win_T *wp, int linenr,
 
     int attr_id = 0;
 
-    if (hl_attrs || vt_fg != -1 || vt_bg != -1) {
+    if (hl_attrs ||!fg_default || !bg_default) {
       attr_id = hl_get_term_attr(&(HlAttrs) {
         .cterm_ae_attr = (int16_t)hl_attrs,
         .cterm_fg_color = vt_fg_idx,
@@ -1082,8 +1073,8 @@ static void fetch_row(Terminal *term, int row, int end_col)
   term->textbuf[line_len] = 0;
 }
 
-static void fetch_cell(Terminal *term, int row, int col,
+static bool fetch_cell(Terminal *term, int row, int col,
-    VTermScreenCell *cell)
+                       VTermScreenCell *cell)
 {
   if (row < 0) {
     ScrollbackLine *sbrow = term->sb_buffer[-row - 1];
@@ -1094,13 +1085,14 @@ static void fetch_cell(Terminal *term, int row, int col,
       *cell = (VTermScreenCell) {
         .chars = { 0 },
         .width = 1,
-        .bg = default_vt_bg_rgb
       };
+      return false;
     }
   } else {
     vterm_screen_get_cell(term->vts, (VTermPos){.row = row, .col = col},
         cell);
   }
+  return true;
 }
 
 // queue a terminal instance for refresh

test/functional/terminal/tui_spec.lua

@@ -640,11 +640,11 @@ describe('TUI', function()
     screen:set_option('rgb', true)
     screen:set_default_attr_ids({
       [1] = {reverse = true},
-      [2] = {foreground = 13},
+      [2] = {foreground = tonumber('0x4040ff')},
       [3] = {bold = true, reverse = true},
       [4] = {bold = true},
-      [5] = {reverse = true, foreground = 4},
+      [5] = {reverse = true, foreground = tonumber('0xe0e000')},
-      [6] = {foreground = 4},
+      [6] = {foreground = tonumber('0xe0e000')},
       [7] = {reverse = true, foreground = Screen.colors.SeaGreen4},
       [8] = {foreground = Screen.colors.SeaGreen4},
       [9] = {bold = true, foreground = Screen.colors.Blue1},