test: avoid noise in CI logs #30264

Problem:
Since 96128a5076 the test logs have noise from tests that *expect*
failures:

    $NVIM_LOG_FILE: /tmp/cirrus-ci-build/build/.nvimlog
    (last 100 lines)
    ERR 2024-09-04T13:38:45.181 T949.28335.0/c terminfo_start:486: uv_pipe_open failed: no such device or address
    ERR 2024-09-04T13:38:45.181 T949.28335.0/c flush_buf:2527: uv_write failed: bad file descriptor
    ERR 2024-09-04T13:38:45.181 T949.28335.0/c flush_buf:2527: uv_write failed: bad file descriptor
    WRN 2024-09-04T13:43:43.294 ?.35904    server_start:173: Failed to start server: address already in use: /…/Xtest_tmpdir/…/T7159.35895.0
    WRN 2024-09-04T13:43:43.314 ?.35907    server_start:173: Failed to start server: illegal operation on a directory: /
    ERR 2024-09-04T13:43:43.332 ?.35909    socket_watcher_init:60: Host lookup failed: https://example.com

Solution:
Rewrite the test to use `vim.system()`. Set NVIM_LOG_FILE in the child
process to a "throwaway" logfile.

test/functional/vimscript/server_spec.lua

@@ -1,7 +1,6 @@
 local t = require('test.testutil')
 local n = require('test.functional.testnvim')()
 
-local assert_log = t.assert_log
 local eq, neq, eval = t.eq, t.neq, n.eval
 local clear, fn, api = n.clear, n.fn, n.api
 local matches = t.matches
@@ -88,7 +87,7 @@ describe('server', function()
     }
     eq(0, eval("serverstop('')"))
     eq(0, eval("serverstop('bogus-socket-name')"))
-    assert_log('Not listening on bogus%-socket%-name', testlog, 10)
+    t.assert_log('Not listening on bogus%-socket%-name', testlog, 10)
   end)
 
   it('parses endpoints', function()
@@ -122,7 +121,7 @@ describe('server', function()
     if status then
       table.insert(expected, v4)
       pcall(fn.serverstart, v4) -- exists already; ignore
-      assert_log('Failed to start server: address already in use: 127%.0%.0%.1', testlog, 10)
+      t.assert_log('Failed to start server: address already in use: 127%.0%.0%.1', testlog, 10)
     end
 
     local v6 = '::1:12345'
@@ -130,7 +129,7 @@ describe('server', function()
     if status then
       table.insert(expected, v6)
       pcall(fn.serverstart, v6) -- exists already; ignore
-      assert_log('Failed to start server: address already in use: ::1', testlog, 10)
+      t.assert_log('Failed to start server: address already in use: ::1', testlog, 10)
     end
     eq(expected, fn.serverlist())
     clear_serverlist()
@@ -173,24 +172,43 @@ end)
 
 describe('startup --listen', function()
   it('validates', function()
-    clear()
+    os.remove(testlog)
+    clear { env = { NVIM_LOG_FILE = testlog } }
 
     -- Tests args with and without "--headless".
     local function _test(args, expected)
-      -- XXX: clear v:shell_error, sigh...
+      local function run(cmd)
-      fn.system({ n.nvim_prog, '-es', '+qall!' })
+        return n.exec_lua(function(cmd_)
-      assert(0 == eval('v:shell_error'))
+          return vim
-      local cmd = vim.list_extend({ unpack(n.nvim_argv) }, vim.list_extend({ '--headless' }, args))
+            .system(cmd_, {
-      local output = fn.system(cmd)
+              text = true,
-      assert(0 ~= eval('v:shell_error'))
+              env = {
-      -- TODO(justinmk): output not properly captured on Windows?
+                -- Avoid noise in the logs; we expect failures for these tests.
-      if is_os('win') then
+                NVIM_LOG_FILE = testlog,
-        return
+              },
+            })
+            :wait()
+        end, cmd) --[[@as vim.SystemCompleted]]
       end
-      matches(expected, output)
+
-      matches(expected, fn.system(vim.list_extend({ unpack(n.nvim_argv) }, args)))
+      local cmd = vim.list_extend({ n.nvim_prog, '+qall!', '--headless' }, args)
+      local r = run(cmd)
+      eq(1, r.code)
+      matches(expected, (r.stderr .. r.stdout):gsub('\\n', ' '))
+
+      if is_os('win') then
+        return -- On Windows, output without --headless is garbage.
+      end
+      table.remove(cmd, 3) -- Remove '--headless'.
+      assert(not vim.tbl_contains(cmd, '--headless'))
+      r = run(cmd)
+      eq(1, r.code)
+      matches(expected, (r.stderr .. r.stdout):gsub('\\n', ' '))
     end
 
+    t.assert_nolog('Failed to start server', testlog, 100)
+    t.assert_nolog('Host lookup failed', testlog, 100)
+
     _test({ '--listen' }, 'nvim.*: Argument missing after: "%-%-listen"')
     _test({ '--listen2' }, 'nvim.*: Garbage after option argument: "%-%-listen2"')
     _test({ '--listen', n.eval('v:servername') }, 'nvim.*: Failed to %-%-listen: ".* already .*"')
@@ -198,10 +216,12 @@ describe('startup --listen', function()
     _test(
       { '--listen', 'https://example.com' },
       ('nvim.*: Failed to %%-%%-listen: "%s"'):format(
-        (is_os('mac') or is_os('win')) and 'unknown node or service'
+        is_os('mac') and 'unknown node or service' or 'service not available for socket type'
-          or 'service not available for socket type'
       )
     )
+
+    t.assert_log('Failed to start server', testlog, 100)
+    t.assert_log('Host lookup failed', testlog, 100)
   end)
 
   it('sets v:servername, overrides $NVIM_LISTEN_ADDRESS', function()

test/testutil.lua

@@ -143,7 +143,7 @@ end
 ---
 ---@param pat (string) Lua pattern to match lines in the log file
 ---@param logfile? (string) Full path to log file (default=$NVIM_LOG_FILE)
----@param nrlines? (number) Search up to this many log lines
+---@param nrlines? (number) Search up to this many log lines (default 10)
 ---@param inverse? (boolean) Assert that the pattern does NOT match.
 function M.assert_log(pat, logfile, nrlines, inverse)
   logfile = logfile or os.getenv('NVIM_LOG_FILE') or '.nvimlog'