perf(lsp): reduce polling handles for workspace/didChangeWatchedFiles (#23500)

Co-authored-by: Lewis Russell <lewis6991@gmail.com>

runtime/lua/vim/_watch.lua

@@ -11,6 +11,7 @@ M.FileChangeType = vim.tbl_add_reverse_lookup({
 --- Joins filepath elements by static '/' separator
 ---
 ---@param ... (string) The path elements.
+---@return string
 local function filepath_join(...)
   return table.concat({ ... }, '/')
 end
@@ -36,7 +37,7 @@ end
 ---     - uvflags (table|nil)
 ---                Same flags as accepted by |uv.fs_event_start()|
 ---@param callback (function) The function called when new events
----@returns (function) A function to stop the watch
+---@return (function) A function to stop the watch
 function M.watch(path, opts, callback)
   vim.validate({
     path = { path, 'string', false },
@@ -75,10 +76,25 @@ end
 
 local default_poll_interval_ms = 2000
 
+--- @class watch.Watches
+--- @field is_dir boolean
+--- @field children? table<string,watch.Watches>
+--- @field cancel? fun()
+--- @field started? boolean
+--- @field handle? uv_fs_poll_t
+
+--- @class watch.PollOpts
+--- @field interval? integer
+--- @field include_pattern? userdata
+--- @field exclude_pattern? userdata
+
 ---@private
 --- Implementation for poll, hiding internally-used parameters.
 ---
----@param watches (table|nil) A tree structure to maintain state for recursive watches.
+---@param path string
+---@param opts watch.PollOpts
+---@param callback fun(patch: string, filechangetype: integer)
+---@param watches (watch.Watches|nil) A tree structure to maintain state for recursive watches.
 ---     - handle (uv_fs_poll_t)
 ---               The libuv handle
 ---     - cancel (function)
@@ -88,15 +104,36 @@ local default_poll_interval_ms = 2000
 ---               be invoked recursively)
 ---     - children (table|nil)
 ---               A mapping of directory entry name to its recursive watches
---      - started (boolean|nil)
+---     - started (boolean|nil)
---                Whether or not the watcher has first been initialized. Used
+---               Whether or not the watcher has first been initialized. Used
---                to prevent a flood of Created events on startup.
+---               to prevent a flood of Created events on startup.
+---@return fun() Cancel function
 local function poll_internal(path, opts, callback, watches)
   path = vim.fs.normalize(path)
   local interval = opts and opts.interval or default_poll_interval_ms
   watches = watches or {
     is_dir = true,
   }
+  watches.cancel = function()
+    if watches.children then
+      for _, w in pairs(watches.children) do
+        w.cancel()
+      end
+    end
+    if watches.handle then
+      stop(watches.handle)
+    end
+  end
+
+  local function incl_match()
+    return not opts.include_pattern or opts.include_pattern:match(path) ~= nil
+  end
+  local function excl_match()
+    return opts.exclude_pattern and opts.exclude_pattern:match(path) ~= nil
+  end
+  if not watches.is_dir and not incl_match() or excl_match() then
+    return watches.cancel
+  end
 
   if not watches.handle then
     local poll, new_err = vim.uv.new_fs_poll()
@@ -120,18 +157,9 @@ local function poll_internal(path, opts, callback, watches)
     end
   end
 
-  watches.cancel = function()
-    if watches.children then
-      for _, w in pairs(watches.children) do
-        w.cancel()
-      end
-    end
-    stop(watches.handle)
-  end
-
   if watches.is_dir then
     watches.children = watches.children or {}
-    local exists = {}
+    local exists = {} --- @type table<string,true>
     for name, ftype in vim.fs.dir(path) do
       exists[name] = true
       if not watches.children[name] then
@@ -143,16 +171,18 @@ local function poll_internal(path, opts, callback, watches)
       end
     end
 
-    local newchildren = {}
+    local newchildren = {} ---@type table<string,watch.Watches>
     for name, watch in pairs(watches.children) do
       if exists[name] then
         newchildren[name] = watch
       else
         watch.cancel()
         watches.children[name] = nil
+        if watch.handle then
           callback(path .. '/' .. name, M.FileChangeType.Deleted)
         end
       end
+    end
     watches.children = newchildren
   end
 
@@ -168,6 +198,15 @@ end
 ---@param opts (table|nil) Additional options
 ---     - interval (number|nil)
 ---                Polling interval in ms as passed to |uv.fs_poll_start()|. Defaults to 2000.
+---     - include_pattern (LPeg pattern|nil)
+---                An |lpeg| pattern. Only changes to files whose full paths match the pattern
+---                will be reported. Only matches against non-directoriess, all directories will
+---                be watched for new potentially-matching files. exclude_pattern can be used to
+---                filter out directories. When nil, matches any file name.
+---     - exclude_pattern (LPeg pattern|nil)
+---                An |lpeg| pattern. Only changes to files and directories whose full path does
+---                not match the pattern will be reported. Matches against both files and
+---                directories. When nil, matches nothing.
 ---@param callback (function) The function called when new events
 ---@returns (function) A function to stop the watch.
 function M.poll(path, opts, callback)

runtime/lua/vim/lsp/_watchfiles.lua

@@ -1,7 +1,7 @@
 local bit = require('bit')
-local lpeg = require('lpeg')
 local watch = require('vim._watch')
 local protocol = require('vim.lsp.protocol')
+local lpeg = vim.lpeg
 
 local M = {}
 
@@ -107,6 +107,13 @@ local to_lsp_change_type = {
   [watch.FileChangeType.Deleted] = protocol.FileChangeType.Deleted,
 }
 
+--- Default excludes the same as VSCode's `files.watcherExclude` setting.
+--- https://github.com/microsoft/vscode/blob/eef30e7165e19b33daa1e15e92fa34ff4a5df0d3/src/vs/workbench/contrib/files/browser/files.contribution.ts#L261
+---@type Lpeg pattern
+M._poll_exclude_pattern = parse('**/.git/{objects,subtree-cache}/**')
+  + parse('**/node_modules/*/**')
+  + parse('**/.hg/store/**')
+
 --- Registers the workspace/didChangeWatchedFiles capability dynamically.
 ---
 ---@param reg table LSP Registration object.
@@ -122,10 +129,10 @@ function M.register(reg, ctx)
   then
     return
   end
-  local watch_regs = {}
+  local watch_regs = {} --- @type table<string,{pattern:userdata,kind:integer}>
   for _, w in ipairs(reg.registerOptions.watchers) do
     local relative_pattern = false
-    local glob_patterns = {}
+    local glob_patterns = {} --- @type {baseUri:string, pattern: string}[]
     if type(w.globPattern) == 'string' then
       for _, folder in ipairs(client.workspace_folders) do
         table.insert(glob_patterns, { baseUri = folder.uri, pattern = w.globPattern })
@@ -135,7 +142,7 @@ function M.register(reg, ctx)
       table.insert(glob_patterns, w.globPattern)
     end
     for _, glob_pattern in ipairs(glob_patterns) do
-      local base_dir = nil
+      local base_dir = nil ---@type string?
       if type(glob_pattern.baseUri) == 'string' then
         base_dir = glob_pattern.baseUri
       elseif type(glob_pattern.baseUri) == 'table' then
@@ -144,6 +151,7 @@ function M.register(reg, ctx)
       assert(base_dir, "couldn't identify root of watch")
       base_dir = vim.uri_to_fname(base_dir)
 
+      ---@type integer
       local kind = w.kind
         or protocol.WatchKind.Create + protocol.WatchKind.Change + protocol.WatchKind.Delete
 
@@ -153,8 +161,8 @@ function M.register(reg, ctx)
         pattern = lpeg.P(base_dir .. '/') * pattern
       end
 
-      table.insert(watch_regs, {
+      watch_regs[base_dir] = watch_regs[base_dir] or {}
-        base_dir = base_dir,
+      table.insert(watch_regs[base_dir], {
         pattern = pattern,
         kind = kind,
       })
@@ -163,12 +171,12 @@ function M.register(reg, ctx)
 
   local callback = function(base_dir)
     return function(fullpath, change_type)
-      for _, w in ipairs(watch_regs) do
+      for _, w in ipairs(watch_regs[base_dir]) do
         change_type = to_lsp_change_type[change_type]
         -- e.g. match kind with Delete bit (0b0100) to Delete change_type (3)
         local kind_mask = bit.lshift(1, change_type - 1)
         local change_type_match = bit.band(w.kind, kind_mask) == kind_mask
-        if base_dir == w.base_dir and M._match(w.pattern, fullpath) and change_type_match then
+        if M._match(w.pattern, fullpath) and change_type_match then
           local change = {
             uri = vim.uri_from_fname(fullpath),
             type = change_type,
@@ -198,16 +206,26 @@ function M.register(reg, ctx)
     end
   end
 
-  local watching = {}
+  for base_dir, watches in pairs(watch_regs) do
-  for _, w in ipairs(watch_regs) do
+    local include_pattern = vim.iter(watches):fold(lpeg.P(false), function(acc, w)
-    if not watching[w.base_dir] then
+      return acc + w.pattern
-      watching[w.base_dir] = true
+    end)
+
     table.insert(
       cancels[client_id][reg.id],
-        M._watchfunc(w.base_dir, { uvflags = { recursive = true } }, callback(w.base_dir))
+      M._watchfunc(base_dir, {
+        uvflags = {
+          recursive = true,
+        },
+        -- include_pattern will ensure the pattern from *any* watcher definition for the
+        -- base_dir matches. This first pass prevents polling for changes to files that
+        -- will never be sent to the LSP server. A second pass in the callback is still necessary to
+        -- match a *particular* pattern+kind pair.
+        include_pattern = include_pattern,
+        exclude_pattern = M._poll_exclude_pattern,
+      }, callback(base_dir))
     )
   end
-  end
 end
 
 --- Unregisters the workspace/didChangeWatchedFiles capability dynamically.

test/functional/lua/watch_spec.lua

@@ -107,6 +107,7 @@ describe('vim._watch', function()
       local result = exec_lua(
         [[
         local root_dir = ...
+        local lpeg = vim.lpeg
 
         local events = {}
 
@@ -118,7 +119,13 @@ describe('vim._watch', function()
           assert(vim.wait(poll_wait_ms, function() return #events == expected_events end), 'Timed out waiting for expected number of events. Current events seen so far: ' .. vim.inspect(events))
         end
 
-        local stop = vim._watch.poll(root_dir, { interval = poll_interval_ms }, function(path, change_type)
+        local incl = lpeg.P(root_dir) * lpeg.P("/file")^-1
+        local excl = lpeg.P(root_dir..'/file.unwatched')
+        local stop = vim._watch.poll(root_dir, {
+            interval = poll_interval_ms,
+            include_pattern = incl,
+            exclude_pattern = excl,
+          }, function(path, change_type)
           table.insert(events, { path = path, change_type = change_type })
         end)
 
@@ -127,12 +134,17 @@ describe('vim._watch', function()
         local watched_path = root_dir .. '/file'
         local watched, err = io.open(watched_path, 'w')
         assert(not err, err)
+        local unwatched_path = root_dir .. '/file.unwatched'
+        local unwatched, err = io.open(unwatched_path, 'w')
+        assert(not err, err)
 
         expected_events = expected_events + 2
         wait_for_events()
 
         watched:close()
         os.remove(watched_path)
+        unwatched:close()
+        os.remove(unwatched_path)
 
         expected_events = expected_events + 2
         wait_for_events()