startup: handle autoload and lua packages during startup

¡NO HAY BANDA!

src/nvim/api/private/helpers.h

@@ -52,7 +52,8 @@
     .type = kObjectTypeLuaRef, \
     .data.luaref = r })
 
-#define NIL ((Object) {.type = kObjectTypeNil})
+#define NIL ((Object)OBJECT_INIT)
+#define NULL_STRING ((String)STRING_INIT)
 
 #define PUT(dict, k, v) \
   kv_push(dict, ((KeyValuePair) { .key = cstr_to_string(k), .value = v }))

src/nvim/api/vim.c

@@ -741,7 +741,11 @@ Integer nvim_strwidth(String text, Error *err)
 ArrayOf(String) nvim_list_runtime_paths(void)
   FUNC_API_SINCE(1)
 {
+  // TODO(bfredl): this should just work:
+  // return nvim_get_runtime_file(NULL_STRING, true);
+
   Array rv = ARRAY_DICT_INIT;
+
   char_u *rtp = p_rtp;
 
   if (*rtp == NUL) {
@@ -788,22 +792,29 @@ ArrayOf(String) nvim_list_runtime_paths(void)
 /// @param name pattern of files to search for
 /// @param all whether to return all matches or only the first
 /// @return list of absolute paths to the found files
-ArrayOf(String) nvim_get_runtime_file(String name, Boolean all)
+ArrayOf(String) nvim_get_runtime_file(String name, Boolean all, Error *err)
   FUNC_API_SINCE(7)
 {
   Array rv = ARRAY_DICT_INIT;
-  if (!name.data) {
+
+  // TODO(bfredl):
+  if (name.size == 0) {
+    api_set_error(err, kErrorTypeValidation, "not yet implemented");
     return rv;
   }
+
   int flags = DIP_START | (all ? DIP_ALL : 0);
-  do_in_runtimepath((char_u *)name.data, flags, find_runtime_cb, &rv);
+  do_in_runtimepath(name.size ? (char_u *)name.data : NULL,
+                    flags, find_runtime_cb, &rv);
   return rv;
 }
 
 static void find_runtime_cb(char_u *fname, void *cookie)
 {
   Array *rv = (Array *)cookie;
-  ADD(*rv, STRING_OBJ(cstr_to_string((char *)fname)));
+  if (fname != NULL) {
+    ADD(*rv, STRING_OBJ(cstr_to_string((char *)fname)));
+  }
 }
 
 String nvim__get_lib_dir(void)

src/nvim/ex_cmds2.c

@@ -2363,7 +2363,7 @@ void ex_compiler(exarg_T *eap)
     do_unlet(S_LEN("b:current_compiler"), true);
 
     snprintf((char *)buf, bufsize, "compiler/%s.vim", eap->arg);
-    if (source_runtime(buf, DIP_ALL) == FAIL) {
+    if (source_in_path(p_rtp, buf, DIP_ALL) == FAIL) {
       EMSG2(_("E666: compiler not supported: %s"), eap->arg);
     }
     xfree(buf);
@@ -2581,6 +2581,7 @@ int do_in_runtimepath(char_u *name, int flags, DoInRuntimepathCB callback,
 /// return FAIL when no file could be sourced, OK otherwise.
 int source_runtime(char_u *name, int flags)
 {
+  flags |= (flags & DIP_NORTP) ? 0 : DIP_START;
   return source_in_path(p_rtp, name, flags);
 }
 

src/nvim/lua/executor.c

@@ -549,14 +549,6 @@ static lua_State *nlua_enter(void)
     // stack: (empty)
     lua_getglobal(lstate, "vim");
     // stack: vim
-    lua_getfield(lstate, -1, "_update_package_paths");
-    // stack: vim, vim._update_package_paths
-    if (lua_pcall(lstate, 0, 0, 0)) {
-      // stack: vim, error
-      nlua_error(lstate, _("E5117: Error while updating package paths: %.*s"));
-      // stack: vim
-    }
-    // stack: vim
     lua_pop(lstate, 1);
     // stack: (empty)
     last_p_rtp = (const void *)p_rtp;
@@ -564,14 +556,6 @@ static lua_State *nlua_enter(void)
   return lstate;
 }
 
-/// Force an update of lua's package paths if runtime path has changed.
-bool nlua_update_package_path(void)
-{
-  lua_State *const lstate = nlua_enter();
-
-  return !!lstate;
-}
-
 static void nlua_print_event(void **argv)
 {
   char *str = argv[0];

src/nvim/lua/vim.lua

@@ -92,67 +92,48 @@ function vim._os_proc_children(ppid)
   return children
 end
 
--- TODO(ZyX-I): Create compatibility layer.
---{{{1 package.path updater function
--- Last inserted paths. Used to clear out items from package.[c]path when they
--- are no longer in &runtimepath.
-local last_nvim_paths = {}
-function vim._update_package_paths()
-  local cur_nvim_paths = {}
-  local rtps = vim.api.nvim_list_runtime_paths()
-  local sep = package.config:sub(1, 1)
-  for _, key in ipairs({'path', 'cpath'}) do
-    local orig_str = package[key] .. ';'
-    local pathtrails_ordered = {}
-    local orig = {}
-    -- Note: ignores trailing item without trailing `;`. Not using something
-    -- simpler in order to preserve empty items (stand for default path).
-    for s in orig_str:gmatch('[^;]*;') do
-      s = s:sub(1, -2)  -- Strip trailing semicolon
-      orig[#orig + 1] = s
-    end
-    if key == 'path' then
-      -- /?.lua and /?/init.lua
-      pathtrails_ordered = {sep .. '?.lua', sep .. '?' .. sep .. 'init.lua'}
-    else
-      local pathtrails = {}
-      for _, s in ipairs(orig) do
-        -- Find out path patterns. pathtrail should contain something like
-        -- /?.so, \?.dll. This allows not to bother determining what correct
-        -- suffixes are.
-        local pathtrail = s:match('[/\\][^/\\]*%?.*$')
-        if pathtrail and not pathtrails[pathtrail] then
-          pathtrails[pathtrail] = true
-          pathtrails_ordered[#pathtrails_ordered + 1] = pathtrail
-        end
-      end
-    end
-    local new = {}
-    for _, rtp in ipairs(rtps) do
-      if not rtp:match(';') then
-        for _, pathtrail in pairs(pathtrails_ordered) do
-          local new_path = rtp .. sep .. 'lua' .. pathtrail
-          -- Always keep paths from &runtimepath at the start:
-          -- append them here disregarding orig possibly containing one of them.
-          new[#new + 1] = new_path
-          cur_nvim_paths[new_path] = true
-        end
-      end
-    end
-    for _, orig_path in ipairs(orig) do
-      -- Handle removing obsolete paths originating from &runtimepath: such
-      -- paths either belong to cur_nvim_paths and were already added above or
-      -- to last_nvim_paths and should not be added at all if corresponding
-      -- entry was removed from &runtimepath list.
-      if not (cur_nvim_paths[orig_path] or last_nvim_paths[orig_path]) then
-        new[#new + 1] = orig_path
-      end
-    end
-    package[key] = table.concat(new, ';')
+local pathtrails = {}
+vim._so_trails = {}
+for s in  (package.cpath..';'):gmatch('[^;]*;') do
+    s = s:sub(1, -2)  -- Strip trailing semicolon
+  -- Find out path patterns. pathtrail should contain something like
+  -- /?.so, \?.dll. This allows not to bother determining what correct
+  -- suffixes are.
+  local pathtrail = s:match('[/\\][^/\\]*%?.*$')
+  if pathtrail and not pathtrails[pathtrail] then
+    pathtrails[pathtrail] = true
+    table.insert(vim._so_trails, pathtrail)
   end
-  last_nvim_paths = cur_nvim_paths
 end
 
+function vim._load_package(name)
+  -- tricky: when debugging this function we must let vim.inspect
+  -- module to be loaded first:
+  --local inspect = (name == "vim.inspect") and tostring or vim.inspect
+
+  local basename = name:gsub('%.', '/')
+  local paths = {"lua/"..basename..".lua", "lua/"..basename.."/init.lua"}
+  for _,path in ipairs(paths) do
+    local found = vim.api.nvim_get_runtime_file(path, false)
+    if #found > 0 then
+      return loadfile(found[1])
+    end
+  end
+
+  for _,trail in ipairs(vim._so_trails) do
+    local path = "lua/"..trail:gsub('?',basename)
+    local found = vim.api.nvim_get_runtime_file(path, false)
+    if #found > 0 then
+      return package.loadlib(found[1])
+    end
+  end
+  return nil
+end
+
+table.insert(package.loaders, 1, vim._load_package)
+
+-- TODO(ZyX-I): Create compatibility layer.
+
 --- Return a human-readable representation of the given object.
 ---
 --@see https://github.com/kikito/inspect.lua

src/nvim/option.c

@@ -3364,10 +3364,6 @@ ambw_end:
     if (!parse_winhl_opt(curwin)) {
       errmsg = e_invarg;
     }
-  } else if (varp == &p_rtp) {  // 'runtimepath'
-    if (!nlua_update_package_path()) {
-      errmsg = (char_u *)N_("E970: Failed to initialize lua interpreter");
-    }
   } else {
     // Options that are a list of flags.
     p = NULL;

src/nvim/os/os_defs.h

@@ -16,10 +16,11 @@
 #define BASENAMELEN (NAME_MAX - 5)
 
 // Use the system path length if it makes sense.
-#if defined(PATH_MAX) && (PATH_MAX > 1024)
+# define DEFAULT_MAXPATHL 4096
+#if defined(PATH_MAX) && (PATH_MAX > DEFAULT_MAXPATHL)
 # define MAXPATHL PATH_MAX
 #else
-# define MAXPATHL 1024
+# define MAXPATHL DEFAULT_MAXPATHL
 #endif
 
 // Command-processing buffer. Use large buffers for all platforms.

src/nvim/syntax.c

@@ -4297,7 +4297,7 @@ static void syn_cmd_include(exarg_T *eap, int syncing)
   prev_toplvl_grp = curwin->w_s->b_syn_topgrp;
   curwin->w_s->b_syn_topgrp = sgl_id;
   if (source ? do_source(eap->arg, false, DOSO_NONE) == FAIL
-             : source_runtime(eap->arg, DIP_ALL) == FAIL) {
+             : source_in_path(p_rtp, eap->arg, DIP_ALL) == FAIL) {
     EMSG2(_(e_notopen), eap->arg);
   }
   curwin->w_s->b_syn_topgrp = prev_toplvl_grp;

test/functional/core/startup_spec.lua

@@ -7,6 +7,7 @@ local ok = helpers.ok
 local eq = helpers.eq
 local matches = helpers.matches
 local eval = helpers.eval
+local exec_lua = helpers.exec_lua
 local feed = helpers.feed
 local funcs = helpers.funcs
 local mkdir = helpers.mkdir
@@ -305,6 +306,27 @@ describe('startup', function()
                                '+q' })
     eq('[\'+q\'] 1', out)
   end)
+
+  local function pack_clear(cmd)
+    clear('--cmd', 'set packpath=test/functional/fixtures', '--cmd', cmd)
+  end
+
+
+  it("handles &packpath during startup", function()
+    pack_clear [[ let g:x = bar#test() ]]
+    eq(-3, eval 'g:x')
+
+    pack_clear [[ lua _G.y = require'bar'.doit() ]]
+    eq(9003, exec_lua [[ return _G.y ]])
+  end)
+
+  it("handles :packadd during startup", function()
+    pack_clear [[ packadd! bonus | let g:x = bonus#secret() ]]
+    eq('halloj', eval 'g:x')
+
+    pack_clear [[ packadd! bonus | lua _G.y = require'bonus'.launch() ]]
+    eq('CPE 1704 TKS', exec_lua [[ return _G.y ]])
+  end)
 end)
 
 describe('sysinit', function()

test/functional/ex_cmds/packadd_spec.lua

@@ -1,74 +0,0 @@
-local helpers = require('test.functional.helpers')(after_each)
-
-local clear = helpers.clear
-local eq = helpers.eq
-local exec_lua = helpers.exec_lua
-
-describe('packadd', function()
-  before_each(function()
-    -- Primarily taken from test/functional/legacy/packadd_spec.lua
-    clear()
-    exec_lua [[
-      TopDirectory = vim.fn.expand(vim.fn.getcwd() .. '/Xdir_lua')
-      PlugDirectory = TopDirectory .. '/pack/mine/opt/mytest'
-
-      vim.o.packpath = TopDirectory
-
-      function FindPathsContainingDir(dir)
-        return vim.fn.filter(
-          vim.split(package.path, ';'),
-          function(k, v)
-            return string.find(v, 'mytest') ~= nil
-          end
-        )
-      end
-    ]]
-  end)
-
-  after_each(function()
-    exec_lua [[
-      vim.fn.delete(TopDirectory, 'rf')
-    ]]
-  end)
-
-  it('should immediately update package.path in lua', function()
-    local count_of_paths = exec_lua [[
-      vim.fn.mkdir(PlugDirectory .. '/lua/', 'p')
-
-      local num_paths_before = #FindPathsContainingDir('mytest')
-
-      vim.cmd("packadd mytest")
-
-      local num_paths_after = #FindPathsContainingDir('mytest')
-
-      return { num_paths_before, num_paths_after }
-    ]]
-
-    eq({0, 2}, count_of_paths)
-  end)
-
-  it('should immediately update package.path in lua even if lua directory does not exist', function()
-    local count_of_paths = exec_lua [[
-      vim.fn.mkdir(PlugDirectory .. '/plugin/', 'p')
-
-      local num_paths_before = #FindPathsContainingDir('mytest')
-
-      vim.cmd("packadd mytest")
-
-      local num_paths_after = #FindPathsContainingDir('mytest')
-
-      return { num_paths_before, num_paths_after }
-    ]]
-
-    eq({0, 2}, count_of_paths)
-  end)
-
-  it('should error for invalid paths', function()
-    local count_of_paths = exec_lua [[
-      local ok, err = pcall(vim.cmd, "packadd asdf")
-      return ok
-    ]]
-
-    eq(false, count_of_paths)
-  end)
-end)

test/functional/fixtures/pack/foo/opt/bonus/autoload/bonus.vim

@@ -0,0 +1,3 @@
+func bonus#secret()
+  return "halloj"
+endfunc

test/functional/fixtures/pack/foo/opt/bonus/lua/bonus.lua

@@ -0,0 +1 @@
+return {launch=function() return "CPE 1704 TKS" end}

test/functional/fixtures/pack/foo/start/bar/autoload/bar.vim

@@ -0,0 +1,3 @@
+func bar#test()
+  return -3
+endfunc

test/functional/fixtures/pack/foo/start/bar/lua/bar.lua

@@ -0,0 +1 @@
+return {doit=function() return 9003 end}

test/functional/lua/overrides_spec.lua

@@ -285,119 +285,6 @@ describe('debug.debug', function()
   end)
 end)
 
-describe('package.path/package.cpath', function()
-  local sl = alter_slashes
-
-  local function get_new_paths(sufs, runtimepaths)
-    runtimepaths = runtimepaths or meths.list_runtime_paths()
-    local new_paths = {}
-    local sep = package.config:sub(1, 1)
-    for _, v in ipairs(runtimepaths) do
-      for _, suf in ipairs(sufs) do
-        new_paths[#new_paths + 1] = v .. sep .. 'lua' .. suf
-      end
-    end
-    return new_paths
-  end
-  local function eval_lua(expr, ...)
-    return meths.exec_lua('return '..expr, {...})
-  end
-  local function set_path(which, value)
-    return exec_lua('package[select(1, ...)] = select(2, ...)', which, value)
-  end
-
-  it('contains directories from &runtimepath on first invocation', function()
-    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'})
-    local new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
-
-    local new_cpaths = get_new_paths(iswin() and {'\\?.dll'} or {'/?.so'})
-    local new_cpaths_str = table.concat(new_cpaths, ';')
-    eq(new_cpaths_str, eval_lua('package.cpath'):sub(1, #new_cpaths_str))
-  end)
-  it('puts directories from &runtimepath always at the start', function()
-    meths.set_option('runtimepath', 'a,b')
-    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {'a', 'b'})
-    local new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
-
-    set_path('path', sl'foo/?.lua;foo/?/init.lua;' .. new_paths_str)
-
-    neq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
-
-    command('set runtimepath+=c')
-    new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {'a', 'b', 'c'})
-    new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
-  end)
-  it('understands uncommon suffixes', function()
-    set_path('cpath', './?/foo/bar/baz/x.nlua')
-    meths.set_option('runtimepath', 'a')
-    local new_paths = get_new_paths({'/?/foo/bar/baz/x.nlua'}, {'a'})
-    local new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str, eval_lua('package.cpath'):sub(1, #new_paths_str))
-
-    set_path('cpath', './yyy?zzz/x')
-    meths.set_option('runtimepath', 'b')
-    new_paths = get_new_paths({'/yyy?zzz/x'}, {'b'})
-    new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str, eval_lua('package.cpath'):sub(1, #new_paths_str))
-
-    set_path('cpath', './yyy?zzz/123?ghi/x')
-    meths.set_option('runtimepath', 'b')
-    new_paths = get_new_paths({'/yyy?zzz/123?ghi/x'}, {'b'})
-    new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str, eval_lua('package.cpath'):sub(1, #new_paths_str))
-  end)
-  it('preserves empty items', function()
-    local many_empty_path = ';;;;;;'
-    local many_empty_cpath = ';;;;;;./?.luaso'
-    set_path('path', many_empty_path)
-    set_path('cpath', many_empty_cpath)
-    meths.set_option('runtimepath', 'a')
-    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {'a'})
-    local new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str .. ';' .. many_empty_path, eval_lua('package.path'))
-    local new_cpaths = get_new_paths({'/?.luaso'}, {'a'})
-    local new_cpaths_str = table.concat(new_cpaths, ';')
-    eq(new_cpaths_str .. ';' .. many_empty_cpath, eval_lua('package.cpath'))
-  end)
-  it('preserves empty value', function()
-    set_path('path', '')
-    meths.set_option('runtimepath', 'a')
-    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {'a'})
-    local new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str .. ';', eval_lua('package.path'))
-  end)
-  it('purges out all additions if runtimepath is set to empty', function()
-    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'})
-    local new_paths_str = table.concat(new_paths, ';')
-    local path = eval_lua('package.path')
-    eq(new_paths_str, path:sub(1, #new_paths_str))
-
-    local new_cpaths = get_new_paths(iswin() and {'\\?.dll'} or {'/?.so'})
-    local new_cpaths_str = table.concat(new_cpaths, ';')
-    local cpath = eval_lua('package.cpath')
-    eq(new_cpaths_str, cpath:sub(1, #new_cpaths_str))
-
-    meths.set_option('runtimepath', '')
-    eq(path:sub(#new_paths_str + 2, -1), eval_lua('package.path'))
-    eq(cpath:sub(#new_cpaths_str + 2, -1), eval_lua('package.cpath'))
-  end)
-  it('works with paths with escaped commas', function()
-    meths.set_option('runtimepath', '\\,')
-    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {','})
-    local new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
-  end)
-  it('ignores paths with semicolons', function()
-    meths.set_option('runtimepath', 'foo;bar,\\,')
-    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {','})
-    local new_paths_str = table.concat(new_paths, ';')
-    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
-  end)
-end)
-
 describe('os.getenv', function()
   it('returns nothing for undefined env var', function()
     eq(NIL, funcs.luaeval('os.getenv("XTEST_1")'))