lua: Add paths from &runtimepath to package.path and package.cpath

runtime/doc/if_lua.txt

@@ -9,7 +9,32 @@ Lua Interface to Nvim					*lua* *Lua*
                                       Type <M-]> to see the table of contents.
 
 ==============================================================================
-1. Commands						*lua-commands*
+1. Importing modules					*lua-require*
+
+Neovim lua interface automatically adjusts `package.path` and `package.cpath` 
+according to effective &runtimepath value.  Adjustment happens after each time 
+'runtimepath' is changed, `package.path` and `package.cpath` are adjusted by 
+prepending directories from 'runtimepath' each suffixed by `/lua` and 
+`?`-containing suffixes from `package.path` and `package.cpath`. I.e. when 
+'runtimepath' option contains `/foo` and `package.path` contains only 
+`./a?d/j/g.nlua;./?.lua;/bar/?.lua` the resulting `package.path` after 
+adjustments will look like this: >
+
+	/foo/lua/?.lua;/foo/lua/a?d/j/g.nlua;./a?d/j/g.nlua;./?.lua;/bar/?.lua
+
+Note that code have taken everything starting from the last path component 
+from existing paths containing a question mark as a `?`-containing suffix, but 
+only applied unique suffixes.
+
+Note 2: even though adjustments happens automatically Neovim does not track 
+current values of `package.path` or `package.cpath`. If you happened to delete 
+some paths from there you need to reset 'runtimepath' to make them readded.
+
+Note 3: paths from 'runtimepath' which contain semicolons cannot be put into 
+`package.[c]path` and thus are ignored.
+
+==============================================================================
+2. Commands						*lua-commands*
 
 							*:lua*
 :[range]lua {chunk}

runtime/doc/vim_diff.txt

@@ -244,6 +244,8 @@ Lua interface (|if_lua.txt|):
   while calling lua chunk: [string "<VimL compiled string>"]:1: TEST” in 
   Neovim.
 - Lua has direct access to Nvim |API| via `vim.api`.
+- Lua package.path and package.cpath are automatically updated according to
+  'runtimepath': |lua-require|.
 - Currently, most legacy Vim features are missing.
 
 |input()| and |inputdialog()| gained support for each other’s features (return 

src/nvim/lua/executor.c

@@ -14,6 +14,7 @@
 #include "nvim/api/vim.h"
 #include "nvim/vim.h"
 #include "nvim/ex_getln.h"
+#include "nvim/ex_cmds2.h"
 #include "nvim/message.h"
 #include "nvim/memline.h"
 #include "nvim/buffer_defs.h"
@@ -284,7 +285,9 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
 ///
 /// Crashes NeoVim if initialization fails. Should be called once per lua
 /// interpreter instance.
-static lua_State *init_lua(void)
+///
+/// @return New lua interpreter instance.
+static lua_State *nlua_init(void)
   FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT
 {
   lua_State *lstate = luaL_newstate();
@@ -297,7 +300,43 @@ static lua_State *init_lua(void)
   return lstate;
 }
 
-static lua_State *global_lstate = NULL;
+/// Enter lua interpreter
+///
+/// Calls nlua_init() if needed. Is responsible for pre-lua call initalization
+/// like updating `package.[c]path` with directories derived from &runtimepath.
+///
+/// @return Interprter instance to use. Will either be initialized now or taken
+///         from previous initalization.
+static lua_State *nlua_enter(void)
+  FUNC_ATTR_NONNULL_RET FUNC_ATTR_WARN_UNUSED_RESULT
+{
+  static lua_State *global_lstate = NULL;
+  if (global_lstate == NULL) {
+    global_lstate = nlua_init();
+  }
+  lua_State *const lstate = global_lstate;
+  // Last used p_rtp value. Must not be dereferenced because value pointed to
+  // may already be freed. Used to check whether &runtimepath option value
+  // changed.
+  static const void *last_p_rtp = NULL;
+  if (last_p_rtp != (const void *)p_rtp) {
+    // stack: (empty)
+    lua_getglobal(lstate, "vim");
+    // stack: vim
+    lua_getfield(lstate, -1, "_update_package_paths");
+    // stack: vim, vim._update_package_paths
+    if (lua_pcall(lstate, 0, 0, 0)) {
+      // stack: vim, error
+      nlua_error(lstate, _("E5117: Error while updating package paths: %.*s"));
+      // stack: vim
+    }
+    // stack: vim
+    lua_pop(lstate, 1);
+    // stack: (empty)
+    last_p_rtp = (const void *)p_rtp;
+  }
+  return lstate;
+}
 
 /// Execute lua string
 ///
@@ -308,11 +347,7 @@ static lua_State *global_lstate = NULL;
 void executor_exec_lua(const String str, typval_T *const ret_tv)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (global_lstate == NULL) {
-    global_lstate = init_lua();
-  }
-
-  NLUA_CALL_C_FUNCTION_2(global_lstate, nlua_exec_lua_string, 0,
+  NLUA_CALL_C_FUNCTION_2(nlua_enter(), nlua_exec_lua_string, 0,
                          (void *)&str, ret_tv);
 }
 
@@ -551,11 +586,7 @@ void executor_eval_lua(const String str, typval_T *const arg,
                        typval_T *const ret_tv)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (global_lstate == NULL) {
-    global_lstate = init_lua();
-  }
-
-  NLUA_CALL_C_FUNCTION_3(global_lstate, nlua_eval_lua_string, 0,
+  NLUA_CALL_C_FUNCTION_3(nlua_enter(), nlua_eval_lua_string, 0,
                          (void *)&str, arg, ret_tv);
 }
 
@@ -570,12 +601,8 @@ void executor_eval_lua(const String str, typval_T *const arg,
 /// @return Return value of the execution.
 Object executor_exec_lua_api(const String str, const Array args, Error *err)
 {
-  if (global_lstate == NULL) {
-    global_lstate = init_lua();
-  }
-
   Object retval = NIL;
-  NLUA_CALL_C_FUNCTION_4(global_lstate, nlua_exec_lua_string_api, 0,
+  NLUA_CALL_C_FUNCTION_4(nlua_enter(), nlua_exec_lua_string_api, 0,
                          (void *)&str, (void *)&args, &retval, err);
   return retval;
 }
@@ -609,9 +636,6 @@ void ex_lua(exarg_T *const eap)
 void ex_luado(exarg_T *const eap)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (global_lstate == NULL) {
-    global_lstate = init_lua();
-  }
   if (u_save(eap->line1 - 1, eap->line2 + 1) == FAIL) {
     EMSG(_("cannot save undo information"));
     return;
@@ -621,7 +645,7 @@ void ex_luado(exarg_T *const eap)
     .data = (char *)eap->arg,
   };
   const linenr_T range[] = { eap->line1, eap->line2 };
-  NLUA_CALL_C_FUNCTION_2(global_lstate, nlua_exec_luado_string, 0,
+  NLUA_CALL_C_FUNCTION_2(nlua_enter(), nlua_exec_luado_string, 0,
                          (void *)&cmd, (void *)range);
 }
 
@@ -633,9 +657,6 @@ void ex_luado(exarg_T *const eap)
 void ex_luafile(exarg_T *const eap)
   FUNC_ATTR_NONNULL_ALL
 {
-  if (global_lstate == NULL) {
-    global_lstate = init_lua();
-  }
-  NLUA_CALL_C_FUNCTION_1(global_lstate, nlua_exec_lua_file, 0,
+  NLUA_CALL_C_FUNCTION_1(nlua_enter(), nlua_exec_lua_file, 0,
                          (void *)eap->arg);
 }

src/nvim/lua/vim.lua

@@ -1,2 +1,57 @@
 -- TODO(ZyX-I): Create compatibility layer.
-return {}
+--{{{1 package.path updater function
+-- Last inserted paths. Used to clear out items from package.[c]path when they
+-- are no longer in &runtimepath.
+local last_nvim_paths = {}
+local function _update_package_paths()
+  local cur_nvim_paths = {}
+  local rtps = vim.api.nvim_list_runtime_paths()
+  local sep = package.config:sub(1, 1)
+  for _, key in ipairs({'path', 'cpath'}) do
+    local orig_str = package[key] .. ';'
+    local pathtrails = {}
+    local pathtrails_ordered = {}
+    local orig = {}
+    -- Note: ignores trailing item without trailing `;`. Not using something
+    -- simpler in order to preserve empty items (stand for default path).
+    for s in orig_str:gmatch('[^;]*;') do
+      s = s:sub(1, -2)  -- Strip trailing semicolon
+      orig[#orig + 1] = s
+      -- Find out path patterns. pathtrail should contain something like
+      -- /?.so, /?/init.lua, /?.lua. This allows not to bother determining what
+      -- correct suffixes are.
+      local pathtrail = s:match('[/\\][^/\\]*%?.*$')
+      if pathtrail and not pathtrails[pathtrail] then
+        pathtrails[pathtrail] = true
+        pathtrails_ordered[#pathtrails_ordered + 1] = pathtrail
+      end
+    end
+    local new = {}
+    for _, rtp in ipairs(rtps) do
+      if not rtp:match(';') then
+        for _, pathtrail in pairs(pathtrails_ordered) do
+          local new_path = rtp .. sep .. 'lua' .. pathtrail
+          -- Always keep paths from &runtimepath at the start:
+          -- append them here disregarding orig possibly containing one of them.
+          new[#new + 1] = new_path
+          cur_nvim_paths[new_path] = true
+        end
+      end
+    end
+    for _, orig_path in ipairs(orig) do
+      -- Handle removing obsolete paths originating from &runtimepath: such
+      -- paths either belong to cur_nvim_paths and were already added above or
+      -- to last_nvim_paths and should not be added at all if corresponding
+      -- entry was removed from &runtimepath list.
+      if not (cur_nvim_paths[orig_path] or last_nvim_paths[orig_path]) then
+        new[#new + 1] = orig_path
+      end
+    end
+    package[key] = table.concat(new, ';')
+  end
+  last_nvim_paths = cur_nvim_paths
+end
+--{{{1 Module definition
+return {
+  _update_package_paths = _update_package_paths,
+}

test/functional/helpers.lua

@@ -579,6 +579,24 @@ local function missing_provider(provider)
   end
 end
 
+local function alter_slashes(obj)
+  if not iswin() then
+    return obj
+  end
+  if type(obj) == 'string' then
+    local ret = obj:gsub('/', '\\')
+    return ret
+  elseif type(obj) == 'table' then
+    local ret = {}
+    for k, v in pairs(obj) do
+      ret[k] = alter_slashes(v)
+    end
+    return ret
+  else
+    assert(false, 'Could only alter slashes for tables of strings and strings')
+  end
+end
+
 local module = {
   prepend_argv = prepend_argv,
   clear = clear,
@@ -646,6 +664,7 @@ local module = {
   NIL = mpack.NIL,
   get_pathsep = get_pathsep,
   missing_provider = missing_provider,
+  alter_slashes = alter_slashes,
 }
 
 return function(after_each)

test/functional/lua/overrides_spec.lua

@@ -3,14 +3,17 @@ local helpers = require('test.functional.helpers')(after_each)
 local Screen = require('test.functional.ui.screen')
 
 local eq = helpers.eq
+local neq = helpers.neq
 local NIL = helpers.NIL
 local feed = helpers.feed
 local clear = helpers.clear
 local funcs = helpers.funcs
 local meths = helpers.meths
+local iswin = helpers.iswin
 local command = helpers.command
 local write_file = helpers.write_file
 local redir_exec = helpers.redir_exec
+local alter_slashes = helpers.alter_slashes
 
 local screen
 
@@ -173,3 +176,116 @@ describe('debug.debug', function()
     ]])
   end)
 end)
+
+describe('package.path/package.cpath', function()
+  local sl = alter_slashes
+
+  local function get_new_paths(sufs, runtimepaths)
+    runtimepaths = runtimepaths or meths.list_runtime_paths()
+    local new_paths = {}
+    for _, v in ipairs(runtimepaths) do
+      for _, suf in ipairs(sufs) do
+        new_paths[#new_paths + 1] = v .. suf:sub(1, 1) .. 'lua' .. suf
+      end
+    end
+    return new_paths
+  end
+  local function execute_lua(cmd, ...)
+    return meths.execute_lua(cmd, {...})
+  end
+  local function eval_lua(expr, ...)
+    return meths.execute_lua('return ' .. expr, {...})
+  end
+  local function set_path(which, value)
+    return execute_lua('package[select(1, ...)] = select(2, ...)', which, value)
+  end
+
+  it('contains directories from &runtimepath on first invocation', function()
+    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'})
+    local new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+
+    local new_cpaths = get_new_paths(iswin() and {'\\?.dll'} or {'/?.so'})
+    local new_cpaths_str = table.concat(new_cpaths, ';')
+    eq(new_cpaths_str, eval_lua('package.cpath'):sub(1, #new_cpaths_str))
+  end)
+  it('puts directories from &runtimepath always at the start', function()
+    meths.set_option('runtimepath', 'a,b')
+    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {'a', 'b'})
+    local new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+
+    set_path('path', sl'foo/?.lua;foo/?/init.lua;' .. new_paths_str)
+
+    neq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+
+    command('set runtimepath+=c')
+    new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {'a', 'b', 'c'})
+    new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+  end)
+  it('understands uncommon suffixes', function()
+    set_path('path', './?/foo/bar/baz/x.nlua')
+    meths.set_option('runtimepath', 'a')
+    local new_paths = get_new_paths({'/?/foo/bar/baz/x.nlua'}, {'a'})
+    local new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+
+    set_path('path', './yyy?zzz/x')
+    meths.set_option('runtimepath', 'b')
+    new_paths = get_new_paths({'/yyy?zzz/x'}, {'b'})
+    new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+
+    set_path('path', './yyy?zzz/123?ghi/x')
+    meths.set_option('runtimepath', 'b')
+    new_paths = get_new_paths({'/yyy?zzz/123?ghi/x'}, {'b'})
+    new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+  end)
+  it('preserves empty items', function()
+    local many_empty_path = ';;;;;;'
+    local many_empty_cpath = ';;;;;;./?.luaso'
+    set_path('path', many_empty_path)
+    set_path('cpath', many_empty_cpath)
+    meths.set_option('runtimepath', 'a')
+    -- No suffixes known, so can’t add anything
+    eq(many_empty_path, eval_lua('package.path'))
+    local new_paths = get_new_paths({'/?.luaso'}, {'a'})
+    local new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str .. ';' .. many_empty_cpath, eval_lua('package.cpath'))
+  end)
+  it('preserves empty value', function()
+    set_path('path', '')
+    meths.set_option('runtimepath', 'a')
+    -- No suffixes known, so can’t add anything
+    eq('', eval_lua('package.path'))
+  end)
+  it('purges out all additions if runtimepath is set to empty', function()
+    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'})
+    local new_paths_str = table.concat(new_paths, ';')
+    local path = eval_lua('package.path')
+    eq(new_paths_str, path:sub(1, #new_paths_str))
+
+    local new_cpaths = get_new_paths(iswin() and {'\\?.dll'} or {'/?.so'})
+    local new_cpaths_str = table.concat(new_cpaths, ';')
+    local cpath = eval_lua('package.cpath')
+    eq(new_cpaths_str, cpath:sub(1, #new_cpaths_str))
+
+    meths.set_option('runtimepath', '')
+    eq(path:sub(#new_paths_str + 2, -1), eval_lua('package.path'))
+    eq(cpath:sub(#new_cpaths_str + 2, -1), eval_lua('package.cpath'))
+  end)
+  it('works with paths with escaped commas', function()
+    meths.set_option('runtimepath', '\\,')
+    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {','})
+    local new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+  end)
+  it('ignores paths with semicolons', function()
+    meths.set_option('runtimepath', 'foo;bar,\\,')
+    local new_paths = get_new_paths(sl{'/?.lua', '/?/init.lua'}, {','})
+    local new_paths_str = table.concat(new_paths, ';')
+    eq(new_paths_str, eval_lua('package.path'):sub(1, #new_paths_str))
+  end)
+end)