Merge pull request #15632 from bfredl/rtptest

runtime: always use DIP_START and remove duplication of start packages in &rtp

runtime/autoload/remote/host.vim

@@ -114,7 +114,7 @@ function! s:RegistrationCommands(host) abort
   let host_id = a:host.'-registration-clone'
   call remote#host#RegisterClone(host_id, a:host)
   let pattern = s:plugin_patterns[a:host]
-  let paths = globpath(&rtp, 'rplugin/'.a:host.'/'.pattern, 1, 1)
+  let paths = nvim_get_runtime_file('rplugin/'.a:host.'/'.pattern, 1)
   let paths = map(paths, 'tr(resolve(v:val),"\\","/")') " Normalize slashes #4795
   let paths = uniq(sort(paths))
   if empty(paths)

runtime/doc/options.txt

@@ -4328,7 +4328,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 
 				*'packpath'* *'pp'*
 'packpath' 'pp'		string	(default: see 'runtimepath')
-	Directories used to find packages.  See |packages|.
+	Directories used to find packages.  See |packages| and |rtp-packages|.
 
 
 						*'paragraphs'* *'para'*
@@ -4887,9 +4887,11 @@ A jump table for the options with a short description can be found at |Q_op|.
 	   ordering.  This is for preferences to overrule or add to the 
 	   distributed defaults or system-wide settings (rarely needed).
 
-	More entries are added when using |packages|.  If it gets very long
-	then `:set rtp` will be truncated, use `:echo &rtp` to see the full
-	string.
+							*rtp-packages*
+	"start" packages will additionally be used to search for runtime files
+	after these, but package entries are not visible in `:set rtp`.
+	See |runtime-search-path| for more information. "opt" packages
+	will be explicitly added to &rtp when |:packadd| is used.
 
 	Note that, unlike 'path', no wildcards like "**" are allowed.  Normal
 	wildcards are allowed, but can significantly slow down searching for

runtime/doc/repeat.txt

@@ -516,16 +516,26 @@ You would now have these files under ~/.local/share/nvim/site:
 	pack/foo/start/foobar/syntax/some.vim
 	pack/foo/opt/foodebug/plugin/debugger.vim
 
-When Vim starts up, after processing your .vimrc, it scans all directories in
-'packpath' for plugins under the "pack/*/start" directory.  First all those
-directories are added to 'runtimepath'.  Then all the plugins are loaded.
-See |packload-two-steps| for how these two steps can be useful.
+							*runtime-search-path*
+When runtime files are searched for, first all paths in 'runtimepath' are
+searched, then all "pack/*/start/*" dirs are searched. However, package entries
+are not visible in `:set rtp` or `echo &rtp`, as the final concatenated path
+would be too long and get truncated. To list all used directories, use
+|nvim_list_runtime_paths()|. In addition |nvim_get_runtime_file()| can be used
+to query for specific files or sub-folders within the runtime path. For
+instance to list all runtime dirs and packages with lua paths, use >
 
-In the example Vim will find "pack/foo/start/foobar/plugin/foo.vim" and adds
-"~/.local/share/nvim/site/pack/foo/start/foobar" to 'runtimepath'.
+    :echo nvim_get_runtime_file("lua/", v:true)
+
+<When Vim starts up, after processing your .vimrc, it scans all directories in
+'packpath' for plugins under the "pack/*/start" directory, and all the plugins
+are loaded.
+
+In the example Vim will find "pack/foo/start/foobar/plugin/foo.vim" and load it.
 
 If the "foobar" plugin kicks in and sets the 'filetype' to "some", Vim will
-find the syntax/some.vim file, because its directory is in 'runtimepath'.
+find the syntax/some.vim file, because its directory is in the runtime search
+path.
 
 Vim will also load ftdetect files, if there are any.
 
@@ -536,7 +546,7 @@ is used.
 Loading packages automatically will not happen if loading plugins is disabled,
 see |load-plugins|.
 
-To load packages earlier, so that 'runtimepath' gets updated: >
+To load packages earlier, so that plugin/ files are sourced:
 	:packloadall
 This also works when loading plugins is disabled.  The automatic loading will
 only happen once.
@@ -664,8 +674,8 @@ found automatically.  Your package would have these files:
 <	pack/foo/start/lib/autoload/foolib.vim >
 		func foolib#getit()
 
-This works, because loading packages will first add all found directories to
-'runtimepath' before sourcing the plugins.
+This works, because start packages will be used to look for autoload files,
+when sourcing the plugins.
 
 ==============================================================================
 Debugging scripts					*debug-scripts*

src/nvim/api/vim.c

@@ -750,47 +750,10 @@ Integer nvim_strwidth(String text, Error *err)
 /// Gets the paths contained in 'runtimepath'.
 ///
 /// @return List of paths
-ArrayOf(String) nvim_list_runtime_paths(void)
+ArrayOf(String) nvim_list_runtime_paths(Error *err)
   FUNC_API_SINCE(1)
 {
-  // TODO(bfredl): this should just work:
-  // return nvim_get_runtime_file(NULL_STRING, true);
-
-  Array rv = ARRAY_DICT_INIT;
-
-  char_u *rtp = p_rtp;
-
-  if (*rtp == NUL) {
-    // No paths
-    return rv;
-  }
-
-  // Count the number of paths in rtp
-  while (*rtp != NUL) {
-    if (*rtp == ',') {
-      rv.size++;
-    }
-    rtp++;
-  }
-  rv.size++;
-
-  // Allocate memory for the copies
-  rv.items = xmalloc(sizeof(*rv.items) * rv.size);
-  // Reset the position
-  rtp = p_rtp;
-  // Start copying
-  for (size_t i = 0; i < rv.size; i++) {
-    rv.items[i].type = kObjectTypeString;
-    rv.items[i].data.string.data = xmalloc(MAXPATHL);
-    // Copy the path from 'runtimepath' to rv.items[i]
-    size_t length = copy_option_part(&rtp,
-                                     (char_u *)rv.items[i].data.string.data,
-                                     MAXPATHL,
-                                     ",");
-    rv.items[i].data.string.size = length;
-  }
-
-  return rv;
+  return nvim_get_runtime_file(NULL_STRING, true, err);
 }
 
 /// Find files in runtime directories
@@ -802,10 +765,6 @@ ArrayOf(String) nvim_list_runtime_paths(void)
 ///
 /// It is not an error to not find any files. An empty array is returned then.
 ///
-/// To find a directory, `name` must end with a forward slash, like
-/// "rplugin/python/". Without the slash it would instead look for an ordinary
-/// file called "rplugin/python".
-///
 /// @param name pattern of files to search for
 /// @param all whether to return all matches or only the first
 /// @return list of absolute paths to the found files
@@ -815,11 +774,7 @@ ArrayOf(String) nvim_get_runtime_file(String name, Boolean all, Error *err)
 {
   Array rv = ARRAY_DICT_INIT;
 
-  int flags = DIP_START | (all ? DIP_ALL : 0);
-
-  if (name.size == 0 || name.data[name.size-1] == '/') {
-    flags |= DIP_DIR;
-  }
+  int flags = DIP_DIRFILE | (all ? DIP_ALL : 0);
 
   do_in_runtimepath((char_u *)(name.size ? name.data : ""),
                     flags, find_runtime_cb, &rv);

src/nvim/ex_cmds2.c

@@ -1641,10 +1641,10 @@ void ex_compiler(exarg_T *eap)
     do_unlet(S_LEN("b:current_compiler"), true);
 
     snprintf((char *)buf, bufsize, "compiler/%s.vim", eap->arg);
-    if (source_in_path(p_rtp, buf, DIP_ALL) == FAIL) {
+    if (source_runtime(buf, DIP_ALL) == FAIL) {
       // Try lua compiler
       snprintf((char *)buf, bufsize, "compiler/%s.lua", eap->arg);
-      if (source_in_path(p_rtp, buf, DIP_ALL) == FAIL) {
+      if (source_runtime(buf, DIP_ALL) == FAIL) {
         EMSG2(_("E666: compiler not supported: %s"), eap->arg);
       }
     }

src/nvim/main.c

@@ -1352,23 +1352,10 @@ static void load_plugins(void)
     char_u *const plugin_pattern_vim = (char_u *)"plugin/**/*.vim";  // NOLINT
     char_u *const plugin_pattern_lua = (char_u *)"plugin/**/*.lua";  // NOLINT
 
-    // First add all package directories to 'runtimepath', so that their
-    // autoload directories can be found.  Only if not done already with a
-    // :packloadall command.
-    // Make a copy of 'runtimepath', so that source_runtime does not use the
-    // pack directories.
-    if (!did_source_packages) {
-      rtp_copy = vim_strsave(p_rtp);
-      add_pack_start_dirs();
-    }
-
-    source_in_path(rtp_copy == NULL ? p_rtp : rtp_copy,
-                   plugin_pattern_vim,
-                   DIP_ALL | DIP_NOAFTER);
-    source_in_path(rtp_copy == NULL ? p_rtp : rtp_copy,
-                   plugin_pattern_lua,
-                   DIP_ALL | DIP_NOAFTER);
-    TIME_MSG("loading plugins");
+    // don't use source_runtime() yet so we can check for :packloadall below
+    source_in_path(p_rtp, plugin_pattern_vim, DIP_ALL | DIP_NOAFTER);
+    source_in_path(p_rtp, plugin_pattern_lua, DIP_ALL | DIP_NOAFTER);
+    TIME_MSG("loading rtp plugins");
     xfree(rtp_copy);
 
     // Only source "start" packages if not done already with a :packloadall

src/nvim/runtime.c

@@ -100,10 +100,8 @@ int do_in_path(char_u *path, char_u *name, int flags,
       }
 
       if (name == NULL) {
-        (*callback)(buf, (void *)&cookie);
-        if (!did_one) {
-          did_one = (cookie == NULL);
-        }
+        (*callback)(buf, cookie);
+        did_one = true;
       } else if (buflen + STRLEN(name) + 2 < MAXPATHL) {
         add_pathsep((char *)buf);
         tail = buf + STRLEN(buf);
@@ -122,10 +120,11 @@ int do_in_path(char_u *path, char_u *name, int flags,
             verbose_leave();
           }
 
+          int ew_flags = ((flags & DIP_DIR) ? EW_DIR : EW_FILE)
+                         | (flags & DIP_DIRFILE) ? (EW_DIR|EW_FILE) : 0;
+
           // Expand wildcards, invoke the callback for each match.
-          if (gen_expand_wildcards(1, &buf, &num_files, &files,
-                                   (flags & DIP_DIR) ? EW_DIR : EW_FILE)
-              == OK) {
+          if (gen_expand_wildcards(1, &buf, &num_files, &files, ew_flags) == OK) {
             for (i = 0; i < num_files; i++) {
               (*callback)(files[i], cookie);
               did_one = true;
@@ -169,28 +168,36 @@ int do_in_path_and_pp(char_u *path, char_u *name, int flags,
                       DoInRuntimepathCB callback, void *cookie)
 {
   int done = FAIL;
+  if (!(flags & (DIP_NOAFTER | DIP_AFTER))) {
+    done = do_in_path_and_pp(path, name, flags | DIP_NOAFTER, callback, cookie);
+    if (done == OK && !(flags & DIP_ALL)) {
+      return done;
+    }
+    flags |= DIP_AFTER;
+  }
 
   if ((flags & DIP_NORTP) == 0) {
-    done = do_in_path(path, name, flags, callback, cookie);
+    done |= do_in_path(path, (name && !*name) ? NULL : name, flags, callback, cookie);
   }
 
   if ((done == FAIL || (flags & DIP_ALL)) && (flags & DIP_START)) {
-    char *start_dir = "pack/*/start/*/%s";  // NOLINT
-    size_t len = STRLEN(start_dir) + STRLEN(name);
-    char_u *s = xmallocz(len);
+    char *start_dir = "pack/*/start/*/%s%s";  // NOLINT
+    size_t len = STRLEN(start_dir) + STRLEN(name) + 6;
+    char_u *s = xmallocz(len);  // TODO(bfredl): get rid of random allocations
+    char *suffix = (flags & DIP_AFTER) ? "after/" : "";
 
-    vim_snprintf((char *)s, len, start_dir, name);
-    done = do_in_path(p_pp, s, flags, callback, cookie);
+    vim_snprintf((char *)s, len, start_dir, suffix, name);
+    done |= do_in_path(p_pp, s, flags & ~DIP_AFTER, callback, cookie);
 
     xfree(s);
 
-    if (done == FAIL|| (flags & DIP_ALL)) {
-      start_dir = "start/*/%s";  // NOLINT
-      len = STRLEN(start_dir) + STRLEN(name);
+    if (done == FAIL || (flags & DIP_ALL)) {
+      start_dir = "start/*/%s%s";  // NOLINT
+      len = STRLEN(start_dir) + STRLEN(name) + 6;
       s = xmallocz(len);
 
-      vim_snprintf((char *)s, len, start_dir, name);
-      done = do_in_path(p_pp, s, flags, callback, cookie);
+      vim_snprintf((char *)s, len, start_dir, suffix, name);
+      done |= do_in_path(p_pp, s, flags & ~DIP_AFTER, callback, cookie);
 
       xfree(s);
     }
@@ -202,7 +209,7 @@ int do_in_path_and_pp(char_u *path, char_u *name, int flags,
     char_u *s = xmallocz(len);
 
     vim_snprintf((char *)s, len, opt_dir, name);
-    done = do_in_path(p_pp, s, flags, callback, cookie);
+    done |= do_in_path(p_pp, s, flags, callback, cookie);
 
     xfree(s);
 
@@ -212,7 +219,7 @@ int do_in_path_and_pp(char_u *path, char_u *name, int flags,
       s = xmallocz(len);
 
       vim_snprintf((char *)s, len, opt_dir, name);
-      done = do_in_path(p_pp, s, flags, callback, cookie);
+      done |= do_in_path(p_pp, s, flags, callback, cookie);
 
       xfree(s);
     }
@@ -222,10 +229,9 @@ int do_in_path_and_pp(char_u *path, char_u *name, int flags,
 }
 
 /// Just like do_in_path_and_pp(), using 'runtimepath' for "path".
-int do_in_runtimepath(char_u *name, int flags, DoInRuntimepathCB callback,
-                      void *cookie)
+int do_in_runtimepath(char_u *name, int flags, DoInRuntimepathCB callback, void *cookie)
 {
-  return do_in_path_and_pp(p_rtp, name, flags, callback, cookie);
+  return do_in_path_and_pp(p_rtp, name, flags | DIP_START, callback, cookie);
 }
 
 /// Source the file "name" from all directories in 'runtimepath'.
@@ -481,15 +487,6 @@ static void add_opt_pack_plugin(char_u *fname, void *cookie)
   add_pack_plugin(true, fname, cookie);
 }
 
-/// Add all packages in the "start" directory to 'runtimepath'.
-void add_pack_start_dirs(void)
-{
-  do_in_path(p_pp, (char_u *)"pack/*/start/*", DIP_ALL + DIP_DIR,  // NOLINT
-             add_start_pack_plugin, &APP_ADD_DIR);
-  do_in_path(p_pp, (char_u *)"start/*", DIP_ALL + DIP_DIR,  // NOLINT
-             add_start_pack_plugin, &APP_ADD_DIR);
-}
-
 /// Load plugins from all packages in the "start" directory.
 void load_start_packages(void)
 {
@@ -508,7 +505,6 @@ void ex_packloadall(exarg_T *eap)
     // First do a round to add all directories to 'runtimepath', then load
     // the plugins. This allows for plugins to use an autoload directory
     // of another plugin.
-    add_pack_start_dirs();
     load_start_packages();
   }
 }

src/nvim/syntax.c

@@ -4394,7 +4394,7 @@ static void syn_cmd_include(exarg_T *eap, int syncing)
   curwin->w_s->b_syn_topgrp = sgl_id;
   if (source
       ? do_source(eap->arg, false, DOSO_NONE) == FAIL
-      : source_in_path(p_rtp, eap->arg, DIP_ALL) == FAIL) {
+      : source_runtime(eap->arg, DIP_ALL) == FAIL) {
     EMSG2(_(e_notopen), eap->arg);
   }
   curwin->w_s->b_syn_topgrp = prev_toplvl_grp;

src/nvim/testdir/test_startup.vim

@@ -102,7 +102,7 @@ func Test_pack_in_rtp_when_plugins_run()
   if RunVim(before, after, '')
 
     let lines = filter(readfile('Xtestout'), '!empty(v:val)')
-    call assert_match('Xhere[/\\]pack[/\\]foo[/\\]start[/\\]foobar', get(lines, 0))
+    call assert_match('runtimepath=Xhere', get(lines, 0))
     call assert_match('autoloaded foo', get(lines, 1))
   endif
 

src/nvim/vim.h

@@ -315,6 +315,7 @@ enum { FOLD_TEXT_LEN = 51 };  //!< buffer size for get_foldtext()
 #define DIP_NOAFTER 0x40  // skip "after" directories
 #define DIP_AFTER   0x80  // only use "after" directories
 #define DIP_LUA  0x100    // also use ".lua" files
+#define DIP_DIRFILE 0x200  // find both files and directories
 
 // Lowest number used for window ID. Cannot have this many windows per tab.
 #define LOWEST_WIN_ID 1000

test/functional/api/vim_spec.lua

@@ -1591,7 +1591,10 @@ describe('API', function()
       eq({'a', '', 'b'}, meths.list_runtime_paths())
       meths.set_option('runtimepath', ',a,b')
       eq({'', 'a', 'b'}, meths.list_runtime_paths())
+      -- trailing , is ignored, use ,, if you really really want $CWD
       meths.set_option('runtimepath', 'a,b,')
+      eq({'a', 'b'}, meths.list_runtime_paths())
+      meths.set_option('runtimepath', 'a,b,,')
       eq({'a', 'b', ''}, meths.list_runtime_paths())
     end)
     it('truncates too long paths', function()
@@ -2012,8 +2015,13 @@ describe('API', function()
       ok(endswith(val[1], p"autoload/remote/define.vim")
          or endswith(val[1], p"autoload/remote/host.vim"))
 
-      eq({}, meths.get_runtime_file("lua", true))
-      eq({}, meths.get_runtime_file("lua/vim", true))
+      val = meths.get_runtime_file("lua", true)
+      eq(1, #val)
+      ok(endswith(val[1], p"lua"))
+
+      val = meths.get_runtime_file("lua/vim", true)
+      eq(1, #val)
+      ok(endswith(val[1], p"lua/vim"))
     end)
 
     it('can find directories', function()

test/functional/core/startup_spec.lua

@@ -310,7 +310,7 @@ describe('startup', function()
   end)
 
   local function pack_clear(cmd)
-    clear('--cmd', 'set packpath=test/functional/fixtures', '--cmd', cmd)
+    clear{args={'--cmd', 'set packpath=test/functional/fixtures', '--cmd', cmd}, env={XDG_CONFIG_HOME='test/functional/fixtures/'}}
   end
 
 
@@ -348,6 +348,16 @@ describe('startup', function()
     pack_clear [[ packadd! bonus | lua _G.y = require'bonus'.launch() ]]
     eq('CPE 1704 TKS', exec_lua [[ return _G.y ]])
   end)
+
+  it("handles the correct order with start packages and after/", function()
+    pack_clear [[ lua _G.test_loadorder = {} vim.cmd "runtime! filen.lua" ]]
+    eq({'ordinary', 'FANCY', 'ordinary after', 'FANCY after'}, exec_lua [[ return _G.test_loadorder ]])
+  end)
+
+  it("handles the correct order with opt packages and after/", function()
+    pack_clear [[ lua _G.test_loadorder = {} vim.cmd "packadd! superspecial\nruntime! filen.lua" ]]
+    eq({'ordinary', 'SuperSpecial', 'FANCY', 'SuperSpecial after', 'ordinary after', 'FANCY after'}, exec_lua [[ return _G.test_loadorder ]])
+  end)
 end)
 
 describe('sysinit', function()
@@ -504,6 +514,7 @@ describe('runtime:', function()
   local xenv = { XDG_CONFIG_HOME=xconfig, XDG_DATA_HOME=xdata }
 
   setup(function()
+    rmdir(xhome)
     mkdir_p(xconfig .. pathsep .. 'nvim')
     mkdir_p(xdata)
   end)
@@ -524,7 +535,7 @@ describe('runtime:', function()
     rmdir(plugin_folder_path)
   end)
 
-  it('loads plugin/*.lua from start plugins', function()
+  it('loads plugin/*.lua from start packages', function()
     local plugin_path = table.concat({xconfig, 'nvim', 'pack', 'catagory',
     'start', 'test_plugin'}, pathsep)
     local plugin_folder_path = table.concat({plugin_path, 'plugin'}, pathsep)
@@ -552,6 +563,27 @@ describe('runtime:', function()
     rmdir(plugin_path)
   end)
 
+  it('loads plugin/*.lua from site packages', function()
+    local nvimdata = iswin() and "nvim-data" or "nvim"
+    local plugin_path = table.concat({xdata, nvimdata, 'site', 'pack', 'xa', 'start', 'yb'}, pathsep)
+    local plugin_folder_path = table.concat({plugin_path, 'plugin'}, pathsep)
+    local plugin_after_path = table.concat({plugin_path, 'after', 'plugin'}, pathsep)
+    local plugin_file_path = table.concat({plugin_folder_path, 'plugin.lua'}, pathsep)
+    local plugin_after_file_path = table.concat({plugin_after_path, 'helloo.lua'}, pathsep)
+
+    mkdir_p(plugin_folder_path)
+    write_file(plugin_file_path, [[table.insert(_G.lista, "unos")]])
+    mkdir_p(plugin_after_path)
+    write_file(plugin_after_file_path, [[table.insert(_G.lista, "dos")]])
+
+    clear{ args_rm={'-u'}, args={'--cmd', 'lua _G.lista = {}'}, env=xenv }
+
+    eq({'unos', 'dos'}, exec_lua "return _G.lista")
+
+    rmdir(plugin_path)
+  end)
+
+
   it('loads ftdetect/*.lua', function()
     local ftdetect_folder = table.concat({xconfig, 'nvim', 'ftdetect'}, pathsep)
     local ftdetect_file = table.concat({ftdetect_folder , 'new-ft.lua'}, pathsep)

test/functional/fixtures/nvim/after/filen.lua

@@ -0,0 +1 @@
+table.insert(_G.test_loadorder, "ordinary after")

test/functional/fixtures/nvim/filen.lua

@@ -0,0 +1 @@
+table.insert(_G.test_loadorder, "ordinary")

test/functional/fixtures/pack/foo/opt/superspecial/after/filen.lua

@@ -0,0 +1 @@
+table.insert(_G.test_loadorder, "SuperSpecial after")

test/functional/fixtures/pack/foo/opt/superspecial/filen.lua

@@ -0,0 +1 @@
+table.insert(_G.test_loadorder, "SuperSpecial")

test/functional/fixtures/pack/foo/start/fancyplugin/after/filen.lua

@@ -0,0 +1 @@
+table.insert(_G.test_loadorder, "FANCY after")

test/functional/fixtures/pack/foo/start/fancyplugin/filen.lua

@@ -0,0 +1 @@
+table.insert(_G.test_loadorder, "FANCY")