refactor(lua): cleanup and docs for threads

runtime/doc/lua.txt

@@ -568,6 +568,26 @@ Example: TCP echo-server                                *tcp-server*
     end)
     print('TCP echo-server listening on port: '..server:getsockname().port)
 
+
+Multithreading						*lua-loop-threading*
+
+Plugins can perform work in separate (os-level) threads using the threading
+APIs in luv, for instance `vim.loop.new_thread`. Note that every thread
+gets its own separate lua interpreter state, with no access to lua globals
+in the main thread. Neither can the state of the editor (buffers, windows,
+etc) be directly accessed from threads.
+
+A subset of the `vim.*` API is available in threads. This includes:
+
+- `vim.loop` with a separate event loop per thread.
+- `vim.mpack` and `vim.json` (useful for serializing messages between threads)
+- `require` in threads can use lua packages from the global |lua-package-path|
+- `print()` and `vim.inspect`
+- `vim.diff`
+- most utility functions in `vim.*` for working with pure lua values
+  like `vim.split`, `vim.tbl_*`, `vim.list_*`, and so on.
+- `vim.is_thread()` returns true from a non-main thread.
+
 ------------------------------------------------------------------------------
 VIM.HIGHLIGHT                                           *lua-highlight*
 

runtime/lua/vim/_load_package.lua

@@ -45,4 +45,5 @@ function vim._load_package(name)
   return nil
 end
 
-table.insert(package.loaders, 1, vim._load_package)
+-- Insert vim._load_package after the preloader at position 2
+table.insert(package.loaders, 2, vim._load_package)

src/nvim/lua/stdlib.c

@@ -474,6 +474,9 @@ static int nlua_stricmp(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
 void nlua_state_add_stdlib(lua_State *const lstate, bool is_thread)
 {
   if (!is_thread) {
+    // TODO(bfredl): some of basic string functions should already be
+    // (or be easy to make) threadsafe
+
     // stricmp
     lua_pushcfunction(lstate, &nlua_stricmp);
     lua_setfield(lstate, -2, "stricmp");

src/nvim/main.c

@@ -157,7 +157,7 @@ void early_init(mparm_T *paramp)
   eval_init();          // init global variables
   init_path(argv0 ? argv0 : "nvim");
   init_normal_cmds();   // Init the table of Normal mode commands.
-  runtime_search_path_init();
+  runtime_init();
   highlight_init();
 
 #ifdef WIN32

src/nvim/os/fs.c

@@ -64,6 +64,9 @@ void fs_init(void)
   uv_mutex_init_recursive(&fs_loop_mutex);
 }
 
+/// TODO(bfredl): some of these operations should
+/// be possible to do the private libuv loop of the
+/// thread, instead of contending the global fs loop
 void fs_loop_lock(void)
 {
   uv_mutex_lock(&fs_loop_mutex);

src/nvim/runtime.c

@@ -27,21 +27,11 @@ static RuntimeSearchPath runtime_search_path;
 static RuntimeSearchPath runtime_search_path_thread;
 static uv_mutex_t runtime_search_path_mutex;
 
-void runtime_search_path_init(void)
+void runtime_init(void)
 {
   uv_mutex_init(&runtime_search_path_mutex);
 }
 
-void runtime_search_path_lock(void)
-{
-  uv_mutex_lock(&runtime_search_path_mutex);
-}
-
-void runtime_search_path_unlock(void)
-{
-  uv_mutex_unlock(&runtime_search_path_mutex);
-}
-
 /// ":runtime [what] {name}"
 void ex_runtime(exarg_T *eap)
 {
@@ -330,8 +320,9 @@ ArrayOf(String) runtime_get_named(bool lua, Array pat, bool all)
 {
   int ref;
   RuntimeSearchPath path = runtime_search_path_get_cached(&ref);
+  static char buf[MAXPATHL];
 
-  ArrayOf(String) rv = runtime_get_named_common(lua, pat, all, path);
+  ArrayOf(String) rv = runtime_get_named_common(lua, pat, all, path, buf, sizeof buf);
 
   runtime_search_path_unref(path, &ref);
   return rv;
@@ -339,18 +330,19 @@ ArrayOf(String) runtime_get_named(bool lua, Array pat, bool all)
 
 ArrayOf(String) runtime_get_named_thread(bool lua, Array pat, bool all)
 {
-  runtime_search_path_lock();
+  // TODO(bfredl): avoid contention between multiple worker threads?
-  ArrayOf(String) rv = runtime_get_named_common(lua, pat, all, runtime_search_path_thread);
+  uv_mutex_lock(&runtime_search_path_mutex);
-  runtime_search_path_unlock();
+  static char buf[MAXPATHL];
+  ArrayOf(String) rv = runtime_get_named_common(lua, pat, all, runtime_search_path_thread,
+                                                buf, sizeof buf);
+  uv_mutex_unlock(&runtime_search_path_mutex);
   return rv;
 }
 
 ArrayOf(String) runtime_get_named_common(bool lua, Array pat, bool all,
-                                         RuntimeSearchPath path)
+                                         RuntimeSearchPath path, char *buf, size_t buf_len)
 {
   ArrayOf(String) rv = ARRAY_DICT_INIT;
-  size_t buf_len = MAXPATHL;
-  char *buf = xmalloc(MAXPATHL);
   for (size_t i = 0; i < kv_size(path); i++) {
     SearchPathItem *item = &kv_A(path, i);
     if (lua) {
@@ -380,7 +372,6 @@ ArrayOf(String) runtime_get_named_common(bool lua, Array pat, bool all,
     }
   }
 done:
-  xfree(buf);
   return rv;
 }
 
@@ -614,10 +605,10 @@ void runtime_search_path_validate(void)
     runtime_search_path = runtime_search_path_build();
     runtime_search_path_valid = true;
     runtime_search_path_ref = NULL;  // initially unowned
-    runtime_search_path_lock();
+    uv_mutex_lock(&runtime_search_path_mutex);
     runtime_search_path_free(runtime_search_path_thread);
     runtime_search_path_thread = copy_runtime_search_path(runtime_search_path);
-    runtime_search_path_unlock();
+    uv_mutex_unlock(&runtime_search_path_mutex);
   }
 }
 

test/functional/lua/thread_spec.lua

@@ -26,13 +26,12 @@ describe('thread', function()
   end)
 
   it('entry func is executed in protected mode', function()
-    local code = [[
+    exec_lua [[
       local thread = vim.loop.new_thread(function()
         error('Error in thread entry func')
       end)
       vim.loop.thread_join(thread)
     ]]
-    exec_lua(code)
 
     screen:expect([[
                                                         |
@@ -51,7 +50,7 @@ describe('thread', function()
   end)
 
   it('callback is executed in protected mode', function()
-    local code = [[
+    exec_lua [[
       local thread = vim.loop.new_thread(function()
         local timer = vim.loop.new_timer()
         local function ontimeout()
@@ -64,7 +63,6 @@ describe('thread', function()
       end)
       vim.loop.thread_join(thread)
     ]]
-    exec_lua(code)
 
     screen:expect([[
                                                         |
@@ -83,35 +81,33 @@ describe('thread', function()
   end)
 
   describe('print', function()
-    it('work', function()
+    it('works', function()
-    local code = [[
+      exec_lua [[
-      local thread = vim.loop.new_thread(function()
+        local thread = vim.loop.new_thread(function()
-        print('print in thread')
+          print('print in thread')
-      end)
+        end)
-      vim.loop.thread_join(thread)
+        vim.loop.thread_join(thread)
-    ]]
+      ]]
-    exec_lua(code)
-
-    screen:expect([[
-      ^                                                  |
-      {1:~                                                 }|
-      {1:~                                                 }|
-      {1:~                                                 }|
-      {1:~                                                 }|
-      {1:~                                                 }|
-      {1:~                                                 }|
-      {1:~                                                 }|
-      {1:~                                                 }|
-      print in thread                                   |
-    ]])
 
+      screen:expect([[
+        ^                                                  |
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        {1:~                                                 }|
+        print in thread                                   |
+      ]])
     end)
   end)
 
   describe('vim.*', function()
     before_each(function()
       clear()
-      local code = [[
+      exec_lua [[
         Thread_Test = {}
 
         Thread_Test.entry_func = function(async, entry_str, args)
@@ -140,11 +136,10 @@ describe('thread', function()
           return self
         end
       ]]
-      exec_lua(code)
     end)
 
     it('is_thread', function()
-      local code = [[
+      exec_lua [[
         local entry = function(async)
           async:send(vim.is_thread())
         end
@@ -154,13 +149,12 @@ describe('thread', function()
         local thread_test = Thread_Test.new(entry, on_async)
         thread_test:do_test()
       ]]
-      exec_lua(code)
 
       eq({'notification', 'result', {true}}, next_msg())
     end)
 
     it('loop', function()
-      local code = [[
+      exec_lua [[
         local entry = function(async)
           async:send(vim.loop.version())
         end
@@ -170,7 +164,6 @@ describe('thread', function()
         local thread_test = Thread_Test.new(entry, on_async)
         thread_test:do_test()
       ]]
-      exec_lua(code)
 
       local msg = next_msg()
       eq(msg[1], 'notification')
@@ -178,7 +171,7 @@ describe('thread', function()
     end)
 
     it('mpack', function()
-      local code = [[
+      exec_lua [[
         local entry = function(async)
           async:send(vim.mpack.encode({33, vim.NIL, 'text'}))
         end
@@ -188,13 +181,12 @@ describe('thread', function()
         local thread_test = Thread_Test.new(entry, on_async)
         thread_test:do_test()
       ]]
-      exec_lua(code)
 
       eq({'notification', 'result', {{33, NIL, 'text'}}}, next_msg())
     end)
 
     it('json', function()
-      local code = [[
+      exec_lua [[
         local entry = function(async)
         async:send(vim.json.encode({33, vim.NIL, 'text'}))
         end
@@ -204,13 +196,12 @@ describe('thread', function()
         local thread_test = Thread_Test.new(entry, on_async)
         thread_test:do_test()
       ]]
-      exec_lua(code)
 
       eq({'notification', 'result', {{33, NIL, 'text'}}}, next_msg())
     end)
 
     it('diff', function()
-      local code = [[
+      exec_lua [[
         local entry = function(async)
           async:send(vim.diff('Hello\n', 'Helli\n'))
         end
@@ -220,7 +211,6 @@ describe('thread', function()
         local thread_test = Thread_Test.new(entry, on_async)
         thread_test:do_test()
       ]]
-      exec_lua(code)
 
       eq({'notification', 'result',
           {table.concat({
@@ -238,9 +228,9 @@ describe('threadpool', function()
   before_each(clear)
 
   it('is_thread', function()
-    eq(false, exec_lua('return vim.is_thread()'))
+    eq(false, exec_lua [[return vim.is_thread()]])
 
-    local code = [[
+    exec_lua [[
       local work_fn = function()
         return vim.is_thread()
       end
@@ -250,19 +240,18 @@ describe('threadpool', function()
       local work = vim.loop.new_work(work_fn, after_work_fn)
       work:queue()
     ]]
-    exec_lua(code)
 
     eq({'notification', 'result', {true}}, next_msg())
   end)
 
   it('with invalid argument', function()
-    local code = [[
+    local status = pcall_err(exec_lua, [[
       local work = vim.loop.new_thread(function() end, function() end)
       work:queue({})
-    ]]
+    ]])
 
     eq([[Error executing lua: [string "<nvim>"]:0: Error: thread arg not support type 'function' at 1]],
-      pcall_err(exec_lua, code))
+       status)
   end)
 
   it('with invalid return value', function()
@@ -276,11 +265,10 @@ describe('threadpool', function()
       [5] = {bold = true},
     })
 
-    local code = [[
+    exec_lua [[
       local work = vim.loop.new_work(function() return {} end, function() end)
       work:queue()
     ]]
-    exec_lua(code)
 
     screen:expect([[
                                                         |
@@ -299,7 +287,7 @@ describe('threadpool', function()
   describe('vim.*', function()
     before_each(function()
       clear()
-      local code = [[
+      exec_lua [[
         Threadpool_Test = {}
 
         Threadpool_Test.work_fn = function(work_fn_str, args)
@@ -322,11 +310,10 @@ describe('threadpool', function()
           return self
         end
       ]]
-      exec_lua(code)
     end)
 
     it('loop', function()
-      local code = [[
+      exec_lua [[
         local work_fn = function()
           return vim.loop.version()
         end
@@ -336,7 +323,6 @@ describe('threadpool', function()
         local threadpool_test = Threadpool_Test.new(work_fn, after_work_fn)
         threadpool_test:do_test()
       ]]
-      exec_lua(code)
 
       local msg = next_msg()
       eq(msg[1], 'notification')
@@ -344,7 +330,7 @@ describe('threadpool', function()
     end)
 
     it('mpack', function()
-      local code = [[
+      exec_lua [[
         local work_fn = function()
           local var = vim.mpack.encode({33, vim.NIL, 'text'})
           return var
@@ -355,13 +341,12 @@ describe('threadpool', function()
         local threadpool_test = Threadpool_Test.new(work_fn, after_work_fn)
         threadpool_test:do_test()
       ]]
-      exec_lua(code)
 
       eq({'notification', 'result', {{33, NIL, 'text'}}}, next_msg())
     end)
 
     it('json', function()
-      local code = [[
+      exec_lua [[
         local work_fn = function()
           local var = vim.json.encode({33, vim.NIL, 'text'})
           return var
@@ -372,13 +357,12 @@ describe('threadpool', function()
         local threadpool_test = Threadpool_Test.new(work_fn, after_work_fn)
         threadpool_test:do_test()
       ]]
-      exec_lua(code)
 
       eq({'notification', 'result', {{33, NIL, 'text'}}}, next_msg())
     end)
 
     it('work', function()
-      local code = [[
+      exec_lua [[
         local work_fn = function()
           return vim.diff('Hello\n', 'Helli\n')
         end
@@ -388,7 +372,6 @@ describe('threadpool', function()
         local threadpool_test = Threadpool_Test.new(work_fn, after_work_fn)
         threadpool_test:do_test()
       ]]
-      exec_lua(code)
 
       eq({'notification', 'result',
           {table.concat({