tree-sitter: implement query functionality and highlighting prototype [skip.lint]

runtime/doc/lua.txt

@@ -594,6 +594,102 @@ tsnode:named_descendant_for_range(start_row, start_col, end_row, end_col)
 	Get the smallest named node within this node that spans the given
 	range of (row, column) positions
 
+Query methods						*lua-treesitter-query*
+
+Tree-sitter queries are supported, with some limitations. Currently, the only
+supported match predicate is `eq?` (both comparing a capture against a string
+and two captures against each other).
+
+vim.treesitter.parse_query(lang, query)
+						*vim.treesitter.parse_query(()*
+	Parse the query as a string. (If the query is in a file, the caller
+        should read the contents into a string before calling).
+
+query:iter_captures(node, bufnr, start_row, end_row)
+							*query:iter_captures()*
+	Iterate over all captures from all matches inside a `node`.
+	`bufnr` is needed if the query contains predicates, then the caller
+	must ensure to use a freshly parsed tree consistent with the current
+	text of the buffer. `start_row` and `end_row` can be used to limit
+	matches inside a row range (this is typically used with root node
+	as the node, i e to get syntax highlight matches in the current
+	viewport)
+
+	The iterator returns two values, a numeric id identifying the capture
+	and the captured node. The following example shows how to get captures
+	by name:
+>
+	for id, node in query:iter_captures(tree:root(), bufnr, first, last) do
+	  local name = query.captures[id] -- name of the capture in the query
+	  -- typically useful info about the node:
+	  local type = node:type() -- type of the captured node
+	  local row1, col1, row2, col2 = node:range() -- range of the capture
+	  ... use the info here ...
+	end
+<
+query:iter_matches(node, bufnr, start_row, end_row)
+							*query:iter_matches()*
+	Iterate over all matches within a node. The arguments are the same as
+	for |query:iter_captures()| but the iterated values are different:
+	an (1-based) index of the pattern in the query, and a table mapping
+	capture indices to nodes. If the query has more than one pattern
+	the capture table might be sparse, and e.g. `pairs` should be used and not
+	`ipairs`. Here an example iterating over all captures in
+	every match:
+>
+	for pattern, match in cquery:iter_matches(tree:root(), bufnr, first, last) do
+	  for id,node in pairs(match) do
+	    local name = query.captures[id]
+	    -- `node` was captured by the `name` capture in the match
+	    ... use the info here ...
+	  end
+	end
+>
+Treesitter syntax highlighting (WIP)		    *lua-treesitter-highlight*
+
+NOTE: This is a partially implemented feature, and not usable as a default
+solution yet. What is documented here is a temporary interface indented
+for those who want to experiment with this feature and contribute to
+its development.
+
+Highlights are defined in the same query format as in the tree-sitter highlight
+crate, which some limitations and additions. Set a highlight query for a
+buffer with this code: >
+
+    local query = [[
+      "for" @keyword
+      "if" @keyword
+      "return" @keyword
+
+      (string_literal) @string
+      (number_literal) @number
+      (comment) @comment
+
+      (preproc_function_def name: (identifier) @function)
+
+      ; ... more definitions
+    ]]
+
+    highlighter = vim.treesitter.TSHighlighter.new(query, bufnr, lang)
+    -- alternatively, to use the current buffer and its filetype:
+    -- highlighter = vim.treesitter.TSHighlighter.new(query)
+
+    -- Don't recreate the highlighter for the same buffer, instead
+    -- modify the query like this:
+    local query2 = [[ ... ]]
+    highlighter:set_query(query2)
+
+As mentioned above the supported predicate is currently only `eq?`. `match?`
+predicates behave like matching always fails. As an addition a capture which
+begin with an upper-case letter like `@WarningMsg` will map directly to this
+highlight group, if defined. Also if the predicate begins with upper-case and
+contains a dot only the part before the first will be interpreted as the
+highlight group. As an example, this warns of a binary expression with two
+identical identifiers, highlighting both as |hl-WarningMsg|: >
+
+    ((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right)
+     (eq? @WarningMsg.left @WarningMsg.right))
+
 ------------------------------------------------------------------------------
 VIM							*lua-builtin*
 

runtime/lua/vim/treesitter.lua

@@ -12,9 +12,13 @@ function Parser:parse()
   if self.valid then
     return self.tree
   end
-  self.tree = self._parser:parse_buf(self.bufnr)
+  local changes
+  self.tree, changes = self._parser:parse_buf(self.bufnr)
   self.valid = true
-  return self.tree
+  for _, cb in ipairs(self.change_cbs) do
+    cb(changes)
+  end
+  return self.tree, changes
 end
 
 function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_size)
@@ -26,17 +30,28 @@ function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_
   self.valid = false
 end
 
-local module = {
+local M = {
   add_language=vim._ts_add_language,
   inspect_language=vim._ts_inspect_language,
+  parse_query = vim._ts_parse_query,
 }
 
-function module.create_parser(bufnr, ft, id)
+setmetatable(M, {
+  __index = function (t, k)
+      if k == "TSHighlighter" then
+        t[k] = require'vim.tshighlighter'
+        return t[k]
+      end
+   end
+ })
+
+function M.create_parser(bufnr, ft, id)
   if bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
-  local self = setmetatable({bufnr=bufnr, valid=false}, Parser)
+  local self = setmetatable({bufnr=bufnr, lang=ft, valid=false}, Parser)
   self._parser = vim._create_ts_parser(ft)
+  self.change_cbs = {}
   self:parse()
     -- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
     -- using it.
@@ -55,7 +70,7 @@ function module.create_parser(bufnr, ft, id)
   return self
 end
 
-function module.get_parser(bufnr, ft)
+function M.get_parser(bufnr, ft, cb)
   if bufnr == nil or bufnr == 0 then
     bufnr = a.nvim_get_current_buf()
   end
@@ -65,9 +80,98 @@ function module.get_parser(bufnr, ft)
   local id = tostring(bufnr)..'_'..ft
 
   if parsers[id] == nil then
-    parsers[id] = module.create_parser(bufnr, ft, id)
+    parsers[id] = M.create_parser(bufnr, ft, id)
+  end
+  if cb ~= nil then
+    table.insert(parsers[id].change_cbs, cb)
   end
   return parsers[id]
 end
 
-return module
+-- query: pattern matching on trees
+-- predicate matching is implemented in lua
+local Query = {}
+Query.__index = Query
+
+function M.parse_query(lang, query)
+  local self = setmetatable({}, Query)
+  self.query = vim._ts_parse_query(lang, query)
+  self.info = self.query:inspect()
+  self.captures = self.info.captures
+  return self
+end
+
+local function get_node_text(node, bufnr)
+  local start_row, start_col, end_row, end_col = node:range()
+  if start_row ~= end_row then
+    return nil
+  end
+  local line = a.nvim_buf_get_lines(bufnr, start_row, start_row+1, true)[1]
+  return string.sub(line, start_col+1, end_col)
+end
+
+local function match_preds(match, preds, bufnr)
+  for _, pred in pairs(preds) do
+    if pred[1] == "eq?" then
+      local node = match[pred[2]]
+      local node_text = get_node_text(node, bufnr)
+
+      local str
+      if type(pred[3]) == "string" then
+        -- (eq? @aa "foo")
+        str = pred[3]
+      else
+        -- (eq? @aa @bb)
+        str = get_node_text(match[pred[3]], bufnr)
+      end
+
+      if node_text ~= str or str == nil then
+        return false
+      end
+    else
+      return false
+    end
+  end
+  return true
+end
+
+function Query:iter_captures(node, bufnr, start, stop)
+  if bufnr == 0 then
+    bufnr = vim.api.nvim_get_current_buf()
+  end
+  local raw_iter = node:_rawquery(self.query,true,start,stop)
+  local function iter()
+    local capture, captured_node, match = raw_iter()
+    if match ~= nil then
+      local preds = self.info.patterns[match.pattern]
+      local active = match_preds(match, preds, bufnr)
+      match.active = active
+      if not active then
+        return iter() -- tail call: try next match
+      end
+    end
+    return capture, captured_node
+  end
+  return iter
+end
+
+function Query:iter_matches(node, bufnr, start, stop)
+  if bufnr == 0 then
+    bufnr = vim.api.nvim_get_current_buf()
+  end
+  local raw_iter = node:_rawquery(self.query,false,start,stop)
+  local function iter()
+    local pattern, match = raw_iter()
+    if match ~= nil then
+      local preds = self.info.patterns[pattern]
+      local active = (not preds) or match_preds(match, preds, bufnr)
+      if not active then
+        return iter() -- tail call: try next match
+      end
+    end
+    return pattern, match
+  end
+  return iter
+end
+
+return M

runtime/lua/vim/tshighlighter.lua

@@ -0,0 +1,142 @@
+local a = vim.api
+
+-- support reload for quick experimentation
+local TSHighlighter = rawget(vim.treesitter, 'TSHighlighter') or {}
+TSHighlighter.__index = TSHighlighter
+
+-- These are conventions defined by tree-sitter, though it
+-- needs to be user extensible also.
+-- TODO(bfredl): this is very much incomplete, we will need to
+-- go through a few tree-sitter provided queries and decide
+-- on translations that makes the most sense.
+TSHighlighter.hl_map = {
+    keyword="Keyword",
+    string="String",
+    type="Type",
+    comment="Comment",
+    constant="Constant",
+    operator="Operator",
+    number="Number",
+    label="Label",
+    ["function"]="Function",
+    ["function.special"]="Function",
+}
+
+function TSHighlighter.new(query, bufnr, ft)
+  local self = setmetatable({}, TSHighlighter)
+  self.parser = vim.treesitter.get_parser(bufnr, ft, function(...) self:on_change(...) end)
+  self.buf = self.parser.bufnr
+  -- TODO(bfredl): perhaps on_start should be called uncondionally, instead for only on mod?
+  local tree = self.parser:parse()
+  self.root = tree:root()
+  self:set_query(query)
+  self.edit_count = 0
+  self.redraw_count = 0
+  self.line_count = {}
+  a.nvim_buf_set_option(self.buf, "syntax", "")
+  a.nvim__buf_set_luahl(self.buf, {
+    on_start=function(...) return self:on_start(...) end,
+    on_window=function(...) return self:on_window(...) end,
+    on_line=function(...) return self:on_line(...) end,
+  })
+
+  -- Tricky: if syntax hasn't been enabled, we need to reload color scheme
+  -- but use synload.vim rather than syntax.vim to not enable
+  -- syntax FileType autocmds. Later on we should integrate with the
+  -- `:syntax` and `set syntax=...` machinery properly.
+  if vim.g.syntax_on ~= 1 then
+    vim.api.nvim_command("runtime! syntax/synload.vim")
+  end
+  return self
+end
+
+function TSHighlighter:set_query(query)
+  if type(query) == "string" then
+    query = vim.treesitter.parse_query(self.parser.lang, query)
+  end
+  self.query = query
+
+  self.id_map = {}
+  for i, capture in ipairs(self.query.captures) do
+    local hl = 0
+    local firstc = string.sub(capture, 1, 1)
+    local hl_group = self.hl_map[capture]
+    if firstc ~= string.lower(firstc) then
+      hl_group = vim.split(capture, '.', true)[1]
+    end
+    if hl_group then
+      hl = a.nvim_get_hl_id_by_name(hl_group)
+    end
+    self.id_map[i] = hl
+  end
+end
+
+function TSHighlighter:on_change(changes)
+  for _, ch in ipairs(changes or {}) do
+    a.nvim__buf_redraw_range(self.buf, ch[1], ch[3]+1)
+  end
+  self.edit_count = self.edit_count + 1
+end
+
+function TSHighlighter:on_start(_, _buf, _tick)
+  local tree = self.parser:parse()
+  self.root = tree:root()
+end
+
+function TSHighlighter:on_window(_, _win, _buf, _topline, botline)
+  self.iter = nil
+  self.active_nodes = {}
+  self.nextrow = 0
+  self.botline = botline
+  self.redraw_count = self.redraw_count + 1
+end
+
+function TSHighlighter:on_line(_, _win, buf, line)
+  if self.iter == nil then
+    self.iter = self.query:iter_captures(self.root,buf,line,self.botline)
+  end
+  while line >= self.nextrow do
+    local capture, node, match = self.iter()
+    local active = true
+    if capture == nil then
+      break
+    end
+    if match ~= nil then
+      active = self:run_pred(match)
+      match.active = active
+    end
+    local start_row, start_col, end_row, end_col = node:range()
+    local hl = self.id_map[capture]
+    if hl > 0 and active then
+      if start_row == line and end_row == line then
+        a.nvim__put_attr(hl, start_col, end_col)
+      elseif end_row >= line then
+        -- TODO(bfredl): this is quite messy. Togheter with multiline bufhl we should support
+        -- luahl generating multiline highlights (and other kinds of annotations)
+        self.active_nodes[{hl=hl, start_row=start_row, start_col=start_col, end_row=end_row, end_col=end_col}] = true
+      end
+    end
+    if start_row > line then
+      self.nextrow = start_row
+    end
+  end
+  for node,_ in pairs(self.active_nodes) do
+    if node.start_row <= line and node.end_row >= line then
+      local start_col, end_col = node.start_col, node.end_col
+      if node.start_row < line then
+        start_col = 0
+      end
+      if node.end_row > line then
+        end_col = 9000
+      end
+      a.nvim__put_attr(node.hl, start_col, end_col)
+    end
+    if node.end_row <= line then
+      self.active_nodes[node] = nil
+    end
+  end
+  self.line_count[line] = (self.line_count[line] or 0) + 1
+  --return tostring(self.line_count[line])
+end
+
+return TSHighlighter

src/nvim/api/buffer.c

@@ -169,21 +169,21 @@ Boolean nvim_buf_attach(uint64_t channel_id,
         goto error;
       }
       cb.on_lines = v->data.luaref;
-      v->data.integer = LUA_NOREF;
+      v->data.luaref = LUA_NOREF;
     } else if (is_lua && strequal("on_changedtick", k.data)) {
       if (v->type != kObjectTypeLuaRef) {
         api_set_error(err, kErrorTypeValidation, "callback is not a function");
         goto error;
       }
       cb.on_changedtick = v->data.luaref;
-      v->data.integer = LUA_NOREF;
+      v->data.luaref = LUA_NOREF;
     } else if (is_lua && strequal("on_detach", k.data)) {
       if (v->type != kObjectTypeLuaRef) {
         api_set_error(err, kErrorTypeValidation, "callback is not a function");
         goto error;
       }
       cb.on_detach = v->data.luaref;
-      v->data.integer = LUA_NOREF;
+      v->data.luaref = LUA_NOREF;
     } else if (is_lua && strequal("utf_sizes", k.data)) {
       if (v->type != kObjectTypeBoolean) {
         api_set_error(err, kErrorTypeValidation, "utf_sizes must be boolean");
@@ -231,6 +231,90 @@ Boolean nvim_buf_detach(uint64_t channel_id,
   return true;
 }
 
+static void buf_clear_luahl(buf_T *buf, bool force)
+{
+  if (buf->b_luahl || force) {
+    executor_free_luaref(buf->b_luahl_start);
+    executor_free_luaref(buf->b_luahl_window);
+    executor_free_luaref(buf->b_luahl_line);
+    executor_free_luaref(buf->b_luahl_end);
+  }
+  buf->b_luahl_start = LUA_NOREF;
+  buf->b_luahl_window = LUA_NOREF;
+  buf->b_luahl_line = LUA_NOREF;
+  buf->b_luahl_end = LUA_NOREF;
+}
+
+/// Unstabilized interface for defining syntax hl in lua.
+///
+/// This is not yet safe for general use, lua callbacks will need to
+/// be restricted, like textlock and probably other stuff.
+///
+/// The API on_line/nvim__put_attr is quite raw and not intended to be the
+/// final shape. Ideally this should operate on chunks larger than a single
+/// line to reduce interpreter overhead, and generate annotation objects
+/// (bufhl/virttext) on the fly but using the same representation.
+void nvim__buf_set_luahl(uint64_t channel_id, Buffer buffer,
+                         DictionaryOf(LuaRef) opts, Error *err)
+  FUNC_API_LUA_ONLY
+{
+  buf_T *buf = find_buffer_by_handle(buffer, err);
+
+  if (!buf) {
+    return;
+  }
+
+  redraw_buf_later(buf, NOT_VALID);
+  buf_clear_luahl(buf, false);
+
+  for (size_t i = 0; i < opts.size; i++) {
+    String k = opts.items[i].key;
+    Object *v = &opts.items[i].value;
+    if (strequal("on_start", k.data)) {
+      if (v->type != kObjectTypeLuaRef) {
+        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+        goto error;
+      }
+      buf->b_luahl_start = v->data.luaref;
+      v->data.luaref = LUA_NOREF;
+    } else if (strequal("on_window", k.data)) {
+      if (v->type != kObjectTypeLuaRef) {
+        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+        goto error;
+      }
+      buf->b_luahl_window = v->data.luaref;
+      v->data.luaref = LUA_NOREF;
+    } else if (strequal("on_line", k.data)) {
+      if (v->type != kObjectTypeLuaRef) {
+        api_set_error(err, kErrorTypeValidation, "callback is not a function");
+        goto error;
+      }
+      buf->b_luahl_line = v->data.luaref;
+      v->data.luaref = LUA_NOREF;
+    } else {
+      api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
+      goto error;
+    }
+  }
+  buf->b_luahl = true;
+  return;
+error:
+  buf_clear_luahl(buf, true);
+  buf->b_luahl = false;
+}
+
+void nvim__buf_redraw_range(Buffer buffer, Integer first, Integer last,
+                            Error *err)
+  FUNC_API_LUA_ONLY
+{
+  buf_T *buf = find_buffer_by_handle(buffer, err);
+  if (!buf) {
+    return;
+  }
+
+  redraw_buf_range_later(buf, (linenr_T)first+1, (linenr_T)last);
+}
+
 /// Sets a buffer line
 ///
 /// @deprecated use nvim_buf_set_lines instead.
@@ -1112,7 +1196,6 @@ Array nvim_buf_get_extmarks(Buffer buffer, Integer ns_id, Object start,
         return rv;
       }
       limit = v->data.integer;
-      v->data.integer = LUA_NOREF;
     } else {
       api_set_error(err, kErrorTypeValidation, "unexpected key: %s", k.data);
       return rv;

src/nvim/api/private/helpers.h

@@ -60,6 +60,12 @@
 #define ADD(array, item) \
   kv_push(array, item)
 
+#define FIXED_TEMP_ARRAY(name, fixsize) \
+  Array name = ARRAY_DICT_INIT; \
+  Object name##__items[fixsize]; \
+  args.size = fixsize; \
+  args.items = name##__items; \
+
 #define STATIC_CSTR_AS_STRING(s) ((String) {.data = s, .size = sizeof(s) - 1})
 
 /// Create a new String instance, putting data in allocated memory

src/nvim/api/vim.c

@@ -189,6 +189,15 @@ Dictionary nvim_get_hl_by_id(Integer hl_id, Boolean rgb, Error *err)
   return hl_get_attr_by_id(attrcode, rgb, err);
 }
 
+/// Gets a highlight group by name
+///
+/// similar to |hlID()|, but allocates a new ID if not present.
+Integer nvim_get_hl_id_by_name(String name)
+  FUNC_API_SINCE(7)
+{
+  return syn_check_group((const char_u *)name.data, (int)name.size);
+}
+
 /// Sends input-keys to Nvim, subject to various quirks controlled by `mode`
 /// flags. This is a blocking call, unlike |nvim_input()|.
 ///
@@ -2546,3 +2555,27 @@ Array nvim__inspect_cell(Integer grid, Integer row, Integer col, Error *err)
   }
   return ret;
 }
+
+/// Set attrs in nvim__buf_set_lua_hl callbacks
+///
+/// TODO(bfredl): This is rather pedestrian. The final
+/// interface should probably be derived from a reformed
+/// bufhl/virttext interface with full support for multi-line
+/// ranges etc
+void nvim__put_attr(Integer id, Integer c0, Integer c1)
+  FUNC_API_LUA_ONLY
+{
+  if (!lua_attr_active) {
+    return;
+  }
+  if (id == 0 || syn_get_final_id((int)id) == 0) {
+    return;
+  }
+  int attr = syn_id2attr((int)id);
+  c0 = MAX(c0, 0);
+  c1 = MIN(c1, (Integer)lua_attr_bufsize);
+  for (Integer c = c0; c < c1; c++) {
+    lua_attr_buf[c] = (sattr_T)hl_combine_attr(lua_attr_buf[c], (int)attr);
+  }
+  return;
+}

src/nvim/buffer_defs.h

@@ -832,6 +832,12 @@ struct file_buffer {
   // The number for times the current line has been flushed in the memline.
   int flush_count;
 
+  bool b_luahl;
+  LuaRef b_luahl_start;
+  LuaRef b_luahl_window;
+  LuaRef b_luahl_line;
+  LuaRef b_luahl_end;
+
   int b_diff_failed;    // internal diff failed for this buffer
 };
 

src/nvim/buffer_updates.c

@@ -157,7 +157,7 @@ void buf_updates_unregister_all(buf_T *buf)
       args.items[0] = BUFFER_OBJ(buf->handle);
 
       textlock++;
-      executor_exec_lua_cb(cb.on_detach, "detach", args, false);
+      executor_exec_lua_cb(cb.on_detach, "detach", args, false, NULL);
       textlock--;
     }
     free_update_callbacks(cb);
@@ -265,7 +265,7 @@ void buf_updates_send_changes(buf_T *buf,
         args.items[7] = INTEGER_OBJ((Integer)deleted_codeunits);
       }
       textlock++;
-      Object res = executor_exec_lua_cb(cb.on_lines, "lines", args, true);
+      Object res = executor_exec_lua_cb(cb.on_lines, "lines", args, true, NULL);
       textlock--;
 
       if (res.type == kObjectTypeBoolean && res.data.boolean == true) {
@@ -293,10 +293,7 @@ void buf_updates_changedtick(buf_T *buf)
     BufUpdateCallbacks cb = kv_A(buf->update_callbacks, i);
     bool keep = true;
     if (cb.on_changedtick != LUA_NOREF) {
-      Array args = ARRAY_DICT_INIT;
+      FIXED_TEMP_ARRAY(args, 2);
-      Object items[2];
-      args.size = 2;
-      args.items = items;
 
       // the first argument is always the buffer handle
       args.items[0] = BUFFER_OBJ(buf->handle);
@@ -306,7 +303,7 @@ void buf_updates_changedtick(buf_T *buf)
 
       textlock++;
       Object res = executor_exec_lua_cb(cb.on_changedtick, "changedtick",
-                                        args, true);
+                                        args, true, NULL);
       textlock--;
 
       if (res.type == kObjectTypeBoolean && res.data.boolean == true) {

src/nvim/func_attr.h

@@ -211,6 +211,8 @@
 # define FUNC_API_NOEXPORT
 /// API function not exposed in VimL/eval.
 # define FUNC_API_REMOTE_ONLY
+/// API function not exposed in VimL/remote.
+# define FUNC_API_LUA_ONLY
 /// API function introduced at the given API level.
 # define FUNC_API_SINCE(X)
 /// API function deprecated since the given API level.

src/nvim/generators/c_grammar.lua

@@ -42,6 +42,7 @@ local c_proto = Ct(
   (fill * Cg((P('FUNC_API_FAST') * Cc(true)), 'fast') ^ -1) *
   (fill * Cg((P('FUNC_API_NOEXPORT') * Cc(true)), 'noexport') ^ -1) *
   (fill * Cg((P('FUNC_API_REMOTE_ONLY') * Cc(true)), 'remote_only') ^ -1) *
+  (fill * Cg((P('FUNC_API_LUA_ONLY') * Cc(true)), 'lua_only') ^ -1) *
   (fill * Cg((P('FUNC_API_REMOTE_IMPL') * Cc(true)), 'remote_impl') ^ -1) *
   (fill * Cg((P('FUNC_API_BRIDGE_IMPL') * Cc(true)), 'bridge_impl') ^ -1) *
   (fill * Cg((P('FUNC_API_COMPOSITOR_IMPL') * Cc(true)), 'compositor_impl') ^ -1) *

src/nvim/generators/gen_api_dispatch.lua

@@ -192,7 +192,7 @@ end
 -- the real API.
 for i = 1, #functions do
   local fn = functions[i]
-  if fn.impl_name == nil then
+  if fn.impl_name == nil and not fn.lua_only then
     local args = {}
 
     output:write('Object handle_'..fn.name..'(uint64_t channel_id, Array args, Error *error)')
@@ -310,12 +310,13 @@ void msgpack_rpc_init_method_table(void)
 
 for i = 1, #functions do
   local fn = functions[i]
-  output:write('  msgpack_rpc_add_method_handler('..
+  if not fn.lua_only then
-               '(String) {.data = "'..fn.name..'", '..
+      output:write('  msgpack_rpc_add_method_handler('..
-               '.size = sizeof("'..fn.name..'") - 1}, '..
+                   '(String) {.data = "'..fn.name..'", '..
-               '(MsgpackRpcRequestHandler) {.fn = handle_'..  (fn.impl_name or fn.name)..
+                   '.size = sizeof("'..fn.name..'") - 1}, '..
-               ', .fast = '..tostring(fn.fast)..'});\n')
+                   '(MsgpackRpcRequestHandler) {.fn = handle_'..  (fn.impl_name or fn.name)..
-
+                   ', .fast = '..tostring(fn.fast)..'});\n')
+  end
 end
 
 output:write('\n}\n\n')

src/nvim/generators/gen_eval.lua

@@ -25,7 +25,7 @@ local gperfpipe = io.open(funcsfname .. '.gperf', 'wb')
 local funcs = require('eval').funcs
 local metadata = mpack.unpack(io.open(metadata_file, 'rb'):read("*all"))
 for _,fun in ipairs(metadata) do
-  if not fun.remote_only then
+  if not (fun.remote_only or fun.lua_only) then
     funcs[fun.name] = {
       args=#fun.parameters,
       func='api_wrapper',

src/nvim/globals.h

@@ -126,6 +126,13 @@ typedef off_t off_T;
  */
 EXTERN int mod_mask INIT(= 0x0);                /* current key modifiers */
 
+
+// TODO(bfredl): for the final interface this should find a more suitable
+// location.
+EXTERN sattr_T *lua_attr_buf INIT(= NULL);
+EXTERN size_t lua_attr_bufsize INIT(= 0);
+EXTERN bool lua_attr_active INIT(= false);
+
 /*
  * Cmdline_row is the row where the command line starts, just below the
  * last window.

src/nvim/lua/executor.c

@@ -835,7 +835,7 @@ Object executor_exec_lua_api(const String str, const Array args, Error *err)
 }
 
 Object executor_exec_lua_cb(LuaRef ref, const char *name, Array args,
-                            bool retval)
+                            bool retval, Error *err)
 {
   lua_State *const lstate = nlua_enter();
   nlua_pushref(lstate, ref);
@@ -845,16 +845,24 @@ Object executor_exec_lua_cb(LuaRef ref, const char *name, Array args,
   }
 
   if (lua_pcall(lstate, (int)args.size+1, retval ? 1 : 0, 0)) {
-    // TODO(bfredl): callbacks:s might not always be msg-safe, for instance
+    // if err is passed, the caller will deal with the error.
-    // lua callbacks for redraw events. Later on let the caller deal with the
+    if (err) {
-    // error instead.
+      size_t len;
-    nlua_error(lstate, _("Error executing lua callback: %.*s"));
+      const char *errstr = lua_tolstring(lstate, -1, &len);
+      api_set_error(err, kErrorTypeException,
+                    "Error executing lua: %.*s", (int)len, errstr);
+    } else {
+      nlua_error(lstate, _("Error executing lua callback: %.*s"));
+    }
     return NIL;
   }
-  Error err = ERROR_INIT;
 
   if (retval) {
-    return nlua_pop_Object(lstate, false, &err);
+    Error dummy = ERROR_INIT;
+    if (err == NULL) {
+      err = &dummy;
+    }
+    return nlua_pop_Object(lstate, false, err);
   } else {
     return NIL;
   }
@@ -1007,4 +1015,7 @@ static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
 
   lua_pushcfunction(lstate, tslua_inspect_lang);
   lua_setfield(lstate, -2, "_ts_inspect_language");
+
+  lua_pushcfunction(lstate, ts_lua_parse_query);
+  lua_setfield(lstate, -2, "_ts_parse_query");
 }

src/nvim/lua/treesitter.c

@@ -26,6 +26,11 @@ typedef struct {
   TSTree *tree;  // internal tree, used for editing/reparsing
 } TSLua_parser;
 
+typedef struct {
+  TSQueryCursor *cursor;
+  int predicated_match;
+} TSLua_cursor;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "lua/treesitter.c.generated.h"
 #endif
@@ -66,6 +71,20 @@ static struct luaL_Reg node_meta[] = {
   { "descendant_for_range", node_descendant_for_range },
   { "named_descendant_for_range", node_named_descendant_for_range },
   { "parent", node_parent },
+  { "_rawquery", node_rawquery },
+  { NULL, NULL }
+};
+
+static struct luaL_Reg query_meta[] = {
+  { "__gc", query_gc },
+  { "__tostring", query_tostring },
+  { "inspect", query_inspect },
+  { NULL, NULL }
+};
+
+// cursor is not exposed, but still needs garbage collection
+static struct luaL_Reg querycursor_meta[] = {
+  { "__gc", querycursor_gc },
   { NULL, NULL }
 };
 
@@ -96,6 +115,8 @@ void tslua_init(lua_State *L)
   build_meta(L, "treesitter_parser", parser_meta);
   build_meta(L, "treesitter_tree", tree_meta);
   build_meta(L, "treesitter_node", node_meta);
+  build_meta(L, "treesitter_query", query_meta);
+  build_meta(L, "treesitter_querycursor", querycursor_meta);
 }
 
 int tslua_register_lang(lua_State *L)
@@ -276,13 +297,33 @@ static int parser_parse_buf(lua_State *L)
   }
   TSInput input = { payload, input_cb, TSInputEncodingUTF8 };
   TSTree *new_tree = ts_parser_parse(p->parser, p->tree, input);
+
+  uint32_t n_ranges = 0;
+  TSRange *changed = p->tree ? ts_tree_get_changed_ranges(p->tree, new_tree,
+                                                          &n_ranges) : NULL;
   if (p->tree) {
     ts_tree_delete(p->tree);
   }
   p->tree = new_tree;
 
   tslua_push_tree(L, p->tree);
-  return 1;
+
+  lua_createtable(L, n_ranges, 0);
+  for (size_t i = 0; i < n_ranges; i++) {
+    lua_createtable(L, 4, 0);
+    lua_pushinteger(L, changed[i].start_point.row);
+    lua_rawseti(L, -2, 1);
+    lua_pushinteger(L, changed[i].start_point.column);
+    lua_rawseti(L, -2, 2);
+    lua_pushinteger(L, changed[i].end_point.row);
+    lua_rawseti(L, -2, 3);
+    lua_pushinteger(L, changed[i].end_point.column);
+    lua_rawseti(L, -2, 4);
+
+    lua_rawseti(L, -2, i+1);
+  }
+  xfree(changed);
+  return 2;
 }
 
 static int parser_tree(lua_State *L)
@@ -383,7 +424,7 @@ static int tree_root(lua_State *L)
     return 0;
   }
   TSNode root = ts_tree_root_node(tree);
-  push_node(L, root);
+  push_node(L, root, 1);
   return 1;
 }
 
@@ -394,18 +435,19 @@ static int tree_root(lua_State *L)
 /// top of stack must either be the tree this node belongs to or another node
 /// of the same tree! This value is not popped. Can only be called inside a
 /// cfunction with the tslua environment.
-static void push_node(lua_State *L, TSNode node)
+static void push_node(lua_State *L, TSNode node, int uindex)
 {
+  assert(uindex > 0 || uindex < -LUA_MINSTACK);
   if (ts_node_is_null(node)) {
-    lua_pushnil(L);  // [src, nil]
+    lua_pushnil(L);  // [nil]
     return;
   }
-  TSNode *ud = lua_newuserdata(L, sizeof(TSNode));  // [src, udata]
+  TSNode *ud = lua_newuserdata(L, sizeof(TSNode));  // [udata]
   *ud = node;
-  lua_getfield(L, LUA_REGISTRYINDEX, "treesitter_node");  // [src, udata, meta]
+  lua_getfield(L, LUA_REGISTRYINDEX, "treesitter_node");  // [udata, meta]
-  lua_setmetatable(L, -2);  // [src, udata]
+  lua_setmetatable(L, -2);  // [udata]
-  lua_getfenv(L, -2);  // [src, udata, reftable]
+  lua_getfenv(L, uindex);  // [udata, reftable]
-  lua_setfenv(L, -2);  // [src, udata]
+  lua_setfenv(L, -2);  // [udata]
 }
 
 static bool node_check(lua_State *L, TSNode *res)
@@ -586,8 +628,7 @@ static int node_child(lua_State *L)
   long num = lua_tointeger(L, 2);
   TSNode child = ts_node_child(node, (uint32_t)num);
 
-  lua_pushvalue(L, 1);
+  push_node(L, child, 1);
-  push_node(L, child);
   return 1;
 }
 
@@ -600,8 +641,7 @@ static int node_named_child(lua_State *L)
   long num = lua_tointeger(L, 2);
   TSNode child = ts_node_named_child(node, (uint32_t)num);
 
-  lua_pushvalue(L, 1);
+  push_node(L, child, 1);
-  push_node(L, child);
   return 1;
 }
 
@@ -617,8 +657,7 @@ static int node_descendant_for_range(lua_State *L)
                  (uint32_t)lua_tointeger(L, 5) };
   TSNode child = ts_node_descendant_for_point_range(node, start, end);
 
-  lua_pushvalue(L, 1);
+  push_node(L, child, 1);
-  push_node(L, child);
   return 1;
 }
 
@@ -634,8 +673,7 @@ static int node_named_descendant_for_range(lua_State *L)
                  (uint32_t)lua_tointeger(L, 5) };
   TSNode child = ts_node_named_descendant_for_point_range(node, start, end);
 
-  lua_pushvalue(L, 1);
+  push_node(L, child, 1);
-  push_node(L, child);
   return 1;
 }
 
@@ -646,7 +684,254 @@ static int node_parent(lua_State *L)
     return 0;
   }
   TSNode parent = ts_node_parent(node);
-  push_node(L, parent);
+  push_node(L, parent, 1);
   return 1;
 }
 
+/// assumes the match table being on top of the stack
+static void set_match(lua_State *L, TSQueryMatch *match, int nodeidx)
+{
+  for (int i = 0; i < match->capture_count; i++) {
+    push_node(L, match->captures[i].node, nodeidx);
+    lua_rawseti(L, -2, match->captures[i].index+1);
+  }
+}
+
+static int query_next_match(lua_State *L)
+{
+  TSLua_cursor *ud = lua_touserdata(L, lua_upvalueindex(1));
+  TSQueryCursor *cursor = ud->cursor;
+
+  TSQuery *query = query_check(L, lua_upvalueindex(3));
+  TSQueryMatch match;
+  if (ts_query_cursor_next_match(cursor, &match)) {
+    lua_pushinteger(L, match.pattern_index+1);  // [index]
+    lua_createtable(L, ts_query_capture_count(query), 2);  // [index, match]
+    set_match(L, &match, lua_upvalueindex(2));
+    return 2;
+  }
+  return 0;
+}
+
+
+static int query_next_capture(lua_State *L)
+{
+  TSLua_cursor *ud = lua_touserdata(L, lua_upvalueindex(1));
+  TSQueryCursor *cursor = ud->cursor;
+
+  TSQuery *query = query_check(L, lua_upvalueindex(3));
+
+  if (ud->predicated_match > -1) {
+    lua_getfield(L, lua_upvalueindex(4), "active");
+    bool active = lua_toboolean(L, -1);
+    lua_pop(L, 1);
+    if (!active) {
+      ts_query_cursor_remove_match(cursor, ud->predicated_match);
+    }
+    ud->predicated_match = -1;
+  }
+
+  TSQueryMatch match;
+  uint32_t capture_index;
+  if (ts_query_cursor_next_capture(cursor, &match, &capture_index)) {
+    TSQueryCapture capture = match.captures[capture_index];
+
+    lua_pushinteger(L, capture.index+1);  // [index]
+    push_node(L, capture.node, lua_upvalueindex(2));  // [index, node]
+
+    uint32_t n_pred;
+    ts_query_predicates_for_pattern(query, match.pattern_index, &n_pred);
+    if (n_pred > 0 && capture_index == 0) {
+      lua_pushvalue(L, lua_upvalueindex(4));  // [index, node, match]
+      set_match(L, &match, lua_upvalueindex(2));
+      lua_pushinteger(L, match.pattern_index+1);
+      lua_setfield(L, -2, "pattern");
+
+      if (match.capture_count > 1) {
+        ud->predicated_match = match.id;
+        lua_pushboolean(L, false);
+        lua_setfield(L, -2, "active");
+      }
+      return 3;
+    }
+    return 2;
+  }
+  return 0;
+}
+
+static int node_rawquery(lua_State *L)
+{
+  TSNode node;
+  if (!node_check(L, &node)) {
+    return 0;
+  }
+  TSQuery *query = query_check(L, 2);
+  // TODO(bfredl): these are expensive allegedly,
+  // use a reuse list later on?
+  TSQueryCursor *cursor = ts_query_cursor_new();
+  ts_query_cursor_exec(cursor, query, node);
+
+  bool captures = lua_toboolean(L, 3);
+
+  if (lua_gettop(L) >= 4) {
+    int start = luaL_checkinteger(L, 4);
+    int end = lua_gettop(L) >= 5 ? luaL_checkinteger(L, 5) : MAXLNUM;
+    ts_query_cursor_set_point_range(cursor,
+                                    (TSPoint){ start, 0 }, (TSPoint){ end, 0 });
+  }
+
+  TSLua_cursor *ud = lua_newuserdata(L, sizeof(*ud));  // [udata]
+  ud->cursor = cursor;
+  ud->predicated_match = -1;
+
+  lua_getfield(L, LUA_REGISTRYINDEX, "treesitter_querycursor");
+  lua_setmetatable(L, -2);  // [udata]
+  lua_pushvalue(L, 1);  // [udata, node]
+
+  // include query separately, as to keep a ref to it for gc
+  lua_pushvalue(L, 2);  // [udata, node, query]
+
+  if (captures) {
+    // placeholder for match state
+    lua_createtable(L, ts_query_capture_count(query), 2);  // [u, n, q, match]
+    lua_pushcclosure(L, query_next_capture, 4);  // [closure]
+  } else {
+    lua_pushcclosure(L, query_next_match, 3);  // [closure]
+  }
+
+  return 1;
+}
+
+static int querycursor_gc(lua_State *L)
+{
+  TSLua_cursor *ud = luaL_checkudata(L, 1, "treesitter_querycursor");
+  ts_query_cursor_delete(ud->cursor);
+  return 0;
+}
+
+// Query methods
+
+int ts_lua_parse_query(lua_State *L)
+{
+  if (lua_gettop(L) < 2 || !lua_isstring(L, 1) || !lua_isstring(L, 2)) {
+    return luaL_error(L, "string expected");
+  }
+
+  const char *lang_name = lua_tostring(L, 1);
+  TSLanguage *lang = pmap_get(cstr_t)(langs, lang_name);
+  if (!lang) {
+    return luaL_error(L, "no such language: %s", lang_name);
+  }
+
+  size_t len;
+  const char *src = lua_tolstring(L, 2, &len);
+
+  uint32_t error_offset;
+  TSQueryError error_type;
+  TSQuery *query = ts_query_new(lang, src, len, &error_offset, &error_type);
+
+  if (!query) {
+    return luaL_error(L, "query: %s at position %d",
+                      query_err_string(error_type), (int)error_offset);
+  }
+
+  TSQuery **ud = lua_newuserdata(L, sizeof(TSQuery *));  // [udata]
+  *ud = query;
+  lua_getfield(L, LUA_REGISTRYINDEX, "treesitter_query");  // [udata, meta]
+  lua_setmetatable(L, -2);  // [udata]
+  return 1;
+}
+
+
+static const char *query_err_string(TSQueryError err) {
+  switch (err) {
+    case TSQueryErrorSyntax: return "invalid syntax";
+    case TSQueryErrorNodeType: return "invalid node type";
+    case TSQueryErrorField: return "invalid field";
+    case TSQueryErrorCapture: return "invalid capture";
+    default: return "error";
+  }
+}
+
+static TSQuery *query_check(lua_State *L, int index)
+{
+  TSQuery **ud = luaL_checkudata(L, index, "treesitter_query");
+  return *ud;
+}
+
+static int query_gc(lua_State *L)
+{
+  TSQuery *query = query_check(L, 1);
+  if (!query) {
+    return 0;
+  }
+
+  ts_query_delete(query);
+  return 0;
+}
+
+static int query_tostring(lua_State *L)
+{
+  lua_pushstring(L, "<query>");
+  return 1;
+}
+
+static int query_inspect(lua_State *L)
+{
+  TSQuery *query = query_check(L, 1);
+  if (!query) {
+    return 0;
+  }
+
+  uint32_t n_pat = ts_query_pattern_count(query);
+  lua_createtable(L, 0, 2);  // [retval]
+  lua_createtable(L, n_pat, 1);  // [retval, patterns]
+  for (size_t i = 0; i < n_pat; i++) {
+    uint32_t len;
+    const TSQueryPredicateStep *step = ts_query_predicates_for_pattern(query,
+                                                                       i, &len);
+    if (len == 0) {
+      continue;
+    }
+    lua_createtable(L, len/4, 1);  // [retval, patterns, pat]
+    lua_createtable(L, 3, 0);  // [retval, patterns, pat, pred]
+    int nextpred = 1;
+    int nextitem = 1;
+    for (size_t k = 0; k < len; k++) {
+      if (step[k].type == TSQueryPredicateStepTypeDone) {
+        lua_rawseti(L, -2, nextpred++);  // [retval, patterns, pat]
+        lua_createtable(L, 3, 0);  // [retval, patterns, pat, pred]
+        nextitem = 1;
+        continue;
+      }
+
+      if (step[k].type == TSQueryPredicateStepTypeString) {
+        uint32_t strlen;
+        const char *str = ts_query_string_value_for_id(query, step[k].value_id,
+                                                       &strlen);
+        lua_pushlstring(L, str, strlen);  // [retval, patterns, pat, pred, item]
+      } else if (step[k].type == TSQueryPredicateStepTypeCapture) {
+        lua_pushnumber(L, step[k].value_id+1);  // [..., pat, pred, item]
+      } else {
+        abort();
+      }
+      lua_rawseti(L, -2, nextitem++);  // [retval, patterns, pat, pred]
+    }
+    // last predicate should have ended with TypeDone
+    lua_pop(L, 1);  // [retval, patters, pat]
+    lua_rawseti(L, -2, i+1);  // [retval, patterns]
+  }
+  lua_setfield(L, -2, "patterns");  // [retval]
+
+  uint32_t n_captures = ts_query_capture_count(query);
+  lua_createtable(L, n_captures, 0);  // [retval, captures]
+  for (size_t i = 0; i < n_captures; i++) {
+    uint32_t strlen;
+    const char *str = ts_query_capture_name_for_id(query, i, &strlen);
+    lua_pushlstring(L, str, strlen);  // [retval, captures, capture]
+    lua_rawseti(L, -2, i+1);
+  }
+  lua_setfield(L, -2, "captures");  // [retval]
+
+  return 1;
+}

src/nvim/screen.c

@@ -116,6 +116,8 @@
 #include "nvim/window.h"
 #include "nvim/os/time.h"
 #include "nvim/api/private/helpers.h"
+#include "nvim/api/vim.h"
+#include "nvim/lua/executor.h"
 
 #define MB_FILLER_CHAR '<'  /* character used when a double-width character
                              * doesn't fit. */
@@ -232,6 +234,22 @@ void redraw_buf_line_later(buf_T *buf,  linenr_T line)
   }
 }
 
+void redraw_buf_range_later(buf_T *buf,  linenr_T firstline, linenr_T lastline)
+{
+  FOR_ALL_WINDOWS_IN_TAB(wp, curtab) {
+    if (wp->w_buffer == buf
+        && lastline >= wp->w_topline && firstline < wp->w_botline) {
+      if (wp->w_redraw_top == 0 || wp->w_redraw_top > firstline) {
+          wp->w_redraw_top = firstline;
+      }
+      if (wp->w_redraw_bot == 0 || wp->w_redraw_bot < lastline) {
+          wp->w_redraw_bot = lastline;
+      }
+      redraw_win_later(wp, VALID);
+    }
+  }
+}
+
 /*
  * Changed something in the current window, at buffer line "lnum", that
  * requires that line and possibly other lines to be redrawn.
@@ -477,6 +495,19 @@ int update_screen(int type)
       if (wwp == wp && syntax_present(wp)) {
         syn_stack_apply_changes(wp->w_buffer);
       }
+
+      buf_T *buf = wp->w_buffer;
+      if (buf->b_luahl && buf->b_luahl_window != LUA_NOREF) {
+        Error err = ERROR_INIT;
+        FIXED_TEMP_ARRAY(args, 2);
+        args.items[0] = BUFFER_OBJ(buf->handle);
+        args.items[1] = INTEGER_OBJ(display_tick);
+        executor_exec_lua_cb(buf->b_luahl_start, "start", args, false, &err);
+        if (ERROR_SET(&err)) {
+          ELOG("error in luahl start: %s", err.msg);
+          api_clear_error(&err);
+        }
+      }
     }
   }
 
@@ -1181,7 +1212,27 @@ static void win_update(win_T *wp)
   idx = 0;              /* first entry in w_lines[].wl_size */
   row = 0;
   srow = 0;
-  lnum = wp->w_topline;         /* first line shown in window */
+  lnum = wp->w_topline;  // first line shown in window
+
+  if (buf->b_luahl && buf->b_luahl_window != LUA_NOREF) {
+    Error err = ERROR_INIT;
+    FIXED_TEMP_ARRAY(args, 4);
+    linenr_T knownmax = ((wp->w_valid & VALID_BOTLINE)
+                         ? wp->w_botline
+                         : (wp->w_topline + wp->w_height_inner));
+    args.items[0] = WINDOW_OBJ(wp->handle);
+    args.items[1] = BUFFER_OBJ(buf->handle);
+    args.items[2] = INTEGER_OBJ(wp->w_topline-1);
+    args.items[3] = INTEGER_OBJ(knownmax);
+    // TODO(bfredl): we could allow this callback to change mod_top, mod_bot.
+    // For now the "start" callback is expected to use nvim__buf_redraw_range.
+    executor_exec_lua_cb(buf->b_luahl_window, "window", args, false, &err);
+    if (ERROR_SET(&err)) {
+      ELOG("error in luahl window: %s", err.msg);
+      api_clear_error(&err);
+    }
+  }
+
   for (;; ) {
     /* stop updating when reached the end of the window (check for _past_
      * the end of the window is at the end of the loop) */
@@ -2229,6 +2280,8 @@ win_line (
 
   row = startrow;
 
+  char *luatext = NULL;
+
   if (!number_only) {
     // To speed up the loop below, set extra_check when there is linebreak,
     // trailing white space and/or syntax processing to be done.
@@ -2454,6 +2507,41 @@ win_line (
   line = ml_get_buf(wp->w_buffer, lnum, FALSE);
   ptr = line;
 
+  buf_T *buf = wp->w_buffer;
+  if (buf->b_luahl && buf->b_luahl_line != LUA_NOREF) {
+    size_t size = STRLEN(line);
+    if (lua_attr_bufsize < size) {
+      xfree(lua_attr_buf);
+      lua_attr_buf = xcalloc(size, sizeof(*lua_attr_buf));
+      lua_attr_bufsize = size;
+    } else if (lua_attr_buf) {
+      memset(lua_attr_buf, 0, size * sizeof(*lua_attr_buf));
+    }
+    Error err = ERROR_INIT;
+    // TODO(bfredl): build a macro for the "static array" pattern
+    // in buf_updates_send_changes?
+    FIXED_TEMP_ARRAY(args, 3);
+    args.items[0] = WINDOW_OBJ(wp->handle);
+    args.items[1] = BUFFER_OBJ(buf->handle);
+    args.items[2] = INTEGER_OBJ(lnum-1);
+    lua_attr_active = true;
+    extra_check = true;
+    Object o = executor_exec_lua_cb(buf->b_luahl_line, "line",
+                                    args, true, &err);
+    lua_attr_active = false;
+    if (o.type == kObjectTypeString) {
+      // TODO(bfredl): this is a bit of a hack. A final API should use an
+      // "unified" interface where luahl can add both bufhl and virttext
+      luatext = o.data.string.data;
+      do_virttext = true;
+    } else if (ERROR_SET(&err)) {
+      ELOG("error in luahl line: %s", err.msg);
+      luatext = err.msg;
+      do_virttext = true;
+      api_clear_error(&err);
+    }
+  }
+
   if (has_spell && !number_only) {
     // For checking first word with a capital skip white space.
     if (cap_col == 0) {
@@ -3429,6 +3517,10 @@ win_line (
           }
         }
 
+        if (buf->b_luahl && v > 0 && v < (long)lua_attr_bufsize+1) {
+          char_attr = hl_combine_attr(char_attr, lua_attr_buf[v-1]);
+        }
+
         if (wp->w_buffer->terminal) {
           char_attr = hl_combine_attr(term_attrs[vcol], char_attr);
         }
@@ -3917,8 +4009,14 @@ win_line (
         int rightmost_vcol = 0;
         int i;
 
-        VirtText virt_text = do_virttext ? bufhl_info.line->virt_text
+        VirtText virt_text;
-                                        : (VirtText)KV_INITIAL_VALUE;
+        if (luatext) {
+          virt_text = (VirtText)KV_INITIAL_VALUE;
+          kv_push(virt_text, ((VirtTextChunk){ .text = luatext, .hl_id = 0 }));
+        } else {
+          virt_text = do_virttext ? bufhl_info.line->virt_text
+                                  : (VirtText)KV_INITIAL_VALUE;
+        }
         size_t virt_pos = 0;
         LineState s = LINE_STATE((char_u *)"");
         int virt_attr = 0;
@@ -4319,6 +4417,7 @@ win_line (
   }
 
   xfree(p_extra_free);
+  xfree(luatext);
   return row;
 }
 

test/functional/api/highlight_spec.lua

@@ -4,6 +4,9 @@ local Screen = require('test.functional.ui.screen')
 local eq, eval = helpers.eq, helpers.eval
 local command = helpers.command
 local meths = helpers.meths
+local funcs = helpers.funcs
+local pcall_err = helpers.pcall_err
+local ok = helpers.ok
 
 describe('API: highlight',function()
   local expected_rgb = {
@@ -110,4 +113,20 @@ describe('API: highlight',function()
        meths.get_hl_by_name('cursorline', 0));
 
   end)
+
+  it('nvim_get_hl_id_by_name', function()
+    -- precondition: use a hl group that does not yet exist
+    eq('Invalid highlight name: Shrubbery', pcall_err(meths.get_hl_by_name, "Shrubbery", true))
+    eq(0, funcs.hlID("Shrubbery"))
+
+    local hl_id = meths.get_hl_id_by_name("Shrubbery")
+    ok(hl_id > 0)
+    eq(hl_id, funcs.hlID("Shrubbery"))
+
+    command('hi Shrubbery guifg=#888888 guibg=#888888')
+    eq({foreground=tonumber("0x888888"), background=tonumber("0x888888")},
+       meths.get_hl_by_id(hl_id, true))
+    eq({foreground=tonumber("0x888888"), background=tonumber("0x888888")},
+       meths.get_hl_by_name("Shrubbery", true))
+  end)
 end)

test/functional/lua/treesitter_spec.lua

@@ -1,5 +1,6 @@
 -- Test suite for testing interactions with API bindings
 local helpers = require('test.functional.helpers')(after_each)
+local Screen = require('test.functional.ui.screen')
 
 local clear = helpers.clear
 local eq = helpers.eq
@@ -26,122 +27,371 @@ describe('treesitter API', function()
        pcall_err(exec_lua, "parser = vim.treesitter.inspect_language('borklang')"))
   end)
 
+end)
+
+describe('treesitter API with C parser', function()
   local ts_path = os.getenv("TREE_SITTER_DIR")
 
-  describe('with C parser', function()
+  -- The tests after this requires an actual parser
-    if ts_path == nil then
+  if ts_path == nil then
-      it("works", function() pending("TREE_SITTER_PATH not set, skipping treesitter parser tests") end)
+    it("works", function() pending("TREE_SITTER_PATH not set, skipping treesitter parser tests") end)
-      return
+    return
-    end
+  end
 
-    before_each(function()
+  before_each(function()
-      local path = ts_path .. '/bin/c'..(iswin() and '.dll' or '.so')
+    local path = ts_path .. '/bin/c'..(iswin() and '.dll' or '.so')
-      exec_lua([[
+    exec_lua([[
-        local path = ...
+      local path = ...
-        vim.treesitter.add_language(path,'c')
+      vim.treesitter.add_language(path,'c')
-      ]], path)
+    ]], path)
-    end)
+  end)
 
-    it('parses buffer', function()
+  it('parses buffer', function()
-      insert([[
+    insert([[
-        int main() {
+      int main() {
-          int x = 3;
+        int x = 3;
-        }]])
+      }]])
 
-      exec_lua([[
+    exec_lua([[
-        parser = vim.treesitter.get_parser(0, "c")
+      parser = vim.treesitter.get_parser(0, "c")
-        tree = parser:parse()
+      tree = parser:parse()
-        root = tree:root()
+      root = tree:root()
-        lang = vim.treesitter.inspect_language('c')
+      lang = vim.treesitter.inspect_language('c')
-      ]])
+    ]])
 
-      eq("<tree>", exec_lua("return tostring(tree)"))
+    eq("<tree>", exec_lua("return tostring(tree)"))
-      eq("<node translation_unit>", exec_lua("return tostring(root)"))
+    eq("<node translation_unit>", exec_lua("return tostring(root)"))
-      eq({0,0,3,0}, exec_lua("return {root:range()}"))
+    eq({0,0,3,0}, exec_lua("return {root:range()}"))
 
-      eq(1, exec_lua("return root:child_count()"))
+    eq(1, exec_lua("return root:child_count()"))
-      exec_lua("child = root:child(0)")
+    exec_lua("child = root:child(0)")
-      eq("<node function_definition>", exec_lua("return tostring(child)"))
+    eq("<node function_definition>", exec_lua("return tostring(child)"))
-      eq({0,0,2,1}, exec_lua("return {child:range()}"))
+    eq({0,0,2,1}, exec_lua("return {child:range()}"))
 
-      eq("function_definition", exec_lua("return child:type()"))
+    eq("function_definition", exec_lua("return child:type()"))
-      eq(true, exec_lua("return child:named()"))
+    eq(true, exec_lua("return child:named()"))
-      eq("number", type(exec_lua("return child:symbol()")))
+    eq("number", type(exec_lua("return child:symbol()")))
-      eq({'function_definition', true}, exec_lua("return lang.symbols[child:symbol()]"))
+    eq({'function_definition', true}, exec_lua("return lang.symbols[child:symbol()]"))
 
-      exec_lua("anon = root:descendant_for_range(0,8,0,9)")
+    exec_lua("anon = root:descendant_for_range(0,8,0,9)")
-      eq("(", exec_lua("return anon:type()"))
+    eq("(", exec_lua("return anon:type()"))
-      eq(false, exec_lua("return anon:named()"))
+    eq(false, exec_lua("return anon:named()"))
-      eq("number", type(exec_lua("return anon:symbol()")))
+    eq("number", type(exec_lua("return anon:symbol()")))
-      eq({'(', false}, exec_lua("return lang.symbols[anon:symbol()]"))
+    eq({'(', false}, exec_lua("return lang.symbols[anon:symbol()]"))
 
-      exec_lua("descendant = root:descendant_for_range(1,2,1,12)")
+    exec_lua("descendant = root:descendant_for_range(1,2,1,12)")
-      eq("<node declaration>", exec_lua("return tostring(descendant)"))
+    eq("<node declaration>", exec_lua("return tostring(descendant)"))
-      eq({1,2,1,12}, exec_lua("return {descendant:range()}"))
+    eq({1,2,1,12}, exec_lua("return {descendant:range()}"))
-      eq("(declaration type: (primitive_type) declarator: (init_declarator declarator: (identifier) value: (number_literal)))", exec_lua("return descendant:sexpr()"))
+    eq("(declaration type: (primitive_type) declarator: (init_declarator declarator: (identifier) value: (number_literal)))", exec_lua("return descendant:sexpr()"))
 
-      eq(true, exec_lua("return child == child"))
+    eq(true, exec_lua("return child == child"))
-      -- separate lua object, but represents same node
+    -- separate lua object, but represents same node
-      eq(true, exec_lua("return child == root:child(0)"))
+    eq(true, exec_lua("return child == root:child(0)"))
-      eq(false, exec_lua("return child == descendant2"))
+    eq(false, exec_lua("return child == descendant2"))
-      eq(false, exec_lua("return child == nil"))
+    eq(false, exec_lua("return child == nil"))
-      eq(false, exec_lua("return child == tree"))
+    eq(false, exec_lua("return child == tree"))
 
-      feed("2G7|ay")
+    feed("2G7|ay")
-      exec_lua([[
+    exec_lua([[
-        tree2 = parser:parse()
+      tree2 = parser:parse()
-        root2 = tree2:root()
+      root2 = tree2:root()
-        descendant2 = root2:descendant_for_range(1,2,1,13)
+      descendant2 = root2:descendant_for_range(1,2,1,13)
-      ]])
+    ]])
-      eq(false, exec_lua("return tree2 == tree1"))
+    eq(false, exec_lua("return tree2 == tree1"))
-      eq(false, exec_lua("return root2 == root"))
+    eq(false, exec_lua("return root2 == root"))
-      eq("<node declaration>", exec_lua("return tostring(descendant2)"))
+    eq("<node declaration>", exec_lua("return tostring(descendant2)"))
-      eq({1,2,1,13}, exec_lua("return {descendant2:range()}"))
+    eq({1,2,1,13}, exec_lua("return {descendant2:range()}"))
 
-      -- orginal tree did not change
+    -- orginal tree did not change
-      eq({1,2,1,12}, exec_lua("return {descendant:range()}"))
+    eq({1,2,1,12}, exec_lua("return {descendant:range()}"))
 
-      -- unchanged buffer: return the same tree
+    -- unchanged buffer: return the same tree
-      eq(true, exec_lua("return parser:parse() == tree2"))
+    eq(true, exec_lua("return parser:parse() == tree2"))
-    end)
+  end)
 
-    it('inspects language', function()
+    local test_text = [[
-        local keys, fields, symbols = unpack(exec_lua([[
+void ui_refresh(void)
-          local lang = vim.treesitter.inspect_language('c')
+{
-          local keys, symbols = {}, {}
+  int width = INT_MAX, height = INT_MAX;
-          for k,_ in pairs(lang) do
+  bool ext_widgets[kUIExtCount];
-            keys[k] = true
+  for (UIExtension i = 0; (int)i < kUIExtCount; i++) {
-          end
+    ext_widgets[i] = true;
+  }
 
-          -- symbols array can have "holes" and is thus not a valid msgpack array
+  bool inclusive = ui_override();
-          -- but we don't care about the numbers here (checked in the parser test)
+  for (size_t i = 0; i < ui_count; i++) {
-          for _, v in pairs(lang.symbols) do
+    UI *ui = uis[i];
-            table.insert(symbols, v)
+    width = MIN(ui->width, width);
-          end
+    height = MIN(ui->height, height);
-          return {keys, lang.fields, symbols}
+    foo = BAR(ui->bazaar, bazaar);
-        ]]))
+    for (UIExtension j = 0; (int)j < kUIExtCount; j++) {
+      ext_widgets[j] &= (ui->ui_ext[j] || inclusive);
+    }
+  }
+}]]
 
-        eq({fields=true, symbols=true}, keys)
+  local query = [[
+    ((call_expression function: (identifier) @minfunc (argument_list (identifier) @min_id)) (eq? @minfunc "MIN"))
+    "for" @keyword
+    (primitive_type) @type
+    (field_expression argument: (identifier) @fieldarg)
+  ]]
 
-        local fset = {}
+  it('support query and iter by capture', function()
-        for _,f in pairs(fields) do
+    insert(test_text)
-          eq("string", type(f))
+
-          fset[f] = true
+    local res = exec_lua([[
+      cquery = vim.treesitter.parse_query("c", ...)
+      parser = vim.treesitter.get_parser(0, "c")
+      tree = parser:parse()
+      res = {}
+      for cid, node in cquery:iter_captures(tree:root(), 0, 7, 14) do
+        -- can't transmit node over RPC. just check the name and range
+        table.insert(res, {cquery.captures[cid], node:type(), node:range()})
+      end
+      return res
+    ]], query)
+
+    eq({
+      { "type", "primitive_type", 8, 2, 8, 6 },
+      { "keyword", "for", 9, 2, 9, 5 },
+      { "type", "primitive_type", 9, 7, 9, 13 },
+      { "minfunc", "identifier", 11, 12, 11, 15 },
+      { "fieldarg", "identifier", 11, 16, 11, 18 },
+      { "min_id", "identifier", 11, 27, 11, 32 },
+      { "minfunc", "identifier", 12, 13, 12, 16 },
+      { "fieldarg", "identifier", 12, 17, 12, 19 },
+      { "min_id", "identifier", 12, 29, 12, 35 },
+      { "fieldarg", "identifier", 13, 14, 13, 16 }
+    }, res)
+  end)
+
+  it('support query and iter by match', function()
+    insert(test_text)
+
+    local res = exec_lua([[
+      cquery = vim.treesitter.parse_query("c", ...)
+      parser = vim.treesitter.get_parser(0, "c")
+      tree = parser:parse()
+      res = {}
+      for pattern, match in cquery:iter_matches(tree:root(), 0, 7, 14) do
+        -- can't transmit node over RPC. just check the name and range
+        local mrepr = {}
+        for cid,node in pairs(match) do
+          table.insert(mrepr, {cquery.captures[cid], node:type(), node:range()})
         end
-        eq(true, fset["directive"])
+        table.insert(res, {pattern, mrepr})
-        eq(true, fset["initializer"])
+      end
+      return res
+    ]], query)
 
-        local has_named, has_anonymous
+    eq({
-        for _,s in pairs(symbols) do
+      { 3, { { "type", "primitive_type", 8, 2, 8, 6 } } },
-          eq("string", type(s[1]))
+      { 2, { { "keyword", "for", 9, 2, 9, 5 } } },
-          eq("boolean", type(s[2]))
+      { 3, { { "type", "primitive_type", 9, 7, 9, 13 } } },
-          if s[1] == "for_statement" and s[2] == true then
+      { 4, { { "fieldarg", "identifier", 11, 16, 11, 18 } } },
-            has_named = true
+      { 1, { { "minfunc", "identifier", 11, 12, 11, 15 }, { "min_id", "identifier", 11, 27, 11, 32 } } },
-          elseif s[1] == "|=" and s[2] == false then
+      { 4, { { "fieldarg", "identifier", 12, 17, 12, 19 } } },
-            has_anonymous = true
+      { 1, { { "minfunc", "identifier", 12, 13, 12, 16 }, { "min_id", "identifier", 12, 29, 12, 35 } } },
-          end
+      { 4, { { "fieldarg", "identifier", 13, 14, 13, 16 } } }
+    }, res)
+  end)
+
+  it('supports highlighting', function()
+    local hl_text = [[
+/// Schedule Lua callback on main loop's event queue
+static int nlua_schedule(lua_State *const lstate)
+{
+  if (lua_type(lstate, 1) != LUA_TFUNCTION
+      || lstate != lstate) {
+    lua_pushliteral(lstate, "vim.schedule: expected function");
+    return lua_error(lstate);
+  }
+
+  LuaRef cb = nlua_ref(lstate, 1);
+
+  multiqueue_put(main_loop.events, nlua_schedule_event,
+                 1, (void *)(ptrdiff_t)cb);
+  return 0;
+}]]
+
+    local hl_query = [[
+(ERROR) @ErrorMsg
+
+"if" @keyword
+"else" @keyword
+"for" @keyword
+"return" @keyword
+
+"const" @type
+"static" @type
+"struct" @type
+"enum" @type
+"extern" @type
+
+(string_literal) @string
+
+(number_literal) @number
+(char_literal) @string
+
+; TODO(bfredl): overlapping matches are unreliable,
+; we need a proper priority mechanism
+;(type_identifier) @type
+((type_identifier) @Special (eq? @Special "LuaRef"))
+
+(primitive_type) @type
+(sized_type_specifier) @type
+
+((binary_expression left: (identifier) @WarningMsg.left right: (identifier) @WarningMsg.right) (eq? @WarningMsg.left @WarningMsg.right))
+
+(comment) @comment
+]]
+
+    local screen = Screen.new(65, 18)
+    screen:attach()
+    screen:set_default_attr_ids({
+      [1] = {bold = true, foreground = Screen.colors.Blue1},
+      [2] = {foreground = Screen.colors.Blue1},
+      [3] = {bold = true, foreground = Screen.colors.SeaGreen4},
+      [4] = {bold = true, foreground = Screen.colors.Brown},
+      [5] = {foreground = Screen.colors.Magenta},
+      [6] = {foreground = Screen.colors.Red},
+      [7] = {foreground = Screen.colors.SlateBlue},
+      [8] = {foreground = Screen.colors.Grey100, background = Screen.colors.Red},
+      [9] = {foreground = Screen.colors.Magenta, background = Screen.colors.Red},
+      [10] = {foreground = Screen.colors.Red, background = Screen.colors.Red},
+
+    })
+
+    insert(hl_text)
+    screen:expect{grid=[[
+      /// Schedule Lua callback on main loop's event queue             |
+      static int nlua_schedule(lua_State *const lstate)                |
+      {                                                                |
+        if (lua_type(lstate, 1) != LUA_TFUNCTION                       |
+            || lstate != lstate) {                                     |
+          lua_pushliteral(lstate, "vim.schedule: expected function");  |
+          return lua_error(lstate);                                    |
+        }                                                              |
+                                                                       |
+        LuaRef cb = nlua_ref(lstate, 1);                               |
+                                                                       |
+        multiqueue_put(main_loop.events, nlua_schedule_event,          |
+                       1, (void *)(ptrdiff_t)cb);                      |
+        return 0;                                                      |
+      ^}                                                                |
+      {1:~                                                                }|
+      {1:~                                                                }|
+                                                                       |
+    ]]}
+
+    exec_lua([[
+      local TSHighlighter = vim.treesitter.TSHighlighter
+      local query = ...
+      test_hl = TSHighlighter.new(query, 0, "c")
+    ]], hl_query)
+    screen:expect{grid=[[
+      {2:/// Schedule Lua callback on main loop's event queue}             |
+      {3:static} {3:int} nlua_schedule(lua_State *{3:const} lstate)                |
+      {                                                                |
+        {4:if} (lua_type(lstate, {5:1}) != LUA_TFUNCTION                       |
+            || {6:lstate} != {6:lstate}) {                                     |
+          lua_pushliteral(lstate, {5:"vim.schedule: expected function"});  |
+          {4:return} lua_error(lstate);                                    |
+        }                                                              |
+                                                                       |
+        {7:LuaRef} cb = nlua_ref(lstate, {5:1});                               |
+                                                                       |
+        multiqueue_put(main_loop.events, nlua_schedule_event,          |
+                       {5:1}, ({3:void} *)(ptrdiff_t)cb);                      |
+        {4:return} {5:0};                                                      |
+      ^}                                                                |
+      {1:~                                                                }|
+      {1:~                                                                }|
+                                                                       |
+    ]]}
+
+    feed('7Go*/<esc>')
+    screen:expect{grid=[[
+      {2:/// Schedule Lua callback on main loop's event queue}             |
+      {3:static} {3:int} nlua_schedule(lua_State *{3:const} lstate)                |
+      {                                                                |
+        {4:if} (lua_type(lstate, {5:1}) != LUA_TFUNCTION                       |
+            || {6:lstate} != {6:lstate}) {                                     |
+          lua_pushliteral(lstate, {5:"vim.schedule: expected function"});  |
+          {4:return} lua_error(lstate);                                    |
+      {8:*^/}                                                               |
+        }                                                              |
+                                                                       |
+        {7:LuaRef} cb = nlua_ref(lstate, {5:1});                               |
+                                                                       |
+        multiqueue_put(main_loop.events, nlua_schedule_event,          |
+                       {5:1}, ({3:void} *)(ptrdiff_t)cb);                      |
+        {4:return} {5:0};                                                      |
+      }                                                                |
+      {1:~                                                                }|
+                                                                       |
+    ]]}
+
+    feed('3Go/*<esc>')
+    screen:expect{grid=[[
+      {2:/// Schedule Lua callback on main loop's event queue}             |
+      {3:static} {3:int} nlua_schedule(lua_State *{3:const} lstate)                |
+      {                                                                |
+      {2:/^*}                                                               |
+      {2:  if (lua_type(lstate, 1) != LUA_TFUNCTION}                       |
+      {2:      || lstate != lstate) {}                                     |
+      {2:    lua_pushliteral(lstate, "vim.schedule: expected function");}  |
+      {2:    return lua_error(lstate);}                                    |
+      {2:*/}                                                               |
+        }                                                              |
+                                                                       |
+        {7:LuaRef} cb = nlua_ref(lstate, {5:1});                               |
+                                                                       |
+        multiqueue_put(main_loop.events, nlua_schedule_event,          |
+                       {5:1}, ({3:void} *)(ptrdiff_t)cb);                      |
+        {4:return} {5:0};                                                      |
+      {8:}}                                                                |
+                                                                       |
+    ]]}
+  end)
+
+  it('inspects language', function()
+      local keys, fields, symbols = unpack(exec_lua([[
+        local lang = vim.treesitter.inspect_language('c')
+        local keys, symbols = {}, {}
+        for k,_ in pairs(lang) do
+          keys[k] = true
         end
-        eq({true,true}, {has_named,has_anonymous})
+
-    end)
+        -- symbols array can have "holes" and is thus not a valid msgpack array
+        -- but we don't care about the numbers here (checked in the parser test)
+        for _, v in pairs(lang.symbols) do
+          table.insert(symbols, v)
+        end
+        return {keys, lang.fields, symbols}
+      ]]))
+
+      eq({fields=true, symbols=true}, keys)
+
+      local fset = {}
+      for _,f in pairs(fields) do
+        eq("string", type(f))
+        fset[f] = true
+      end
+      eq(true, fset["directive"])
+      eq(true, fset["initializer"])
+
+      local has_named, has_anonymous
+      for _,s in pairs(symbols) do
+        eq("string", type(s[1]))
+        eq("boolean", type(s[2]))
+        if s[1] == "for_statement" and s[2] == true then
+          has_named = true
+        elseif s[1] == "|=" and s[2] == false then
+          has_anonymous = true
+        end
+      end
+      eq({true,true}, {has_named,has_anonymous})
   end)
 end)