Merge pull request #22837 from clason/ts-sync

build(deps): bump tree-sitter-viml to HEAD
fix(tests): adapt treesitter/highlight_spec priority test
fix(treesitter): update queries from nvim-treesitter
refactor(treesitter)!: rename help parser to vimdoc
docs(treesitter): add query injections

cmake.deps/CMakeLists.txt

@@ -191,11 +191,11 @@ set(TREESITTER_C_SHA256 af66fde03feb0df4faf03750102a0d265b007e5d957057b6b293c131
 set(TREESITTER_LUA_URL https://github.com/MunifTanjim/tree-sitter-lua/archive/v0.0.14.tar.gz)
 set(TREESITTER_LUA_SHA256 930d0370dc15b66389869355c8e14305b9ba7aafd36edbfdb468c8023395016d)
 
-set(TREESITTER_VIM_URL https://github.com/vigoux/tree-sitter-viml/archive/55ff1b080c09edeced9b748cf4c16d0b49d17fb9.tar.gz)
+set(TREESITTER_VIM_URL https://github.com/vigoux/tree-sitter-viml/archive/e39a7bbcfdcfc7900629962b785c7e14503ae590.tar.gz)
-set(TREESITTER_VIM_SHA256 1b1cd39e33c8fb02fa7fe3977e844883c2a8508a7edd621f2d21e39a9aeefa92)
+set(TREESITTER_VIM_SHA256 7ca85fa1a5a9e4d057ff3b7ae53d13d31371973e734ada87a83f3f6cbe9c0e32)
 
-set(TREESITTER_HELP_URL https://github.com/neovim/tree-sitter-vimdoc/archive/v1.3.0.tar.gz)
+set(TREESITTER_VIMDOC_URL https://github.com/neovim/tree-sitter-vimdoc/archive/b2ec4ec5f7be24cb6f7ccffafd7204477fe5784a.tar.gz)
-set(TREESITTER_HELP_SHA256 f33f6d49c7d71feb2fd68ef2b2684da150f9f8e486ad9726213631d673942331)
+set(TREESITTER_VIMDOC_SHA256 0689a57d455243de6c6a6c8737a8ce137e225eb8f32676a7037f7dd13dfaec5d)
 
 set(TREESITTER_QUERY_URL https://github.com/nvim-treesitter/tree-sitter-query/archive/v0.1.0.tar.gz)
 set(TREESITTER_QUERY_SHA256 e2b806f80e8bf1c4f4e5a96248393fe6622fc1fc6189d6896d269658f67f914c)

cmake.deps/cmake/BuildTreesitterParsers.cmake

@@ -38,9 +38,9 @@ BuildTSParser(
   CMAKE_FILE TreesitterParserCMakeLists.txt)
 
 BuildTSParser(
-  LANG help
+  LANG vimdoc
-  URL ${TREESITTER_HELP_URL}
+  URL ${TREESITTER_VIMDOC_URL}
-  SHA256 ${TREESITTER_HELP_SHA256}
+  SHA256 ${TREESITTER_VIMDOC_SHA256}
   CMAKE_FILE TreesitterParserCMakeLists.txt)
 
 BuildTSParser(

runtime/doc/news.txt

@@ -62,6 +62,10 @@ The following changes may require adaptations in user config or plugins.
 
 • |nvim_exec()| is now deprecated in favor of |nvim_exec2()|.
 
+• `help` treesitter parser was renamed to `vimdoc`. The only user-visible
+  change is that language-specific highlight groups need to be renamed from
+  `@foo.help` to `@foo.vimdoc`.
+
 ==============================================================================
 NEW FEATURES                                                    *news-features*
 

runtime/doc/treesitter.txt

@@ -195,7 +195,7 @@ treesitter queries from Lua.
 TREESITTER QUERY PREDICATES                            *treesitter-predicates*
 
 Predicates are special scheme nodes that are evaluated to conditionally capture
-nodes. For example, the `eq?` predicate can be used as follows: >
+nodes. For example, the `eq?` predicate can be used as follows: >query
 
     ((identifier) @foo (#eq? @foo "foo"))
 <
@@ -204,13 +204,13 @@ to only match identifier corresponding to the `"foo"` text.
 The following predicates are built in:
 
     `eq?`                                            *treesitter-predicate-eq?*
-        Match a string against the text corresponding to a node: >
+        Match a string against the text corresponding to a node: >query
             ((identifier) @foo (#eq? @foo "foo"))
             ((node1) @left (node2) @right (#eq? @left @right))
 <
     `match?`                                      *treesitter-predicate-match?*
     `vim-match?`                              *treesitter-predicate-vim-match?*
-         Match a |regexp| against the text corresponding to a node: >
+         Match a |regexp| against the text corresponding to a node: >query
             ((identifier) @constant (#match? @constant "^[A-Z_]+$"))
 <         Note: The `^` and `$` anchors will match the start and end of the
                node's text.
@@ -220,13 +220,14 @@ The following predicates are built in:
          similar to `match?`
 
     `contains?`                                *treesitter-predicate-contains?*
-        Match a string against parts of the text corresponding to a node: >
+        Match a string against parts of the text corresponding to a node:
+        >query
             ((identifier) @foo (#contains? @foo "foo"))
             ((identifier) @foo-bar (#contains? @foo-bar "foo" "bar"))
 <
     `any-of?`                                    *treesitter-predicate-any-of?*
         Match any of the given strings against the text corresponding to
-        a node: >
+        a node: >query
             ((identifier) @foo (#any-of? @foo "foo" "bar"))
 <
         This is the recommended way to check if the node matches one of many
@@ -243,7 +244,7 @@ Use |vim.treesitter.query.list_predicates()| to list all available predicates.
 TREESITTER QUERY DIRECTIVES                            *treesitter-directives*
 
 Treesitter directives store metadata for a node or match and perform side
-effects. For example, the `set!` directive sets metadata on the match or node: >
+effects. For example, the `set!` directive sets metadata on the match or node: >query
 
         ((identifier) @foo (#set! "type" "parameter"))
 <
@@ -259,7 +260,7 @@ The following directives are built in:
             {key}
             {value}
 
-        Examples: >
+        Examples: >query
             ((identifier) @foo (#set! @foo "kind" "parameter"))
             ((node1) @left (node2) @right (#set! "type" "pair"))
 <
@@ -275,7 +276,7 @@ The following directives are built in:
             {end_row}
             {end_col}
 
-        Example: >
+        Example: >query
             ((identifier) @constant (#offset! @constant 0 1 0 -1))
 <
 
@@ -304,7 +305,8 @@ currently supported modeline alternatives:
         a base depends on your 'runtimepath' value.
 
 Note: These modeline comments must be at the top of the query, but can be
-repeated, for example, the following two modeline blocks are both valid: >
+repeated, for example, the following two modeline blocks are both valid:
+>query
 
     ;; inherits: foo,bar
     ;; extends
@@ -318,13 +320,13 @@ TREESITTER SYNTAX HIGHLIGHTING                          *treesitter-highlight*
 
 Syntax highlighting is specified through queries named `highlights.scm`,
 which match a |TSNode| in the parsed |TSTree| to a `capture` that can be
-assigned a highlight group. For example, the query >
+assigned a highlight group. For example, the query >query
 
     (parameters (identifier) @parameter)
 <
 matches any `identifier` node inside a function `parameter` node (e.g., the
 `bar` in `foo(bar)`) to the capture named `@parameter`. It is also possible to
-match literal expressions (provided the parser returns them): >
+match literal expressions (provided the parser returns them): >query
 
     "return" @keyword.return
 <
@@ -409,7 +411,7 @@ The following captures are linked by default to standard |group-name|s:
                                                   *treesitter-highlight-spell*
 The special `@spell` capture can be used to indicate that a node should be
 spell checked by Nvim's builtin |spell| checker. For example, the following
-capture marks comments as to be checked: >
+capture marks comments as to be checked: >query
 
     (comment) @spell
 <
@@ -420,14 +422,14 @@ There is also `@nospell` which disables spellchecking regions with `@spell`.
 Treesitter highlighting supports |conceal| via the `conceal` metadata. By
 convention, nodes to be concealed are captured as `@conceal`, but any capture
 can be used. For example, the following query can be used to hide code block
-delimiters in Markdown: >
+delimiters in Markdown: >query
 
-    (fenced_code_block_delimiter) @conceal (#set! conceal "")
+    (fenced_code_block_delimiter @conceal (#set! conceal ""))
 <
 It is also possible to replace a node with a single character, which (unlike
 legacy syntax) can be given a custom highlight. For example, the following
 (ill-advised) query replaces the `!=` operator by a Unicode glyph, which is
-still highlighted the same as other operators: >
+still highlighted the same as other operators: >query
 
     "!=" @operator (#set! conceal "≠")
 <
@@ -438,9 +440,10 @@ Treesitter uses |nvim_buf_set_extmark()| to set highlights with a default
 priority of 100. This enables plugins to set a highlighting priority lower or
 higher than tree-sitter. It is also possible to change the priority of an
 individual query pattern manually by setting its `"priority"` metadata
-attribute: >
+attribute: >query
 
-    (super_important_node) @ImportantHighlight (#set! "priority" 105)
+    ((super_important_node) @superimportant (#set! "priority" 105))
+<
 
 ==============================================================================
 TREESITTER LANGUAGE INJECTIONS                *treesitter-language-injections*

runtime/lua/vim/treesitter/language.lua

@@ -4,7 +4,9 @@ local a = vim.api
 local M = {}
 
 ---@type table<string,string>
-local ft_to_lang = {}
+local ft_to_lang = {
+  help = 'vimdoc',
+}
 
 --- Get the filetypes associated with the parser named {lang}.
 --- @param lang string Name of parser

runtime/queries/c/highlights.scm

@@ -1,22 +1,19 @@
-(identifier) @variable
+; Lower priority to prefer @parameter when identifier appears in parameter_declaration.
+((identifier) @variable (#set! "priority" 95))
 
 [
-  "const"
   "default"
   "enum"
-  "extern"
-  "inline"
-  "return"
-  "sizeof"
-  "static"
   "struct"
   "typedef"
   "union"
-  "volatile"
   "goto"
-  "register"
 ] @keyword
 
+"sizeof" @keyword.operator
+
+"return" @keyword.return
+
 [
   "while"
   "for"
@@ -32,7 +29,6 @@
  "switch"
 ] @conditional
 
-"#define" @constant.macro
 [
   "#if"
   "#ifdef"
@@ -41,10 +37,18 @@
   "#elif"
   "#endif"
   (preproc_directive)
-] @keyword
+] @preproc
+
+"#define" @define
 
 "#include" @include
 
+[ ";" ":" "," ] @punctuation.delimiter
+
+"..." @punctuation.special
+
+[ "(" ")" "[" "]" "{" "}"] @punctuation.bracket
+
 [
   "="
 
@@ -62,6 +66,7 @@
   ">>"
 
   "->"
+  "."
 
   "<"
   "<="
@@ -88,35 +93,25 @@
   "++"
 ] @operator
 
+;; Make sure the comma operator is given a highlight group after the comma
+;; punctuator so the operator is highlighted properly.
+(comma_expression [ "," ] @operator)
+
 [
  (true)
  (false)
 ] @boolean
 
-[ "." ";" ":" "," ] @punctuation.delimiter
+(conditional_expression [ "?" ":" ] @conditional.ternary)
-
-(conditional_expression [ "?" ":" ] @conditional)
-
-
-[ "(" ")" "[" "]" "{" "}"] @punctuation.bracket
 
 (string_literal) @string
-(string_literal) @spell
 (system_lib_string) @string
+(escape_sequence) @string.escape
 
 (null) @constant.builtin
 (number_literal) @number
-(char_literal) @number
+(char_literal) @character
 
-(call_expression
-  function: (identifier) @function)
-(call_expression
-  function: (field_expression
-    field: (field_identifier) @function))
-(function_declarator
-  declarator: (identifier) @function)
-(preproc_function_def
-  name: (identifier) @function.macro)
 [
  (preproc_arg)
  (preproc_defined)
@@ -126,18 +121,32 @@
 (statement_identifier) @label
 
 [
-(type_identifier)
+ (type_identifier)
-(primitive_type)
+ (sized_type_specifier)
-(sized_type_specifier)
+ (type_descriptor)
-(type_descriptor)
+] @type
- ] @type
 
-(declaration (type_qualifier) @type)
+(storage_class_specifier) @storageclass
-(cast_expression type: (type_descriptor) @type)
+
-(sizeof_expression value: (parenthesized_expression (identifier) @type))
+(type_qualifier) @type.qualifier
+
+(linkage_specification
+  "extern" @storageclass)
+
+(type_definition
+  declarator: (type_identifier) @type.definition)
+
+(primitive_type) @type.builtin
 
 ((identifier) @constant
- (#match? @constant "^[A-Z][A-Z0-9_]+$"))
+ (#lua-match? @constant "^[A-Z][A-Z0-9_]+$"))
+(enumerator
+  name: (identifier) @constant)
+(case_statement
+  value: (identifier) @constant)
+
+((identifier) @constant.builtin
+    (#any-of? @constant.builtin "stderr" "stdin" "stdout"))
 
 ;; Preproc def / undef
 (preproc_def
@@ -147,9 +156,20 @@
   argument: (_) @constant
   (#eq? @_u "#undef"))
 
+(call_expression
+  function: (identifier) @function.call)
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function.call))
+(function_declarator
+  declarator: (identifier) @function)
+(preproc_function_def
+  name: (identifier) @function.macro)
 
-(comment) @comment
+(comment) @comment @spell
-(comment) @spell
+
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^/[*][*][^*].*[*]/$"))
 
 ;; Parameters
 (parameter_declaration
@@ -158,7 +178,20 @@
 (parameter_declaration
   declarator: (pointer_declarator) @parameter)
 
-(preproc_params
+(preproc_params (identifier) @parameter)
-  (identifier)) @parameter
+
+[
+  "__attribute__"
+  "__cdecl"
+  "__clrcall"
+  "__stdcall"
+  "__fastcall"
+  "__thiscall"
+  "__vectorcall"
+  "_unaligned"
+  "__unaligned"
+  "__declspec"
+  (attribute_declaration)
+] @attribute
 
 (ERROR) @error

runtime/queries/c/injections.scm

@@ -1,5 +1,5 @@
-((preproc_arg) @injection.content
+; ((preproc_arg) @injection.content
- (#set! injection.language "c"))
+;  (#set! injection.language "c"))
 
 ; ((comment) @injection.content
 ;  (#set! injection.language "comment"))

runtime/queries/lua/highlights.scm

@@ -8,8 +8,6 @@
  "local"
 ] @keyword
 
-(label_statement) @label
-
 (break_statement) @keyword
 
 (do_statement
@@ -109,6 +107,7 @@
 [
   ";"
   ":"
+  "::"
   ","
   "."
 ] @punctuation.delimiter
@@ -129,12 +128,21 @@
 (identifier) @variable
 
 ((identifier) @variable.builtin
- (#eq? @variable.builtin "self"))
+ (#any-of? @variable.builtin "_G" "_VERSION" "debug" "io" "jit" "math" "os" "package" "self" "string" "table" "utf8"))
+
+((identifier) @keyword.coroutine
+  (#eq? @keyword.coroutine "coroutine"))
 
 (variable_list
-  attribute: (attribute
+   attribute: (attribute
-    (["<" ">"] @punctuation.bracket
+     (["<" ">"] @punctuation.bracket
-     (identifier) @attribute)))
+      (identifier) @attribute)))
+
+;; Labels
+
+(label_statement (identifier) @label)
+
+(goto_statement (identifier) @label)
 
 ;; Constants
 
@@ -172,7 +180,7 @@
 (function_call name: (dot_index_expression field: (identifier) @function.call))
 (function_declaration name: (dot_index_expression field: (identifier) @function))
 
-(method_index_expression method: (identifier) @method)
+(method_index_expression method: (identifier) @method.call)
 
 (function_call
   (identifier) @function.builtin
@@ -180,20 +188,27 @@
     ;; built-in functions in Lua 5.1
     "assert" "collectgarbage" "dofile" "error" "getfenv" "getmetatable" "ipairs"
     "load" "loadfile" "loadstring" "module" "next" "pairs" "pcall" "print"
-    "rawequal" "rawget" "rawset" "require" "select" "setfenv" "setmetatable"
+    "rawequal" "rawget" "rawlen" "rawset" "require" "select" "setfenv" "setmetatable"
-    "tonumber" "tostring" "type" "unpack" "xpcall"))
+    "tonumber" "tostring" "type" "unpack" "xpcall"
+    "__add" "__band" "__bnot" "__bor" "__bxor" "__call" "__concat" "__div" "__eq" "__gc"
+    "__idiv" "__index" "__le" "__len" "__lt" "__metatable" "__mod" "__mul" "__name" "__newindex"
+    "__pairs" "__pow" "__shl" "__shr" "__sub" "__tostring" "__unm"))
 
 ;; Others
 
-(comment) @comment
+(comment) @comment @spell
-(comment) @spell
 
-(hash_bang_line) @comment
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^[-][-][-]"))
+
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^[-][-](%s?)@"))
+
+(hash_bang_line) @preproc
 
 (number) @number
 
-(string) @string
+(string) @string @spell
-(string) @spell
 
 ;; Error
 (ERROR) @error

runtime/queries/lua/injections.scm

@@ -11,18 +11,18 @@
   name: (_) @_vimcmd_identifier
   arguments: (arguments (string content: _ @injection.content)))
   (#set! injection.language "vim")
-  (#any-of? @_vimcmd_identifier "vim.cmd" "vim.api.nvim_command" "vim.api.nvim_exec" "vim.api.nvim_cmd"))
+  (#any-of? @_vimcmd_identifier "vim.cmd" "vim.api.nvim_command" "vim.api.nvim_exec2" "vim.api.nvim_cmd"))
 
 ((function_call
   name: (_) @_vimcmd_identifier
   arguments: (arguments (string content: _ @injection.content) .))
   (#set! injection.language "query")
-  (#eq? @_vimcmd_identifier "vim.treesitter.query.set_query"))
+  (#any-of? @_vimcmd_identifier "vim.treesitter.query.set" "vim.treesitter.query.parse"))
 
-; ;; highlight string as query if starts with `;; query`
+;; highlight string as query if starts with `;; query`
-; ((string ("string_content") @injection.content)
+((string ("string_content") @injection.content)
-;  (#set! injection.language "query")
+ (#set! injection.language "query")
-;  (#lua-match? @injection.content "^%s*;+%s?query"))
+ (#lua-match? @injection.content "^%s*;+%s?query"))
 
 ; ((comment) @injection.content
 ;  (#set! injection.language "comment"))

runtime/queries/vim/highlights.scm

@@ -271,7 +271,7 @@
   ":"
 ] @punctuation.delimiter
 
-(ternary_expression ["?" ":"] @conditional)
+(ternary_expression ["?" ":"] @conditional.ternary)
 
 ; Options
 ((set_value) @number

scripts/gen_help_html.lua

@@ -624,7 +624,7 @@ local function get_helptags(help_dir)
   return m
 end
 
--- Use the help.so parser defined in the build, not whatever happens to be installed on the system.
+-- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system.
 local function ensure_runtimepath()
   if not vim.o.runtimepath:find('build/lib/nvim/') then
     vim.cmd[[set runtimepath^=./build/lib/nvim/]]
@@ -643,8 +643,8 @@ local function parse_buf(fname)
     buf = fname
     vim.cmd('sbuffer '..tostring(fname))          -- Buffer number.
   end
-  -- vim.treesitter.require_language('help', './build/lib/nvim/parser/help.so')
+  -- vim.treesitter.require_language('help', './build/lib/nvim/parser/vimdoc.so')
-  local lang_tree = vim.treesitter.get_parser(buf, 'help')
+  local lang_tree = vim.treesitter.get_parser(buf)
   return lang_tree, buf
 end
 

test/functional/treesitter/highlight_spec.lua

@@ -575,14 +575,14 @@ describe('treesitter highlighting', function()
 
     exec_lua [[
       local parser = vim.treesitter.get_parser(0, "c")
-      test_hl = vim.treesitter.highlighter.new(parser, {queries = {c = hl_query..'\n((translation_unit) @Error (set! "priority" 101))\n'}})
+      test_hl = vim.treesitter.highlighter.new(parser, {queries = {c = hl_query..'\n((translation_unit) @constant (#set! "priority" 101))\n'}})
     ]]
-    -- expect everything to have Error highlight
+    -- expect everything to have Constant highlight
     screen:expect{grid=[[
       {12:int}{8: x = INT_MAX;}                                                 |
-      {8:#define READ_STRING(x, y) (}{12:char_u}{8: *)read_string((x), (}{12:size_t}{8:)(y))}|
+      {8:#define READ_STRING(x, y) (char_u *)read_string((x), (size_t)(y))}|
-      {8:#define foo }{12:void}{8: main() { \}                                      |
+      {8:#define foo void main() { \}                                      |
-      {8:              }{12:return}{8: 42;  \}                                      |
+      {8:              return 42;  \}                                      |
       {8:            }}                                                    |
       ^                                                                 |
       {1:~                                                                }|
@@ -599,13 +599,13 @@ describe('treesitter highlighting', function()
                                                                        |
     ]], attr_ids={
       [1] = {bold = true, foreground = Screen.colors.Blue1};
-      [8] = {foreground = Screen.colors.Grey100, background = Screen.colors.Red};
+      [8] = {foreground = Screen.colors.Magenta1};
       -- bold will not be overwritten at the moment
-      [12] = {background = Screen.colors.Red, bold = true, foreground = Screen.colors.Grey100};
+      [12] = {bold = true, foreground = Screen.colors.Magenta1};
     }}
 
     eq({
-      {capture='Error', metadata = { priority='101' }, lang='c' };
+      {capture='constant', metadata = { priority='101' }, lang='c' };
       {capture='type', metadata = { }, lang='c' };
     }, exec_lua [[ return vim.treesitter.get_captures_at_pos(0, 0, 2) ]])
     end)