Merge pull request #22837 from clason/ts-sync

build(deps): bump tree-sitter-viml to HEAD fix(tests): adapt treesitter/highlight_spec priority test fix(treesitter): update queries from nvim-treesitter refactor(treesitter)!: rename help parser to vimdoc docs(treesitter): add query injections
2025-02-25 18:55:25 -06:00 · 2023-04-01 15:39:25 +02:00 · 2023-04-01 15:39:25 +02:00 · 19110eddb9
commit 19110eddb9
parent 4f66530af1 438c3419df
14 changed files with 157 additions and 100 deletions
--- a/cmake.deps/CMakeLists.txt
+++ b/cmake.deps/CMakeLists.txt
@ -191,11 +191,11 @@ set(TREESITTER_C_SHA256 af66fde03feb0df4faf03750102a0d265b007e5d957057b6b293c131
 set(TREESITTER_LUA_URL https://github.com/MunifTanjim/tree-sitter-lua/archive/v0.0.14.tar.gz)
 set(TREESITTER_LUA_SHA256 930d0370dc15b66389869355c8e14305b9ba7aafd36edbfdb468c8023395016d)

-set(TREESITTER_VIM_URL https://github.com/vigoux/tree-sitter-viml/archive/55ff1b080c09edeced9b748cf4c16d0b49d17fb9.tar.gz)
-set(TREESITTER_VIM_SHA256 1b1cd39e33c8fb02fa7fe3977e844883c2a8508a7edd621f2d21e39a9aeefa92)
+set(TREESITTER_VIM_URL https://github.com/vigoux/tree-sitter-viml/archive/e39a7bbcfdcfc7900629962b785c7e14503ae590.tar.gz)
+set(TREESITTER_VIM_SHA256 7ca85fa1a5a9e4d057ff3b7ae53d13d31371973e734ada87a83f3f6cbe9c0e32)

-set(TREESITTER_HELP_URL https://github.com/neovim/tree-sitter-vimdoc/archive/v1.3.0.tar.gz)
-set(TREESITTER_HELP_SHA256 f33f6d49c7d71feb2fd68ef2b2684da150f9f8e486ad9726213631d673942331)
+set(TREESITTER_VIMDOC_URL https://github.com/neovim/tree-sitter-vimdoc/archive/b2ec4ec5f7be24cb6f7ccffafd7204477fe5784a.tar.gz)
+set(TREESITTER_VIMDOC_SHA256 0689a57d455243de6c6a6c8737a8ce137e225eb8f32676a7037f7dd13dfaec5d)

 set(TREESITTER_QUERY_URL https://github.com/nvim-treesitter/tree-sitter-query/archive/v0.1.0.tar.gz)
 set(TREESITTER_QUERY_SHA256 e2b806f80e8bf1c4f4e5a96248393fe6622fc1fc6189d6896d269658f67f914c)
--- a/cmake.deps/cmake/BuildTreesitterParsers.cmake
+++ b/cmake.deps/cmake/BuildTreesitterParsers.cmake
@ -38,9 +38,9 @@ BuildTSParser(
  CMAKE_FILE TreesitterParserCMakeLists.txt)

 BuildTSParser(
-  LANG help
-  URL ${TREESITTER_HELP_URL}
-  SHA256 ${TREESITTER_HELP_SHA256}
+  LANG vimdoc
+  URL ${TREESITTER_VIMDOC_URL}
+  SHA256 ${TREESITTER_VIMDOC_SHA256}
  CMAKE_FILE TreesitterParserCMakeLists.txt)

 BuildTSParser(
--- a/runtime/doc/news.txt
+++ b/runtime/doc/news.txt
@ -62,6 +62,10 @@ The following changes may require adaptations in user config or plugins.

 • |nvim_exec()| is now deprecated in favor of |nvim_exec2()|.

+• `help` treesitter parser was renamed to `vimdoc`. The only user-visible
+  change is that language-specific highlight groups need to be renamed from
+  `@foo.help` to `@foo.vimdoc`.
+
 ==============================================================================
 NEW FEATURES                                                    *news-features*

--- a/runtime/doc/treesitter.txt
+++ b/runtime/doc/treesitter.txt
@ -195,7 +195,7 @@ treesitter queries from Lua.
 TREESITTER QUERY PREDICATES                            *treesitter-predicates*

 Predicates are special scheme nodes that are evaluated to conditionally capture
-nodes. For example, the `eq?` predicate can be used as follows: >
+nodes. For example, the `eq?` predicate can be used as follows: >query

    ((identifier) @foo (#eq? @foo "foo"))
 <
@ -204,13 +204,13 @@ to only match identifier corresponding to the `"foo"` text.
 The following predicates are built in:

    `eq?`                                            *treesitter-predicate-eq?*
-        Match a string against the text corresponding to a node: >
+        Match a string against the text corresponding to a node: >query
            ((identifier) @foo (#eq? @foo "foo"))
            ((node1) @left (node2) @right (#eq? @left @right))
 <
    `match?`                                      *treesitter-predicate-match?*
    `vim-match?`                              *treesitter-predicate-vim-match?*
-         Match a |regexp| against the text corresponding to a node: >
+         Match a |regexp| against the text corresponding to a node: >query
            ((identifier) @constant (#match? @constant "^[A-Z_]+$"))
 <         Note: The `^` and `$` anchors will match the start and end of the
               node's text.
@ -220,13 +220,14 @@ The following predicates are built in:
         similar to `match?`

    `contains?`                                *treesitter-predicate-contains?*
-        Match a string against parts of the text corresponding to a node: >
+        Match a string against parts of the text corresponding to a node:
+        >query
            ((identifier) @foo (#contains? @foo "foo"))
            ((identifier) @foo-bar (#contains? @foo-bar "foo" "bar"))
 <
    `any-of?`                                    *treesitter-predicate-any-of?*
        Match any of the given strings against the text corresponding to
-        a node: >
+        a node: >query
            ((identifier) @foo (#any-of? @foo "foo" "bar"))
 <
        This is the recommended way to check if the node matches one of many
@ -243,7 +244,7 @@ Use |vim.treesitter.query.list_predicates()| to list all available predicates.
 TREESITTER QUERY DIRECTIVES                            *treesitter-directives*

 Treesitter directives store metadata for a node or match and perform side
-effects. For example, the `set!` directive sets metadata on the match or node: >
+effects. For example, the `set!` directive sets metadata on the match or node: >query

        ((identifier) @foo (#set! "type" "parameter"))
 <
@ -259,7 +260,7 @@ The following directives are built in:
            {key}
            {value}

-        Examples: >
+        Examples: >query
            ((identifier) @foo (#set! @foo "kind" "parameter"))
            ((node1) @left (node2) @right (#set! "type" "pair"))
 <
@ -275,7 +276,7 @@ The following directives are built in:
            {end_row}
            {end_col}

-        Example: >
+        Example: >query
            ((identifier) @constant (#offset! @constant 0 1 0 -1))
 <

@ -304,7 +305,8 @@ currently supported modeline alternatives:
        a base depends on your 'runtimepath' value.

 Note: These modeline comments must be at the top of the query, but can be
-repeated, for example, the following two modeline blocks are both valid: >
+repeated, for example, the following two modeline blocks are both valid:
+>query

    ;; inherits: foo,bar
    ;; extends
@ -318,13 +320,13 @@ TREESITTER SYNTAX HIGHLIGHTING                          *treesitter-highlight*

 Syntax highlighting is specified through queries named `highlights.scm`,
 which match a |TSNode| in the parsed |TSTree| to a `capture` that can be
-assigned a highlight group. For example, the query >
+assigned a highlight group. For example, the query >query

    (parameters (identifier) @parameter)
 <
 matches any `identifier` node inside a function `parameter` node (e.g., the
 `bar` in `foo(bar)`) to the capture named `@parameter`. It is also possible to
-match literal expressions (provided the parser returns them): >
+match literal expressions (provided the parser returns them): >query

    "return" @keyword.return
 <
@ -409,7 +411,7 @@ The following captures are linked by default to standard |group-name|s:
                                                  *treesitter-highlight-spell*
 The special `@spell` capture can be used to indicate that a node should be
 spell checked by Nvim's builtin |spell| checker. For example, the following
-capture marks comments as to be checked: >
+capture marks comments as to be checked: >query

    (comment) @spell
 <
@ -420,14 +422,14 @@ There is also `@nospell` which disables spellchecking regions with `@spell`.
 Treesitter highlighting supports |conceal| via the `conceal` metadata. By
 convention, nodes to be concealed are captured as `@conceal`, but any capture
 can be used. For example, the following query can be used to hide code block
-delimiters in Markdown: >
+delimiters in Markdown: >query

-    (fenced_code_block_delimiter) @conceal (#set! conceal "")
+    (fenced_code_block_delimiter @conceal (#set! conceal ""))
 <
 It is also possible to replace a node with a single character, which (unlike
 legacy syntax) can be given a custom highlight. For example, the following
 (ill-advised) query replaces the `!=` operator by a Unicode glyph, which is
-still highlighted the same as other operators: >
+still highlighted the same as other operators: >query

    "!=" @operator (#set! conceal "≠")
 <
@ -438,9 +440,10 @@ Treesitter uses |nvim_buf_set_extmark()| to set highlights with a default
 priority of 100. This enables plugins to set a highlighting priority lower or
 higher than tree-sitter. It is also possible to change the priority of an
 individual query pattern manually by setting its `"priority"` metadata
-attribute: >
+attribute: >query

-    (super_important_node) @ImportantHighlight (#set! "priority" 105)
+    ((super_important_node) @superimportant (#set! "priority" 105))
+<

 ==============================================================================
 TREESITTER LANGUAGE INJECTIONS                *treesitter-language-injections*
--- a/runtime/lua/vim/treesitter/language.lua
+++ b/runtime/lua/vim/treesitter/language.lua
@ -4,7 +4,9 @@ local a = vim.api
 local M = {}

 ---@type table<string,string>
-local ft_to_lang = {}
+local ft_to_lang = {
+  help = 'vimdoc',
+}

 --- Get the filetypes associated with the parser named {lang}.
 --- @param lang string Name of parser
--- a/runtime/queries/c/highlights.scm
+++ b/runtime/queries/c/highlights.scm
@ -1,22 +1,19 @@
-(identifier) @variable
+; Lower priority to prefer @parameter when identifier appears in parameter_declaration.
+((identifier) @variable (#set! "priority" 95))

 [
-  "const"
  "default"
  "enum"
-  "extern"
-  "inline"
-  "return"
-  "sizeof"
-  "static"
  "struct"
  "typedef"
  "union"
-  "volatile"
  "goto"
-  "register"
 ] @keyword

+"sizeof" @keyword.operator
+
+"return" @keyword.return
+
 [
  "while"
  "for"
@ -32,7 +29,6 @@
 "switch"
 ] @conditional

-"#define" @constant.macro
 [
  "#if"
  "#ifdef"
@ -41,10 +37,18 @@
  "#elif"
  "#endif"
  (preproc_directive)
-] @keyword
+] @preproc
+
+"#define" @define

 "#include" @include

+[ ";" ":" "," ] @punctuation.delimiter
+
+"..." @punctuation.special
+
+[ "(" ")" "[" "]" "{" "}"] @punctuation.bracket
+
 [
  "="

@ -62,6 +66,7 @@
  ">>"

  "->"
+  "."

  "<"
  "<="
@ -88,35 +93,25 @@
  "++"
 ] @operator

+;; Make sure the comma operator is given a highlight group after the comma
+;; punctuator so the operator is highlighted properly.
+(comma_expression [ "," ] @operator)
+
 [
 (true)
 (false)
 ] @boolean

-[ "." ";" ":" "," ] @punctuation.delimiter
-
-(conditional_expression [ "?" ":" ] @conditional)
-
-
-[ "(" ")" "[" "]" "{" "}"] @punctuation.bracket
+(conditional_expression [ "?" ":" ] @conditional.ternary)

 (string_literal) @string
-(string_literal) @spell
 (system_lib_string) @string
+(escape_sequence) @string.escape

 (null) @constant.builtin
 (number_literal) @number
-(char_literal) @number
+(char_literal) @character

-(call_expression
-  function: (identifier) @function)
-(call_expression
-  function: (field_expression
-    field: (field_identifier) @function))
-(function_declarator
-  declarator: (identifier) @function)
-(preproc_function_def
-  name: (identifier) @function.macro)
 [
 (preproc_arg)
 (preproc_defined)
@ -126,18 +121,32 @@
 (statement_identifier) @label

 [
-(type_identifier)
-(primitive_type)
-(sized_type_specifier)
-(type_descriptor)
- ] @type
+ (type_identifier)
+ (sized_type_specifier)
+ (type_descriptor)
+] @type

-(declaration (type_qualifier) @type)
-(cast_expression type: (type_descriptor) @type)
-(sizeof_expression value: (parenthesized_expression (identifier) @type))
+(storage_class_specifier) @storageclass
+
+(type_qualifier) @type.qualifier
+
+(linkage_specification
+  "extern" @storageclass)
+
+(type_definition
+  declarator: (type_identifier) @type.definition)
+
+(primitive_type) @type.builtin

 ((identifier) @constant
- (#match? @constant "^[A-Z][A-Z0-9_]+$"))
+ (#lua-match? @constant "^[A-Z][A-Z0-9_]+$"))
+(enumerator
+  name: (identifier) @constant)
+(case_statement
+  value: (identifier) @constant)
+
+((identifier) @constant.builtin
+    (#any-of? @constant.builtin "stderr" "stdin" "stdout"))

 ;; Preproc def / undef
 (preproc_def
@ -147,9 +156,20 @@
  argument: (_) @constant
  (#eq? @_u "#undef"))

+(call_expression
+  function: (identifier) @function.call)
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function.call))
+(function_declarator
+  declarator: (identifier) @function)
+(preproc_function_def
+  name: (identifier) @function.macro)

-(comment) @comment
-(comment) @spell
+(comment) @comment @spell
+
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^/[*][*][^*].*[*]/$"))

 ;; Parameters
 (parameter_declaration
@ -158,7 +178,20 @@
 (parameter_declaration
  declarator: (pointer_declarator) @parameter)

-(preproc_params
-  (identifier)) @parameter
+(preproc_params (identifier) @parameter)
+
+[
+  "__attribute__"
+  "__cdecl"
+  "__clrcall"
+  "__stdcall"
+  "__fastcall"
+  "__thiscall"
+  "__vectorcall"
+  "_unaligned"
+  "__unaligned"
+  "__declspec"
+  (attribute_declaration)
+] @attribute

 (ERROR) @error
--- a/runtime/queries/c/injections.scm
+++ b/runtime/queries/c/injections.scm
@ -1,5 +1,5 @@
-((preproc_arg) @injection.content
- (#set! injection.language "c"))
+; ((preproc_arg) @injection.content
+;  (#set! injection.language "c"))

 ; ((comment) @injection.content
 ;  (#set! injection.language "comment"))
--- a/runtime/queries/lua/highlights.scm
+++ b/runtime/queries/lua/highlights.scm
@ -8,8 +8,6 @@
 "local"
 ] @keyword

-(label_statement) @label
-
 (break_statement) @keyword

 (do_statement
@ -109,6 +107,7 @@
 [
  ";"
  ":"
+  "::"
  ","
  "."
 ] @punctuation.delimiter
@ -129,12 +128,21 @@
 (identifier) @variable

 ((identifier) @variable.builtin
- (#eq? @variable.builtin "self"))
+ (#any-of? @variable.builtin "_G" "_VERSION" "debug" "io" "jit" "math" "os" "package" "self" "string" "table" "utf8"))
+
+((identifier) @keyword.coroutine
+  (#eq? @keyword.coroutine "coroutine"))

 (variable_list
-  attribute: (attribute
-    (["<" ">"] @punctuation.bracket
-     (identifier) @attribute)))
+   attribute: (attribute
+     (["<" ">"] @punctuation.bracket
+      (identifier) @attribute)))
+
+;; Labels
+
+(label_statement (identifier) @label)
+
+(goto_statement (identifier) @label)

 ;; Constants

@ -172,7 +180,7 @@
 (function_call name: (dot_index_expression field: (identifier) @function.call))
 (function_declaration name: (dot_index_expression field: (identifier) @function))

-(method_index_expression method: (identifier) @method)
+(method_index_expression method: (identifier) @method.call)

 (function_call
  (identifier) @function.builtin
@ -180,20 +188,27 @@
    ;; built-in functions in Lua 5.1
    "assert" "collectgarbage" "dofile" "error" "getfenv" "getmetatable" "ipairs"
    "load" "loadfile" "loadstring" "module" "next" "pairs" "pcall" "print"
-    "rawequal" "rawget" "rawset" "require" "select" "setfenv" "setmetatable"
-    "tonumber" "tostring" "type" "unpack" "xpcall"))
+    "rawequal" "rawget" "rawlen" "rawset" "require" "select" "setfenv" "setmetatable"
+    "tonumber" "tostring" "type" "unpack" "xpcall"
+    "__add" "__band" "__bnot" "__bor" "__bxor" "__call" "__concat" "__div" "__eq" "__gc"
+    "__idiv" "__index" "__le" "__len" "__lt" "__metatable" "__mod" "__mul" "__name" "__newindex"
+    "__pairs" "__pow" "__shl" "__shr" "__sub" "__tostring" "__unm"))

 ;; Others

-(comment) @comment
-(comment) @spell
+(comment) @comment @spell

-(hash_bang_line) @comment
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^[-][-][-]"))
+
+((comment) @comment.documentation
+  (#lua-match? @comment.documentation "^[-][-](%s?)@"))
+
+(hash_bang_line) @preproc

 (number) @number

-(string) @string
-(string) @spell
+(string) @string @spell

 ;; Error
 (ERROR) @error
--- a/runtime/queries/lua/injections.scm
+++ b/runtime/queries/lua/injections.scm
@ -11,18 +11,18 @@
  name: (_) @_vimcmd_identifier
  arguments: (arguments (string content: _ @injection.content)))
  (#set! injection.language "vim")
-  (#any-of? @_vimcmd_identifier "vim.cmd" "vim.api.nvim_command" "vim.api.nvim_exec" "vim.api.nvim_cmd"))
+  (#any-of? @_vimcmd_identifier "vim.cmd" "vim.api.nvim_command" "vim.api.nvim_exec2" "vim.api.nvim_cmd"))

 ((function_call
  name: (_) @_vimcmd_identifier
  arguments: (arguments (string content: _ @injection.content) .))
  (#set! injection.language "query")
-  (#eq? @_vimcmd_identifier "vim.treesitter.query.set_query"))
+  (#any-of? @_vimcmd_identifier "vim.treesitter.query.set" "vim.treesitter.query.parse"))

-; ;; highlight string as query if starts with `;; query`
-; ((string ("string_content") @injection.content)
-;  (#set! injection.language "query")
-;  (#lua-match? @injection.content "^%s*;+%s?query"))
+;; highlight string as query if starts with `;; query`
+((string ("string_content") @injection.content)
+ (#set! injection.language "query")
+ (#lua-match? @injection.content "^%s*;+%s?query"))

 ; ((comment) @injection.content
 ;  (#set! injection.language "comment"))
--- a/runtime/queries/vim/highlights.scm
+++ b/runtime/queries/vim/highlights.scm
@ -271,7 +271,7 @@
  ":"
 ] @punctuation.delimiter

-(ternary_expression ["?" ":"] @conditional)
+(ternary_expression ["?" ":"] @conditional.ternary)

 ; Options
 ((set_value) @number
--- a/runtime/queries/vimdoc/highlights.scm
+++ b/runtime/queries/vimdoc/highlights.scm
--- a/runtime/queries/vimdoc/injections.scm
+++ b/runtime/queries/vimdoc/injections.scm
--- a/scripts/gen_help_html.lua
+++ b/scripts/gen_help_html.lua
@ -624,7 +624,7 @@ local function get_helptags(help_dir)
  return m
 end

-- Use the help.so parser defined in the build, not whatever happens to be installed on the system.
+-- Use the vimdoc parser defined in the build, not whatever happens to be installed on the system.
 local function ensure_runtimepath()
  if not vim.o.runtimepath:find('build/lib/nvim/') then
    vim.cmd[[set runtimepath^=./build/lib/nvim/]]
@ -643,8 +643,8 @@ local function parse_buf(fname)
    buf = fname
    vim.cmd('sbuffer '..tostring(fname))          -- Buffer number.
  end
-  -- vim.treesitter.require_language('help', './build/lib/nvim/parser/help.so')
-  local lang_tree = vim.treesitter.get_parser(buf, 'help')
+  -- vim.treesitter.require_language('help', './build/lib/nvim/parser/vimdoc.so')
+  local lang_tree = vim.treesitter.get_parser(buf)
  return lang_tree, buf
 end

--- a/test/functional/treesitter/highlight_spec.lua
+++ b/test/functional/treesitter/highlight_spec.lua
@ -575,14 +575,14 @@ describe('treesitter highlighting', function()

    exec_lua [[
      local parser = vim.treesitter.get_parser(0, "c")
-      test_hl = vim.treesitter.highlighter.new(parser, {queries = {c = hl_query..'\n((translation_unit) @Error (set! "priority" 101))\n'}})
+      test_hl = vim.treesitter.highlighter.new(parser, {queries = {c = hl_query..'\n((translation_unit) @constant (#set! "priority" 101))\n'}})
    ]]
-    -- expect everything to have Error highlight
+    -- expect everything to have Constant highlight
    screen:expect{grid=[[
      {12:int}{8: x = INT_MAX;}                                                 |
-      {8:#define READ_STRING(x, y) (}{12:char_u}{8: *)read_string((x), (}{12:size_t}{8:)(y))}|
-      {8:#define foo }{12:void}{8: main() { \}                                      |
-      {8:              }{12:return}{8: 42;  \}                                      |
+      {8:#define READ_STRING(x, y) (char_u *)read_string((x), (size_t)(y))}|
+      {8:#define foo void main() { \}                                      |
+      {8:              return 42;  \}                                      |
      {8:            }}                                                    |
      ^                                                                 |
      {1:~                                                                }|
@ -599,13 +599,13 @@ describe('treesitter highlighting', function()
                                                                       |
    ]], attr_ids={
      [1] = {bold = true, foreground = Screen.colors.Blue1};
-      [8] = {foreground = Screen.colors.Grey100, background = Screen.colors.Red};
+      [8] = {foreground = Screen.colors.Magenta1};
      -- bold will not be overwritten at the moment
-      [12] = {background = Screen.colors.Red, bold = true, foreground = Screen.colors.Grey100};
+      [12] = {bold = true, foreground = Screen.colors.Magenta1};
    }}

    eq({
-      {capture='Error', metadata = { priority='101' }, lang='c' };
+      {capture='constant', metadata = { priority='101' }, lang='c' };
      {capture='type', metadata = { }, lang='c' };
    }, exec_lua [[ return vim.treesitter.get_captures_at_pos(0, 0, 2) ]])
    end)