core: evaluate #lua-match? predicates in tree-sitter queries (#1620)

The bundled typescript, typescriptreact, and zig highlight queries come
from nvim-treesitter and use the `#lua-match?` predicate.
web-tree-sitter does not evaluate it, so the predicate matched every
node: each lowercase identifier also got the `type` and `constant`
captures, and every block comment got `comment.documentation`. With a
theme that colors these groups differently, every identifier in a
TypeScript or Zig code block was drawn as a constant.

The tree-sitter worker now rewrites `#lua-match?` (and its `not-` and
`any-` forms) to `#match?` when it loads a query, so bundled queries and
queries that apps register are both covered. It translates only the Lua
pattern items that the bundled queries use: literals, anchors at either
end, sets of literals and `%d`, `.` (which also matches a newline in
Lua), `*`, and `+`. Any other pattern, or one that RegExp rejects, keeps
`#lua-match?` and behaves as before, so a query never fails to load
because of the rewrite. That covers about 220 of the 385 uses in
nvim-treesitter.

A table in `client.test.ts` checks the captures for TypeScript, TSX,
Zig, and doc comments, and a second table checks the translation on a
registered query, including patterns that stay untranslated (from
nvim-treesitter's kitty and clojure queries) and must still load. Each
of 12 breaks of the translation (dropping the guard, the `not-`/`any-`
mapping, `%d`, `.`, the anchors or quantifiers, or accepting `-`, `(` or
`%`) fails one of the tables.

Fixes #1613

packages/core/src/lib/tree-sitter/client.test.ts

@@ -470,22 +470,40 @@ describeClient("TreeSitterClient", () => {
     expect(client.getAllBuffers()).toHaveLength(0)
   })
 
-  test("should parse JSX only for react filetypes", async () => {
+  test("should parse JSX only for react filetypes and evaluate #lua-match? predicates", async () => {
     await client.initialize()
 
     const jsx = 'const view = <Box title="x">hi</Box>'
-    const rows = [
+    const rows: { filetype: string; source: string; expected: string[]; absent?: string[] }[] = [
       { filetype: "javascriptreact", source: jsx, expected: ["18:title=property", "32:Box=constructor"] },
-      { filetype: "typescriptreact", source: jsx, expected: ["18:title=tag.attribute", "32:Box=tag"] },
+      {
+        filetype: "typescriptreact",
+        source: jsx,
+        expected: ["18:title=tag.attribute", "32:Box=tag"],
+        absent: ["6:view=type", "6:view=constant"],
+      },
       { filetype: "typescript", source: "const n = <number>value", expected: ["11:number=type.builtin"] },
+      {
+        filetype: "typescript",
+        source: "const n = /* c */ 1",
+        expected: ["6:n=variable", "10:/* c */=comment"],
+        absent: ["6:n=type", "6:n=constant", "10:/* c */=comment.documentation"],
+      },
+      {
+        filetype: "typescript",
+        source: "const N_1 = /** d\n */ 1",
+        expected: ["6:N_1=variable", "6:N_1=type", "6:N_1=constant", "12:/** d\n */=comment.documentation"],
+      },
+      { filetype: "zig", source: "const n = 1", expected: ["6:n=variable"], absent: ["6:n=type", "6:n=constant"] },
     ]
 
-    for (const { filetype, source, expected } of rows) {
+    for (const { filetype, source, expected, absent = [] } of rows) {
       const { highlights = [] } = await client.highlightOnce(source, filetype)
       const tokens = highlights.map(([start, end, group]) => `${start}:${source.slice(start, end)}=${group}`)
 
       expect(tokens).toContain("0:const=keyword")
       expect(expected.filter((token) => !tokens.includes(token))).toEqual([])
+      expect(absent.filter((token) => tokens.includes(token))).toEqual([])
       expect(tokens.filter((token) => token.endsWith("=string.regexp"))).toEqual([])
     }
   })
@@ -542,7 +560,27 @@ describeClient("TreeSitterClient", () => {
 
   test("should support local file paths for parser configuration", async () => {
     const testQueryPath = join(dataPath, `test-highlights-${Date.now()}.scm`)
-    const simpleQuery = "(identifier) @variable"
+    // Lua pattern predicate → identifiers of testCode that it captures. A pattern the worker does not translate
+    // (the last rows; the first two are from nvim-treesitter's kitty and clojure queries) stays #lua-match?, which
+    // web-tree-sitter ignores, so it captures everything; the query must still load.
+    const all = "a9 ab A_2 _x $y aXz"
+    const luaPredicates: [string, string][] = [
+      ['lua-match? "^[a-z][%d]$"', "a9"],
+      ['lua-match? "^[A-Z_][A-Z%d_]+$"', "A_2"],
+      ['lua-match? "^a.z$"', "aXz"],
+      ['not-lua-match? "^a"', "A_2 _x $y"],
+      ['not-any-lua-match? "^a"', "A_2 _x $y"],
+      ['any-lua-match? "b$"', "ab"],
+      ['lua-match? "^-"', all],
+      ['lua-match? "^-%>[^>].*"', all],
+      ['lua-match? "^%u"', all],
+      ['lua-match? "^(a)?$"', all],
+      ['lua-match? "^[a"', all],
+      ['lua-match? "[z-a]"', all],
+    ]
+    const simpleQuery = luaPredicates
+      .map(([predicate], i) => `((identifier) @p${i} (#${predicate.replace(" ", ` @p${i} `)}))`)
+      .join("\n")
     const javascriptParser = (await getParsers()).find((parser) => parser.filetype === "javascript")
     if (!javascriptParser) {
       throw new Error("Expected the default JavaScript parser")
@@ -567,12 +605,16 @@ describeClient("TreeSitterClient", () => {
       const hasAliasParser = await client.preloadParser("test-lang-react")
       expect(hasAliasParser).toBe(true)
 
-      const testCode = "const myVariable = 42;"
+      const testCode = "a9; ab; A_2; _x; $y; aXz;"
       const result = await client.highlightOnce(testCode, "test-lang")
       const aliasResult = await client.highlightOnce(testCode, "test-lang-react")
+      const captured = luaPredicates.map(([predicate], i) => {
+        const names = result.highlights!.filter(([, , group]) => group === `p${i}`)
+        return [predicate, names.map(([start, end]) => testCode.slice(start, end)).join(" ")]
+      })
 
-      expect(result.highlights).toBeDefined()
-      expect(aliasResult.highlights).toBeDefined()
+      expect(captured).toEqual(luaPredicates)
+      expect(aliasResult.highlights).toEqual(result.highlights)
       expect(result.error).toBeUndefined()
       expect(aliasResult.error).toBeUndefined()
       expect(result.warning).toBeUndefined()

packages/core/src/lib/tree-sitter/parser.worker.ts

@@ -54,6 +54,25 @@ interface ReusableParserState {
   }
 }
 
+// web-tree-sitter ignores the #lua-match? predicates of nvim-treesitter queries, so they would match every node.
+// Rewrites one to #match? when its Lua pattern uses only what the bundled queries use: literals, an anchor at
+// either end, sets of literals and %d, "." (which also matches a newline in Lua), "*", and "+". Any other
+// pattern, or one RegExp rejects, keeps #lua-match? so the query still loads.
+function translateLuaMatch(query: string): string {
+  return query.replace(
+    /#(not-)?(any-)?lua-match\?(\s+@[\w.-]+\s+)"(\^?(?:(?:[\w/!:#.]|\[\^?(?:[\w*-]|%d)+\])[*+]?)*\$?)"/g,
+    (predicate, not = "", any = "", capture: string, pattern: string) => {
+      const source = pattern.replaceAll("%d", "0-9").replaceAll(".", "[^]")
+      try {
+        new RegExp(source)
+      } catch {
+        return predicate
+      }
+      return `#${any}${not}match?${capture}"${source}"`
+    },
+  )
+}
+
 class ParserWorker {
   private bufferParsers: Map<number, ParserState> = new Map()
   private filetypeParserOptions: Map<string, FiletypeParserOptions> = new Map()
@@ -81,7 +100,7 @@ class ParserWorker {
     if (!this.tsDataPath) {
       return ""
     }
-    return DownloadUtils.fetchHighlightQueries(sources, this.tsDataPath, filetype)
+    return translateLuaMatch(await DownloadUtils.fetchHighlightQueries(sources, this.tsDataPath, filetype))
   }
 
   async initialize({ dataPath, treeSitterWasmPath }: { dataPath: string; treeSitterWasmPath?: string }) {