From af187ab52e3d43042243093ece4072296197b101 Mon Sep 17 00:00:00 2001 From: Simon Klee Date: Sat, 10 Oct 2026 14:54:52 +0200 Subject: [PATCH] core: evaluate #lua-match? predicates in tree-sitter queries (#1620) The bundled typescript, typescriptreact, and zig highlight queries come from nvim-treesitter and use the `#lua-match?` predicate. web-tree-sitter does not evaluate it, so the predicate matched every node: each lowercase identifier also got the `type` and `constant` captures, and every block comment got `comment.documentation`. With a theme that colors these groups differently, every identifier in a TypeScript or Zig code block was drawn as a constant. The tree-sitter worker now rewrites `#lua-match?` (and its `not-` and `any-` forms) to `#match?` when it loads a query, so bundled queries and queries that apps register are both covered. It translates only the Lua pattern items that the bundled queries use: literals, anchors at either end, sets of literals and `%d`, `.` (which also matches a newline in Lua), `*`, and `+`. Any other pattern, or one that RegExp rejects, keeps `#lua-match?` and behaves as before, so a query never fails to load because of the rewrite. That covers about 220 of the 385 uses in nvim-treesitter. A table in `client.test.ts` checks the captures for TypeScript, TSX, Zig, and doc comments, and a second table checks the translation on a registered query, including patterns that stay untranslated (from nvim-treesitter's kitty and clojure queries) and must still load. Each of 12 breaks of the translation (dropping the guard, the `not-`/`any-` mapping, `%d`, `.`, the anchors or quantifiers, or accepting `-`, `(` or `%`) fails one of the tables. Fixes #1613 --- .../core/src/lib/tree-sitter/client.test.ts | 58 ++++++++++++++++--- .../core/src/lib/tree-sitter/parser.worker.ts | 21 ++++++- 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/packages/core/src/lib/tree-sitter/client.test.ts b/packages/core/src/lib/tree-sitter/client.test.ts index c1c57f9e..d99eed7c 100644 --- a/packages/core/src/lib/tree-sitter/client.test.ts +++ b/packages/core/src/lib/tree-sitter/client.test.ts @@ -470,22 +470,40 @@ describeClient("TreeSitterClient", () => { expect(client.getAllBuffers()).toHaveLength(0) }) - test("should parse JSX only for react filetypes", async () => { + test("should parse JSX only for react filetypes and evaluate #lua-match? predicates", async () => { await client.initialize() const jsx = 'const view = hi' - const rows = [ + const rows: { filetype: string; source: string; expected: string[]; absent?: string[] }[] = [ { filetype: "javascriptreact", source: jsx, expected: ["18:title=property", "32:Box=constructor"] }, - { filetype: "typescriptreact", source: jsx, expected: ["18:title=tag.attribute", "32:Box=tag"] }, + { + filetype: "typescriptreact", + source: jsx, + expected: ["18:title=tag.attribute", "32:Box=tag"], + absent: ["6:view=type", "6:view=constant"], + }, { filetype: "typescript", source: "const n = value", expected: ["11:number=type.builtin"] }, + { + filetype: "typescript", + source: "const n = /* c */ 1", + expected: ["6:n=variable", "10:/* c */=comment"], + absent: ["6:n=type", "6:n=constant", "10:/* c */=comment.documentation"], + }, + { + filetype: "typescript", + source: "const N_1 = /** d\n */ 1", + expected: ["6:N_1=variable", "6:N_1=type", "6:N_1=constant", "12:/** d\n */=comment.documentation"], + }, + { filetype: "zig", source: "const n = 1", expected: ["6:n=variable"], absent: ["6:n=type", "6:n=constant"] }, ] - for (const { filetype, source, expected } of rows) { + for (const { filetype, source, expected, absent = [] } of rows) { const { highlights = [] } = await client.highlightOnce(source, filetype) const tokens = highlights.map(([start, end, group]) => `${start}:${source.slice(start, end)}=${group}`) expect(tokens).toContain("0:const=keyword") expect(expected.filter((token) => !tokens.includes(token))).toEqual([]) + expect(absent.filter((token) => tokens.includes(token))).toEqual([]) expect(tokens.filter((token) => token.endsWith("=string.regexp"))).toEqual([]) } }) @@ -542,7 +560,27 @@ describeClient("TreeSitterClient", () => { test("should support local file paths for parser configuration", async () => { const testQueryPath = join(dataPath, `test-highlights-${Date.now()}.scm`) - const simpleQuery = "(identifier) @variable" + // Lua pattern predicate → identifiers of testCode that it captures. A pattern the worker does not translate + // (the last rows; the first two are from nvim-treesitter's kitty and clojure queries) stays #lua-match?, which + // web-tree-sitter ignores, so it captures everything; the query must still load. + const all = "a9 ab A_2 _x $y aXz" + const luaPredicates: [string, string][] = [ + ['lua-match? "^[a-z][%d]$"', "a9"], + ['lua-match? "^[A-Z_][A-Z%d_]+$"', "A_2"], + ['lua-match? "^a.z$"', "aXz"], + ['not-lua-match? "^a"', "A_2 _x $y"], + ['not-any-lua-match? "^a"', "A_2 _x $y"], + ['any-lua-match? "b$"', "ab"], + ['lua-match? "^-"', all], + ['lua-match? "^-%>[^>].*"', all], + ['lua-match? "^%u"', all], + ['lua-match? "^(a)?$"', all], + ['lua-match? "^[a"', all], + ['lua-match? "[z-a]"', all], + ] + const simpleQuery = luaPredicates + .map(([predicate], i) => `((identifier) @p${i} (#${predicate.replace(" ", ` @p${i} `)}))`) + .join("\n") const javascriptParser = (await getParsers()).find((parser) => parser.filetype === "javascript") if (!javascriptParser) { throw new Error("Expected the default JavaScript parser") @@ -567,12 +605,16 @@ describeClient("TreeSitterClient", () => { const hasAliasParser = await client.preloadParser("test-lang-react") expect(hasAliasParser).toBe(true) - const testCode = "const myVariable = 42;" + const testCode = "a9; ab; A_2; _x; $y; aXz;" const result = await client.highlightOnce(testCode, "test-lang") const aliasResult = await client.highlightOnce(testCode, "test-lang-react") + const captured = luaPredicates.map(([predicate], i) => { + const names = result.highlights!.filter(([, , group]) => group === `p${i}`) + return [predicate, names.map(([start, end]) => testCode.slice(start, end)).join(" ")] + }) - expect(result.highlights).toBeDefined() - expect(aliasResult.highlights).toBeDefined() + expect(captured).toEqual(luaPredicates) + expect(aliasResult.highlights).toEqual(result.highlights) expect(result.error).toBeUndefined() expect(aliasResult.error).toBeUndefined() expect(result.warning).toBeUndefined() diff --git a/packages/core/src/lib/tree-sitter/parser.worker.ts b/packages/core/src/lib/tree-sitter/parser.worker.ts index 50bb6161..9f85d85c 100644 --- a/packages/core/src/lib/tree-sitter/parser.worker.ts +++ b/packages/core/src/lib/tree-sitter/parser.worker.ts @@ -54,6 +54,25 @@ interface ReusableParserState { } } +// web-tree-sitter ignores the #lua-match? predicates of nvim-treesitter queries, so they would match every node. +// Rewrites one to #match? when its Lua pattern uses only what the bundled queries use: literals, an anchor at +// either end, sets of literals and %d, "." (which also matches a newline in Lua), "*", and "+". Any other +// pattern, or one RegExp rejects, keeps #lua-match? so the query still loads. +function translateLuaMatch(query: string): string { + return query.replace( + /#(not-)?(any-)?lua-match\?(\s+@[\w.-]+\s+)"(\^?(?:(?:[\w/!:#.]|\[\^?(?:[\w*-]|%d)+\])[*+]?)*\$?)"/g, + (predicate, not = "", any = "", capture: string, pattern: string) => { + const source = pattern.replaceAll("%d", "0-9").replaceAll(".", "[^]") + try { + new RegExp(source) + } catch { + return predicate + } + return `#${any}${not}match?${capture}"${source}"` + }, + ) +} + class ParserWorker { private bufferParsers: Map = new Map() private filetypeParserOptions: Map = new Map() @@ -81,7 +100,7 @@ class ParserWorker { if (!this.tsDataPath) { return "" } - return DownloadUtils.fetchHighlightQueries(sources, this.tsDataPath, filetype) + return translateLuaMatch(await DownloadUtils.fetchHighlightQueries(sources, this.tsDataPath, filetype)) } async initialize({ dataPath, treeSitterWasmPath }: { dataPath: string; treeSitterWasmPath?: string }) {