core: add input parser conformance vectors (#1623)

The stdin parser gets a group of byte sequences wrong on terminals
without
the Kitty keyboard protocol. Escape followed by another sequence (a
mouse
report, a reply, a paste, Alt+key) loses both parts; Alt with
punctuation or
a non-ASCII character gives a key with an empty name or garbage keys; a
wheel
release ends a drag; an OSC string cut short by a new sequence swallows
it;
an unknown CSI sequence becomes an empty-name key; and an X10 report
with a
0 byte or a control byte gives wrong coordinates.

The fix will land in the native parser port, so this change does not
touch
the TypeScript parser. It adds the correct behavior as data:
`stdin-parser.conformance.json` holds 38 rows of byte steps (with
explicit
Escape-timeout steps) and the events each must produce, and its `format`
field defines the notation and string encoding so a native test can load
the same file. The 32 rows that main gets wrong run as `test.todo`; the
6
rows it gets right are normal tests. Every row also runs one byte at a
time.
`bun test --todo src/lib/stdin-parser.test.ts` fails a todo row that
starts
to pass, which is how the port shows a fixed row. Existing tests that
pin
today's behavior are marked so the port knows to flip them.

Each todo row fails on main with exactly the wrong output reported in
the
issue, and breaking the parser six ways (Alt prefix, timer boundary,
ESC-less
mouse recovery, SGR motion) fails a normal row each time.

Refs #1614; the native parser port closes it.

.oxfmtrc.json

@@ -7,6 +7,7 @@
     "**/.zig-cache/**",
     "**/zig-pkg/**",
     "packages/core/src/lib/tree-sitter/default-parsers.ts",
-    "packages/web/public/recordings/*.json"
+    "packages/web/public/recordings/*.json",
+    "packages/core/src/lib/stdin-parser.conformance.json"
   ]
 }

packages/core/src/lib/parse.keypress-kitty.test.ts

@@ -4,6 +4,7 @@ import { parseKeypress, type ParseKeypressOptions } from "./parse.keypress.js"
 test("parseKeypress - Kitty keyboard protocol disabled by default", () => {
   // Kitty sequences should fall back to regular parsing when disabled
   const result = parseKeypress("\x1b[97u")!
+  // 044: pins main; the conformance vectors make an unrecognized CSI sequence a response, not a key.
   expect(result.name).toBe("")
   expect(result.code).toBeUndefined()
 })
@@ -236,6 +237,7 @@ test("parseKeypress - Kitty keyboard invalid codepoint", () => {
   const options: ParseKeypressOptions = { useKittyKeyboard: true }
   const result = parseKeypress("\x1b[1114112u", options)! // Invalid codepoint > 0x10FFFF
   // Should fall back to regular parsing when Kitty fails
+  // 044: pins main; the conformance vectors make an unrecognized CSI sequence a response, not a key.
   expect(result.name).toBe("")
   expect(result.ctrl).toBe(true)
   expect(result.meta).toBe(true)

packages/core/src/lib/parse.mouse.test.ts

@@ -314,6 +314,7 @@ describe("MouseParser SGR mode", () => {
     test("scroll release (m) is not classified as scroll", () => {
       // Some terminals send release for scroll too; the parser should not
       // report that as a scroll event.
+      // 044: pins main; the conformance vectors make a wheel release a response with no mouse event.
       const e = parser.parseMouseEvent(encodeSGR(64, 10, 5, false))!
       expect(e.type).not.toBe("scroll")
     })

packages/core/src/lib/stdin-parser.conformance.json

@@ -0,0 +1,51 @@
+{
+  "format": [
+    "Input parser conformance vectors. A row is [name, steps, expected, issue?].",
+    "A step is one read: a string (its UTF-8 bytes), an array of byte values, or null (the Escape timeout passes).",
+    "After the last step the timeout passes once more. The result must not change when each step arrives one byte at a time.",
+    "expected lists every event in order: key(NAME ctrl meta shift release), mouse(TYPE X,Y bBUTTON), response(SEQUENCE), paste(TEXT).",
+    "NAME, SEQUENCE, and TEXT are JSON strings as JavaScript JSON.stringify writes them (C0 controls as lowercase \\u00xx, non-ASCII literal).",
+    "SEQUENCE is the response bytes read as latin1 (one code point per byte); TEXT is the paste bytes read as UTF-8.",
+    "A row with an issue id is wrong on main; its test is a todo until the native port fixes it."
+  ],
+  "rows": [
+    ["ESC + SGR mouse motion", ["\u001b\u001b[<35;10;5M"], ["key(\"escape\")", "mouse(move 9,4 b0)"], "044"],
+    ["ESC, then SGR mouse motion in a 2nd read", ["\u001b", "\u001b[<35;10;5M"], ["key(\"escape\")", "mouse(move 9,4 b0)"], "044"],
+    ["ESC + X10 press", ["\u001b\u001b[M !!"], ["key(\"escape\")", "mouse(down 0,0 b0)"], "044"],
+    ["ESC + Kitty flags reply", ["\u001b\u001b[?1u"], ["key(\"escape\")", "response(\"\\u001b[?1u\")"], "044"],
+    ["ESC + bracketed paste", ["\u001b\u001b[200~hi\u001b[201~"], ["key(\"escape\")", "paste(\"hi\")"], "044"],
+    ["ESC + OSC 11 reply", ["\u001b\u001b]11;rgb:0000/0000/0000\u0007"], ["key(\"escape\")", "response(\"\\u001b]11;rgb:0000/0000/0000\\u0007\")"], "044"],
+    ["ESC + cursor position report", ["\u001b\u001b[12;1R"], ["key(\"escape\")", "response(\"\\u001b[12;1R\")"], "044"],
+    ["ESC + DA1 reply", ["\u001b\u001b[?62;22c"], ["key(\"escape\")", "response(\"\\u001b[?62;22c\")"], "044"],
+    ["ESC + focus-in", ["\u001b\u001b[I"], ["key(\"escape\")", "response(\"\\u001b[I\")"], "044"],
+    ["ESC + Alt+a", ["\u001b\u001ba"], ["key(\"escape\")", "key(\"a\" meta)"], "044"],
+    ["ESC ESC ESC", ["\u001b\u001b\u001b"], ["key(\"escape\")", "key(\"escape\" meta)"], "044"],
+    ["ESC + mouse, ESC + reply", ["\u001b\u001b[<0;1;1M\u001b\u001b[?1u"], ["key(\"escape\")", "mouse(down 0,0 b0)", "key(\"escape\")", "response(\"\\u001b[?1u\")"], "044"],
+    ["ESC + Alt+!", ["\u001b\u001b!"], ["key(\"escape\")", "key(\"!\" meta)"], "044"],
+    ["Alt+.", ["\u001b."], ["key(\".\" meta)"], "044"],
+    ["Alt+/", ["\u001b/"], ["key(\"/\" meta)"], "044"],
+    ["Alt+,", ["\u001b,"], ["key(\",\" meta)"], "044"],
+    ["Alt+;", ["\u001b;"], ["key(\";\" meta)"], "044"],
+    ["Alt+<", ["\u001b<"], ["key(\"<\" meta)"], "044"],
+    ["Alt+>", ["\u001b>"], ["key(\">\" meta)"], "044"],
+    ["Alt+!", ["\u001b!"], ["key(\"!\" meta)"], "044"],
+    ["8-bit Alt+@ (0xC0), then A", [[192, 65]], ["key(\"@\" meta)", "key(\"a\" shift)"], "044"],
+    ["Alt+é", ["\u001bé"], ["key(\"é\" meta)"], "044"],
+    ["Alt+ж", ["\u001bж"], ["key(\"ж\" meta)"], "044"],
+    ["Alt+中", ["\u001b中"], ["key(\"中\" meta)"], "044"],
+    ["SGR wheel-up release", ["\u001b[<64;11;6m"], ["response(\"\\u001b[<64;11;6m\")"], "044"],
+    ["SGR wheel-down release", ["\u001b[<65;11;6m"], ["response(\"\\u001b[<65;11;6m\")"], "044"],
+    ["press, drag, wheel-up, its release, drag", ["\u001b[<0;1;1M\u001b[<32;4;1M\u001b[<64;4;1M\u001b[<64;4;1m\u001b[<32;9;1M"], ["mouse(down 0,0 b0)", "mouse(drag 3,0 b0)", "mouse(scroll 3,0 b0)", "response(\"\\u001b[<64;4;1m\")", "mouse(drag 8,0 b0)"], "044"],
+    ["X10 press at column and row 223 (byte 0)", [[27, 91, 77, 32, 0, 0]], ["mouse(down 223,223 b0)"], "044"],
+    ["X10 report cut short by ESC [ A", ["\u001b[M \u001b[A"], ["response(\"\\u001b[M \")", "key(\"up\")"], "044"],
+    ["OSC string cut short by ESC [ A", ["\u001b]foo\u001b[A"], ["response(\"\\u001b]foo\")", "key(\"up\")"], "044"],
+    ["unrecognized CSI ESC [ < 0 M", ["\u001b[<0M"], ["response(\"\\u001b[<0M\")"], "044"],
+    ["unrecognized CSI ESC [ h", ["\u001b[h"], ["response(\"\\u001b[h\")"], "044"],
+    ["SGR mouse motion", ["\u001b[<35;10;5M"], ["mouse(move 9,4 b0)"]],
+    ["Alt+a", ["\u001ba"], ["key(\"a\" meta)"]],
+    ["ESC ESC (Alt+Escape)", ["\u001b\u001b"], ["key(\"escape\" meta)"]],
+    ["ESC ESC [A (Alt+Up)", ["\u001b\u001b[A"], ["key(\"up\" meta)"]],
+    ["ESC, timeout, ESC, timeout, ESC", ["\u001b", null, "\u001b", null, "\u001b"], ["key(\"escape\")", "key(\"escape\")", "key(\"escape\")"]],
+    ["ESC, timeout, SGR wheel-up without ESC", ["\u001b", null, "[<64;38;15M"], ["key(\"escape\")", "mouse(scroll 37,14 b0)"]]
+  ]
+}

packages/core/src/lib/stdin-parser.test.ts

@@ -4,6 +4,7 @@ import { ManualClock } from "../testing/manual-clock.js"
 import type { Clock, TimerHandle } from "./clock.js"
 import type { ScrollInfo } from "./parse.mouse.js"
 import { StdinParser, type StdinEvent, type StdinParserOptions } from "./stdin-parser.js"
+import conformance from "./stdin-parser.conformance.json" with { type: "json" }
 
 type KeySnap = {
   type: "key"
@@ -172,6 +173,43 @@ function assertChunkInvariant(input: Uint8Array, opts?: StdinParserOptions) {
   }
 }
 
+// Conformance vectors for the native parser port (issue 044); the JSON file's `format` field defines them.
+// `bun test --todo src/lib/stdin-parser.test.ts` also runs the todo rows: a todo row that passes fails that run,
+// so remove its issue id once the port fixes it.
+type ConformanceRow = [name: string, steps: Array<string | number[] | null>, expected: string[], issue?: string]
+const CONFORMANCE = conformance.rows as ConformanceRow[]
+
+function notation(event: Snap): string {
+  switch (event.type) {
+    case "key": {
+      const flags = (["ctrl", "meta", "shift"] as const).filter((flag) => event[flag]).map((flag) => ` ${flag}`)
+      const kind = event.eventType === "press" ? "" : ` ${event.eventType}`
+      return `key(${JSON.stringify(event.name)}${flags.join("")}${kind})`
+    }
+    case "mouse":
+      return `mouse(${event.event.type} ${event.event.x},${event.event.y} b${event.event.button})`
+    case "response":
+      return `response(${JSON.stringify(event.sequence)})`
+    case "paste":
+      return `paste(${JSON.stringify(Buffer.from(event.bytes).toString())})`
+  }
+}
+
+function runConformance(steps: ConformanceRow[1]): string[] {
+  const { parser, clock } = createTimedParser()
+  const out: string[] = []
+  try {
+    for (const step of [...steps, null]) {
+      if (step === null) clock.advance(TEST_TIMEOUT_MS)
+      else parser.push(buf(step))
+      out.push(...snap(parser).map(notation))
+    }
+    return out
+  } finally {
+    parser.destroy()
+  }
+}
+
 describe("StdinParser", () => {
   describe("printable ASCII", () => {
     test("lowercase a-z", () => {
@@ -2101,25 +2139,6 @@ describe("StdinParser", () => {
       }
     })
 
-    test("multiple sequential timeouts", () => {
-      const { parser, clock } = createTimedParser()
-      try {
-        parser.push(Buffer.from("\x1b"))
-        clock.advance(10)
-        expect(snap(parser)).toEqual([k("escape", { raw: "\x1b" })])
-
-        parser.push(Buffer.from("\x1b"))
-        clock.advance(10)
-        expect(snap(parser)).toEqual([k("escape", { raw: "\x1b" })])
-
-        parser.push(Buffer.from("\x1b"))
-        clock.advance(10)
-        expect(snap(parser)).toEqual([k("escape", { raw: "\x1b" })])
-      } finally {
-        parser.destroy()
-      }
-    })
-
     test("custom timeout delay", () => {
       const { parser, clock } = createTimedParser({ timeoutMs: 50 })
       try {
@@ -2490,6 +2509,7 @@ describe("StdinParser", () => {
       }
     })
 
+    // The first, second, and fourth rows pin behavior on main that the 044 conformance vectors replace.
     table([
       ["CSI with unknown final byte produces empty-name key", "\x1b[h", [k("", { raw: "\x1b[h" })]],
       ["ESC followed by punctuation stays one empty-name key", "\x1b!", [k("", { raw: "\x1b!" })]],
@@ -2523,6 +2543,18 @@ describe("StdinParser", () => {
     })
   })
 
+  describe("conformance vectors", () => {
+    for (const [name, steps, expected, issue] of CONFORMANCE) {
+      const bytewise = steps.flatMap((step) => (step === null ? [null] : Array.from(buf(step), (byte) => [byte])))
+      const run = () => {
+        expect(runConformance(steps)).toEqual(expected)
+        expect(runConformance(bytewise)).toEqual(expected)
+      }
+      if (issue) test.todo(`${name} (${issue})`, run)
+      else test(name, run)
+    }
+  })
+
   describe("timer/clock disagreement race condition", () => {
     test("timeout callback flushes even when now() reports slightly less elapsed time than timeoutMs", () => {
       const inner = new ManualClock()