fix(parser): do not omit < token opening type argument list (#26328)

Fix tokens produced when `<<` opens a type argument list. e.g.:

```typescript
type Bar = ReturnType<<T>(x: T) => number>;
```

Previously only 1 `<` token was produced between `ReturnType` and `T`. This PR corrects that so 2 `<` tokens are produced.

TS-ESLint get this wrong too - hence why this wasn't noticed before, as our conformance tests compare to TS-ESLint.

PR submitted to fix TS-ESLint: https://github.com/typescript-eslint/typescript-eslint/pull/12821

In meantime, this PR skips the erroneously failing ESTree tokens conformance tests. We can can re-enable those tests once the fix lands in TS-ESLint.

crates/oxc_parser/src/js/expression.rs

@@ -926,9 +926,9 @@ impl<'a, C: Config> ParserImpl<'a, C> {
                             self,
                         );
                     } else {
-                        // `re_lex_as_typescript_l_angle` may have popped the original token
-                        // (e.g. `<<`) from the collected token stream. Rewind restored the
-                        // parser's current token, so write it back to the stream.
+                        // `re_lex_as_typescript_l_angle` may have overwritten the original `<<`
+                        // in the collected token stream with the single `<` it re-lexed.
+                        // Rewind restored the parser's current token, so write it back over that `<`.
                         // This is a no-op when tokens are statically disabled (`NoTokensLexerConfig`).
                         self.lexer.rewrite_last_collected_token(self.token);
                         return lhs;
@@ -1127,9 +1127,9 @@ impl<'a, C: Config> ParserImpl<'a, C> {
                     if let Some(args) = self.parse_type_arguments_in_expression() {
                         type_arguments = Some(args);
                     } else {
-                        // `re_lex_as_typescript_l_angle` may have popped the original token
-                        // (e.g. `<<`) from the collected token stream. Rewind restored the
-                        // parser's current token, so write it back to the stream.
+                        // `re_lex_as_typescript_l_angle` may have overwritten the original `<<`
+                        // in the collected token stream with the single `<` it re-lexed.
+                        // Rewind restored the parser's current token, so write it back over that `<`.
                         // This is a no-op when tokens are statically disabled (`NoTokensLexerConfig`).
                         self.lexer.rewrite_last_collected_token(self.token);
                     }

crates/oxc_parser/src/lexer/mod.rs

@@ -332,8 +332,9 @@ impl<'a, C: Config> Lexer<'a, C> {
 
     /// Overwrite the last token in the collected token stream.
     ///
-    /// Used to restore a token that was popped by `re_lex_as_typescript_l_angle`
-    /// when `try_parse` fails and rewinds.
+    /// Used by `re_lex_as_typescript_l_angle` to replace a compound token (`<<`, `<=` or `<<=`)
+    /// with the single `<` it re-lexed, and to restore the compound token when `try_parse`
+    /// fails and rewinds.
     #[inline]
     pub(crate) fn rewrite_last_collected_token(&mut self, token: Token) {
         // Make this function a no-op when tokens are statically disabled (`NoTokensLexerConfig`)

crates/oxc_parser/src/lexer/typescript.rs

@@ -10,24 +10,25 @@ impl<C: Config> Lexer<'_, C> {
     /// The lexer eagerly produces the compound token, but the parser needs a single `<`
     /// to speculatively try parsing type arguments.
     ///
-    /// The compound token was pushed to the collected token stream *before* `try_parse`
-    /// created its checkpoint (since it was the "current" token at that point).
-    /// This means checkpoint's `tokens_len` includes it, and `truncate` on rewind
-    /// won't remove it. So we must pop it here. If the speculative parse succeeds,
-    /// the individual `<` and subsequent tokens replace it naturally. If it fails
-    /// and `try_parse` rewinds, the caller (`expression.rs`) restores the original
-    /// compound token via `rewrite_last_collected_token`.
+    /// Only `<<` reaches here in code which parses. `<=` and `<<=` arrive only from the type-context
+    /// callers in `ts/types.rs` and always go on to fail, for the reason documented on
+    /// `parse_type_arguments_in_expression`, which rejects them before reaching here.
     ///
-    /// The remaining characters after the first `<` (e.g. the second `<` in `<<`)
-    /// will be lexed as separate tokens on subsequent `next_token` calls.
+    /// The compound token was pushed to the collected token stream *before* `try_parse`
+    /// created its checkpoint (since it was the "current" token at that point), so it is
+    /// the last token in the stream. Overwrite it there with the single `<`. The remaining
+    /// characters after the first `<` (e.g. the second `<` in `<<`) are lexed as separate
+    /// tokens on subsequent `next_token` calls, so the stream ends up with all of them.
+    ///
+    /// If the speculative parse fails and `try_parse` rewinds, `truncate` leaves this `<`
+    /// as the last collected token, and the caller (`expression.rs`) restores the original
+    /// compound token over it via `rewrite_last_collected_token`.
     pub(crate) fn re_lex_as_typescript_l_angle(&mut self, offset: u32) -> Token {
         self.token.set_start(self.offset() - offset);
         self.source.back(offset as usize - 1);
-        if self.config.tokens() {
-            let popped = self.tokens.pop();
-            debug_assert!(popped.is_some());
-        }
-        self.finish_re_lex(Kind::LAngle)
+        let token = self.finish_re_lex(Kind::LAngle);
+        self.rewrite_last_collected_token(token);
+        token
     }
 
     /// Re-tokenize `>>` or `>>>` to `>`.

napi/parser/test/parse-raw.test.ts

@@ -210,9 +210,21 @@ describeRangeParent.concurrent("range & parent TypeScript", () => {
   );
 });
 
+// TS-ESLint drops the first `<` of a `<<` which opens a type argument list,
+// e.g. `ReturnType<<T>(x: T) => number>`. Oxc emits both `<` tokens.
+// These fixtures' ASTs match TS-ESLint, so only their token tests are skipped.
+// https://github.com/typescript-eslint/typescript-eslint/issues/12820
+const TS_TOKENS_SKIP_PATHS = new Set([
+  "tests/cases/compiler/importTypeWithUnparenthesizedGenericFunctionParsed.ts.md",
+  "tests/cases/compiler/parseGenericArrowRatherThanLeftShift.ts.md",
+]);
+const tsTokensFixturePaths = tsFixturePaths.filter((path) => !TS_TOKENS_SKIP_PATHS.has(path));
+
 describeTokens.concurrent("tokens TypeScript", () => {
   // oxlint-disable-next-line jest/expect-expect
-  it.each(tsFixturePaths)("%s", (path) => runCaseInWorker(TEST_TYPE_TS | TEST_TYPE_TOKENS, path));
+  it.each(tsTokensFixturePaths)("%s", (path) =>
+    runCaseInWorker(TEST_TYPE_TS | TEST_TYPE_TOKENS, path),
+  );
 });
 
 // Check lazy deserialization doesn't throw

tasks/coverage/snapshots/estree_typescript_tokens.snap

@@ -1,5 +1,5 @@
 commit: b465fdbf
 
 estree_typescript_tokens Summary:
-AST Parsed     : 9723/9723 (100.00%)
-Positive Passed: 9723/9723 (100.00%)
+AST Parsed     : 9721/9721 (100.00%)
+Positive Passed: 9721/9721 (100.00%)

tasks/coverage/src/tools.rs

@@ -907,8 +907,18 @@ static TS_SKIP_PATHS: &[&str] = &[
     "typescript/tests/cases/conformance/es6/templates/templateStringMultiline3.ts",
 ];
 
+// Additional skip paths for the TypeScript ESTree *token* tests.
+// These fixtures' ASTs match TS-ESLint, so only the token comparison is skipped.
+static TS_TOKENS_SKIP_PATHS: &[&str] = &[
+    // TS-ESLint drops the first `<` of a `<<` which opens a type argument list,
+    // e.g. `ReturnType<<T>(x: T) => number>`. Oxc correctly emits both `<` tokens.
+    // <https://github.com/typescript-eslint/typescript-eslint/issues/12820>
+    "typescript/tests/cases/compiler/importTypeWithUnparenthesizedGenericFunctionParsed.ts",
+    "typescript/tests/cases/compiler/parseGenericArrowRatherThanLeftShift.ts",
+];
+
 pub fn run_estree_typescript(files: &[TypeScriptFile]) -> Vec<CoverageResult> {
-    run_estree_typescript_impl(files, "AST", RuntimeParserConfig::default(), |ret| {
+    run_estree_typescript_impl(files, "AST", &[], RuntimeParserConfig::default(), |ret| {
         let mut program = ret.program;
         let source_text = program.source_text;
         Utf8ToUtf16::new(source_text).convert_program_with_ascending_order_checks(&mut program);
@@ -917,23 +927,30 @@ pub fn run_estree_typescript(files: &[TypeScriptFile]) -> Vec<CoverageResult> {
 }
 
 pub fn run_estree_typescript_tokens(files: &[TypeScriptFile]) -> Vec<CoverageResult> {
-    run_estree_typescript_impl(files, "TOKENS", RuntimeParserConfig::new(true), |ret| {
-        let ParserReturn { program, tokens, .. } = ret;
-        let source_text = program.source_text;
-        let span_converter = Utf8ToUtf16::new(source_text);
-        to_estree_tokens_pretty_json(
-            &tokens,
-            &program,
-            source_text,
-            &span_converter,
-            ESTreeTokenOptions::new(true),
-        )
-    })
+    run_estree_typescript_impl(
+        files,
+        "TOKENS",
+        TS_TOKENS_SKIP_PATHS,
+        RuntimeParserConfig::new(true),
+        |ret| {
+            let ParserReturn { program, tokens, .. } = ret;
+            let source_text = program.source_text;
+            let span_converter = Utf8ToUtf16::new(source_text);
+            to_estree_tokens_pretty_json(
+                &tokens,
+                &program,
+                source_text,
+                &span_converter,
+                ESTreeTokenOptions::new(true),
+            )
+        },
+    )
 }
 
 fn run_estree_typescript_impl(
     files: &[TypeScriptFile],
-    section_key: &'static str,
+    section_key: &str,
+    extra_skip_paths: &[&str],
     parser_config: RuntimeParserConfig,
     get_json: impl for<'a> Fn(ParserReturn<'a>) -> String + Sync,
 ) -> Vec<CoverageResult> {
@@ -943,7 +960,11 @@ fn run_estree_typescript_impl(
             if test_file.should_fail {
                 return None;
             }
-            if test_file.path.to_str().is_some_and(|p| TS_SKIP_PATHS.contains(&p)) {
+            if test_file
+                .path
+                .to_str()
+                .is_some_and(|p| TS_SKIP_PATHS.contains(&p) || extra_skip_paths.contains(&p))
+            {
                 return None;
             }