fix(parser): fix debug assert failure when lexer error with tokens enabled (#26229)

`Lexer::finalize_tokens` discards the last collected token - the `Eof` that terminates every token stream - and debug asserted that it was `Eof`.

Turns out that assumption does not always hold. Where the lexer cannot lex any further it emits `Kind::Undetermined`, and no `Eof` ever follows, so the last token is `Undetermined` and the debug assertion fails. This can be triggered by an unterminated string, for example:

```js
foo();
'unterminated
```

Like `Eof`, `Undetermined` is not a "real" token, so either way it's fine to remove it from the token stream. The fix is just to relax the debug assertion to accept either.

Release behaviour is unchanged - it was only a debug assert. Only came across this while running the conformance tests which compare current lexer to the new one (`oxc_lexer`).

apps/oxlint/test/fixtures/tokens/files/unterminated_string.js

@@ -0,0 +1,4 @@
+// Lexing ends in an error, so the token stream ends with `Undetermined` rather than `Eof`.
+// This test case ensures that doesn't cause a panic.
+foo();
+'unterminated

apps/oxlint/test/fixtures/tokens/output.snap.md

@@ -2845,8 +2845,15 @@
    :  ^
    `----
 
-Found 0 warnings and 251 errors.
-Finished in Xms on 8 files with 1 rules using X threads.
+  x Unterminated string
+   ,-[files/unterminated_string.js:4:1]
+ 3 | foo();
+ 4 | 'unterminated
+   : ^^^^^^^^^^^^^^
+   `----
+
+Found 0 warnings and 252 errors.
+Finished in Xms on 9 files with 1 rules using X threads.
 ```
 
 # stderr

crates/oxc_parser/src/lexer/mod.rs

@@ -370,10 +370,16 @@ impl<'a, C: Config> Lexer<'a, C> {
     /// Called at very end of parsing.
     pub(crate) fn finalize_tokens(&mut self) -> ArenaVec<'a, Token> {
         if self.config.tokens() {
-            // Tokens are enabled. Discard last token, which is `Eof`.
+            // Tokens are enabled. Discard the last token, which marks the end of input.
+            // Usually that is `Eof`. Where the lexer cannot lex any further - an unterminated string,
+            // for example - it emits `Undetermined` instead, and no `Eof` ever follows.
+            // The parser treats the two alike, and neither is a real token, so both are discarded.
             let mut tokens = self.take_tokens();
             let last_token = tokens.pop();
-            debug_assert!(last_token.is_some_and(|token| token.kind() == Kind::Eof));
+            debug_assert!(
+                last_token
+                    .is_some_and(|token| matches!(token.kind(), Kind::Eof | Kind::Undetermined))
+            );
             tokens
         } else {
             // Tokens are disabled. Just return an empty vec.

crates/oxc_parser/src/lib.rs

@@ -1274,4 +1274,34 @@ mod test {
         assert!(ret.diagnostics.is_empty());
         assert_eq!(ret.program.body.len(), 2);
     }
+
+    // Where the lexer cannot lex any further it emits `Undetermined`, and no `Eof` follows.
+    // That final token ends the input just as `Eof` does, so it is discarded, and discarding it
+    // must not take the real token before it with it.
+    #[test]
+    fn tokens_when_lexing_ends_in_error() {
+        let allocator = Allocator::default();
+        let ret = Parser::new(&allocator, "foo();\n'unterminated", SourceType::default())
+            .with_config(config::TokensParserConfig)
+            .parse();
+
+        assert!(!ret.panicked);
+        assert_eq!(ret.diagnostics.len(), 1);
+        assert_eq!(ret.diagnostics.first().unwrap().to_string(), "Unterminated string");
+
+        let tokens = ret
+            .tokens
+            .iter()
+            .map(|token| (token.kind(), token.start(), token.end()))
+            .collect::<Vec<_>>();
+        assert_eq!(
+            tokens,
+            [
+                (Kind::Ident, 0, 3),
+                (Kind::LParen, 3, 4),
+                (Kind::RParen, 4, 5),
+                (Kind::Semicolon, 5, 6),
+            ]
+        );
+    }
 }