perf(parser): remove token field from LexerCheckpoint (#26350)

Small perf optimization to parser.

`Lexer::token` is the token currently being built. Every method which produces a token ends in `finish_next_inner` or `finish_re_lex`, and both reset it to `Token::default()`. So between lexer calls it is always `Token::default()`, so there is no point in saving and restoring it - it's always a no-op.

Remove the `token` field from `LexerCheckpoint`, which reduces its size by 16 bytes.

Add a debug asserts in `checkpoint` and `checkpoint_with_error_recovery` checking this invariant.

crates/oxc_parser/src/lexer/mod.rs

@@ -50,7 +50,6 @@ use trivia_builder::TriviaBuilder;
 #[derive(Debug)]
 pub struct LexerCheckpoint<'a> {
     source_position: SourcePosition<'a>,
-    token: Token,
     errors_snapshot: ErrorSnapshot<'a>,
     tokens_len: usize,
     pure_comments: Option<(usize, usize)>,
@@ -81,6 +80,15 @@ pub struct Lexer<'a, C: Config> {
 
     source_type: SourceType,
 
+    /// The token currently being built.
+    ///
+    /// Between lexer calls, `token` is always `Token::default()`. Every method which produces a token
+    /// ends in `finish_next_inner` or `finish_re_lex`, and both reset it.
+    ///
+    /// The one exception is before the first token is lexed, when it is `Token::new_on_new_line()`.
+    /// `checkpoint` and `rewind` can never see that state, because the parser lexes the first token
+    /// before it does anything else, and they are the only things which depend on `token` being
+    /// `Token::default()`. Both assert it.
     token: Token,
 
     pub(crate) errors: Vec<ParserDiagnostic<'a>>,
@@ -188,14 +196,17 @@ impl<'a, C: Config> Lexer<'a, C> {
     /// Creates a checkpoint storing the current lexer state.
     /// Use `rewind` to restore the lexer to the state stored in the checkpoint.
     pub fn checkpoint(&self) -> LexerCheckpoint<'a> {
+        // `token` is always `Token::default()` here, so it does not need to be saved. See `token` field docs.
+        debug_assert_eq!(self.token, Token::default());
+
         let errors_snapshot = if self.errors.is_empty() {
             ErrorSnapshot::Empty
         } else {
             ErrorSnapshot::Count(self.errors.len())
         };
+
         LexerCheckpoint {
             source_position: self.source.position(),
-            token: self.token,
             errors_snapshot,
             tokens_len: self.tokens.len(),
             pure_comments: self.trivia_builder.previous_token_pure_comments(),
@@ -206,14 +217,17 @@ impl<'a, C: Config> Lexer<'a, C> {
     /// Create a checkpoint that can handle error popping.
     /// This is more expensive as it clones the errors vector.
     pub(crate) fn checkpoint_with_error_recovery(&self) -> LexerCheckpoint<'a> {
+        // `token` is always `Token::default()` here, so it does not need to be saved. See `token` field docs.
+        debug_assert_eq!(self.token, Token::default());
+
         let errors_snapshot = if self.errors.is_empty() {
             ErrorSnapshot::Empty
         } else {
             ErrorSnapshot::Full(self.errors.clone())
         };
+
         LexerCheckpoint {
             source_position: self.source.position(),
-            token: self.token,
             errors_snapshot,
             tokens_len: self.tokens.len(),
             pure_comments: self.trivia_builder.previous_token_pure_comments(),
@@ -223,14 +237,17 @@ impl<'a, C: Config> Lexer<'a, C> {
 
     /// Rewinds the lexer to the same state as when the passed in `checkpoint` was created.
     pub fn rewind(&mut self, checkpoint: LexerCheckpoint<'a>) {
+        // `token` is always `Token::default()` here, so it does not need to be restored. See `token` field docs.
+        debug_assert_eq!(self.token, Token::default());
+
         match checkpoint.errors_snapshot {
             ErrorSnapshot::Empty => self.errors.clear(),
             ErrorSnapshot::Count(len) => self.errors.truncate(len),
             ErrorSnapshot::Full(errors) => self.errors = errors,
         }
+
         self.tokens.truncate(checkpoint.tokens_len);
         self.source.set_position(checkpoint.source_position);
-        self.token = checkpoint.token;
         self.trivia_builder.set_pure_comments(checkpoint.pure_comments);
         self.trivia_builder.set_no_side_effects_comments(checkpoint.no_side_effects_comments);
     }

crates/oxc_parser/src/lexer/token.rs

@@ -52,7 +52,7 @@ const _: () = {
     assert!(is_valid_shift::<bool>(HAS_SEPARATOR_SHIFT));
 };
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, PartialEq, Eq)]
 #[repr(transparent)]
 pub struct Token(u128);