fix(regular_expression): reject oversized backreferences (#26055)

## Summary

- keep decimal backreference values as `u64` until capture-count validation
- reject oversized indexed references in Unicode and Unicode Sets modes
- preserve Annex B fallback behavior in non-Unicode mode
- cover `u32::MAX`, `u32::MAX + 1`, and the reported wrap-to-1 regression

crates/oxc_regular_expression/src/parser/mod.rs

@@ -289,6 +289,33 @@ mod test {
         }
     }
 
+    #[test]
+    fn oversized_decimal_backreferences() {
+        let allocator = Allocator::default();
+        let patterns = [r"()\4294967295", r"()\4294967296", r"()\4294967297"];
+
+        for pattern_text in patterns {
+            for flags_text in ["u", "v"] {
+                assert!(
+                    LiteralParser::new(
+                        &allocator,
+                        pattern_text,
+                        Some(flags_text),
+                        Options::default(),
+                    )
+                    .parse()
+                    .is_err(),
+                    "/{pattern_text}/{flags_text} should reject an out-of-range backreference"
+                );
+            }
+
+            let pattern = LiteralParser::new(&allocator, pattern_text, None, Options::default())
+                .parse()
+                .expect("Annex B decimal escape fallback should parse");
+            assert_eq!(pattern.to_string(), pattern_text);
+        }
+    }
+
     #[test]
     fn should_handle_empty() {
         let allocator = Allocator::default();

crates/oxc_regular_expression/src/parser/pattern_parser/pattern_parser_impl.rs

@@ -474,13 +474,14 @@ impl<'a> PatternParser<'a> {
             if self.state.unicode_mode {
                 // [SS:EE] AtomEscape :: DecimalEscape
                 // It is a Syntax Error if the CapturingGroupNumber of DecimalEscape is strictly greater than CountLeftCapturingParensWithin(the Pattern containing AtomEscape).
-                if self.state.num_of_capturing_groups < index {
+                if u64::from(self.state.num_of_capturing_groups) < index {
                     return Err(diagnostics::invalid_indexed_reference(
                         self.span_factory.create(span_start, self.reader.offset()),
                         self.state.num_of_capturing_groups as usize,
                     ));
                 }
 
+                let index = u32::try_from(index).expect("validated capture index must fit in u32");
                 return Ok(Some(ast::Term::IndexedReference(ArenaBox::new_in(
                     ast::IndexedReference {
                         span: self.span_factory.create(span_start, self.reader.offset()),
@@ -490,7 +491,8 @@ impl<'a> PatternParser<'a> {
                 ))));
             }
 
-            if index <= self.state.num_of_capturing_groups {
+            if index <= u64::from(self.state.num_of_capturing_groups) {
+                let index = u32::try_from(index).expect("validated capture index must fit in u32");
                 return Ok(Some(ast::Term::IndexedReference(ArenaBox::new_in(
                     ast::IndexedReference {
                         span: self.span_factory.create(span_start, self.reader.offset()),
@@ -1776,14 +1778,13 @@ impl<'a> PatternParser<'a> {
     // DecimalEscape ::
     //   NonZeroDigit DecimalDigits[~Sep][opt] [lookahead ∉ DecimalDigit]
     // ```
-    fn consume_decimal_escape(&mut self) -> Result<Option<u32>> {
+    fn consume_decimal_escape(&mut self) -> Result<Option<u64>> {
         let checkpoint = self.reader.checkpoint();
 
         if let Some(index) = self.consume_decimal_digits()? {
             // \0 is CharacterEscape, not DecimalEscape
             if index != 0 {
-                #[expect(clippy::cast_possible_truncation)]
-                return Ok(Some(index as u32));
+                return Ok(Some(index));
             }
 
             self.reader.rewind(checkpoint);