mirror of
https://github.com/aphrody-labs/oxc.git
synced 2026-10-15 22:59:17 +00:00
feat(str): add JSStr, JSChar and JSStrBuilder types (#26435)
Part of #26242. This PR adds the `JSStr`, `JSChar` and `JSStrBuilder` types, along with many methods that are all used in follow-up PRs, which we can verify by checking them.
This commit is contained in:
1 parent
2455cd4818
commit
02331c8f70
6 files changed
+1330
-3
No files matched your search
@@ -12,6 +12,7 @@ on:
|
||||
- "crates/oxc_ast/src/generated/utf8_to_utf16_converter.rs"
|
||||
- "crates/oxc_data_structures/**"
|
||||
- "crates/oxc_parser/**"
|
||||
- "crates/oxc_str/**"
|
||||
- "crates/oxc_traverse/**"
|
||||
- ".github/workflows/miri.yml"
|
||||
push:
|
||||
@@ -23,6 +24,7 @@ on:
|
||||
- "crates/oxc_ast/src/generated/utf8_to_utf16_converter.rs"
|
||||
- "crates/oxc_data_structures/**"
|
||||
- "crates/oxc_parser/**"
|
||||
- "crates/oxc_str/**"
|
||||
- "crates/oxc_traverse/**"
|
||||
- ".github/workflows/miri.yml"
|
||||
|
||||
@@ -44,7 +46,7 @@ jobs:
|
||||
- uses: ./.github/actions/check-changes
|
||||
id: filter
|
||||
with:
|
||||
packages: oxc_allocator,oxc_ast,oxc_data_structures,oxc_parser,oxc_transformer
|
||||
packages: oxc_allocator,oxc_ast,oxc_data_structures,oxc_parser,oxc_str,oxc_transformer
|
||||
paths: .github/workflows/miri.yml
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
pr-number: ${{ github.event.pull_request.number }}
|
||||
@@ -67,5 +69,5 @@ jobs:
|
||||
env:
|
||||
MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-disable-isolation
|
||||
run: |
|
||||
cargo +"${MIRI_TOOLCHAIN}" miri test --all-features -p oxc_allocator -p oxc_ast -p oxc_data_structures
|
||||
cargo +"${MIRI_TOOLCHAIN}" miri test --all-features -p oxc_allocator -p oxc_ast -p oxc_data_structures -p oxc_str
|
||||
cargo +"${MIRI_TOOLCHAIN}" miri test -p oxc_parser -p oxc_transformer
|
||||
@@ -0,0 +1,84 @@
|
||||
/// A JavaScript string code point, including lone surrogates.
|
||||
///
|
||||
/// The value is always in `0..=0x10_FFFF`. Unlike [`char`], this includes the
|
||||
/// surrogate range `0xD800..=0xDFFF`. A supplementary character is one `JSChar`,
|
||||
/// even though it occupies two UTF-16 code units.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
#[repr(transparent)]
|
||||
pub struct JSChar(u32);
|
||||
|
||||
impl JSChar {
|
||||
/// Construct a code point, returning `None` if `value > 0x10_FFFF`.
|
||||
#[inline]
|
||||
pub const fn from_u32(value: u32) -> Option<Self> {
|
||||
if value <= 0x10_FFFF { Some(Self(value)) } else { None }
|
||||
}
|
||||
|
||||
/// Return the code point's numeric value.
|
||||
#[inline]
|
||||
pub const fn to_u32(self) -> u32 {
|
||||
self.0
|
||||
}
|
||||
|
||||
/// Return a Unicode scalar value, or `None` for a lone surrogate.
|
||||
#[inline]
|
||||
pub const fn to_char(self) -> Option<char> {
|
||||
if self.is_surrogate() {
|
||||
None
|
||||
} else {
|
||||
// SAFETY: `JSChar` is at most 0x10_FFFF, and we excluded surrogates.
|
||||
Some(unsafe { char::from_u32_unchecked(self.0) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct a code point without checking its range.
|
||||
///
|
||||
/// # Safety
|
||||
/// `value` must be at most `0x10_FFFF`.
|
||||
#[inline]
|
||||
pub(super) const unsafe fn from_u32_unchecked(value: u32) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) const fn from_code_unit(unit: u16) -> Self {
|
||||
Self(unit as u32)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) const fn is_surrogate(self) -> bool {
|
||||
self.0 >= 0xD800 && self.0 <= 0xDFFF
|
||||
}
|
||||
|
||||
/// Encode one code point. The caller handles pairing adjacent surrogates.
|
||||
#[inline]
|
||||
#[expect(clippy::cast_possible_truncation, reason = "each byte is masked or range-checked")]
|
||||
pub(super) fn encode(self, buffer: &mut [u8; 4]) -> &[u8] {
|
||||
let value = self.0;
|
||||
let len = match value {
|
||||
0..=0x7F => {
|
||||
buffer[0] = value as u8;
|
||||
1
|
||||
}
|
||||
0x80..=0x7FF => {
|
||||
buffer[0] = 0xC0 | (value >> 6) as u8;
|
||||
buffer[1] = 0x80 | (value & 0x3F) as u8;
|
||||
2
|
||||
}
|
||||
0x800..=0xFFFF => {
|
||||
buffer[0] = 0xE0 | (value >> 12) as u8;
|
||||
buffer[1] = 0x80 | ((value >> 6) & 0x3F) as u8;
|
||||
buffer[2] = 0x80 | (value & 0x3F) as u8;
|
||||
3
|
||||
}
|
||||
_ => {
|
||||
buffer[0] = 0xF0 | (value >> 18) as u8;
|
||||
buffer[1] = 0x80 | ((value >> 12) & 0x3F) as u8;
|
||||
buffer[2] = 0x80 | ((value >> 6) & 0x3F) as u8;
|
||||
buffer[3] = 0x80 | (value & 0x3F) as u8;
|
||||
4
|
||||
}
|
||||
};
|
||||
&buffer[..len]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
use std::{
|
||||
fmt::{self, Write},
|
||||
hash::{Hash, Hasher},
|
||||
iter::FusedIterator,
|
||||
marker::PhantomData,
|
||||
ptr::NonNull,
|
||||
slice, str,
|
||||
};
|
||||
|
||||
use oxc_allocator::{Allocator, CloneIn, CloneInSemanticIds, Dummy, GetAllocator};
|
||||
|
||||
use crate::{Ident, JSChar, JSStrBuilder, Str};
|
||||
|
||||
/// An immutable JavaScript string borrowed from source text or arena memory.
|
||||
///
|
||||
/// JavaScript strings can contain lone surrogates, which Rust's [`prim@str`] cannot
|
||||
/// represent. `JSStr` stores [canonical WTF-8](https://wtf-8.codeberg.page/):
|
||||
/// Unicode scalar values use UTF-8, lone surrogates use three bytes, and a
|
||||
/// surrogate pair uses one four-byte supplementary character.
|
||||
///
|
||||
/// # Invariants
|
||||
///
|
||||
/// * The bytes are canonical WTF-8. In particular, a lead surrogate encoding
|
||||
/// cannot be immediately followed by a trail surrogate encoding.
|
||||
/// * `has_lone_surrogate` is true if and only if the bytes encode a lone surrogate.
|
||||
/// When false, the bytes are valid UTF-8.
|
||||
/// * The pointer references `len` initialized bytes, valid and immutable for `'a`.
|
||||
/// * The byte length fits in `u32` and does not exceed `isize::MAX`.
|
||||
///
|
||||
/// All constructors maintain these invariants. [`as_bytes`](Self::as_bytes)
|
||||
/// exposes the bytes read-only and [`from_bytes_unchecked`](Self::from_bytes_unchecked)
|
||||
/// re-borrows them under the same invariants; consumers read the value with
|
||||
/// [`as_str`](Self::as_str), [`chars`](Self::chars), or
|
||||
/// [`encode_utf16`](Self::encode_utf16).
|
||||
///
|
||||
/// Equality uses the canonical bytes, and UTF-8 values hash like `str`. Ordering
|
||||
/// by these bytes would differ from JavaScript's UTF-16 ordering, so `JSStr`
|
||||
/// does not implement `Ord`.
|
||||
///
|
||||
/// ```
|
||||
/// use oxc_allocator::Allocator;
|
||||
/// use oxc_str::JSStrBuilder;
|
||||
///
|
||||
/// let allocator = Allocator::new();
|
||||
/// let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
/// builder.push_code_unit(0xD800);
|
||||
/// builder.push('a');
|
||||
/// let value = builder.into_js_str();
|
||||
/// assert_eq!(value.as_str(), None);
|
||||
/// assert_eq!(value.encode_utf16().collect::<Vec<_>>(), [0xD800, 0x61]);
|
||||
/// ```
|
||||
///
|
||||
/// Borrowing source text does not extend its lifetime:
|
||||
/// ```compile_fail
|
||||
/// use oxc_str::JSStr;
|
||||
/// let value;
|
||||
/// {
|
||||
/// let source = String::from("hello");
|
||||
/// value = JSStr::from(source.as_str());
|
||||
/// }
|
||||
/// println!("{value:?}");
|
||||
/// ```
|
||||
///
|
||||
/// Arena-backed strings prevent resetting the arena while still in use:
|
||||
/// ```compile_fail
|
||||
/// use oxc_allocator::Allocator;
|
||||
/// use oxc_str::JSStrBuilder;
|
||||
/// let mut allocator = Allocator::new();
|
||||
/// let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
/// builder.push_code_unit(0xD800);
|
||||
/// let value = builder.into_js_str();
|
||||
/// allocator.reset();
|
||||
/// println!("{value:?}");
|
||||
/// ```
|
||||
#[derive(Clone, Copy)]
|
||||
#[repr(C)]
|
||||
pub struct JSStr<'a> {
|
||||
ptr: NonNull<u8>,
|
||||
len: u32,
|
||||
has_lone_surrogate: bool,
|
||||
_marker: PhantomData<&'a [u8]>,
|
||||
}
|
||||
|
||||
impl JSStr<'static> {
|
||||
/// Return the empty string without allocating.
|
||||
#[inline]
|
||||
pub const fn empty() -> Self {
|
||||
Self::from_str("")
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> JSStr<'a> {
|
||||
/// Borrow UTF-8 bytes. Their validity already proves the WTF-8 invariant.
|
||||
#[inline]
|
||||
const fn from_str(value: &'a str) -> Self {
|
||||
assert!(value.len() <= u32::MAX as usize, "JavaScript string exceeds u32::MAX bytes");
|
||||
// SAFETY: `str` is UTF-8 (thus canonical WTF-8 without surrogates), and
|
||||
// its bytes are immutable and valid for `'a`. The length was checked.
|
||||
unsafe { Self::from_bytes_unchecked(value.as_bytes(), false) }
|
||||
}
|
||||
|
||||
/// Copy UTF-8 text into an arena.
|
||||
///
|
||||
/// # Panics
|
||||
/// Panics if the byte length exceeds `u32::MAX`.
|
||||
#[inline]
|
||||
pub fn from_str_in(value: &str, allocator: &impl GetAllocator<'a>) -> Self {
|
||||
// Check the length before allocating or copying.
|
||||
JSStr::from(value).clone_in(allocator.allocator())
|
||||
}
|
||||
|
||||
/// Concatenate JavaScript strings into an arena.
|
||||
///
|
||||
/// Leading and trailing surrogates pair across string boundaries, including
|
||||
/// empty strings between them. Inputs are copied into the destination arena.
|
||||
///
|
||||
/// # Panics
|
||||
/// Panics if the sum of input byte lengths exceeds `u32::MAX` or `isize::MAX`.
|
||||
#[inline]
|
||||
pub fn from_js_strs_array_in<const N: usize>(
|
||||
strings: [JSStr<'_>; N],
|
||||
allocator: &impl GetAllocator<'a>,
|
||||
) -> Self {
|
||||
let capacity = strings
|
||||
.iter()
|
||||
.try_fold(0usize, |len, value| len.checked_add(value.len()))
|
||||
.expect("JavaScript string capacity overflow");
|
||||
let mut builder = JSStrBuilder::with_capacity_in(capacity, allocator);
|
||||
for value in strings {
|
||||
builder.push_js_str(value);
|
||||
}
|
||||
builder.into_js_str()
|
||||
}
|
||||
|
||||
/// Borrow the value as UTF-8, or return `None` if it contains a lone surrogate.
|
||||
///
|
||||
/// This checks the cached flag in O(1); it does not scan the bytes.
|
||||
#[inline]
|
||||
pub fn as_str(self) -> Option<&'a str> {
|
||||
if self.has_lone_surrogate() {
|
||||
None
|
||||
} else {
|
||||
// SAFETY: By `JSStr`'s invariant, canonical WTF-8 with a false
|
||||
// surrogate flag is valid UTF-8.
|
||||
Some(unsafe { str::from_utf8_unchecked(self.as_bytes()) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the byte length of the WTF-8 representation.
|
||||
///
|
||||
/// Use [`len_utf16`](Self::len_utf16) for JavaScript's string length.
|
||||
#[inline]
|
||||
pub const fn len(self) -> usize {
|
||||
self.len as usize
|
||||
}
|
||||
|
||||
/// Return whether the string is empty.
|
||||
#[inline]
|
||||
pub const fn is_empty(self) -> bool {
|
||||
self.len == 0
|
||||
}
|
||||
|
||||
/// Return whether the string contains a lone surrogate, in O(1).
|
||||
#[inline]
|
||||
pub const fn has_lone_surrogate(self) -> bool {
|
||||
self.has_lone_surrogate
|
||||
}
|
||||
|
||||
/// Count UTF-16 code units, including lone surrogates.
|
||||
///
|
||||
/// This scans the bytes in O(n) time.
|
||||
pub fn len_utf16(self) -> usize {
|
||||
// Every non-continuation byte starts one code point. Four-byte code
|
||||
// points contribute a second code unit. Valid WTF-8 has no other bytes
|
||||
// >= 0xF0, so this counts code units without decoding individual points.
|
||||
self.as_bytes()
|
||||
.iter()
|
||||
.map(|&byte| usize::from(byte & 0xC0 != 0x80) + usize::from(byte >= 0xF0))
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Iterate code points. Paired surrogates yield one supplementary `JSChar`.
|
||||
#[inline]
|
||||
pub fn chars(self) -> impl FusedIterator<Item = JSChar> + Clone + 'a {
|
||||
JSChars { remaining: self.as_bytes() }
|
||||
}
|
||||
|
||||
/// Iterate UTF-16 code units, preserving lone surrogates.
|
||||
#[inline]
|
||||
pub fn encode_utf16(self) -> impl FusedIterator<Item = u16> + Clone + 'a {
|
||||
EncodeUtf16 { chars: JSChars { remaining: self.as_bytes() }, pending: 0 }
|
||||
}
|
||||
|
||||
/// Borrow the underlying canonical WTF-8 bytes.
|
||||
///
|
||||
/// Together with [`has_lone_surrogate`](Self::has_lone_surrogate), the
|
||||
/// bytes are a complete representation: storage held outside an arena can
|
||||
/// keep them and borrow the value back with
|
||||
/// [`from_bytes_unchecked`](Self::from_bytes_unchecked).
|
||||
#[inline]
|
||||
pub fn as_bytes(self) -> &'a [u8] {
|
||||
// SAFETY: `JSStr`'s pointer references `len` initialized bytes, valid
|
||||
// and immutable for `'a`, including the zero-length case.
|
||||
unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len()) }
|
||||
}
|
||||
|
||||
/// Borrow bytes whose encoding and metadata have already been established,
|
||||
/// such as bytes previously taken from [`as_bytes`](Self::as_bytes)
|
||||
/// together with the value's surrogate flag.
|
||||
///
|
||||
/// # Safety
|
||||
/// * `bytes` must be canonical WTF-8: valid WTF-8 in which a lead
|
||||
/// surrogate encoding is never followed by a trail surrogate encoding.
|
||||
/// * `has_lone_surrogate` must exactly describe whether they encode a lone surrogate.
|
||||
/// * `bytes.len()` must fit in `u32`.
|
||||
///
|
||||
/// The reference itself guarantees initialization, immutability, lifetime,
|
||||
/// and the `isize::MAX` bound.
|
||||
#[inline]
|
||||
#[expect(clippy::cast_possible_truncation, reason = "the caller guarantees the length fits")]
|
||||
pub const unsafe fn from_bytes_unchecked(bytes: &'a [u8], has_lone_surrogate: bool) -> Self {
|
||||
Self {
|
||||
ptr: NonNull::from_ref(bytes).cast(),
|
||||
len: bytes.len() as u32,
|
||||
has_lone_surrogate,
|
||||
_marker: PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: The only referenced storage is an immutable byte slice valid for
|
||||
// `'a`. `JSStr` exposes neither mutation nor an allocator reference.
|
||||
unsafe impl Send for JSStr<'_> {}
|
||||
// SAFETY: Sharing `JSStr` only shares immutable bytes, as with `&[u8]`.
|
||||
unsafe impl Sync for JSStr<'_> {}
|
||||
|
||||
impl<'a> From<&'a str> for JSStr<'a> {
|
||||
/// Borrow UTF-8 without allocating. Panics if its byte length exceeds `u32::MAX`.
|
||||
#[inline]
|
||||
fn from(value: &'a str) -> Self {
|
||||
Self::from_str(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<Str<'a>> for JSStr<'a> {
|
||||
#[inline]
|
||||
fn from(value: Str<'a>) -> Self {
|
||||
Self::from(value.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<Ident<'a>> for JSStr<'a> {
|
||||
#[inline]
|
||||
fn from(value: Ident<'a>) -> Self {
|
||||
Self::from(value.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for JSStr<'_> {
|
||||
#[inline]
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.as_bytes() == other.as_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for JSStr<'_> {}
|
||||
|
||||
impl PartialEq<str> for JSStr<'_> {
|
||||
#[inline]
|
||||
fn eq(&self, other: &str) -> bool {
|
||||
self.as_bytes() == other.as_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq<&str> for JSStr<'_> {
|
||||
#[inline]
|
||||
fn eq(&self, other: &&str) -> bool {
|
||||
self == *other
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq<JSStr<'_>> for str {
|
||||
#[inline]
|
||||
fn eq(&self, other: &JSStr<'_>) -> bool {
|
||||
other == self
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq<JSStr<'_>> for &str {
|
||||
#[inline]
|
||||
fn eq(&self, other: &JSStr<'_>) -> bool {
|
||||
other == *self
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq<Ident<'_>> for JSStr<'_> {
|
||||
#[inline]
|
||||
fn eq(&self, other: &Ident<'_>) -> bool {
|
||||
self == other.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq<JSStr<'_>> for Ident<'_> {
|
||||
#[inline]
|
||||
fn eq(&self, other: &JSStr<'_>) -> bool {
|
||||
other == self.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash for JSStr<'_> {
|
||||
#[inline]
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
// Match `str` hashing so UTF-8 values hash like `str` and `Str`.
|
||||
match self.as_str() {
|
||||
Some(value) => value.hash(state),
|
||||
None => self.as_bytes().hash(state),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for JSStr<'_> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_char('"')?;
|
||||
for c in self.chars() {
|
||||
if let Some(c) = c.to_char() {
|
||||
// `char::escape_debug` escapes a quote for `char` literals;
|
||||
// `str`'s Debug prints it plain, and this output is a string.
|
||||
if c == '\'' {
|
||||
f.write_char(c)?;
|
||||
} else {
|
||||
for escaped in c.escape_debug() {
|
||||
f.write_char(escaped)?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
write!(f, "\\u{:04x}", c.to_u32())?;
|
||||
}
|
||||
}
|
||||
f.write_char('"')
|
||||
}
|
||||
}
|
||||
|
||||
impl<'new_alloc> CloneIn<'new_alloc> for JSStr<'_> {
|
||||
type Cloned = JSStr<'new_alloc>;
|
||||
|
||||
#[inline]
|
||||
fn clone_in_impl(
|
||||
&self,
|
||||
_with_semantic_ids: CloneInSemanticIds,
|
||||
allocator: &'new_alloc Allocator,
|
||||
) -> Self::Cloned {
|
||||
let bytes = allocator.alloc_slice_copy(self.as_bytes());
|
||||
// SAFETY: Copying identical bytes preserves canonical WTF-8, the
|
||||
// surrogate flag, and the u32 length bound. Storage belongs to the new arena.
|
||||
unsafe { JSStr::from_bytes_unchecked(bytes, self.has_lone_surrogate()) }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Dummy<'a> for JSStr<'a> {
|
||||
#[inline]
|
||||
fn dummy(_allocator: &'a Allocator) -> Self {
|
||||
JSStr::empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// `remaining` is always canonical WTF-8 starting at a code-point boundary.
|
||||
#[derive(Clone)]
|
||||
struct JSChars<'a> {
|
||||
remaining: &'a [u8],
|
||||
}
|
||||
|
||||
impl Iterator for JSChars<'_> {
|
||||
type Item = JSChar;
|
||||
|
||||
#[inline]
|
||||
fn next(&mut self) -> Option<JSChar> {
|
||||
let (&first, rest) = self.remaining.split_first()?;
|
||||
let (value, len) = if first < 0x80 {
|
||||
(u32::from(first), 0)
|
||||
} else {
|
||||
// SAFETY: `remaining` is complete, valid WTF-8 at a code-point
|
||||
// boundary. Its leading byte therefore determines how many
|
||||
// continuation bytes are present. Their bit patterns and the
|
||||
// encoding's range restrictions guarantee a value <= 0x10FFFF.
|
||||
unsafe {
|
||||
let second = u32::from(*rest.get_unchecked(0) & 0x3F);
|
||||
if first < 0xE0 {
|
||||
((u32::from(first & 0x1F) << 6) | second, 1)
|
||||
} else {
|
||||
let third = u32::from(*rest.get_unchecked(1) & 0x3F);
|
||||
if first < 0xF0 {
|
||||
((u32::from(first & 0x0F) << 12) | (second << 6) | third, 2)
|
||||
} else {
|
||||
let fourth = u32::from(*rest.get_unchecked(2) & 0x3F);
|
||||
((u32::from(first & 7) << 18) | (second << 12) | (third << 6) | fourth, 3)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
// SAFETY: The complete code point has `len` continuation bytes, as
|
||||
// established above. The suffix begins at the next code-point boundary.
|
||||
self.remaining = unsafe { rest.get_unchecked(len..) };
|
||||
// SAFETY: Decoding valid WTF-8 produces a code point <= 0x10FFFF.
|
||||
Some(unsafe { JSChar::from_u32_unchecked(value) })
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let len = self.remaining.len();
|
||||
(len.div_ceil(4), Some(len))
|
||||
}
|
||||
}
|
||||
|
||||
impl FusedIterator for JSChars<'_> {}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct EncodeUtf16<'a> {
|
||||
chars: JSChars<'a>,
|
||||
/// Zero when empty, otherwise a trailing surrogate (which cannot be zero).
|
||||
pending: u16,
|
||||
}
|
||||
|
||||
impl Iterator for EncodeUtf16<'_> {
|
||||
type Item = u16;
|
||||
|
||||
#[inline]
|
||||
#[expect(clippy::cast_possible_truncation, reason = "code units are masked or range-checked")]
|
||||
fn next(&mut self) -> Option<u16> {
|
||||
if self.pending != 0 {
|
||||
let trail = self.pending;
|
||||
self.pending = 0;
|
||||
return Some(trail);
|
||||
}
|
||||
let value = self.chars.next()?.to_u32();
|
||||
if value <= 0xFFFF {
|
||||
Some(value as u16)
|
||||
} else {
|
||||
let offset = value - 0x10000;
|
||||
self.pending = 0xDC00 | (offset & 0x3FF) as u16;
|
||||
Some(0xD800 | (offset >> 10) as u16)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FusedIterator for EncodeUtf16<'_> {}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "js_str/tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,548 @@
|
||||
use std::{
|
||||
collections::hash_map::DefaultHasher,
|
||||
hash::{Hash, Hasher},
|
||||
};
|
||||
|
||||
use oxc_allocator::{Allocator, CloneIn, Dummy};
|
||||
use oxc_data_structures::types::implements;
|
||||
|
||||
use crate::{Ident, JSChar, JSStr, JSStrBuilder, Str};
|
||||
|
||||
fn from_utf16_in<'a>(units: &[u16], allocator: &'a Allocator) -> JSStr<'a> {
|
||||
let mut builder = JSStrBuilder::with_capacity_in(units.len(), &allocator);
|
||||
builder.push_utf16(units);
|
||||
builder.into_js_str()
|
||||
}
|
||||
|
||||
/// Decode the input independently with Rust's UTF-16 decoder. Valid scalar
|
||||
/// values use std's UTF-8 encoder; errors supply the exact unpaired code unit.
|
||||
fn assert_value(value: JSStr<'_>, units: &[u16]) {
|
||||
let mut expected_points = Vec::new();
|
||||
let mut expected_bytes = Vec::new();
|
||||
let mut has_lone = false;
|
||||
for decoded in char::decode_utf16(units.iter().copied()) {
|
||||
match decoded {
|
||||
Ok(c) => {
|
||||
expected_points.push(c as u32);
|
||||
expected_bytes.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes());
|
||||
}
|
||||
Err(error) => {
|
||||
let unit = error.unpaired_surrogate();
|
||||
expected_points.push(u32::from(unit));
|
||||
// All surrogate encodings have the prefix ED and two
|
||||
// continuation bytes carrying the remaining ten bits.
|
||||
expected_bytes.extend([
|
||||
0xED,
|
||||
0x80 | ((unit >> 6) & 0x3F) as u8,
|
||||
0x80 | (unit & 0x3F) as u8,
|
||||
]);
|
||||
has_lone = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(value.encode_utf16().collect::<Vec<_>>(), units);
|
||||
assert_eq!(value.chars().map(JSChar::to_u32).collect::<Vec<_>>(), expected_points);
|
||||
assert_eq!(value.as_bytes(), expected_bytes);
|
||||
assert_eq!(value.len(), expected_bytes.len());
|
||||
assert_eq!(value.len_utf16(), units.len());
|
||||
assert_eq!(value.has_lone_surrogate(), has_lone);
|
||||
let utf8 = String::from_utf16(units).ok();
|
||||
assert_eq!(value.as_str(), utf8.as_deref());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layouts_and_traits() {
|
||||
assert_eq!(size_of::<JSChar>(), 4);
|
||||
assert_eq!(size_of::<JSStr<'_>>(), if cfg!(target_pointer_width = "64") { 16 } else { 12 });
|
||||
assert_eq!(size_of::<Option<JSStr<'_>>>(), size_of::<JSStr<'_>>());
|
||||
assert_eq!(align_of::<JSStr<'_>>(), align_of::<usize>());
|
||||
assert!(implements!(JSStr: Send));
|
||||
assert!(implements!(JSStr: Sync));
|
||||
assert!(implements!(JSStr: Copy));
|
||||
assert!(!implements!(JSStr: std::fmt::Display));
|
||||
assert!(!implements!(JSStr: AsRef<str>));
|
||||
assert!(!implements!(JSStr: std::ops::Deref));
|
||||
assert!(!implements!(JSStr: Ord));
|
||||
assert!(!implements!(JSStrBuilder: Send));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concatenate_js_strings_across_arenas() {
|
||||
let allocator = Allocator::new();
|
||||
let value = {
|
||||
let input_allocator = Allocator::new();
|
||||
let units = [0xDC00, 0x61, 0xD800, 0xDC00, 0x62, 0xD800];
|
||||
for split in 0..=units.len() {
|
||||
let left = from_utf16_in(&units[..split], &input_allocator);
|
||||
let right = from_utf16_in(&units[split..], &input_allocator);
|
||||
let value = JSStr::from_js_strs_array_in([left, JSStr::empty(), right], &&allocator);
|
||||
assert_value(value, &units);
|
||||
}
|
||||
let lead = from_utf16_in(&[0xD800], &input_allocator);
|
||||
let trail = from_utf16_in(&[0xDC00], &input_allocator);
|
||||
JSStr::from_js_strs_array_in([lead, trail], &&allocator)
|
||||
};
|
||||
assert_value(value, &[0xD800, 0xDC00]);
|
||||
assert_eq!(value.as_str(), Some("𐀀"));
|
||||
assert_value(JSStr::from_js_strs_array_in([], &&allocator), &[]);
|
||||
assert_eq!(
|
||||
JSStr::from_js_strs_array_in([JSStr::from("hello"), JSStr::from("é")], &&allocator)
|
||||
.as_str(),
|
||||
Some("helloé")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn js_char_range_and_encoding() {
|
||||
assert_eq!(JSChar::from_u32(0x11_0000), None);
|
||||
assert_eq!(JSChar::from_u32(u32::MAX), None);
|
||||
let boundaries = [
|
||||
0, 0x7F, 0x80, 0x7FF, 0x800, 0xD7FF, 0xD800, 0xDBFF, 0xDC00, 0xDFFF, 0xE000, 0xFFFF,
|
||||
0x10000, 0x10_FFFF,
|
||||
];
|
||||
let values: Box<dyn Iterator<Item = u32>> =
|
||||
if cfg!(miri) { Box::new(boundaries.into_iter()) } else { Box::new(0..=0x10_FFFF) };
|
||||
let mut allocator = Allocator::new();
|
||||
for value in values {
|
||||
allocator.reset();
|
||||
let js_char = JSChar::from_u32(value).unwrap();
|
||||
assert_eq!(js_char.to_u32(), value);
|
||||
assert_eq!(js_char.to_char(), char::from_u32(value));
|
||||
let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
builder.push_js_char(js_char);
|
||||
if let Some(c) = char::from_u32(value) {
|
||||
assert_eq!(builder.into_js_str().as_str(), Some(c.encode_utf8(&mut [0; 4]) as &str));
|
||||
} else {
|
||||
assert_value(builder.into_js_str(), &[u16::try_from(value).unwrap()]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn borrowing_and_arena_conversions() {
|
||||
const EMPTY: JSStr<'_> = JSStr::empty();
|
||||
assert_value(EMPTY, &[]);
|
||||
assert_value(JSStr::from("a𐀀�"), &[0x61, 0xD800, 0xDC00, 0xFFFD]);
|
||||
let source = String::from("hello\0é😀");
|
||||
let value = JSStr::from(source.as_str());
|
||||
assert_eq!(value.as_str().unwrap().as_ptr(), source.as_ptr());
|
||||
assert_eq!(JSStr::from(Str::from(source.as_str())), value);
|
||||
let ident = JSStr::from(Ident::from(source.as_str()));
|
||||
assert_eq!(ident, value);
|
||||
assert_eq!(ident.as_bytes().as_ptr(), source.as_ptr());
|
||||
assert_eq!(value, source.as_str());
|
||||
assert_eq!(source.as_str(), value);
|
||||
assert_eq!(value, *source.as_str());
|
||||
assert_eq!(*source.as_str(), value);
|
||||
assert_ne!(*"other", value);
|
||||
let allocator = Allocator::new();
|
||||
let copied = JSStr::from_str_in(&source, &&allocator);
|
||||
assert_eq!(copied, value);
|
||||
assert_ne!(copied.as_bytes().as_ptr(), source.as_ptr());
|
||||
assert_eq!(JSStr::dummy(&allocator), JSStr::empty());
|
||||
assert!(JSStr::empty().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equality_with_ident() {
|
||||
let allocator = Allocator::new();
|
||||
let ident = Ident::from("foo");
|
||||
assert_eq!(JSStr::from("foo"), ident);
|
||||
assert_eq!(ident, JSStr::from("foo"));
|
||||
assert_ne!(JSStr::from("bar"), ident);
|
||||
assert_ne!(ident, JSStr::from("bar"));
|
||||
let lone = from_utf16_in(&[0x66, 0x6F, 0x6F, 0xD800], &allocator);
|
||||
assert_ne!(lone, ident);
|
||||
assert_ne!(ident, lone);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_survives_original_arena() {
|
||||
let destination = Allocator::new();
|
||||
let units = [0xD800, 0x61, 0xDC00, 0xD83D, 0xDE00];
|
||||
let cloned = {
|
||||
let source = Allocator::new();
|
||||
let value = from_utf16_in(&units, &source);
|
||||
let cloned = value.clone_in(&destination);
|
||||
assert_ne!(value.as_bytes().as_ptr(), cloned.as_bytes().as_ptr());
|
||||
cloned
|
||||
};
|
||||
assert_value(cloned, &units);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_builders_and_into_js_str_do_not_allocate() {
|
||||
let allocator = Allocator::new();
|
||||
let before = allocator.used_bytes();
|
||||
assert_eq!(JSStrBuilder::new_in(&&allocator).into_js_str(), JSStr::empty());
|
||||
assert_eq!(JSStrBuilder::with_capacity_in(0, &&allocator).into_js_str(), JSStr::empty());
|
||||
assert_eq!(allocator.used_bytes(), before);
|
||||
|
||||
let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
builder.push_code_unit(0xD800);
|
||||
let before = allocator.used_bytes();
|
||||
assert_value(builder.into_js_str(), &[0xD800]);
|
||||
assert_eq!(allocator.used_bytes(), before);
|
||||
|
||||
let mut builder = JSStrBuilder::with_capacity_in(64, &&allocator);
|
||||
builder.push_str("hello");
|
||||
builder.push_code_unit(0xD800);
|
||||
let before = allocator.used_bytes();
|
||||
assert_value(builder.into_js_str(), &[0x68, 0x65, 0x6C, 0x6C, 0x6F, 0xD800]);
|
||||
assert_eq!(allocator.used_bytes(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn boundary_pairing_and_empty_appends() {
|
||||
let allocator = Allocator::new();
|
||||
let lead = from_utf16_in(&[0xD800], &allocator);
|
||||
let trail = from_utf16_in(&[0xDC00], &allocator);
|
||||
let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
builder.push_js_str(lead);
|
||||
builder.push_str("");
|
||||
builder.push_utf16(&[]);
|
||||
builder.push_js_str(JSStr::empty());
|
||||
builder.push_js_str(trail);
|
||||
assert_eq!(builder.into_js_str(), JSStr::from("𐀀"));
|
||||
|
||||
for units in [
|
||||
vec![0xD800, 0xD800],
|
||||
vec![0xDC00, 0xD800],
|
||||
vec![0xD800, 0x61],
|
||||
vec![0xD800, 0xDC00],
|
||||
vec![0xD800, 0xDC00, 0xD800],
|
||||
] {
|
||||
let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
for &unit in &units {
|
||||
builder.push_js_str(from_utf16_in(&[unit], &allocator));
|
||||
}
|
||||
assert_value(builder.into_js_str(), &units);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_short_partition() {
|
||||
let alphabet: &[u16] = if cfg!(miri) {
|
||||
&[0x61, 0xD800, 0xDC00]
|
||||
} else {
|
||||
&[
|
||||
0, 0x7F, 0x80, 0x7FF, 0x800, 0xD7FF, 0xD800, 0xDBFF, 0xDC00, 0xDFFF, 0xE000, 0xFFFD,
|
||||
0xFFFF,
|
||||
]
|
||||
};
|
||||
let mut allocator = Allocator::new();
|
||||
for &a in alphabet {
|
||||
for &b in alphabet {
|
||||
for &c in alphabet {
|
||||
allocator.reset();
|
||||
let units = [a, b, c];
|
||||
for first in 0..=3 {
|
||||
for second in first..=3 {
|
||||
let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
for part in [&units[..first], &units[first..second], &units[second..]] {
|
||||
builder.push_js_str(from_utf16_in(part, &allocator));
|
||||
}
|
||||
assert_value(builder.into_js_str(), &units);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hash_of<T: Hash + ?Sized>(value: &T) -> u64 {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
value.hash(&mut hasher);
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf8_hashes_like_str() {
|
||||
for text in ["", "foo", "\u{e9}\u{1f600}"] {
|
||||
let value = JSStr::from(text);
|
||||
assert_eq!(hash_of(&value), hash_of(text));
|
||||
assert_eq!(hash_of(&value), hash_of(&Str::from(text)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equal_lone_surrogate_strings_hash_equal() {
|
||||
let allocator = Allocator::new();
|
||||
let units = [0x61, 0xD800, 0xDFFF];
|
||||
let mut by_unit = JSStrBuilder::new_in(&&allocator);
|
||||
for &unit in &units {
|
||||
by_unit.push_code_unit(unit);
|
||||
}
|
||||
let mut by_js_str = JSStrBuilder::new_in(&&allocator);
|
||||
by_js_str.push_str("a");
|
||||
by_js_str.push_js_str(from_utf16_in(&[0xD800], &allocator));
|
||||
by_js_str.push_js_str(from_utf16_in(&[0xDFFF], &allocator));
|
||||
let (a, b) = (by_unit.into_js_str(), by_js_str.into_js_str());
|
||||
assert_eq!(a, b);
|
||||
assert_eq!(hash_of(&a), hash_of(&b));
|
||||
}
|
||||
|
||||
fn concat<'a>(allocator: &'a Allocator, a: JSStr<'_>, b: JSStr<'_>) -> JSStr<'a> {
|
||||
let mut builder = JSStrBuilder::new_in(&allocator);
|
||||
builder.push_js_str(a);
|
||||
builder.push_js_str(b);
|
||||
builder.into_js_str()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn randomized_appends_and_associativity() {
|
||||
let mut seed = 0xA930_154Bu32;
|
||||
let mut next = || {
|
||||
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
seed
|
||||
};
|
||||
let mut allocator = Allocator::new();
|
||||
for _ in 0..if cfg!(miri) { 12 } else { 2000 } {
|
||||
allocator.reset();
|
||||
let mut expected = Vec::new();
|
||||
let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
for _ in 0..if cfg!(miri) { 6 } else { 32 } {
|
||||
let len = next() as usize % 9;
|
||||
let units = std::iter::repeat_with(|| {
|
||||
let value = next();
|
||||
if value.is_multiple_of(4) {
|
||||
0xD800 | u16::try_from((value >> 8) & 0x7FF).unwrap()
|
||||
} else {
|
||||
u16::try_from((value >> 8) & 0xFFFF).unwrap()
|
||||
}
|
||||
})
|
||||
.take(len)
|
||||
.collect::<Vec<_>>();
|
||||
match next() % 3 {
|
||||
0 => builder.push_utf16(&units),
|
||||
1 => builder.push_js_str(from_utf16_in(&units, &allocator)),
|
||||
_ => {
|
||||
for decoded in char::decode_utf16(units.iter().copied()) {
|
||||
match decoded {
|
||||
Ok(c) => builder.push(c),
|
||||
Err(error) => builder.push_code_unit(error.unpaired_surrogate()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
expected.extend_from_slice(&units);
|
||||
}
|
||||
let value = builder.into_js_str();
|
||||
assert_value(value, &expected);
|
||||
let n = expected.len();
|
||||
let a = from_utf16_in(&expected[..n / 3], &allocator);
|
||||
let b = from_utf16_in(&expected[n / 3..2 * n / 3], &allocator);
|
||||
let c = from_utf16_in(&expected[2 * n / 3..], &allocator);
|
||||
assert_eq!(concat(&allocator, concat(&allocator, a, b), c), value);
|
||||
assert_eq!(concat(&allocator, a, concat(&allocator, b, c)), value);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn growth_with_interleaved_allocations() {
|
||||
let allocator = Allocator::new();
|
||||
let mut builder = JSStrBuilder::with_capacity_in(1, &&allocator);
|
||||
let mut expected = Vec::new();
|
||||
for _ in 0..64 {
|
||||
let units = [0xD800, 0xDC00, 0x61, 0xDFFF, 0xD800];
|
||||
builder.push_utf16(&units);
|
||||
expected.extend(units);
|
||||
allocator.alloc([42u8; 128]);
|
||||
}
|
||||
assert_value(builder.into_js_str(), &expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equality_hash_and_debug() {
|
||||
let allocator = Allocator::new();
|
||||
let units = [0xD800, 0xDC00, 0xD800, 0x61, 0xFFFD];
|
||||
let a = from_utf16_in(&units, &allocator);
|
||||
let b = concat(
|
||||
&allocator,
|
||||
from_utf16_in(&units[..1], &allocator),
|
||||
from_utf16_in(&units[1..], &allocator),
|
||||
);
|
||||
let hash = |value: JSStr<'_>| {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
value.hash(&mut hasher);
|
||||
hasher.finish()
|
||||
};
|
||||
assert_eq!(a, b);
|
||||
assert_eq!(hash(a), hash(b));
|
||||
assert_ne!(a, JSStr::from("𐀀�d800a�"));
|
||||
assert_ne!(a, "𐀀�d800a�");
|
||||
assert_eq!(format!("{a:?}"), "\"𐀀\\ud800a�\"");
|
||||
assert_eq!(format!("{:?}", JSStr::from("\"\n\\")), "\"\\\"\\n\\\\\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloned_iterators_and_fused_end() {
|
||||
let allocator = Allocator::new();
|
||||
let value = from_utf16_in(&[0xD800, 0xDC00, 0xDFFF], &allocator);
|
||||
let mut chars = value.chars();
|
||||
assert_eq!(chars.size_hint(), (2, Some(7)));
|
||||
assert_eq!(chars.next().unwrap().to_u32(), 0x10000);
|
||||
assert_eq!(chars.clone().next().unwrap().to_u32(), 0xDFFF);
|
||||
assert_eq!(chars.next().unwrap().to_u32(), 0xDFFF);
|
||||
assert_eq!(chars.next(), None);
|
||||
assert_eq!(chars.next(), None);
|
||||
let mut units = value.encode_utf16();
|
||||
assert_eq!(units.next(), Some(0xD800));
|
||||
assert_eq!(units.clone().collect::<Vec<_>>(), [0xDC00, 0xDFFF]);
|
||||
assert_eq!(units.next(), Some(0xDC00));
|
||||
assert_eq!(units.next(), Some(0xDFFF));
|
||||
assert_eq!(units.next(), None);
|
||||
assert_eq!(units.next(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "capacity overflow")]
|
||||
fn reject_capacity_overflow() {
|
||||
let allocator = Allocator::new();
|
||||
let _ = JSStrBuilder::with_capacity_in(usize::MAX, &&allocator);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_matches_str_for_utf8() {
|
||||
for text in [
|
||||
"",
|
||||
"plain",
|
||||
"Foo's bar",
|
||||
"quote \" tick ' back \\",
|
||||
"tab\t nl\n nul\0",
|
||||
"\u{e9}\u{1f600}\u{2028}",
|
||||
] {
|
||||
assert_eq!(format!("{:?}", JSStr::from(text)), format!("{text:?}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bytes_round_trip_through_owned_storage() {
|
||||
let allocator = Allocator::new();
|
||||
let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
builder.push_utf16(&[0x61, 0xD800, 0xD83D, 0xDE00]);
|
||||
let value = builder.into_js_str();
|
||||
|
||||
let stored: Box<[u8]> = value.as_bytes().into();
|
||||
let flag = value.has_lone_surrogate();
|
||||
// SAFETY: the bytes and flag come from an existing `JSStr`.
|
||||
let restored = unsafe { JSStr::from_bytes_unchecked(&stored, flag) };
|
||||
assert_eq!(restored, value);
|
||||
assert_eq!(restored.has_lone_surrogate(), flag);
|
||||
assert_eq!(restored.len_utf16(), 4);
|
||||
|
||||
let plain = JSStr::from("plain");
|
||||
// SAFETY: UTF-8 is canonical WTF-8 without lone surrogates.
|
||||
let restored = unsafe { JSStr::from_bytes_unchecked(plain.as_bytes(), false) };
|
||||
assert_eq!(restored.as_str(), Some("plain"));
|
||||
}
|
||||
|
||||
/// Encode UTF-16 as canonical WTF-8 without `JSStrBuilder`. Pairs become
|
||||
/// four-byte supplementary characters and lone surrogates use three bytes.
|
||||
fn reference_wtf8(units: &[u16]) -> Vec<u8> {
|
||||
let mut bytes = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < units.len() {
|
||||
let unit = u32::from(units[i]);
|
||||
let trail = units.get(i + 1).map_or(0, |&unit| u32::from(unit));
|
||||
let point = if (0xD800..=0xDBFF).contains(&unit) && (0xDC00..=0xDFFF).contains(&trail) {
|
||||
i += 2;
|
||||
0x10000 + ((unit - 0xD800) << 10) + (trail - 0xDC00)
|
||||
} else {
|
||||
i += 1;
|
||||
unit
|
||||
};
|
||||
if point < 0x80 {
|
||||
bytes.push((point & 0x7F) as u8);
|
||||
} else if point < 0x800 {
|
||||
bytes.extend([0xC0 | ((point >> 6) & 0x1F) as u8, 0x80 | (point & 0x3F) as u8]);
|
||||
} else if point < 0x10000 {
|
||||
bytes.extend([
|
||||
0xE0 | ((point >> 12) & 0x0F) as u8,
|
||||
0x80 | ((point >> 6) & 0x3F) as u8,
|
||||
0x80 | (point & 0x3F) as u8,
|
||||
]);
|
||||
} else {
|
||||
bytes.extend([
|
||||
0xF0 | ((point >> 18) & 0x07) as u8,
|
||||
0x80 | ((point >> 12) & 0x3F) as u8,
|
||||
0x80 | ((point >> 6) & 0x3F) as u8,
|
||||
0x80 | (point & 0x3F) as u8,
|
||||
]);
|
||||
}
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
/// Check the `JSStr` invariants directly against the bytes.
|
||||
fn assert_invariants(value: JSStr<'_>) {
|
||||
let bytes = value.as_bytes();
|
||||
let units = value.encode_utf16().collect::<Vec<_>>();
|
||||
assert_eq!(bytes, reference_wtf8(&units), "bytes are not canonical WTF-8");
|
||||
let paired = bytes.windows(5).any(|window| {
|
||||
window[0] == 0xED
|
||||
&& (0xA0..=0xAF).contains(&window[1])
|
||||
&& window[3] == 0xED
|
||||
&& (0xB0..=0xBF).contains(&window[4])
|
||||
});
|
||||
assert!(!paired, "lead surrogate encoding followed by a trail encoding: {bytes:x?}");
|
||||
let has_lone = bytes.windows(2).any(|window| window[0] == 0xED && window[1] >= 0xA0);
|
||||
assert_eq!(value.has_lone_surrogate(), has_lone);
|
||||
assert_eq!(value.as_str().is_some(), !has_lone);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_piece_sequence_and_push_method() {
|
||||
const PIECES: &[&[u16]] = &[
|
||||
&[],
|
||||
&[0x61],
|
||||
&[0xD800],
|
||||
&[0xDC00],
|
||||
&[0xDBFF],
|
||||
&[0xDFFF],
|
||||
&[0xD83D, 0xDE00],
|
||||
&[0xD800, 0x61],
|
||||
&[0x61, 0xDC00],
|
||||
&[0xDC00, 0xD800],
|
||||
&[0xE9],
|
||||
];
|
||||
const METHODS: usize = 4;
|
||||
let max_pieces = if cfg!(miri) { 2 } else { 3 };
|
||||
let mut allocator = Allocator::new();
|
||||
let mut sequence = Vec::new();
|
||||
for len in 0..=max_pieces {
|
||||
for index in 0..PIECES.len().pow(len) {
|
||||
sequence.clear();
|
||||
let mut rest = index;
|
||||
for _ in 0..len {
|
||||
sequence.push(PIECES[rest % PIECES.len()]);
|
||||
rest /= PIECES.len();
|
||||
}
|
||||
let expected = sequence.concat();
|
||||
'methods: for methods in 0..METHODS.pow(len) {
|
||||
allocator.reset();
|
||||
let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
let mut rest = methods;
|
||||
for &piece in &sequence {
|
||||
match rest % METHODS {
|
||||
0 => builder.push_utf16(piece),
|
||||
1 => {
|
||||
for &unit in piece {
|
||||
builder.push_code_unit(unit);
|
||||
}
|
||||
}
|
||||
2 => builder.push_js_str(from_utf16_in(piece, &allocator)),
|
||||
_ => match String::from_utf16(piece) {
|
||||
Ok(text) => builder.push_str(&text),
|
||||
Err(_) => continue 'methods,
|
||||
},
|
||||
}
|
||||
rest /= METHODS;
|
||||
}
|
||||
let value = builder.into_js_str();
|
||||
assert_invariants(value);
|
||||
assert_eq!(value.encode_utf16().collect::<Vec<_>>(), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
use oxc_allocator::{ArenaVec, GetAllocator};
|
||||
|
||||
use crate::{JSChar, JSStr};
|
||||
|
||||
/// Build a JavaScript string in an arena, preserving lone surrogates.
|
||||
///
|
||||
/// Appends concatenate UTF-16 values. A leading surrogate at the end of one
|
||||
/// append pairs with a trailing surrogate at the start of the next, including
|
||||
/// across empty appends. [`into_js_str`](Self::into_js_str) returns canonical WTF-8
|
||||
/// without copying the completed buffer.
|
||||
///
|
||||
/// ```
|
||||
/// use oxc_allocator::Allocator;
|
||||
/// use oxc_str::JSStrBuilder;
|
||||
///
|
||||
/// let allocator = Allocator::new();
|
||||
/// let mut builder = JSStrBuilder::new_in(&&allocator);
|
||||
/// builder.push_code_unit(0xD800);
|
||||
/// builder.push_str("");
|
||||
/// builder.push_code_unit(0xDC00);
|
||||
/// assert_eq!(builder.into_js_str().as_str(), Some("𐀀"));
|
||||
/// ```
|
||||
pub struct JSStrBuilder<'a> {
|
||||
/// Canonical WTF-8, excluding a held final leading surrogate.
|
||||
bytes: ArenaVec<'a, u8>,
|
||||
/// Always in 0xD800..=0xDBFF when present, with three bytes of spare capacity
|
||||
/// in `bytes`, so finishing the string never needs to grow the buffer.
|
||||
pending_lead_surrogate: Option<u16>,
|
||||
/// Describes only `bytes`, excluding `pending_lead_surrogate`.
|
||||
has_lone_surrogate: bool,
|
||||
}
|
||||
|
||||
impl<'a> JSStrBuilder<'a> {
|
||||
/// Construct an empty builder without allocating.
|
||||
#[inline]
|
||||
pub fn new_in(allocator: &impl GetAllocator<'a>) -> Self {
|
||||
Self {
|
||||
bytes: ArenaVec::new_in(allocator),
|
||||
pending_lead_surrogate: None,
|
||||
has_lone_surrogate: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserve capacity in bytes. Zero capacity does not allocate.
|
||||
///
|
||||
/// # Panics
|
||||
/// Panics if capacity exceeds `u32::MAX` or `isize::MAX`.
|
||||
#[inline]
|
||||
pub fn with_capacity_in(capacity: usize, allocator: &impl GetAllocator<'a>) -> Self {
|
||||
Self {
|
||||
bytes: ArenaVec::with_capacity_in(capacity, allocator),
|
||||
pending_lead_surrogate: None,
|
||||
has_lone_surrogate: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Append UTF-8 text. Empty text preserves a pending leading surrogate.
|
||||
#[inline]
|
||||
pub fn push_str(&mut self, value: &str) {
|
||||
if value.is_empty() {
|
||||
return;
|
||||
}
|
||||
// Reserve before changing state, so a capacity panic cannot leave a
|
||||
// flushed leading surrogate that a later append would fail to pair.
|
||||
if self.pending_lead_surrogate.is_some() {
|
||||
self.bytes.reserve(value.len() + 3);
|
||||
self.flush_pending();
|
||||
}
|
||||
self.bytes.extend_from_slice_copy(value.as_bytes());
|
||||
}
|
||||
|
||||
/// Append a Unicode scalar value.
|
||||
#[inline]
|
||||
pub fn push(&mut self, value: char) {
|
||||
self.push_str(value.encode_utf8(&mut [0; 4]));
|
||||
}
|
||||
|
||||
/// Append one JavaScript code point, pairing surrogates at the boundary.
|
||||
#[inline]
|
||||
#[expect(clippy::cast_possible_truncation, reason = "leading surrogates fit in u16")]
|
||||
pub fn push_js_char(&mut self, value: JSChar) {
|
||||
let point = value.to_u32();
|
||||
if let Some(lead) = self.pending_lead_surrogate
|
||||
&& (0xDC00..=0xDFFF).contains(&point)
|
||||
{
|
||||
self.bytes.reserve(4);
|
||||
self.append_pair(lead, point as u16);
|
||||
self.pending_lead_surrogate = None;
|
||||
return;
|
||||
}
|
||||
|
||||
let mut buffer = [0; 4];
|
||||
let bytes = value.encode(&mut buffer);
|
||||
let is_lead = (0xD800..=0xDBFF).contains(&point);
|
||||
let additional = self.pending_bytes() + bytes.len();
|
||||
self.bytes.reserve(additional);
|
||||
self.flush_pending();
|
||||
if is_lead {
|
||||
self.pending_lead_surrogate = Some(point as u16);
|
||||
} else {
|
||||
self.bytes.extend_from_slice_copy(bytes);
|
||||
self.has_lone_surrogate |= value.is_surrogate();
|
||||
}
|
||||
}
|
||||
|
||||
/// Append one UTF-16 code unit.
|
||||
///
|
||||
/// Use this for input from a UTF-16 API; use [`push_js_char`](Self::push_js_char)
|
||||
/// for an already decoded JavaScript code point.
|
||||
#[inline]
|
||||
pub fn push_code_unit(&mut self, unit: u16) {
|
||||
self.push_js_char(JSChar::from_code_unit(unit));
|
||||
}
|
||||
|
||||
/// Append potentially ill-formed UTF-16.
|
||||
///
|
||||
/// This accepts code-unit buffers from UTF-16 APIs without replacing lone
|
||||
/// surrogates. A pair may span consecutive calls, including empty buffers.
|
||||
#[inline]
|
||||
pub fn push_utf16(&mut self, units: &[u16]) {
|
||||
for &unit in units {
|
||||
self.push_code_unit(unit);
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a JavaScript string, repairing a surrogate pair at the boundary.
|
||||
///
|
||||
/// UTF-8 inputs are copied without scanning. For inputs with lone surrogates,
|
||||
/// only the boundary encodings change; the interior is copied as bytes.
|
||||
#[inline]
|
||||
pub fn push_js_str(&mut self, value: JSStr<'_>) {
|
||||
if let Some(value) = value.as_str() {
|
||||
self.push_str(value);
|
||||
} else {
|
||||
self.push_js_str_slow(value);
|
||||
}
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
fn push_js_str_slow(&mut self, value: JSStr<'_>) {
|
||||
// A surrogate's WTF-8 encoding is 0xED, then two continuation bytes
|
||||
// holding the low twelve bits.
|
||||
fn decode_surrogate(second: u8, third: u8) -> u16 {
|
||||
0xD000 | (u16::from(second & 0x3F) << 6) | u16::from(third & 0x3F)
|
||||
}
|
||||
|
||||
debug_assert!(value.has_lone_surrogate());
|
||||
|
||||
let mut bytes = value.as_bytes();
|
||||
let additional = bytes.len() + self.pending_bytes();
|
||||
self.bytes.reserve(additional);
|
||||
let mut trimmed = false;
|
||||
|
||||
if let Some(lead) = self.pending_lead_surrogate
|
||||
&& let [0xED, second @ (0xB0..), third, ..] = bytes
|
||||
{
|
||||
self.append_pair(lead, decode_surrogate(*second, *third));
|
||||
self.pending_lead_surrogate = None;
|
||||
bytes = &bytes[3..];
|
||||
trimmed = true;
|
||||
} else {
|
||||
self.flush_pending();
|
||||
}
|
||||
|
||||
if let [.., 0xED, second @ 0xA0..=0xAF, third] = bytes {
|
||||
self.pending_lead_surrogate = Some(decode_surrogate(*second, *third));
|
||||
bytes = &bytes[..bytes.len() - 3];
|
||||
trimmed = true;
|
||||
}
|
||||
|
||||
// Without trimming, the appended bytes still contain a lone surrogate.
|
||||
// Trimming may remove the only one, so inspect the interior in that case.
|
||||
// An already-set output flag needs no further scan.
|
||||
self.has_lone_surrogate = self.has_lone_surrogate
|
||||
|| !trimmed
|
||||
|| bytes.windows(3).any(|bytes| bytes[0] == 0xED && bytes[1] >= 0xA0);
|
||||
self.bytes.extend_from_slice_copy(bytes);
|
||||
}
|
||||
|
||||
/// Consume the builder and return a string without copying or rescanning the buffer.
|
||||
#[inline]
|
||||
pub fn into_js_str(mut self) -> JSStr<'a> {
|
||||
// A pending surrogate always has three bytes of capacity reserved for it.
|
||||
self.flush_pending();
|
||||
let bytes = self.bytes.into_arena_slice();
|
||||
// SAFETY: Appends maintain canonical WTF-8 and the exact surrogate flag.
|
||||
// The final pending surrogate has been flushed. `ArenaVec<u8>` bounds
|
||||
// its length by u32::MAX and isize::MAX; the slice owns the arena lifetime.
|
||||
unsafe { JSStr::from_bytes_unchecked(bytes, self.has_lone_surrogate) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn pending_bytes(&self) -> usize {
|
||||
if self.pending_lead_surrogate.is_some() { 3 } else { 0 }
|
||||
}
|
||||
|
||||
/// Call only after reserving space and deciding the next append cannot pair.
|
||||
#[inline]
|
||||
fn flush_pending(&mut self) {
|
||||
if let Some(lead) = self.pending_lead_surrogate.take() {
|
||||
self.bytes.extend_from_slice_copy(JSChar::from_code_unit(lead).encode(&mut [0; 4]));
|
||||
self.has_lone_surrogate = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// `lead` and `trail` have already been checked at the append boundary.
|
||||
#[inline]
|
||||
fn append_pair(&mut self, lead: u16, trail: u16) {
|
||||
let value = 0x10000 + ((u32::from(lead) - 0xD800) << 10) + (u32::from(trail) - 0xDC00);
|
||||
let c = char::from_u32(value).unwrap();
|
||||
self.bytes.extend_from_slice_copy(c.encode_utf8(&mut [0; 4]).as_bytes());
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,46 @@
|
||||
//! String types for oxc.
|
||||
//!
|
||||
//! This crate provides [`Str`], [`Ident`], and [`CompactStr`] types for efficient string handling.
|
||||
//! This crate provides [`Str`], [`Ident`], and [`CompactStr`] for UTF-8 strings, and
|
||||
//! [`JSStr`], [`JSChar`], and [`JSStrBuilder`] for JavaScript strings, including lone surrogates.
|
||||
//!
|
||||
//! # Naming
|
||||
//!
|
||||
//! Types are named for what a value means in JavaScript, as with [`JSStr`]
|
||||
//! and [`JSChar`]. Storage, encoding, and ownership stay out of public names:
|
||||
//! WTF-8 is an implementation detail, and whether a name borrows or owns its
|
||||
//! text is the type's concern rather than the caller's.
|
||||
//!
|
||||
//! # Consuming `JSStr`
|
||||
//!
|
||||
//! A consumer that needs `str` decides what a lone surrogate means for its
|
||||
//! check, and two policies cover the cases in practice. Checks over names,
|
||||
//! paths, and specifiers decline when [`JSStr::as_str`] returns `None`: a
|
||||
//! fixed name set or a well-known path never contains a lone surrogate, so
|
||||
//! declining changes nothing. Comparisons with fixed names use `JSStr`'s
|
||||
//! `PartialEq<&str>` directly and need no conversion. A value with a lone
|
||||
//! surrogate is not matched against user-configured patterns; rules skip
|
||||
//! that value. A value shown to a person uses the `Debug` form, which
|
||||
//! escapes the surrogate instead of replacing it.
|
||||
//!
|
||||
//! Declining is only safe where skipping the work is conservative, as for an
|
||||
//! optional optimization or a lookup that cannot match. Code that decides
|
||||
//! whether a program is valid, whether a rewrite is safe, or what code to emit
|
||||
//! must handle the value instead of skipping it.
|
||||
|
||||
mod compact_str;
|
||||
mod ident;
|
||||
mod ident_hasher;
|
||||
mod js_char;
|
||||
mod js_str;
|
||||
mod js_str_builder;
|
||||
mod str;
|
||||
|
||||
pub use compact_str::{CompactStr, MAX_INLINE_LEN};
|
||||
pub use ident::{ArenaIdentHashMap, ArenaIdentHashSet, Ident, IdentHashMap, IdentHashSet};
|
||||
pub use ident_hasher::{IdentBuildHasher, IdentHasher};
|
||||
pub use js_char::JSChar;
|
||||
pub use js_str::JSStr;
|
||||
pub use js_str_builder::JSStrBuilder;
|
||||
pub use str::{Str, Str as ArenaStr};
|
||||
|
||||
#[doc(hidden)]
|
||||
|
||||
Reference in new issue
Block a user