feat(codegen): add ascii_only option (#25994)

Closes #17068. Supersedes #18095 / #18118.

Adds `CodegenOptions::ascii_only` (and `codegen.asciiOnly` in
`oxc_minify_napi`): escape every non-ASCII character in string literals,
untagged template literals, regexp literals, directives and identifier
names so the output is 7-bit clean — like esbuild `--charset=ascii` /
terser `format.ascii_only`. Off by default; the default path is
untouched (codegen bench and conformance unchanged).

```js
// Input
let café = "naïve 😀";
let re = /[–—]/u;

// Output with ascii_only: true
let caf\u00E9 = "na\u00EFve \u{1F600}";
let re = /[\u2013\u2014]/u;
```

Characters above U+FFFF use ES2015 code-point escapes. Regex patterns
use escaped surrogate pairs instead; RegExp.prototype.source reflects
the escaped spelling.
Tagged template quasis, JSX names/text/attribute strings, hashbangs and
preserved comments may still contain non-ASCII characters. JavaScript
expressions inside tagged templates and JSX are escaped normally.

Performance; with the option disabled, codegen codspeed benchmarks are
withing noise (<1%). With the option on, performance is only affected
for files that contain non ascii characters.

---

Our use case is a large Vite/Rolldown app served through proxies that
mangle non-ASCII JS (previously handled by terser's `ascii_only`);
#17068 also lists deno_core snapshots and Chrome extension content
scripts.

AI-assisted (Claude Code); I reviewed the code, wrote the tests, and
have been running this in production builds via a vendored
`oxc_codegen`.

---------

Co-authored-by: Samuel Attard <6634592+MarshallOfSound@users.noreply.github.com>
Co-authored-by: Cameron Clark <cameron.clark@hey.com>
Co-authored-by: Dunqing <dengqing0821@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-09-10 17:10:02 +01:00
1 parent d7713ad4cb
commit 9ef028c2ad
14 files changed
+688 -83

No files matched your search

+7
View File
@@ -70,3 +70,10 @@ pn = "pn"
oce = "oce"
ome = "ome"
froms = "froms"
[type.ascii-only-tests]
extend-glob = ["ascii_only.rs"]
[type.ascii-only-tests.extend-words]
# Prefix of café in escaped Unicode test fixtures.
caf = "caf"
+2
View File
@@ -26,6 +26,8 @@ bitflags! {
///
/// When set, TypeScript syntax features are enabled in the output.
const TYPESCRIPT = 1 << 2;
/// The current template literal is tagged, so preserve its raw quasis.
const TAGGED_TEMPLATE = 1 << 3;
}
}
+24 -22
View File
@@ -95,7 +95,7 @@ impl Gen for Directive<'_> {
}
}
quote.print(p);
p.print_str(directive);
p.print_directive_raw(directive);
quote.print(p);
p.print_ascii_byte(b';');
p.print_soft_newline();
@@ -1044,7 +1044,7 @@ impl Gen for ImportAttribute<'_> {
fn r#gen(&self, p: &mut Codegen, _ctx: Context) {
match &self.key {
ImportAttributeKey::Identifier(identifier) => {
p.print_str(identifier.name.as_str());
p.print_name(identifier.name.as_str());
}
ImportAttributeKey::StringLiteral(literal) => {
p.print_string_literal(literal, false);
@@ -1395,7 +1395,7 @@ impl Gen for IdentifierReference<'_> {
let name = p.get_identifier_reference_name(self);
p.print_space_before_identifier();
p.add_source_mapping_for_name(self.span, name);
p.print_str(name);
p.print_name(name);
}
}
@@ -1403,7 +1403,7 @@ impl Gen for IdentifierName<'_> {
fn r#gen(&self, p: &mut Codegen, _ctx: Context) {
p.print_space_before_identifier();
p.add_source_mapping_for_name(self.span, &self.name);
p.print_str(self.name.as_str());
p.print_name(self.name.as_str());
}
}
@@ -1412,7 +1412,7 @@ impl Gen for BindingIdentifier<'_> {
let name = p.get_binding_identifier_name(self);
p.print_space_before_identifier();
p.add_source_mapping_for_name(self.span, name);
p.print_str(name);
p.print_name(name);
}
}
@@ -1420,7 +1420,7 @@ impl Gen for LabelIdentifier<'_> {
fn r#gen(&self, p: &mut Codegen, _ctx: Context) {
p.print_space_before_identifier();
p.add_source_mapping_for_name(self.span, &self.name);
p.print_str(self.name.as_str());
p.print_name(self.name.as_str());
}
}
@@ -1513,10 +1513,10 @@ impl Gen for RegExpLiteral<'_> {
p.print_hard_space();
}
p.print_ascii_byte(b'/');
p.print_str(self.regex.pattern.text.as_str());
p.print_regex_pattern(self.regex.pattern.text.as_str());
p.print_ascii_byte(b'/');
p.print_str(self.regex.flags.to_inline_string().as_str());
p.prev_reg_exp_end = p.code().len();
p.need_space_before_identifier = p.code().len();
}
}
@@ -2214,10 +2214,10 @@ impl Gen for AssignmentTargetPropertyIdentifier<'_> {
self.binding.print(p, ctx);
} else {
// `({x: a} = y);`
p.print_str(self.binding.name.as_str());
p.print_name(self.binding.name.as_str());
p.print_colon();
p.print_soft_space();
p.print_str(ident_name);
p.print_name(ident_name);
}
if let Some(expr) = &self.init {
p.print_soft_space();
@@ -2351,7 +2351,7 @@ impl GenExpr for ImportExpression<'_> {
}
impl Gen for TemplateLiteral<'_> {
fn r#gen(&self, p: &mut Codegen, _ctx: Context) {
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
if self.is_no_substitution_template() {
p.print_property_key_annotation(self.span.start);
}
@@ -2359,14 +2359,16 @@ impl Gen for TemplateLiteral<'_> {
p.print_ascii_byte(b'`');
debug_assert_eq!(self.quasis.len(), self.expressions.len() + 1);
let (first_quasi, remaining_quasis) = self.quasis.split_first().unwrap();
p.print_str_escaping_script_close_tag(first_quasi.value.raw.as_str());
let tagged = ctx.contains(Context::TAGGED_TEMPLATE);
p.print_template_quasi_raw(first_quasi.value.raw.as_str(), tagged);
for (expr, quasi) in self.expressions.iter().zip(remaining_quasis) {
p.print_str("${");
p.print_leading_comments_before_expression(expr);
// Nested expressions start with an empty context, so they do not inherit the tag.
p.print_expression(expr);
p.print_ascii_byte(b'}');
p.add_source_mapping(quasi.span);
p.print_str_escaping_script_close_tag(quasi.value.raw.as_str());
p.print_template_quasi_raw(quasi.value.raw.as_str(), tagged);
}
p.print_ascii_byte(b'`');
}
@@ -2380,7 +2382,7 @@ impl GenExpr for TaggedTemplateExpression<'_> {
if let Some(type_parameters) = &self.type_arguments {
type_parameters.print(p, ctx);
}
self.quasi.print(p, ctx);
self.quasi.print(p, ctx | Context::TAGGED_TEMPLATE);
}
}
@@ -2648,7 +2650,7 @@ impl Gen for JSXIdentifier<'_> {
impl Gen for JSXMemberExpressionObject<'_> {
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
match self {
Self::IdentifierReference(ident) => ident.print(p, ctx),
Self::IdentifierReference(ident) => p.print_jsx_identifier_reference(ident),
Self::MemberExpression(member_expr) => member_expr.print(p, ctx),
Self::ThisExpression(expr) => expr.print(p, ctx),
}
@@ -2667,7 +2669,7 @@ impl Gen for JSXElementName<'_> {
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
match self {
Self::Identifier(identifier) => identifier.print(p, ctx),
Self::IdentifierReference(identifier) => identifier.print(p, ctx),
Self::IdentifierReference(identifier) => p.print_jsx_identifier_reference(identifier),
Self::NamespacedName(namespaced_name) => namespaced_name.print(p, ctx),
Self::MemberExpression(member_expr) => member_expr.print(p, ctx),
Self::ThisExpression(expr) => expr.print(p, ctx),
@@ -3091,7 +3093,7 @@ impl Gen for PrivateIdentifier<'_> {
p.add_source_mapping_for_private_name(self.span, name);
p.print_ascii_byte(b'#');
p.print_str(name);
p.print_name(name);
}
}
@@ -3647,7 +3649,7 @@ impl Gen for TSTemplateLiteralType<'_> {
types.print(p, ctx);
p.print_ascii_byte(b'}');
}
p.print_str(item.value.raw.as_str());
p.print_template_quasi_raw(item.value.raw.as_str(), false);
}
p.print_ascii_byte(b'`');
}
@@ -3930,7 +3932,7 @@ impl Gen for TSImportTypeQualifier<'_> {
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
match self {
TSImportTypeQualifier::Identifier(ident) => {
p.print_str(ident.name.as_str());
p.print_name(ident.name.as_str());
}
TSImportTypeQualifier::QualifiedName(qualified) => {
qualified.print(p, ctx);
@@ -3943,7 +3945,7 @@ impl Gen for TSImportTypeQualifiedName<'_> {
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
self.left.print(p, ctx);
p.print_ascii_byte(b'.');
p.print_str(self.right.name.as_str());
p.print_name(self.right.name.as_str());
}
}
@@ -3964,7 +3966,7 @@ impl Gen for TSIndexSignature<'_> {
p.print_str("readonly ");
}
p.print_ascii_byte(b'[');
p.print_str(self.parameter.name.as_str());
p.print_name(self.parameter.name.as_str());
p.print_colon();
p.print_soft_space();
self.parameter.type_annotation.print(p, ctx);
@@ -4220,7 +4222,7 @@ impl Gen for TSEnumMember<'_> {
p.add_source_mapping(quasi.span);
p.print_str("[`");
p.print_str(quasi.value.raw.as_str());
p.print_template_quasi_raw(quasi.value.raw.as_str(), false);
p.print_str("`]");
}
}
+190 -3
View File
@@ -100,7 +100,9 @@ pub struct Codegen<'a> {
// states
prev_op_end: usize,
prev_reg_exp_end: usize,
/// Output position after a regex or escaped identifier that requires a separating space
/// before a following identifier, even if its last byte is not an identifier character.
need_space_before_identifier: usize,
need_space_before_dot: usize,
print_next_indent_as_space: bool,
binary_expr_stack: Stack<BinaryExpressionVisitor<'a>>,
@@ -148,6 +150,22 @@ pub struct Codegen<'a> {
sourcemap_builder: Option<SourcemapBuilder<'a>>,
}
/// How [`Codegen::print_non_ascii_escaped`] escapes a non-ASCII character, by the grammar
/// of the text being printed.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum NonAsciiEscape {
/// An IdentifierName.
Identifier,
/// A regular expression's pattern source (raw text; an escaping backslash is consumed;
/// astral characters become a surrogate pair, valid with or without the `u`/`v` flags).
RegExp,
/// An untagged template literal quasi (raw text; an escaping backslash is consumed, an
/// LS/PS line continuation is respelled with LF, `</script` is escaped).
TemplateRaw,
/// A directive's raw string text (as `TemplateRaw`, without the `</script` handling).
Directive,
}
impl Default for Codegen<'_> {
fn default() -> Self {
Self::new()
@@ -187,7 +205,7 @@ impl<'a> Codegen<'a> {
class_stack: Stack::with_capacity(4),
next_class_id: ClassId::from_usize(0),
prev_op_end: 0,
prev_reg_exp_end: 0,
need_space_before_identifier: 0,
prev_op: None,
start_of_stmt: 0,
start_of_arrow_expr: 0,
@@ -306,6 +324,166 @@ impl<'a> Codegen<'a> {
self.code.print_str(s);
}
/// Print an identifier name (binding, reference, label, private name, property key),
/// `\u`-escaping non-ASCII characters when [CodegenOptions::ascii_only] is set.
#[inline]
pub fn print_name(&mut self, s: &str) {
if !self.options.ascii_only || s.is_ascii() {
self.code.print_str(s);
} else {
self.print_non_ascii_escaped(s, NonAsciiEscape::Identifier);
self.need_space_before_identifier = self.code.len();
}
}
/// Print `s`, replacing every non-ASCII character with an escape sequence according to
/// `mode` (see [NonAsciiEscape]). ASCII runs are copied through unchanged, except that
/// `</script` is escaped in [NonAsciiEscape::TemplateRaw] mode like everywhere else
/// template text is printed.
#[cold]
pub(crate) fn print_non_ascii_escaped(&mut self, s: &str, mode: NonAsciiEscape) {
let raw = mode != NonAsciiEscape::Identifier;
let mut start = 0;
for (i, ch) in s.char_indices() {
if ch.is_ascii() {
continue;
}
// In raw text (regex source, template raw) the character may itself be the
// target of a backslash: an identity escape `\é` in a non-`u` regex, a
// NonEscapeCharacter `\é` or a LineContinuation `\<LS>` in a template. Emitting
// `\uXXXX` after that backslash would produce `\\uXXXX` — an escaped backslash
// followed by literal text — so the backslash is consumed here instead: `\é` and
// `\u00E9` mean the same in both grammars.
let mut end = i;
let mut escaped = false;
if raw {
let backslashes =
s.as_bytes()[start..i].iter().rev().take_while(|&&b| b == b'\\').count();
escaped = backslashes % 2 == 1;
if escaped {
end = i - 1;
}
}
if start < end {
if mode == NonAsciiEscape::TemplateRaw {
self.print_str_escaping_script_close_tag(&s[start..end]);
} else {
self.code.print_str(&s[start..end]);
}
}
start = i + ch.len_utf8();
// A LineContinuation (`\` + LS/PS) stays a LineContinuation, spelled with LF: it
// contributes nothing to the cooked value either way, and keeping a continuation in
// place means the surrounding characters still lex the same (`$\<LS>{`, `\0\<LS>1`).
let line_continuation = escaped
&& matches!(mode, NonAsciiEscape::TemplateRaw | NonAsciiEscape::Directive)
&& matches!(ch, '\u{2028}' | '\u{2029}');
if line_continuation {
self.print_str("\\\n");
continue;
}
match mode {
NonAsciiEscape::RegExp => self.print_unicode_escape_utf16(ch),
NonAsciiEscape::Identifier
| NonAsciiEscape::TemplateRaw
| NonAsciiEscape::Directive => self.print_unicode_escape(ch),
}
}
if start < s.len() {
if mode == NonAsciiEscape::TemplateRaw {
self.print_str_escaping_script_close_tag(&s[start..]);
} else {
self.code.print_str(&s[start..]);
}
}
}
/// Print `ch` as `\uXXXX`, or as `\u{X…}` above the BMP (ES2015 code point escape:
/// valid in string literals, template literals and identifier names).
#[inline]
pub(crate) fn print_unicode_escape(&mut self, ch: char) {
if let Ok(unit) = u16::try_from(ch as u32) {
self.print_u16_escape(unit);
} else {
self.print_code_point_escape(ch as u32);
}
}
/// Print `ch` as `\uXXXX`, or as an escaped UTF-16 surrogate pair above the BMP. Used in
/// regular expression source, where `\u{…}` requires the `u`/`v` flag. Surrogate pair
/// escapes preserve the original character's matching behavior under the pattern's flags.
#[inline]
fn print_unicode_escape_utf16(&mut self, ch: char) {
let mut units = [0u16; 2];
for unit in ch.encode_utf16(&mut units) {
self.print_u16_escape(*unit);
}
}
#[inline]
fn print_code_point_escape(&mut self, cp: u32) {
const HEX: &[u8; 16] = b"0123456789ABCDEF";
self.print_str("\\u{");
let mut started = false;
for shift in (0..8).rev() {
let nibble = (cp >> (shift * 4)) & 0xF;
if nibble != 0 || started || shift == 0 {
started = true;
self.print_ascii_byte(HEX[nibble as usize]);
}
}
self.print_ascii_byte(b'}');
}
#[inline]
fn print_u16_escape(&mut self, unit: u16) {
const HEX: &[u8; 16] = b"0123456789ABCDEF";
let bytes = [
b'\\',
b'u',
HEX[(unit >> 12) as usize & 0xF],
HEX[(unit >> 8) as usize & 0xF],
HEX[(unit >> 4) as usize & 0xF],
HEX[unit as usize & 0xF],
];
// SAFETY: all 6 bytes are ASCII
unsafe { self.code.print_bytes_unchecked(&bytes) };
}
/// Print a template literal quasi's raw text. Only untagged templates are escaped under
/// [CodegenOptions::ascii_only]: a tag function (e.g. `String.raw`) can observe the raw
/// text, which escaping would change.
#[inline]
pub(crate) fn print_template_quasi_raw(&mut self, raw: &str, tagged: bool) {
if !self.options.ascii_only || tagged || raw.is_ascii() {
self.print_str_escaping_script_close_tag(raw);
} else {
self.print_non_ascii_escaped(raw, NonAsciiEscape::TemplateRaw);
}
}
/// Print a directive's raw string text, escaping non-ASCII characters under
/// [CodegenOptions::ascii_only].
#[inline]
pub(crate) fn print_directive_raw(&mut self, raw: &str) {
if !self.options.ascii_only || raw.is_ascii() {
self.code.print_str(raw);
} else {
self.print_non_ascii_escaped(raw, NonAsciiEscape::Directive);
}
}
/// Print a regular expression's pattern source, escaping non-ASCII characters under
/// [CodegenOptions::ascii_only].
#[inline]
pub(crate) fn print_regex_pattern(&mut self, pattern: &str) {
if !self.options.ascii_only || pattern.is_ascii() {
self.code.print_str(pattern);
} else {
self.print_non_ascii_escaped(pattern, NonAsciiEscape::RegExp);
}
}
/// Push str into the buffer, escaping `</script` to `<\/script`.
#[inline]
pub fn print_str_escaping_script_close_tag(&mut self, s: &str) {
@@ -477,7 +655,7 @@ impl<'a> Codegen<'a> {
fn print_space_before_identifier(&mut self) {
let Some(byte) = self.last_byte() else { return };
if self.prev_reg_exp_end != self.code.len() {
if self.need_space_before_identifier != self.code.len() {
let is_identifier = if byte.is_ascii() {
// Fast path for ASCII (very common case)
is_identifier_part_ascii(byte as char)
@@ -821,6 +999,15 @@ impl<'a> Codegen<'a> {
}
}
/// A component reference in JSX position (`<Foo/>`, `<Foo.Bar/>`). JSX names have no escape
/// syntax, so — unlike [`IdentifierReference`] elsewhere — the (possibly renamed) name is
/// printed verbatim even under `ascii_only`.
fn print_jsx_identifier_reference(&mut self, ident: &IdentifierReference<'_>) {
let name = self.get_identifier_reference_name(ident);
self.add_source_mapping_for_name(ident.span, name);
self.print_str(name);
}
#[inline]
fn get_identifier_reference_name(&self, reference: &IdentifierReference<'a>) -> &'a str {
if let Some(scoping) = &self.scoping
+19
View File
@@ -15,6 +15,23 @@ pub struct CodegenOptions {
/// Default is `false`.
pub minify: bool,
/// Escape non-ASCII characters in string literals, untagged template literals, regular
/// expression literals and identifier names.
///
/// Uses `\uXXXX` for characters up to U+FFFF and `\u{...}` for higher code points.
/// Regular expressions use escaped UTF-16 surrogate pairs for higher code points instead;
/// escaping changes the observable `RegExp.prototype.source` value.
///
/// Code point escapes (`\u{...}`) require ES2015 or later; this option does not provide
/// ES5-compatible output.
///
/// Non-ASCII characters are left unescaped in tagged template quasis (whose raw text is
/// observable), JSX names and text, JSX attribute strings, hashbangs and preserved comments.
/// JavaScript expressions inside tagged templates and JSX are escaped normally.
///
/// Default is `false`.
pub ascii_only: bool,
/// Print comments?
///
/// At present, only some leading comments are preserved.
@@ -50,6 +67,7 @@ impl Default for CodegenOptions {
Self {
single_quote: false,
minify: false,
ascii_only: false,
comments: CommentOptions::default(),
source_map_path: None,
indent_char: IndentChar::default(),
@@ -65,6 +83,7 @@ impl CodegenOptions {
Self {
single_quote: false,
minify: true,
ascii_only: false,
comments: CommentOptions::disabled(),
source_map_path: None,
indent_char: IndentChar::default(),
+61 -17
View File
@@ -90,10 +90,13 @@ impl Codegen<'_> {
allow_backtick,
};
// With `ascii_only`, every UTF-8 lead byte routes to the unicode-escape handler.
let table = if self.options.ascii_only { &ESCAPES_ASCII_ONLY.0 } else { &ESCAPES.0 };
// Loop through bytes.
while let Some(b) = state.peek() {
// Look up whether byte needs escaping
let escape = ESCAPES.0[b as usize];
let escape = table[b as usize];
if escape == Escape::__ {
// No escape required.
// SAFETY: We just checked there's a byte to consume.
@@ -365,6 +368,7 @@ enum Escape {
LS = 15, // LS/PS - U+2028 LINE SEPARATOR or U+2029 PARAGRAPH SEPARATOR (first byte)
NB = 16, // NBSP - Non-breaking space (first byte)
LO = 17, // � - U+FFFD lossy replacement character (first byte)
UC = 18, // any non-ASCII character (UTF-8 lead byte) - `ascii_only` mode
}
/// Struct which ensures content is aligned on 128.
@@ -400,6 +404,20 @@ static ESCAPES: Aligned128<[Escape; 256]> = {
])
};
/// As [`ESCAPES`], but all non-ASCII UTF-8 lead bytes trigger Unicode escaping.
/// Keep `Escape::LO` for 0xEF so surrogate markers are decoded by the same handler
/// in both modes.
static ESCAPES_ASCII_ONLY: Aligned128<[Escape; 256]> = {
let mut table = ESCAPES.0;
let mut i = 0xC0;
while i < 0x100 {
table[i] = Escape::UC;
i += 1;
}
table[0xEF] = Escape::LO;
Aligned128(table)
};
type ByteHandler = unsafe fn(&mut Codegen, &mut PrintStringState);
/// Byte handlers.
@@ -407,10 +425,10 @@ type ByteHandler = unsafe fn(&mut Codegen, &mut PrintStringState);
/// Indexed by `escape as usize - 1` (where `escape` is not `Escape::__`).
/// Must be in same order as discriminants in `Escape`.
///
/// Function pointers are 8 bytes each, so `BYTE_HANDLERS` is 136 bytes in total.
/// Aligned on 128, so first 16 occupy a pair of L1 cache lines.
/// The last will be in separate cache line, but it should be vanishingly rare that it's accessed.
static BYTE_HANDLERS: Aligned128<[ByteHandler; 17]> = Aligned128([
/// On 64-bit targets, the 18 function pointer entries occupy 144 bytes before alignment padding.
/// Aligned on 128, so the first 16 entries occupy a pair of 64-byte cache lines.
/// The remaining two handlers process lossy replacement markers and Unicode escapes.
static BYTE_HANDLERS: Aligned128<[ByteHandler; 18]> = Aligned128([
print_null,
print_bell,
print_backspace,
@@ -428,6 +446,7 @@ static BYTE_HANDLERS: Aligned128<[ByteHandler; 17]> = Aligned128([
print_ls_or_ps,
print_non_breaking_space,
print_lossy_replacement,
print_unicode_escaped,
]);
/// Call byte handler for byte which needs escaping.
@@ -690,21 +709,25 @@ unsafe fn print_lossy_replacement(codegen: &mut Codegen, state: &mut PrintString
if next2 == LOSSY_REPLACEMENT_CHAR_LAST_2_BYTES {
// Get the 4 hex bytes
let bytes = &mut state.bytes;
let hex: [u8; 4] = bytes.as_slice()[3..7].try_into().unwrap();
let mut hex: [u8; 4] = bytes.as_slice()[3..7].try_into().unwrap();
if hex == *b"fffd" {
// Actual lossy replacement character.
// Flush up to and including the lossy replacement character, then skip the 4 hex bytes.
// SAFETY: 0xEF is always the start of a 3-byte Unicode character
unsafe { state.consume_bytes_unchecked(3) };
state.flush(codegen);
// SAFETY: 0xEF is always the start of a 3-byte Unicode character.
// `bytes.as_slice()[3..7]` would have panicked if there weren't 4 more bytes after it.
// All those bytes are ASCII, so this leaves `bytes` on a UTF-8 char boundary.
unsafe { state.consume_bytes_unchecked(4) };
// Start next chunk after the 4 hex bytes
state.start_chunk();
return;
if !codegen.options.ascii_only {
// Flush up to and including the lossy replacement character, then skip the 4 hex bytes.
// SAFETY: 0xEF is always the start of a 3-byte Unicode character
unsafe { state.consume_bytes_unchecked(3) };
state.flush(codegen);
// SAFETY: 0xEF is always the start of a 3-byte Unicode character.
// `bytes.as_slice()[3..7]` would have panicked if there weren't 4 more bytes after it.
// All those bytes are ASCII, so this leaves `bytes` on a UTF-8 char boundary.
unsafe { state.consume_bytes_unchecked(4) };
// Start next chunk after the 4 hex bytes
state.start_chunk();
return;
}
// Match the uppercase hex digits used by `print_unicode_escape`.
hex = *b"FFFD";
}
// Flush text before the lossy replacement character
@@ -730,11 +753,32 @@ unsafe fn print_lossy_replacement(codegen: &mut Codegen, state: &mut PrintString
}
// `lone_surrogates` is `false` or character is some other character starting with 0xEF.
if codegen.options.ascii_only {
// SAFETY: 0xEF is a UTF-8 lead byte, as required by the Unicode escape handler.
unsafe { print_unicode_escaped(codegen, state) };
return;
}
// Advance past the character.
// SAFETY: 0xEF is always the start of a 3-byte Unicode character
unsafe { state.consume_bytes_unchecked(3) };
}
// Non-ASCII UTF-8 lead byte, `ascii_only` mode: print the character as `\uXXXX`
// (or a `\u{...}` code point escape above the BMP).
unsafe fn print_unicode_escaped(codegen: &mut Codegen, state: &mut PrintStringState) {
debug_assert!(state.peek().is_some_and(|b| b >= 0xC0));
// Decode the character at the current position.
// SAFETY: `bytes` is always positioned on a UTF-8 character boundary within a valid `&str`,
// so the remaining slice is valid UTF-8 and non-empty.
let rest = unsafe { std::str::from_utf8_unchecked(state.bytes.as_slice()) };
let ch = rest.chars().next().unwrap();
// SAFETY: consuming exactly one whole character keeps `bytes` on a char boundary.
unsafe { state.flush_and_consume_bytes(codegen, ch.len_utf8()) };
codegen.print_unicode_escape(ch);
}
/// Call a closure while hinting to compiler that this branch is rarely taken.
///
/// "Cold trampoline function", suggested in:
@@ -0,0 +1,246 @@
//! ASCII escaping and exceptions for `CodegenOptions::ascii_only`.
use oxc_allocator::Allocator;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_parser::Parser;
use oxc_semantic::SemanticBuilder;
use oxc_span::SourceType;
use crate::tester::{default_options, test_options};
fn ascii() -> CodegenOptions {
CodegenOptions { ascii_only: true, ..default_options() }
}
fn ascii_min() -> CodegenOptions {
CodegenOptions { ascii_only: true, minify: true, ..CodegenOptions::default() }
}
#[track_caller]
fn test(source: &str, expected: &str) {
assert!(expected.is_ascii(), "expected output must itself be ASCII: {expected:?}");
test_options(source, expected, ascii());
}
#[track_caller]
fn test_minify(source: &str, expected: &str) {
assert!(expected.is_ascii(), "expected output must itself be ASCII: {expected:?}");
test_options(source, expected, ascii_min());
}
#[test]
fn off_by_default() {
test_options("let café = 'naïve';", "let café = \"naïve\";\n", default_options());
}
#[test]
fn string_literals() {
test("let x = 'café';", "let x = \"caf\\u00E9\";\n");
test("let x = '日本語';", "let x = \"\\u65E5\\u672C\\u8A9E\";\n");
// Above the BMP: ES2015 code point escape.
test("let x = '😀';", "let x = \"\\u{1F600}\";\n");
// Existing escapes and line separators retain their values.
test("let x = '\\u00E9';", "let x = \"\\u00E9\";\n");
test("let x = '\u{2028}';", "let x = \"\\u2028\";\n");
// `</script` handling is preserved inside the same literal.
test("let x = 'a</script>é';", "let x = \"a<\\/script>\\u00E9\";\n");
// Directives keep their raw text, with non-ASCII characters escaped in place.
test_minify(
"'use strict'; 'caf\\é'; let x = 'é';",
"\"use strict\";\"caf\\u00E9\";let x=`\\u00E9`;",
);
// A directive containing a LineContinuation is not a Use Strict Directive and must not become one.
test("'use\\\u{2028} strict';", "\"use\\\n strict\";\n");
}
#[test]
fn lone_surrogates_and_replacement_characters() {
for (value, escaped) in [
(r"\uD800\uDBFF \uDC00\uDFFF", r"\ud800\udbff \udc00\udfff"),
("��", r"\uFFFD\uFFFD"),
(r"�\uD800é�\uDC00�", r"\uFFFD\ud800\u00E9\uFFFD\udc00\uFFFD"),
// A three-byte character sharing U+FFFD's lead byte is not a surrogate marker.
(r"\uD800\uFEFF\uFFFF😀", r"\ud800\uFEFF\uFFFF\u{1F600}"),
] {
test(&format!("let x = '{value}';"), &format!("let x = \"{escaped}\";\n"));
test_minify(&format!("let x = '{value}';"), &format!("let x=`{escaped}`;"));
}
}
#[test]
fn identifiers() {
test("let café = 1; café++;", "let caf\\u00E9 = 1;\ncaf\\u00E9++;\n");
// Above the BMP an identifier escape must be a code point escape, not a surrogate pair.
test("let 𠮷 = 1;", "let \\u{20BB7} = 1;\n");
test(
"x.café; x = { café: 1, 'naïve': 2 };",
"x.caf\\u00E9;\nx = {\n\tcaf\\u00E9: 1,\n\t\"na\\u00EFve\": 2\n};\n",
);
test(
"class A { #ñ = 1; m() { return this.#ñ; } }",
"class A {\n\t#\\u00F1 = 1;\n\tm() {\n\t\treturn this.#\\u00F1;\n\t}\n}\n",
);
test("ñ: for (;;) break ñ;", "\\u00F1: for (;;) break \\u00F1;\n");
test(
"export { café as naïve } from 'x';",
"export { caf\\u00E9 as na\\u00EFve } from \"x\";\n",
);
// Unrenamed destructuring keeps its shorthand property.
test("({ ñame } = opts);", "({\\u00F1ame} = opts);\n");
test(
"import x from 'm' with { тип: 'json' };",
"import x from \"m\" with { \\u0442\\u0438\\u043F: \"json\" };\n",
);
}
#[test]
fn renamed_destructuring_assignment() {
let source = "let ñame; ({ ñame } = opts); ({ ñame = fallback } = opts);";
for (name, escaped) in [("a", "a"), ("é", r"\u00E9")] {
for minify in [false, true] {
let allocator = Allocator::default();
let parsed = Parser::new(&allocator, source, SourceType::mjs()).parse();
assert!(parsed.diagnostics.is_empty());
let semantic = SemanticBuilder::new().build(&parsed.program);
assert!(semantic.diagnostics.is_empty());
let mut scoping = semantic.semantic.into_scoping();
let symbol_id = scoping.get_binding(scoping.root_scope_id(), "ñame".into()).unwrap();
scoping.set_symbol_name(symbol_id, name.into());
let code = Codegen::new()
.with_options(CodegenOptions { minify, ..ascii() })
.with_scoping(Some(scoping))
.build(&parsed.program)
.code;
let expected = if minify {
format!(
"let {escaped};({{\\u00F1ame:{escaped}}}=opts);({{\\u00F1ame:{escaped}=fallback}}=opts);"
)
} else {
format!(
"let {escaped};\n({{\\u00F1ame: {escaped}}} = opts);\n({{\\u00F1ame: {escaped} = fallback}} = opts);\n"
)
};
assert_eq!(code, expected);
}
}
}
#[test]
fn minified_identifier_boundaries() {
for (source, expected) in [
("for (𐀀 of xs) {}", r"for(\u{10000} of xs){}"),
("𐀀 in obj; 𐀀 instanceof C;", r"\u{10000} in obj;\u{10000} instanceof C;"),
("𐀀 as T; 𐀀 satisfies T;", r"\u{10000} as T;\u{10000} satisfies T;"),
("import { 𐀀 as x } from 'm';", r#"import{\u{10000} as x}from"m";"#),
("export { x as 𐀀 } from 'm';", r#"export{x as \u{10000}}from"m";"#),
// Punctuation must end the escaped-identifier boundary state.
("𐀀; f(𐀀); class A {}", r"\u{10000};f(\u{10000});class A{}"),
] {
test_minify(source, expected);
}
}
#[test]
fn typescript() {
test("let x: import('m').Тип;", "let x: import(\"m\").\\u0422\\u0438\\u043F;\n");
test("let x: A.Б;", "let x: A.\\u0411;\n");
test("let x: import('m').А.Б;", "let x: import(\"m\").\\u0410.\\u0411;\n");
test(
"interface I { [ключ: string]: number }",
"interface I {\n\t[\\u043A\\u043B\\u044E\\u0447: string]: number;\n}\n",
);
test("type T = `préfixe-${string}`;", "type T = `pr\\u00E9fixe-${string}`;\n");
test("enum E { [`clé`] = 1 }", "enum E {\n\t[`cl\\u00E9`] = 1\n}\n");
}
#[test]
fn regular_expressions() {
test("let r = /café/g;", "let r = /caf\\u00E9/g;\n");
// Regex patterns use surrogate pair escapes, which also work without the `u`/`v` flags.
test("let r = /😀+/u;", "let r = /\\uD83D\\uDE00+/u;\n");
test("let r = /😀/;", "let r = /\\uD83D\\uDE00/;\n");
// An identity-escaped non-ASCII char keeps its meaning: the backslash is consumed
// (`/\é/` and `/\u00E9/` match the same thing; `/\\u00E9/` would not).
test("let r = /[\\–\\—]/;", "let r = /[\\u2013\\u2014]/;\n");
test("let r = /a\\\\é/;", "let r = /a\\\\\\u00E9/;\n");
}
#[test]
fn template_literals() {
test("let x = `café ${y} naïve`;", "let x = `caf\\u00E9 ${y} na\\u00EFve`;\n");
// NonEscapeCharacter: `\é` cooks to `é`; the backslash is consumed so the cooked value is kept.
test("let x = `caf\\é`;", "let x = `caf\\u00E9`;\n");
// LineContinuation with LS: still a LineContinuation (cooks to nothing), spelled with LF so
// the characters around it keep lexing the same way (`$\\<LS>{` must not become `${`).
test("let x = `a\\\u{2028}b`;", "let x = `a\\\nb`;\n");
test("let x = `$\\\u{2028}{`;", "let x = `$\\\n{`;\n");
test("let x = `\\0\\\u{2029}1`;", "let x = `\\0\\\n1`;\n");
test("let x = `a\\\\é`;", "let x = `a\\\\\\u00E9`;\n");
// Exercise both the ASCII chunk before a Unicode escape and the final suffix.
test("let x = `</script>é</script>`;", "let x = `<\\/script>\\u00E9<\\/script>`;\n");
test_minify("let x = `</script>é</script>`;", "let x=`<\\/script>\\u00E9<\\/script>`;");
}
#[test]
fn tagged_template_is_not_escaped() {
// Asserted separately because the expected output is intentionally not ASCII.
test_options("let x = tag`café ${`naïve`}`;", "let x = tag`café ${`na\\u00EFve`}`;\n", ascii());
let source = "tag`é${tag`ü${`ñ`}ö`}ø${`á`}å`; `é`;";
test_options(source, "tag`é${tag`ü${`\\u00F1`}ö`}ø${`\\u00E1`}å`;\n`\\u00E9`;\n", ascii());
test_options(source, "tag`é${tag`ü${`\\u00F1`}ö`}ø${`\\u00E1`}å`;`\\u00E9`;", ascii_min());
}
#[test]
fn jsx_is_not_escaped() {
// JSX names, attribute strings and text are preserved; only embedded JS expressions
// receive Unicode escapes.
test_options(
"<Кнопка label='é' title={'ü'}>ø</Кнопка>;",
"<Кнопка label=\"é\" title={\"\\u00FC\"}>ø</Кнопка>;\n",
ascii(),
);
}
mod sourcemap {
use super::*;
type Position = (u32, u32);
#[track_caller]
fn test(source: &str, minify: bool, mappings: &[(Position, Position)]) {
let allocator = Allocator::default();
let parsed = Parser::new(&allocator, source, SourceType::mjs()).parse();
assert!(parsed.diagnostics.is_empty());
let map = Codegen::new()
.with_options(CodegenOptions { minify, ..ascii() })
.build(&parsed.program)
.map
.expect("sourcemap should be generated");
for &(src, dst) in mappings {
assert!(
map.get_tokens().any(|token| {
(token.get_src_line(), token.get_src_col()) == src
&& (token.get_dst_line(), token.get_dst_col()) == dst
}),
"missing mapping {src:?} -> {dst:?} (minify={minify}) for {source:?}",
);
}
}
#[test]
fn escapes_shift_generated_columns() {
let source = "const café = '😀'; café(next);";
// Source columns count the astral character as two UTF-16 code units.
test(source, false, &[((0, 19), (1, 0)), ((0, 24), (1, 10))]);
test(source, true, &[((0, 19), (0, 28)), ((0, 24), (0, 38))]);
}
#[test]
fn template_separators_collapse_generated_lines() {
for separator in ['\u{2028}', '\u{2029}'] {
let source = format!("const s = `é{separator}${{next}}`; after();");
test(&source, false, &[((1, 2), (0, 25)), ((1, 10), (1, 0))]);
test(&source, true, &[((1, 2), (0, 23)), ((1, 10), (0, 30))]);
}
}
}
+39 -40
View File
@@ -2,7 +2,7 @@
//! * <https://github.com/evanw/esbuild/blob/v0.24.0/internal/js_printer/js_printer_test.go>
//! * <https://github.com/evanw/esbuild/blob/v0.24.0/internal/js_parser/js_parser_test.go>
use crate::tester::{test, test_minify};
use crate::tester::{test, test_ascii, test_minify, test_minify_ascii};
// NOTE: These values are aligned with terser, not esbuild.
#[test]
@@ -879,49 +879,48 @@ fn test_ascii_only() {
test("let π = 'π'", "let π = \"π\";\n");
test("let π_ = 'π'", "let π_ = \"π\";\n");
test("let _π = 'π'", "let _π = \"π\";\n");
// testASCII(t, "let π = 'π'", "let \\u03C0 = \"\\u03C0\";\n");
// testASCII(t, "let π_ = 'π'", "let \\u03C0_ = \"\\u03C0\";\n");
// testASCII(t, "let _π = 'π'", "let _\\u03C0 = \"\\u03C0\";\n");
test_ascii("let π = 'π'", "let \\u03C0 = \"\\u03C0\";\n");
test_ascii("let π_ = 'π'", "let \\u03C0_ = \"\\u03C0\";\n");
test_ascii("let _π = 'π'", "let _\\u03C0 = \"\\u03C0\";\n");
test("let 貓 = '🐈'", "let 貓 = \"🐈\";\n");
test("let 貓abc = '🐈'", "let 貓abc = \"🐈\";\n");
test("let abc貓 = '🐈'", "let abc貓 = \"🐈\";\n");
// testASCII(t, "let 貓 = '🐈'", "let \\u8C93 = \"\\u{1F408}\";\n");
// testASCII(t, "let 貓abc = '🐈'", "let \\u8C93abc = \"\\u{1F408}\";\n");
// testASCII(t, "let abc貓 = '🐈'", "let abc\\u8C93 = \"\\u{1F408}\";\n");
test_ascii("let 貓 = '🐈'", "let \\u8C93 = \"\\u{1F408}\";\n");
test_ascii("let 貓abc = '🐈'", "let \\u8C93abc = \"\\u{1F408}\";\n");
test_ascii("let abc貓 = '🐈'", "let abc\\u8C93 = \"\\u{1F408}\";\n");
// Test a character outside the BMP
test("var 𐀀", "var 𐀀;\n");
test("var \\u{10000}", "var 𐀀;\n");
// testASCII(t, "var 𐀀", "var \\u{10000};\n");
// testASCII(t, "var \\u{10000}", "var \\u{10000};\n");
// testTargetASCII(t, 2015, "'𐀀'", "\"\\u{10000}\";\n");
test_ascii("var 𐀀", "var \\u{10000};\n");
test_ascii("var \\u{10000}", "var \\u{10000};\n");
test_ascii("'𐀀'", "\"\\u{10000}\";\n");
// testTargetASCII(t, 5, "'𐀀'", "\"\\uD800\\uDC00\";\n");
// testTargetASCII(t, 2015, "x.𐀀", "x[\"\\u{10000}\"];\n");
// esbuild rewrites an astral member name to a computed key for ES5; as ES2015+ it stays an identifier.
test_ascii("x.𐀀", "x.\\u{10000};\n");
// testTargetASCII(t, 5, "x.𐀀", "x[\"\\uD800\\uDC00\"];\n");
// Escapes should use consistent case
// testASCII(
// t,
// "var \\u{100a} = {\\u100A: '\\u100A'}",
// "var \\u100A = { \\u100A: \"\\u100A\" };\n",
// );
// testASCII(
// t,
// "var \\u{1000a} = {\\u{1000A}: '\\u{1000A}'}",
// "var \\u{1000A} = { \"\\u{1000A}\": \"\\u{1000A}\" };\n",
// );
test_ascii("var \\u{100a} = {\\u100A: '\\u100A'}", "var \\u100A = { \\u100A: \"\\u100A\" };\n");
// esbuild quotes an astral property key for its ES5 targets; as an ES2015+ identifier it can stay bare.
test_ascii(
"var \\u{1000a} = {\\u{1000A}: '\\u{1000A}'}",
"var \\u{1000A} = { \\u{1000A}: \"\\u{1000A}\" };\n",
);
// These characters should always be escaped
// test( "let x = '\u2028'", "let x = \"\\u2028\";\n");
// test( "let x = '\u2029'", "let x = \"\\u2029\";\n");
// test( "let x = '\uFEFF'", "let x = \"\\uFEFF\";\n");
// Line and paragraph separators are always escaped.
test("let x = '\u{2028}'", "let x = \"\\u2028\";\n");
test("let x = '\u{2029}'", "let x = \"\\u2029\";\n");
// The byte order mark is escaped only in ASCII-only mode.
test("let x = '\u{FEFF}'", "let x = \"\u{FEFF}\";\n");
test_ascii("let x = '\u{FEFF}'", "let x = \"\\uFEFF\";\n");
// There should still be a space before "extends"
// testASCII(t, "class 𐀀 extends π {}", "class \\u{10000} extends \\u03C0 {\n}\n");
// testASCII(t, "(class 𐀀 extends π {})", "(class \\u{10000} extends \\u03C0 {\n});\n");
// test_minifyASCII(t, "class 𐀀 extends π {}", "class \\u{10000} extends \\u03C0{}");
// test_minifyASCII(t, "(class 𐀀 extends π {})", "(class \\u{10000} extends \\u03C0{});");
test_ascii("class 𐀀 extends π {}", "class \\u{10000} extends \\u03C0 {}\n");
test_ascii("(class 𐀀 extends π {})", "(class \\u{10000} extends \\u03C0 {});\n");
test_minify_ascii("class 𐀀 extends π {}", "class \\u{10000} extends \\u03C0{}");
test_minify_ascii("(class 𐀀 extends π {})", "(class \\u{10000} extends \\u03C0{});");
}
#[test]
@@ -973,13 +972,13 @@ fn test_jsx() {
test("<a b={<>{c}</>}/>", "<a b={<>{c}</>} />;\n");
// These can't be escaped because JSX lacks a syntax for escapes
// testJSXASCII(t, "<π/>", "<π />;\n");
// testJSXASCII(t, "<π.𐀀/>", "<π.𐀀 />;\n");
// testJSXASCII(t, "<𐀀.π/>", "<𐀀.π />;\n");
// testJSXASCII(t, "<π>x</π>", "<π>x</π>;\n");
// testJSXASCII(t, "<𐀀>x</𐀀>", "<𐀀>x</𐀀>;\n");
// testJSXASCII(t, "<a π/>", "<a π />;\n");
// testJSXASCII(t, "<a 𐀀/>", "<a 𐀀 />;\n");
test_ascii("<π/>", "<π />;\n");
test_ascii("<π.𐀀/>", "<π.𐀀 />;\n");
test_ascii("<𐀀.π/>", "<𐀀.π />;\n");
test_ascii("<π>x</π>", "<π>x</π>;\n");
test_ascii("<𐀀>x</𐀀>", "<𐀀>x</𐀀>;\n");
test_ascii("<a π/>", "<a π />;\n");
test_ascii("<a 𐀀/>", "<a 𐀀 />;\n");
// JSX text is deliberately not printed as ASCII when JSX preservation is
// enabled. This is because:
@@ -989,10 +988,10 @@ fn test_jsx() {
// c) People do very weird/custom things with JSX that "preserve" shouldn't break
//
// See also: https://github.com/evanw/esbuild/issues/3605
// testJSXASCII(t, "<a b='π'/>", "<a b='π' />;\n");
// testJSXASCII(t, "<a b='𐀀'/>", "<a b='𐀀' />;\n");
// testJSXASCII(t, "<a>π</a>", "<a>π</a>;\n");
// testJSXASCII(t, "<a>𐀀</a>", "<a>𐀀</a>;\n");
test_ascii("<a b='π'/>", "<a b=\"π\" />;\n");
test_ascii("<a b='𐀀'/>", "<a b=\"𐀀\" />;\n");
test_ascii("<a>π</a>", "<a>π</a>;\n");
test_ascii("<a>𐀀</a>", "<a>𐀀</a>;\n");
// testJSXMinify(t, "<a b c={x,y} d='true'/>", "<a b c={(x,y)}d='true'/>;");
// testJSXMinify(t, "<a><b/><c/></a>", "<a><b/><c/></a>;");
@@ -1,4 +1,5 @@
#![expect(clippy::missing_panics_doc, clippy::literal_string_with_formatting_args)]
pub mod ascii_only;
pub mod comments;
pub mod esbuild;
pub mod js;
@@ -24,6 +24,20 @@ pub fn test(source_text: &str, expected: &str) {
test_options(source_text, expected, default_options());
}
#[track_caller]
pub fn test_ascii(source_text: &str, expected: &str) {
test_options(source_text, expected, CodegenOptions { ascii_only: true, ..default_options() });
}
#[track_caller]
pub fn test_minify_ascii(source_text: &str, expected: &str) {
test_options(
source_text,
expected,
CodegenOptions { ascii_only: true, minify: true, ..CodegenOptions::default() },
);
}
#[track_caller]
pub fn test_same(source_text: &str) {
test(source_text, source_text);
+18
View File
@@ -8,6 +8,24 @@ export interface CodegenOptions {
* @default true
*/
removeWhitespace?: boolean
/**
* Escape non-ASCII characters in string literals, untagged template literals, regular
* expression literals and identifier names.
*
* Uses `\uXXXX` for characters up to U+FFFF and `\u{...}` for higher code points.
* Regular expressions use escaped UTF-16 surrogate pairs for higher code points instead;
* escaping changes the observable `RegExp.prototype.source` value.
*
* Code point escapes (`\u{...}`) require ES2015 or later; this option does not provide
* ES5-compatible output.
*
* Non-ASCII characters are left unescaped in tagged template quasis (whose raw text is
* observable), JSX names and text, JSX attribute strings, hashbangs and preserved comments.
* JavaScript expressions inside tagged templates and JSX are escaped normally.
*
* @default false
*/
asciiOnly?: boolean
/**
* How to handle legal comments (comments containing `@license`, `@preserve`, or starting with `//!`/`/*!`).
*
+18
View File
@@ -8,6 +8,24 @@ export interface CodegenOptions {
* @default true
*/
removeWhitespace?: boolean
/**
* Escape non-ASCII characters in string literals, untagged template literals, regular
* expression literals and identifier names.
*
* Uses `\uXXXX` for characters up to U+FFFF and `\u{...}` for higher code points.
* Regular expressions use escaped UTF-16 surrogate pairs for higher code points instead;
* escaping changes the observable `RegExp.prototype.source` value.
*
* Code point escapes (`\u{...}`) require ES2015 or later; this option does not provide
* ES5-compatible output.
*
* Non-ASCII characters are left unescaped in tagged template quasis (whose raw text is
* observable), JSX names and text, JSX attribute strings, hashbangs and preserved comments.
* JavaScript expressions inside tagged templates and JSX are escaped normally.
*
* @default false
*/
asciiOnly?: boolean
/**
* How to handle legal comments (comments containing `@license`, `@preserve`, or starting with `//!`/`/*!`).
*
+19 -1
View File
@@ -391,6 +391,23 @@ pub struct CodegenOptions {
/// @default true
pub remove_whitespace: Option<bool>,
/// Escape non-ASCII characters in string literals, untagged template literals, regular
/// expression literals and identifier names.
///
/// Uses `\uXXXX` for characters up to U+FFFF and `\u{...}` for higher code points.
/// Regular expressions use escaped UTF-16 surrogate pairs for higher code points instead;
/// escaping changes the observable `RegExp.prototype.source` value.
///
/// Code point escapes (`\u{...}`) require ES2015 or later; this option does not provide
/// ES5-compatible output.
///
/// Non-ASCII characters are left unescaped in tagged template quasis (whose raw text is
/// observable), JSX names and text, JSX attribute strings, hashbangs and preserved comments.
/// JavaScript expressions inside tagged templates and JSX are escaped normally.
///
/// @default false
pub ascii_only: Option<bool>,
/// How to handle legal comments (comments containing `@license`, `@preserve`, or starting with `//!`/`/*!`).
///
/// * `"none"` - Do not preserve any legal comments.
@@ -406,7 +423,7 @@ pub struct CodegenOptions {
impl Default for CodegenOptions {
fn default() -> Self {
Self { remove_whitespace: Some(true), legal_comments: None }
Self { remove_whitespace: Some(true), ascii_only: None, legal_comments: None }
}
}
@@ -423,6 +440,7 @@ impl CodegenOptions {
// Need to remove all comments.
oxc_codegen::CodegenOptions { minify: false, ..oxc_codegen::CodegenOptions::minify() }
};
opts.ascii_only = self.ascii_only.unwrap_or(false);
if let Some(legal) = &self.legal_comments {
opts.comments.legal = match legal {
+30
View File
@@ -27,6 +27,36 @@ describe("simple", () => {
expect(ret.code).toBe("function foo() {\n\tvar bar;\n\tbar(undefined);\n}\nfoo();\n");
});
it.each([
// spellchecker:disable-next-line
[true, undefined, "export let caf\\u00E9=`na\\u00EFve \\u2615`;"],
// spellchecker:disable-next-line
[true, true, "export let caf\\u00E9=`na\\u00EFve \\u2615`;"],
// spellchecker:disable-next-line
[true, false, 'export let caf\\u00E9 = "na\\u00EFve \\u2615";\n'],
[false, true, "export let café=`naïve ☕`;"],
[false, false, 'export let café = "naïve ☕";\n'],
[undefined, true, "export let café=`naïve ☕`;"],
[undefined, false, 'export let café = "naïve ☕";\n'],
] as const)(
"codegen.asciiOnly=%s, removeWhitespace=%s",
(asciiOnly, removeWhitespace, expected) => {
const ret = minifySync("test.js", "export let café = 'naïve ☕';", {
codegen: { asciiOnly, removeWhitespace },
});
expect(ret.code).toBe(expected);
expect(ret.errors.length).toBe(0);
},
);
it("preserves Unicode legal comments with codegen.asciiOnly", () => {
const ret = minifySync("test.js", "/*! café 😀 */\nexport let x = 'é';", {
codegen: { asciiOnly: true, legalComments: "inline" },
});
expect(ret.code).toBe("/*! café 😀 */\nexport let x=`\\u00E9`;");
expect(ret.errors.length).toBe(0);
});
it("defaults to esnext", () => {
const code = "try { foo } catch (e) {}";
const ret = minifySync("test.js", code);