mirror of
https://github.com/aphrody-labs/oxc.git
synced 2026-10-15 06:39:17 +00:00
feat(codegen): add ascii_only option (#25994)
Closes #17068. Supersedes #18095 / #18118. Adds `CodegenOptions::ascii_only` (and `codegen.asciiOnly` in `oxc_minify_napi`): escape every non-ASCII character in string literals, untagged template literals, regexp literals, directives and identifier names so the output is 7-bit clean — like esbuild `--charset=ascii` / terser `format.ascii_only`. Off by default; the default path is untouched (codegen bench and conformance unchanged). ```js // Input let café = "naïve 😀"; let re = /[–—]/u; // Output with ascii_only: true let caf\u00E9 = "na\u00EFve \u{1F600}"; let re = /[\u2013\u2014]/u; ``` Characters above U+FFFF use ES2015 code-point escapes. Regex patterns use escaped surrogate pairs instead; RegExp.prototype.source reflects the escaped spelling. Tagged template quasis, JSX names/text/attribute strings, hashbangs and preserved comments may still contain non-ASCII characters. JavaScript expressions inside tagged templates and JSX are escaped normally. Performance; with the option disabled, codegen codspeed benchmarks are withing noise (<1%). With the option on, performance is only affected for files that contain non ascii characters. --- Our use case is a large Vite/Rolldown app served through proxies that mangle non-ASCII JS (previously handled by terser's `ascii_only`); #17068 also lists deno_core snapshots and Chrome extension content scripts. AI-assisted (Claude Code); I reviewed the code, wrote the tests, and have been running this in production builds via a vendored `oxc_codegen`. --------- Co-authored-by: Samuel Attard <6634592+MarshallOfSound@users.noreply.github.com> Co-authored-by: Cameron Clark <cameron.clark@hey.com> Co-authored-by: Dunqing <dengqing0821@gmail.com>
This commit is contained in:
14 files changed
+688
-83
No files matched your search
@@ -70,3 +70,10 @@ pn = "pn"
|
||||
oce = "oce"
|
||||
ome = "ome"
|
||||
froms = "froms"
|
||||
|
||||
[type.ascii-only-tests]
|
||||
extend-glob = ["ascii_only.rs"]
|
||||
|
||||
[type.ascii-only-tests.extend-words]
|
||||
# Prefix of café in escaped Unicode test fixtures.
|
||||
caf = "caf"
|
||||
@@ -26,6 +26,8 @@ bitflags! {
|
||||
///
|
||||
/// When set, TypeScript syntax features are enabled in the output.
|
||||
const TYPESCRIPT = 1 << 2;
|
||||
/// The current template literal is tagged, so preserve its raw quasis.
|
||||
const TAGGED_TEMPLATE = 1 << 3;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ impl Gen for Directive<'_> {
|
||||
}
|
||||
}
|
||||
quote.print(p);
|
||||
p.print_str(directive);
|
||||
p.print_directive_raw(directive);
|
||||
quote.print(p);
|
||||
p.print_ascii_byte(b';');
|
||||
p.print_soft_newline();
|
||||
@@ -1044,7 +1044,7 @@ impl Gen for ImportAttribute<'_> {
|
||||
fn r#gen(&self, p: &mut Codegen, _ctx: Context) {
|
||||
match &self.key {
|
||||
ImportAttributeKey::Identifier(identifier) => {
|
||||
p.print_str(identifier.name.as_str());
|
||||
p.print_name(identifier.name.as_str());
|
||||
}
|
||||
ImportAttributeKey::StringLiteral(literal) => {
|
||||
p.print_string_literal(literal, false);
|
||||
@@ -1395,7 +1395,7 @@ impl Gen for IdentifierReference<'_> {
|
||||
let name = p.get_identifier_reference_name(self);
|
||||
p.print_space_before_identifier();
|
||||
p.add_source_mapping_for_name(self.span, name);
|
||||
p.print_str(name);
|
||||
p.print_name(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1403,7 +1403,7 @@ impl Gen for IdentifierName<'_> {
|
||||
fn r#gen(&self, p: &mut Codegen, _ctx: Context) {
|
||||
p.print_space_before_identifier();
|
||||
p.add_source_mapping_for_name(self.span, &self.name);
|
||||
p.print_str(self.name.as_str());
|
||||
p.print_name(self.name.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1412,7 +1412,7 @@ impl Gen for BindingIdentifier<'_> {
|
||||
let name = p.get_binding_identifier_name(self);
|
||||
p.print_space_before_identifier();
|
||||
p.add_source_mapping_for_name(self.span, name);
|
||||
p.print_str(name);
|
||||
p.print_name(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1420,7 +1420,7 @@ impl Gen for LabelIdentifier<'_> {
|
||||
fn r#gen(&self, p: &mut Codegen, _ctx: Context) {
|
||||
p.print_space_before_identifier();
|
||||
p.add_source_mapping_for_name(self.span, &self.name);
|
||||
p.print_str(self.name.as_str());
|
||||
p.print_name(self.name.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1513,10 +1513,10 @@ impl Gen for RegExpLiteral<'_> {
|
||||
p.print_hard_space();
|
||||
}
|
||||
p.print_ascii_byte(b'/');
|
||||
p.print_str(self.regex.pattern.text.as_str());
|
||||
p.print_regex_pattern(self.regex.pattern.text.as_str());
|
||||
p.print_ascii_byte(b'/');
|
||||
p.print_str(self.regex.flags.to_inline_string().as_str());
|
||||
p.prev_reg_exp_end = p.code().len();
|
||||
p.need_space_before_identifier = p.code().len();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2214,10 +2214,10 @@ impl Gen for AssignmentTargetPropertyIdentifier<'_> {
|
||||
self.binding.print(p, ctx);
|
||||
} else {
|
||||
// `({x: a} = y);`
|
||||
p.print_str(self.binding.name.as_str());
|
||||
p.print_name(self.binding.name.as_str());
|
||||
p.print_colon();
|
||||
p.print_soft_space();
|
||||
p.print_str(ident_name);
|
||||
p.print_name(ident_name);
|
||||
}
|
||||
if let Some(expr) = &self.init {
|
||||
p.print_soft_space();
|
||||
@@ -2351,7 +2351,7 @@ impl GenExpr for ImportExpression<'_> {
|
||||
}
|
||||
|
||||
impl Gen for TemplateLiteral<'_> {
|
||||
fn r#gen(&self, p: &mut Codegen, _ctx: Context) {
|
||||
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
|
||||
if self.is_no_substitution_template() {
|
||||
p.print_property_key_annotation(self.span.start);
|
||||
}
|
||||
@@ -2359,14 +2359,16 @@ impl Gen for TemplateLiteral<'_> {
|
||||
p.print_ascii_byte(b'`');
|
||||
debug_assert_eq!(self.quasis.len(), self.expressions.len() + 1);
|
||||
let (first_quasi, remaining_quasis) = self.quasis.split_first().unwrap();
|
||||
p.print_str_escaping_script_close_tag(first_quasi.value.raw.as_str());
|
||||
let tagged = ctx.contains(Context::TAGGED_TEMPLATE);
|
||||
p.print_template_quasi_raw(first_quasi.value.raw.as_str(), tagged);
|
||||
for (expr, quasi) in self.expressions.iter().zip(remaining_quasis) {
|
||||
p.print_str("${");
|
||||
p.print_leading_comments_before_expression(expr);
|
||||
// Nested expressions start with an empty context, so they do not inherit the tag.
|
||||
p.print_expression(expr);
|
||||
p.print_ascii_byte(b'}');
|
||||
p.add_source_mapping(quasi.span);
|
||||
p.print_str_escaping_script_close_tag(quasi.value.raw.as_str());
|
||||
p.print_template_quasi_raw(quasi.value.raw.as_str(), tagged);
|
||||
}
|
||||
p.print_ascii_byte(b'`');
|
||||
}
|
||||
@@ -2380,7 +2382,7 @@ impl GenExpr for TaggedTemplateExpression<'_> {
|
||||
if let Some(type_parameters) = &self.type_arguments {
|
||||
type_parameters.print(p, ctx);
|
||||
}
|
||||
self.quasi.print(p, ctx);
|
||||
self.quasi.print(p, ctx | Context::TAGGED_TEMPLATE);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2648,7 +2650,7 @@ impl Gen for JSXIdentifier<'_> {
|
||||
impl Gen for JSXMemberExpressionObject<'_> {
|
||||
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
|
||||
match self {
|
||||
Self::IdentifierReference(ident) => ident.print(p, ctx),
|
||||
Self::IdentifierReference(ident) => p.print_jsx_identifier_reference(ident),
|
||||
Self::MemberExpression(member_expr) => member_expr.print(p, ctx),
|
||||
Self::ThisExpression(expr) => expr.print(p, ctx),
|
||||
}
|
||||
@@ -2667,7 +2669,7 @@ impl Gen for JSXElementName<'_> {
|
||||
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
|
||||
match self {
|
||||
Self::Identifier(identifier) => identifier.print(p, ctx),
|
||||
Self::IdentifierReference(identifier) => identifier.print(p, ctx),
|
||||
Self::IdentifierReference(identifier) => p.print_jsx_identifier_reference(identifier),
|
||||
Self::NamespacedName(namespaced_name) => namespaced_name.print(p, ctx),
|
||||
Self::MemberExpression(member_expr) => member_expr.print(p, ctx),
|
||||
Self::ThisExpression(expr) => expr.print(p, ctx),
|
||||
@@ -3091,7 +3093,7 @@ impl Gen for PrivateIdentifier<'_> {
|
||||
|
||||
p.add_source_mapping_for_private_name(self.span, name);
|
||||
p.print_ascii_byte(b'#');
|
||||
p.print_str(name);
|
||||
p.print_name(name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3647,7 +3649,7 @@ impl Gen for TSTemplateLiteralType<'_> {
|
||||
types.print(p, ctx);
|
||||
p.print_ascii_byte(b'}');
|
||||
}
|
||||
p.print_str(item.value.raw.as_str());
|
||||
p.print_template_quasi_raw(item.value.raw.as_str(), false);
|
||||
}
|
||||
p.print_ascii_byte(b'`');
|
||||
}
|
||||
@@ -3930,7 +3932,7 @@ impl Gen for TSImportTypeQualifier<'_> {
|
||||
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
|
||||
match self {
|
||||
TSImportTypeQualifier::Identifier(ident) => {
|
||||
p.print_str(ident.name.as_str());
|
||||
p.print_name(ident.name.as_str());
|
||||
}
|
||||
TSImportTypeQualifier::QualifiedName(qualified) => {
|
||||
qualified.print(p, ctx);
|
||||
@@ -3943,7 +3945,7 @@ impl Gen for TSImportTypeQualifiedName<'_> {
|
||||
fn r#gen(&self, p: &mut Codegen, ctx: Context) {
|
||||
self.left.print(p, ctx);
|
||||
p.print_ascii_byte(b'.');
|
||||
p.print_str(self.right.name.as_str());
|
||||
p.print_name(self.right.name.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3964,7 +3966,7 @@ impl Gen for TSIndexSignature<'_> {
|
||||
p.print_str("readonly ");
|
||||
}
|
||||
p.print_ascii_byte(b'[');
|
||||
p.print_str(self.parameter.name.as_str());
|
||||
p.print_name(self.parameter.name.as_str());
|
||||
p.print_colon();
|
||||
p.print_soft_space();
|
||||
self.parameter.type_annotation.print(p, ctx);
|
||||
@@ -4220,7 +4222,7 @@ impl Gen for TSEnumMember<'_> {
|
||||
p.add_source_mapping(quasi.span);
|
||||
|
||||
p.print_str("[`");
|
||||
p.print_str(quasi.value.raw.as_str());
|
||||
p.print_template_quasi_raw(quasi.value.raw.as_str(), false);
|
||||
p.print_str("`]");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,7 +100,9 @@ pub struct Codegen<'a> {
|
||||
|
||||
// states
|
||||
prev_op_end: usize,
|
||||
prev_reg_exp_end: usize,
|
||||
/// Output position after a regex or escaped identifier that requires a separating space
|
||||
/// before a following identifier, even if its last byte is not an identifier character.
|
||||
need_space_before_identifier: usize,
|
||||
need_space_before_dot: usize,
|
||||
print_next_indent_as_space: bool,
|
||||
binary_expr_stack: Stack<BinaryExpressionVisitor<'a>>,
|
||||
@@ -148,6 +150,22 @@ pub struct Codegen<'a> {
|
||||
sourcemap_builder: Option<SourcemapBuilder<'a>>,
|
||||
}
|
||||
|
||||
/// How [`Codegen::print_non_ascii_escaped`] escapes a non-ASCII character, by the grammar
|
||||
/// of the text being printed.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum NonAsciiEscape {
|
||||
/// An IdentifierName.
|
||||
Identifier,
|
||||
/// A regular expression's pattern source (raw text; an escaping backslash is consumed;
|
||||
/// astral characters become a surrogate pair, valid with or without the `u`/`v` flags).
|
||||
RegExp,
|
||||
/// An untagged template literal quasi (raw text; an escaping backslash is consumed, an
|
||||
/// LS/PS line continuation is respelled with LF, `</script` is escaped).
|
||||
TemplateRaw,
|
||||
/// A directive's raw string text (as `TemplateRaw`, without the `</script` handling).
|
||||
Directive,
|
||||
}
|
||||
|
||||
impl Default for Codegen<'_> {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
@@ -187,7 +205,7 @@ impl<'a> Codegen<'a> {
|
||||
class_stack: Stack::with_capacity(4),
|
||||
next_class_id: ClassId::from_usize(0),
|
||||
prev_op_end: 0,
|
||||
prev_reg_exp_end: 0,
|
||||
need_space_before_identifier: 0,
|
||||
prev_op: None,
|
||||
start_of_stmt: 0,
|
||||
start_of_arrow_expr: 0,
|
||||
@@ -306,6 +324,166 @@ impl<'a> Codegen<'a> {
|
||||
self.code.print_str(s);
|
||||
}
|
||||
|
||||
/// Print an identifier name (binding, reference, label, private name, property key),
|
||||
/// `\u`-escaping non-ASCII characters when [CodegenOptions::ascii_only] is set.
|
||||
#[inline]
|
||||
pub fn print_name(&mut self, s: &str) {
|
||||
if !self.options.ascii_only || s.is_ascii() {
|
||||
self.code.print_str(s);
|
||||
} else {
|
||||
self.print_non_ascii_escaped(s, NonAsciiEscape::Identifier);
|
||||
self.need_space_before_identifier = self.code.len();
|
||||
}
|
||||
}
|
||||
|
||||
/// Print `s`, replacing every non-ASCII character with an escape sequence according to
|
||||
/// `mode` (see [NonAsciiEscape]). ASCII runs are copied through unchanged, except that
|
||||
/// `</script` is escaped in [NonAsciiEscape::TemplateRaw] mode like everywhere else
|
||||
/// template text is printed.
|
||||
#[cold]
|
||||
pub(crate) fn print_non_ascii_escaped(&mut self, s: &str, mode: NonAsciiEscape) {
|
||||
let raw = mode != NonAsciiEscape::Identifier;
|
||||
let mut start = 0;
|
||||
for (i, ch) in s.char_indices() {
|
||||
if ch.is_ascii() {
|
||||
continue;
|
||||
}
|
||||
// In raw text (regex source, template raw) the character may itself be the
|
||||
// target of a backslash: an identity escape `\é` in a non-`u` regex, a
|
||||
// NonEscapeCharacter `\é` or a LineContinuation `\<LS>` in a template. Emitting
|
||||
// `\uXXXX` after that backslash would produce `\\uXXXX` — an escaped backslash
|
||||
// followed by literal text — so the backslash is consumed here instead: `\é` and
|
||||
// `\u00E9` mean the same in both grammars.
|
||||
let mut end = i;
|
||||
let mut escaped = false;
|
||||
if raw {
|
||||
let backslashes =
|
||||
s.as_bytes()[start..i].iter().rev().take_while(|&&b| b == b'\\').count();
|
||||
escaped = backslashes % 2 == 1;
|
||||
if escaped {
|
||||
end = i - 1;
|
||||
}
|
||||
}
|
||||
if start < end {
|
||||
if mode == NonAsciiEscape::TemplateRaw {
|
||||
self.print_str_escaping_script_close_tag(&s[start..end]);
|
||||
} else {
|
||||
self.code.print_str(&s[start..end]);
|
||||
}
|
||||
}
|
||||
start = i + ch.len_utf8();
|
||||
// A LineContinuation (`\` + LS/PS) stays a LineContinuation, spelled with LF: it
|
||||
// contributes nothing to the cooked value either way, and keeping a continuation in
|
||||
// place means the surrounding characters still lex the same (`$\<LS>{`, `\0\<LS>1`).
|
||||
let line_continuation = escaped
|
||||
&& matches!(mode, NonAsciiEscape::TemplateRaw | NonAsciiEscape::Directive)
|
||||
&& matches!(ch, '\u{2028}' | '\u{2029}');
|
||||
if line_continuation {
|
||||
self.print_str("\\\n");
|
||||
continue;
|
||||
}
|
||||
match mode {
|
||||
NonAsciiEscape::RegExp => self.print_unicode_escape_utf16(ch),
|
||||
NonAsciiEscape::Identifier
|
||||
| NonAsciiEscape::TemplateRaw
|
||||
| NonAsciiEscape::Directive => self.print_unicode_escape(ch),
|
||||
}
|
||||
}
|
||||
if start < s.len() {
|
||||
if mode == NonAsciiEscape::TemplateRaw {
|
||||
self.print_str_escaping_script_close_tag(&s[start..]);
|
||||
} else {
|
||||
self.code.print_str(&s[start..]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Print `ch` as `\uXXXX`, or as `\u{X…}` above the BMP (ES2015 code point escape:
|
||||
/// valid in string literals, template literals and identifier names).
|
||||
#[inline]
|
||||
pub(crate) fn print_unicode_escape(&mut self, ch: char) {
|
||||
if let Ok(unit) = u16::try_from(ch as u32) {
|
||||
self.print_u16_escape(unit);
|
||||
} else {
|
||||
self.print_code_point_escape(ch as u32);
|
||||
}
|
||||
}
|
||||
|
||||
/// Print `ch` as `\uXXXX`, or as an escaped UTF-16 surrogate pair above the BMP. Used in
|
||||
/// regular expression source, where `\u{…}` requires the `u`/`v` flag. Surrogate pair
|
||||
/// escapes preserve the original character's matching behavior under the pattern's flags.
|
||||
#[inline]
|
||||
fn print_unicode_escape_utf16(&mut self, ch: char) {
|
||||
let mut units = [0u16; 2];
|
||||
for unit in ch.encode_utf16(&mut units) {
|
||||
self.print_u16_escape(*unit);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn print_code_point_escape(&mut self, cp: u32) {
|
||||
const HEX: &[u8; 16] = b"0123456789ABCDEF";
|
||||
self.print_str("\\u{");
|
||||
let mut started = false;
|
||||
for shift in (0..8).rev() {
|
||||
let nibble = (cp >> (shift * 4)) & 0xF;
|
||||
if nibble != 0 || started || shift == 0 {
|
||||
started = true;
|
||||
self.print_ascii_byte(HEX[nibble as usize]);
|
||||
}
|
||||
}
|
||||
self.print_ascii_byte(b'}');
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn print_u16_escape(&mut self, unit: u16) {
|
||||
const HEX: &[u8; 16] = b"0123456789ABCDEF";
|
||||
let bytes = [
|
||||
b'\\',
|
||||
b'u',
|
||||
HEX[(unit >> 12) as usize & 0xF],
|
||||
HEX[(unit >> 8) as usize & 0xF],
|
||||
HEX[(unit >> 4) as usize & 0xF],
|
||||
HEX[unit as usize & 0xF],
|
||||
];
|
||||
// SAFETY: all 6 bytes are ASCII
|
||||
unsafe { self.code.print_bytes_unchecked(&bytes) };
|
||||
}
|
||||
|
||||
/// Print a template literal quasi's raw text. Only untagged templates are escaped under
|
||||
/// [CodegenOptions::ascii_only]: a tag function (e.g. `String.raw`) can observe the raw
|
||||
/// text, which escaping would change.
|
||||
#[inline]
|
||||
pub(crate) fn print_template_quasi_raw(&mut self, raw: &str, tagged: bool) {
|
||||
if !self.options.ascii_only || tagged || raw.is_ascii() {
|
||||
self.print_str_escaping_script_close_tag(raw);
|
||||
} else {
|
||||
self.print_non_ascii_escaped(raw, NonAsciiEscape::TemplateRaw);
|
||||
}
|
||||
}
|
||||
|
||||
/// Print a directive's raw string text, escaping non-ASCII characters under
|
||||
/// [CodegenOptions::ascii_only].
|
||||
#[inline]
|
||||
pub(crate) fn print_directive_raw(&mut self, raw: &str) {
|
||||
if !self.options.ascii_only || raw.is_ascii() {
|
||||
self.code.print_str(raw);
|
||||
} else {
|
||||
self.print_non_ascii_escaped(raw, NonAsciiEscape::Directive);
|
||||
}
|
||||
}
|
||||
|
||||
/// Print a regular expression's pattern source, escaping non-ASCII characters under
|
||||
/// [CodegenOptions::ascii_only].
|
||||
#[inline]
|
||||
pub(crate) fn print_regex_pattern(&mut self, pattern: &str) {
|
||||
if !self.options.ascii_only || pattern.is_ascii() {
|
||||
self.code.print_str(pattern);
|
||||
} else {
|
||||
self.print_non_ascii_escaped(pattern, NonAsciiEscape::RegExp);
|
||||
}
|
||||
}
|
||||
|
||||
/// Push str into the buffer, escaping `</script` to `<\/script`.
|
||||
#[inline]
|
||||
pub fn print_str_escaping_script_close_tag(&mut self, s: &str) {
|
||||
@@ -477,7 +655,7 @@ impl<'a> Codegen<'a> {
|
||||
fn print_space_before_identifier(&mut self) {
|
||||
let Some(byte) = self.last_byte() else { return };
|
||||
|
||||
if self.prev_reg_exp_end != self.code.len() {
|
||||
if self.need_space_before_identifier != self.code.len() {
|
||||
let is_identifier = if byte.is_ascii() {
|
||||
// Fast path for ASCII (very common case)
|
||||
is_identifier_part_ascii(byte as char)
|
||||
@@ -821,6 +999,15 @@ impl<'a> Codegen<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A component reference in JSX position (`<Foo/>`, `<Foo.Bar/>`). JSX names have no escape
|
||||
/// syntax, so — unlike [`IdentifierReference`] elsewhere — the (possibly renamed) name is
|
||||
/// printed verbatim even under `ascii_only`.
|
||||
fn print_jsx_identifier_reference(&mut self, ident: &IdentifierReference<'_>) {
|
||||
let name = self.get_identifier_reference_name(ident);
|
||||
self.add_source_mapping_for_name(ident.span, name);
|
||||
self.print_str(name);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_identifier_reference_name(&self, reference: &IdentifierReference<'a>) -> &'a str {
|
||||
if let Some(scoping) = &self.scoping
|
||||
|
||||
@@ -15,6 +15,23 @@ pub struct CodegenOptions {
|
||||
/// Default is `false`.
|
||||
pub minify: bool,
|
||||
|
||||
/// Escape non-ASCII characters in string literals, untagged template literals, regular
|
||||
/// expression literals and identifier names.
|
||||
///
|
||||
/// Uses `\uXXXX` for characters up to U+FFFF and `\u{...}` for higher code points.
|
||||
/// Regular expressions use escaped UTF-16 surrogate pairs for higher code points instead;
|
||||
/// escaping changes the observable `RegExp.prototype.source` value.
|
||||
///
|
||||
/// Code point escapes (`\u{...}`) require ES2015 or later; this option does not provide
|
||||
/// ES5-compatible output.
|
||||
///
|
||||
/// Non-ASCII characters are left unescaped in tagged template quasis (whose raw text is
|
||||
/// observable), JSX names and text, JSX attribute strings, hashbangs and preserved comments.
|
||||
/// JavaScript expressions inside tagged templates and JSX are escaped normally.
|
||||
///
|
||||
/// Default is `false`.
|
||||
pub ascii_only: bool,
|
||||
|
||||
/// Print comments?
|
||||
///
|
||||
/// At present, only some leading comments are preserved.
|
||||
@@ -50,6 +67,7 @@ impl Default for CodegenOptions {
|
||||
Self {
|
||||
single_quote: false,
|
||||
minify: false,
|
||||
ascii_only: false,
|
||||
comments: CommentOptions::default(),
|
||||
source_map_path: None,
|
||||
indent_char: IndentChar::default(),
|
||||
@@ -65,6 +83,7 @@ impl CodegenOptions {
|
||||
Self {
|
||||
single_quote: false,
|
||||
minify: true,
|
||||
ascii_only: false,
|
||||
comments: CommentOptions::disabled(),
|
||||
source_map_path: None,
|
||||
indent_char: IndentChar::default(),
|
||||
|
||||
@@ -90,10 +90,13 @@ impl Codegen<'_> {
|
||||
allow_backtick,
|
||||
};
|
||||
|
||||
// With `ascii_only`, every UTF-8 lead byte routes to the unicode-escape handler.
|
||||
let table = if self.options.ascii_only { &ESCAPES_ASCII_ONLY.0 } else { &ESCAPES.0 };
|
||||
|
||||
// Loop through bytes.
|
||||
while let Some(b) = state.peek() {
|
||||
// Look up whether byte needs escaping
|
||||
let escape = ESCAPES.0[b as usize];
|
||||
let escape = table[b as usize];
|
||||
if escape == Escape::__ {
|
||||
// No escape required.
|
||||
// SAFETY: We just checked there's a byte to consume.
|
||||
@@ -365,6 +368,7 @@ enum Escape {
|
||||
LS = 15, // LS/PS - U+2028 LINE SEPARATOR or U+2029 PARAGRAPH SEPARATOR (first byte)
|
||||
NB = 16, // NBSP - Non-breaking space (first byte)
|
||||
LO = 17, // � - U+FFFD lossy replacement character (first byte)
|
||||
UC = 18, // any non-ASCII character (UTF-8 lead byte) - `ascii_only` mode
|
||||
}
|
||||
|
||||
/// Struct which ensures content is aligned on 128.
|
||||
@@ -400,6 +404,20 @@ static ESCAPES: Aligned128<[Escape; 256]> = {
|
||||
])
|
||||
};
|
||||
|
||||
/// As [`ESCAPES`], but all non-ASCII UTF-8 lead bytes trigger Unicode escaping.
|
||||
/// Keep `Escape::LO` for 0xEF so surrogate markers are decoded by the same handler
|
||||
/// in both modes.
|
||||
static ESCAPES_ASCII_ONLY: Aligned128<[Escape; 256]> = {
|
||||
let mut table = ESCAPES.0;
|
||||
let mut i = 0xC0;
|
||||
while i < 0x100 {
|
||||
table[i] = Escape::UC;
|
||||
i += 1;
|
||||
}
|
||||
table[0xEF] = Escape::LO;
|
||||
Aligned128(table)
|
||||
};
|
||||
|
||||
type ByteHandler = unsafe fn(&mut Codegen, &mut PrintStringState);
|
||||
|
||||
/// Byte handlers.
|
||||
@@ -407,10 +425,10 @@ type ByteHandler = unsafe fn(&mut Codegen, &mut PrintStringState);
|
||||
/// Indexed by `escape as usize - 1` (where `escape` is not `Escape::__`).
|
||||
/// Must be in same order as discriminants in `Escape`.
|
||||
///
|
||||
/// Function pointers are 8 bytes each, so `BYTE_HANDLERS` is 136 bytes in total.
|
||||
/// Aligned on 128, so first 16 occupy a pair of L1 cache lines.
|
||||
/// The last will be in separate cache line, but it should be vanishingly rare that it's accessed.
|
||||
static BYTE_HANDLERS: Aligned128<[ByteHandler; 17]> = Aligned128([
|
||||
/// On 64-bit targets, the 18 function pointer entries occupy 144 bytes before alignment padding.
|
||||
/// Aligned on 128, so the first 16 entries occupy a pair of 64-byte cache lines.
|
||||
/// The remaining two handlers process lossy replacement markers and Unicode escapes.
|
||||
static BYTE_HANDLERS: Aligned128<[ByteHandler; 18]> = Aligned128([
|
||||
print_null,
|
||||
print_bell,
|
||||
print_backspace,
|
||||
@@ -428,6 +446,7 @@ static BYTE_HANDLERS: Aligned128<[ByteHandler; 17]> = Aligned128([
|
||||
print_ls_or_ps,
|
||||
print_non_breaking_space,
|
||||
print_lossy_replacement,
|
||||
print_unicode_escaped,
|
||||
]);
|
||||
|
||||
/// Call byte handler for byte which needs escaping.
|
||||
@@ -690,21 +709,25 @@ unsafe fn print_lossy_replacement(codegen: &mut Codegen, state: &mut PrintString
|
||||
if next2 == LOSSY_REPLACEMENT_CHAR_LAST_2_BYTES {
|
||||
// Get the 4 hex bytes
|
||||
let bytes = &mut state.bytes;
|
||||
let hex: [u8; 4] = bytes.as_slice()[3..7].try_into().unwrap();
|
||||
let mut hex: [u8; 4] = bytes.as_slice()[3..7].try_into().unwrap();
|
||||
|
||||
if hex == *b"fffd" {
|
||||
// Actual lossy replacement character.
|
||||
// Flush up to and including the lossy replacement character, then skip the 4 hex bytes.
|
||||
// SAFETY: 0xEF is always the start of a 3-byte Unicode character
|
||||
unsafe { state.consume_bytes_unchecked(3) };
|
||||
state.flush(codegen);
|
||||
// SAFETY: 0xEF is always the start of a 3-byte Unicode character.
|
||||
// `bytes.as_slice()[3..7]` would have panicked if there weren't 4 more bytes after it.
|
||||
// All those bytes are ASCII, so this leaves `bytes` on a UTF-8 char boundary.
|
||||
unsafe { state.consume_bytes_unchecked(4) };
|
||||
// Start next chunk after the 4 hex bytes
|
||||
state.start_chunk();
|
||||
return;
|
||||
if !codegen.options.ascii_only {
|
||||
// Flush up to and including the lossy replacement character, then skip the 4 hex bytes.
|
||||
// SAFETY: 0xEF is always the start of a 3-byte Unicode character
|
||||
unsafe { state.consume_bytes_unchecked(3) };
|
||||
state.flush(codegen);
|
||||
// SAFETY: 0xEF is always the start of a 3-byte Unicode character.
|
||||
// `bytes.as_slice()[3..7]` would have panicked if there weren't 4 more bytes after it.
|
||||
// All those bytes are ASCII, so this leaves `bytes` on a UTF-8 char boundary.
|
||||
unsafe { state.consume_bytes_unchecked(4) };
|
||||
// Start next chunk after the 4 hex bytes
|
||||
state.start_chunk();
|
||||
return;
|
||||
}
|
||||
// Match the uppercase hex digits used by `print_unicode_escape`.
|
||||
hex = *b"FFFD";
|
||||
}
|
||||
|
||||
// Flush text before the lossy replacement character
|
||||
@@ -730,11 +753,32 @@ unsafe fn print_lossy_replacement(codegen: &mut Codegen, state: &mut PrintString
|
||||
}
|
||||
|
||||
// `lone_surrogates` is `false` or character is some other character starting with 0xEF.
|
||||
if codegen.options.ascii_only {
|
||||
// SAFETY: 0xEF is a UTF-8 lead byte, as required by the Unicode escape handler.
|
||||
unsafe { print_unicode_escaped(codegen, state) };
|
||||
return;
|
||||
}
|
||||
|
||||
// Advance past the character.
|
||||
// SAFETY: 0xEF is always the start of a 3-byte Unicode character
|
||||
unsafe { state.consume_bytes_unchecked(3) };
|
||||
}
|
||||
|
||||
// Non-ASCII UTF-8 lead byte, `ascii_only` mode: print the character as `\uXXXX`
|
||||
// (or a `\u{...}` code point escape above the BMP).
|
||||
unsafe fn print_unicode_escaped(codegen: &mut Codegen, state: &mut PrintStringState) {
|
||||
debug_assert!(state.peek().is_some_and(|b| b >= 0xC0));
|
||||
|
||||
// Decode the character at the current position.
|
||||
// SAFETY: `bytes` is always positioned on a UTF-8 character boundary within a valid `&str`,
|
||||
// so the remaining slice is valid UTF-8 and non-empty.
|
||||
let rest = unsafe { std::str::from_utf8_unchecked(state.bytes.as_slice()) };
|
||||
let ch = rest.chars().next().unwrap();
|
||||
// SAFETY: consuming exactly one whole character keeps `bytes` on a char boundary.
|
||||
unsafe { state.flush_and_consume_bytes(codegen, ch.len_utf8()) };
|
||||
codegen.print_unicode_escape(ch);
|
||||
}
|
||||
|
||||
/// Call a closure while hinting to compiler that this branch is rarely taken.
|
||||
///
|
||||
/// "Cold trampoline function", suggested in:
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//! ASCII escaping and exceptions for `CodegenOptions::ascii_only`.
|
||||
|
||||
use oxc_allocator::Allocator;
|
||||
use oxc_codegen::{Codegen, CodegenOptions};
|
||||
use oxc_parser::Parser;
|
||||
use oxc_semantic::SemanticBuilder;
|
||||
use oxc_span::SourceType;
|
||||
|
||||
use crate::tester::{default_options, test_options};
|
||||
|
||||
fn ascii() -> CodegenOptions {
|
||||
CodegenOptions { ascii_only: true, ..default_options() }
|
||||
}
|
||||
|
||||
fn ascii_min() -> CodegenOptions {
|
||||
CodegenOptions { ascii_only: true, minify: true, ..CodegenOptions::default() }
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
fn test(source: &str, expected: &str) {
|
||||
assert!(expected.is_ascii(), "expected output must itself be ASCII: {expected:?}");
|
||||
test_options(source, expected, ascii());
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
fn test_minify(source: &str, expected: &str) {
|
||||
assert!(expected.is_ascii(), "expected output must itself be ASCII: {expected:?}");
|
||||
test_options(source, expected, ascii_min());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn off_by_default() {
|
||||
test_options("let café = 'naïve';", "let café = \"naïve\";\n", default_options());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_literals() {
|
||||
test("let x = 'café';", "let x = \"caf\\u00E9\";\n");
|
||||
test("let x = '日本語';", "let x = \"\\u65E5\\u672C\\u8A9E\";\n");
|
||||
// Above the BMP: ES2015 code point escape.
|
||||
test("let x = '😀';", "let x = \"\\u{1F600}\";\n");
|
||||
// Existing escapes and line separators retain their values.
|
||||
test("let x = '\\u00E9';", "let x = \"\\u00E9\";\n");
|
||||
test("let x = '\u{2028}';", "let x = \"\\u2028\";\n");
|
||||
// `</script` handling is preserved inside the same literal.
|
||||
test("let x = 'a</script>é';", "let x = \"a<\\/script>\\u00E9\";\n");
|
||||
// Directives keep their raw text, with non-ASCII characters escaped in place.
|
||||
test_minify(
|
||||
"'use strict'; 'caf\\é'; let x = 'é';",
|
||||
"\"use strict\";\"caf\\u00E9\";let x=`\\u00E9`;",
|
||||
);
|
||||
// A directive containing a LineContinuation is not a Use Strict Directive and must not become one.
|
||||
test("'use\\\u{2028} strict';", "\"use\\\n strict\";\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lone_surrogates_and_replacement_characters() {
|
||||
for (value, escaped) in [
|
||||
(r"\uD800\uDBFF \uDC00\uDFFF", r"\ud800\udbff \udc00\udfff"),
|
||||
("��", r"\uFFFD\uFFFD"),
|
||||
(r"�\uD800é�\uDC00�", r"\uFFFD\ud800\u00E9\uFFFD\udc00\uFFFD"),
|
||||
// A three-byte character sharing U+FFFD's lead byte is not a surrogate marker.
|
||||
(r"\uD800\uFEFF\uFFFF😀", r"\ud800\uFEFF\uFFFF\u{1F600}"),
|
||||
] {
|
||||
test(&format!("let x = '{value}';"), &format!("let x = \"{escaped}\";\n"));
|
||||
test_minify(&format!("let x = '{value}';"), &format!("let x=`{escaped}`;"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identifiers() {
|
||||
test("let café = 1; café++;", "let caf\\u00E9 = 1;\ncaf\\u00E9++;\n");
|
||||
// Above the BMP an identifier escape must be a code point escape, not a surrogate pair.
|
||||
test("let 𠮷 = 1;", "let \\u{20BB7} = 1;\n");
|
||||
test(
|
||||
"x.café; x = { café: 1, 'naïve': 2 };",
|
||||
"x.caf\\u00E9;\nx = {\n\tcaf\\u00E9: 1,\n\t\"na\\u00EFve\": 2\n};\n",
|
||||
);
|
||||
test(
|
||||
"class A { #ñ = 1; m() { return this.#ñ; } }",
|
||||
"class A {\n\t#\\u00F1 = 1;\n\tm() {\n\t\treturn this.#\\u00F1;\n\t}\n}\n",
|
||||
);
|
||||
test("ñ: for (;;) break ñ;", "\\u00F1: for (;;) break \\u00F1;\n");
|
||||
test(
|
||||
"export { café as naïve } from 'x';",
|
||||
"export { caf\\u00E9 as na\\u00EFve } from \"x\";\n",
|
||||
);
|
||||
// Unrenamed destructuring keeps its shorthand property.
|
||||
test("({ ñame } = opts);", "({\\u00F1ame} = opts);\n");
|
||||
test(
|
||||
"import x from 'm' with { тип: 'json' };",
|
||||
"import x from \"m\" with { \\u0442\\u0438\\u043F: \"json\" };\n",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renamed_destructuring_assignment() {
|
||||
let source = "let ñame; ({ ñame } = opts); ({ ñame = fallback } = opts);";
|
||||
for (name, escaped) in [("a", "a"), ("é", r"\u00E9")] {
|
||||
for minify in [false, true] {
|
||||
let allocator = Allocator::default();
|
||||
let parsed = Parser::new(&allocator, source, SourceType::mjs()).parse();
|
||||
assert!(parsed.diagnostics.is_empty());
|
||||
let semantic = SemanticBuilder::new().build(&parsed.program);
|
||||
assert!(semantic.diagnostics.is_empty());
|
||||
let mut scoping = semantic.semantic.into_scoping();
|
||||
let symbol_id = scoping.get_binding(scoping.root_scope_id(), "ñame".into()).unwrap();
|
||||
scoping.set_symbol_name(symbol_id, name.into());
|
||||
let code = Codegen::new()
|
||||
.with_options(CodegenOptions { minify, ..ascii() })
|
||||
.with_scoping(Some(scoping))
|
||||
.build(&parsed.program)
|
||||
.code;
|
||||
let expected = if minify {
|
||||
format!(
|
||||
"let {escaped};({{\\u00F1ame:{escaped}}}=opts);({{\\u00F1ame:{escaped}=fallback}}=opts);"
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"let {escaped};\n({{\\u00F1ame: {escaped}}} = opts);\n({{\\u00F1ame: {escaped} = fallback}} = opts);\n"
|
||||
)
|
||||
};
|
||||
assert_eq!(code, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minified_identifier_boundaries() {
|
||||
for (source, expected) in [
|
||||
("for (𐀀 of xs) {}", r"for(\u{10000} of xs){}"),
|
||||
("𐀀 in obj; 𐀀 instanceof C;", r"\u{10000} in obj;\u{10000} instanceof C;"),
|
||||
("𐀀 as T; 𐀀 satisfies T;", r"\u{10000} as T;\u{10000} satisfies T;"),
|
||||
("import { 𐀀 as x } from 'm';", r#"import{\u{10000} as x}from"m";"#),
|
||||
("export { x as 𐀀 } from 'm';", r#"export{x as \u{10000}}from"m";"#),
|
||||
// Punctuation must end the escaped-identifier boundary state.
|
||||
("𐀀; f(𐀀); class A {}", r"\u{10000};f(\u{10000});class A{}"),
|
||||
] {
|
||||
test_minify(source, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn typescript() {
|
||||
test("let x: import('m').Тип;", "let x: import(\"m\").\\u0422\\u0438\\u043F;\n");
|
||||
test("let x: A.Б;", "let x: A.\\u0411;\n");
|
||||
test("let x: import('m').А.Б;", "let x: import(\"m\").\\u0410.\\u0411;\n");
|
||||
test(
|
||||
"interface I { [ключ: string]: number }",
|
||||
"interface I {\n\t[\\u043A\\u043B\\u044E\\u0447: string]: number;\n}\n",
|
||||
);
|
||||
test("type T = `préfixe-${string}`;", "type T = `pr\\u00E9fixe-${string}`;\n");
|
||||
test("enum E { [`clé`] = 1 }", "enum E {\n\t[`cl\\u00E9`] = 1\n}\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regular_expressions() {
|
||||
test("let r = /café/g;", "let r = /caf\\u00E9/g;\n");
|
||||
// Regex patterns use surrogate pair escapes, which also work without the `u`/`v` flags.
|
||||
test("let r = /😀+/u;", "let r = /\\uD83D\\uDE00+/u;\n");
|
||||
test("let r = /😀/;", "let r = /\\uD83D\\uDE00/;\n");
|
||||
// An identity-escaped non-ASCII char keeps its meaning: the backslash is consumed
|
||||
// (`/\é/` and `/\u00E9/` match the same thing; `/\\u00E9/` would not).
|
||||
test("let r = /[\\–\\—]/;", "let r = /[\\u2013\\u2014]/;\n");
|
||||
test("let r = /a\\\\é/;", "let r = /a\\\\\\u00E9/;\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn template_literals() {
|
||||
test("let x = `café ${y} naïve`;", "let x = `caf\\u00E9 ${y} na\\u00EFve`;\n");
|
||||
// NonEscapeCharacter: `\é` cooks to `é`; the backslash is consumed so the cooked value is kept.
|
||||
test("let x = `caf\\é`;", "let x = `caf\\u00E9`;\n");
|
||||
// LineContinuation with LS: still a LineContinuation (cooks to nothing), spelled with LF so
|
||||
// the characters around it keep lexing the same way (`$\\<LS>{` must not become `${`).
|
||||
test("let x = `a\\\u{2028}b`;", "let x = `a\\\nb`;\n");
|
||||
test("let x = `$\\\u{2028}{`;", "let x = `$\\\n{`;\n");
|
||||
test("let x = `\\0\\\u{2029}1`;", "let x = `\\0\\\n1`;\n");
|
||||
test("let x = `a\\\\é`;", "let x = `a\\\\\\u00E9`;\n");
|
||||
// Exercise both the ASCII chunk before a Unicode escape and the final suffix.
|
||||
test("let x = `</script>é</script>`;", "let x = `<\\/script>\\u00E9<\\/script>`;\n");
|
||||
test_minify("let x = `</script>é</script>`;", "let x=`<\\/script>\\u00E9<\\/script>`;");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tagged_template_is_not_escaped() {
|
||||
// Asserted separately because the expected output is intentionally not ASCII.
|
||||
test_options("let x = tag`café ${`naïve`}`;", "let x = tag`café ${`na\\u00EFve`}`;\n", ascii());
|
||||
let source = "tag`é${tag`ü${`ñ`}ö`}ø${`á`}å`; `é`;";
|
||||
test_options(source, "tag`é${tag`ü${`\\u00F1`}ö`}ø${`\\u00E1`}å`;\n`\\u00E9`;\n", ascii());
|
||||
test_options(source, "tag`é${tag`ü${`\\u00F1`}ö`}ø${`\\u00E1`}å`;`\\u00E9`;", ascii_min());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jsx_is_not_escaped() {
|
||||
// JSX names, attribute strings and text are preserved; only embedded JS expressions
|
||||
// receive Unicode escapes.
|
||||
test_options(
|
||||
"<Кнопка label='é' title={'ü'}>ø</Кнопка>;",
|
||||
"<Кнопка label=\"é\" title={\"\\u00FC\"}>ø</Кнопка>;\n",
|
||||
ascii(),
|
||||
);
|
||||
}
|
||||
|
||||
mod sourcemap {
|
||||
use super::*;
|
||||
|
||||
type Position = (u32, u32);
|
||||
|
||||
#[track_caller]
|
||||
fn test(source: &str, minify: bool, mappings: &[(Position, Position)]) {
|
||||
let allocator = Allocator::default();
|
||||
let parsed = Parser::new(&allocator, source, SourceType::mjs()).parse();
|
||||
assert!(parsed.diagnostics.is_empty());
|
||||
let map = Codegen::new()
|
||||
.with_options(CodegenOptions { minify, ..ascii() })
|
||||
.build(&parsed.program)
|
||||
.map
|
||||
.expect("sourcemap should be generated");
|
||||
for &(src, dst) in mappings {
|
||||
assert!(
|
||||
map.get_tokens().any(|token| {
|
||||
(token.get_src_line(), token.get_src_col()) == src
|
||||
&& (token.get_dst_line(), token.get_dst_col()) == dst
|
||||
}),
|
||||
"missing mapping {src:?} -> {dst:?} (minify={minify}) for {source:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escapes_shift_generated_columns() {
|
||||
let source = "const café = '😀'; café(next);";
|
||||
// Source columns count the astral character as two UTF-16 code units.
|
||||
test(source, false, &[((0, 19), (1, 0)), ((0, 24), (1, 10))]);
|
||||
test(source, true, &[((0, 19), (0, 28)), ((0, 24), (0, 38))]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn template_separators_collapse_generated_lines() {
|
||||
for separator in ['\u{2028}', '\u{2029}'] {
|
||||
let source = format!("const s = `é{separator}${{next}}`; after();");
|
||||
test(&source, false, &[((1, 2), (0, 25)), ((1, 10), (1, 0))]);
|
||||
test(&source, true, &[((1, 2), (0, 23)), ((1, 10), (0, 30))]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
//! * <https://github.com/evanw/esbuild/blob/v0.24.0/internal/js_printer/js_printer_test.go>
|
||||
//! * <https://github.com/evanw/esbuild/blob/v0.24.0/internal/js_parser/js_parser_test.go>
|
||||
|
||||
use crate::tester::{test, test_minify};
|
||||
use crate::tester::{test, test_ascii, test_minify, test_minify_ascii};
|
||||
|
||||
// NOTE: These values are aligned with terser, not esbuild.
|
||||
#[test]
|
||||
@@ -879,49 +879,48 @@ fn test_ascii_only() {
|
||||
test("let π = 'π'", "let π = \"π\";\n");
|
||||
test("let π_ = 'π'", "let π_ = \"π\";\n");
|
||||
test("let _π = 'π'", "let _π = \"π\";\n");
|
||||
// testASCII(t, "let π = 'π'", "let \\u03C0 = \"\\u03C0\";\n");
|
||||
// testASCII(t, "let π_ = 'π'", "let \\u03C0_ = \"\\u03C0\";\n");
|
||||
// testASCII(t, "let _π = 'π'", "let _\\u03C0 = \"\\u03C0\";\n");
|
||||
test_ascii("let π = 'π'", "let \\u03C0 = \"\\u03C0\";\n");
|
||||
test_ascii("let π_ = 'π'", "let \\u03C0_ = \"\\u03C0\";\n");
|
||||
test_ascii("let _π = 'π'", "let _\\u03C0 = \"\\u03C0\";\n");
|
||||
|
||||
test("let 貓 = '🐈'", "let 貓 = \"🐈\";\n");
|
||||
test("let 貓abc = '🐈'", "let 貓abc = \"🐈\";\n");
|
||||
test("let abc貓 = '🐈'", "let abc貓 = \"🐈\";\n");
|
||||
// testASCII(t, "let 貓 = '🐈'", "let \\u8C93 = \"\\u{1F408}\";\n");
|
||||
// testASCII(t, "let 貓abc = '🐈'", "let \\u8C93abc = \"\\u{1F408}\";\n");
|
||||
// testASCII(t, "let abc貓 = '🐈'", "let abc\\u8C93 = \"\\u{1F408}\";\n");
|
||||
test_ascii("let 貓 = '🐈'", "let \\u8C93 = \"\\u{1F408}\";\n");
|
||||
test_ascii("let 貓abc = '🐈'", "let \\u8C93abc = \"\\u{1F408}\";\n");
|
||||
test_ascii("let abc貓 = '🐈'", "let abc\\u8C93 = \"\\u{1F408}\";\n");
|
||||
|
||||
// Test a character outside the BMP
|
||||
test("var 𐀀", "var 𐀀;\n");
|
||||
test("var \\u{10000}", "var 𐀀;\n");
|
||||
// testASCII(t, "var 𐀀", "var \\u{10000};\n");
|
||||
// testASCII(t, "var \\u{10000}", "var \\u{10000};\n");
|
||||
// testTargetASCII(t, 2015, "'𐀀'", "\"\\u{10000}\";\n");
|
||||
test_ascii("var 𐀀", "var \\u{10000};\n");
|
||||
test_ascii("var \\u{10000}", "var \\u{10000};\n");
|
||||
test_ascii("'𐀀'", "\"\\u{10000}\";\n");
|
||||
// testTargetASCII(t, 5, "'𐀀'", "\"\\uD800\\uDC00\";\n");
|
||||
// testTargetASCII(t, 2015, "x.𐀀", "x[\"\\u{10000}\"];\n");
|
||||
// esbuild rewrites an astral member name to a computed key for ES5; as ES2015+ it stays an identifier.
|
||||
test_ascii("x.𐀀", "x.\\u{10000};\n");
|
||||
// testTargetASCII(t, 5, "x.𐀀", "x[\"\\uD800\\uDC00\"];\n");
|
||||
|
||||
// Escapes should use consistent case
|
||||
// testASCII(
|
||||
// t,
|
||||
// "var \\u{100a} = {\\u100A: '\\u100A'}",
|
||||
// "var \\u100A = { \\u100A: \"\\u100A\" };\n",
|
||||
// );
|
||||
// testASCII(
|
||||
// t,
|
||||
// "var \\u{1000a} = {\\u{1000A}: '\\u{1000A}'}",
|
||||
// "var \\u{1000A} = { \"\\u{1000A}\": \"\\u{1000A}\" };\n",
|
||||
// );
|
||||
test_ascii("var \\u{100a} = {\\u100A: '\\u100A'}", "var \\u100A = { \\u100A: \"\\u100A\" };\n");
|
||||
// esbuild quotes an astral property key for its ES5 targets; as an ES2015+ identifier it can stay bare.
|
||||
test_ascii(
|
||||
"var \\u{1000a} = {\\u{1000A}: '\\u{1000A}'}",
|
||||
"var \\u{1000A} = { \\u{1000A}: \"\\u{1000A}\" };\n",
|
||||
);
|
||||
|
||||
// These characters should always be escaped
|
||||
// test( "let x = '\u2028'", "let x = \"\\u2028\";\n");
|
||||
// test( "let x = '\u2029'", "let x = \"\\u2029\";\n");
|
||||
// test( "let x = '\uFEFF'", "let x = \"\\uFEFF\";\n");
|
||||
// Line and paragraph separators are always escaped.
|
||||
test("let x = '\u{2028}'", "let x = \"\\u2028\";\n");
|
||||
test("let x = '\u{2029}'", "let x = \"\\u2029\";\n");
|
||||
// The byte order mark is escaped only in ASCII-only mode.
|
||||
test("let x = '\u{FEFF}'", "let x = \"\u{FEFF}\";\n");
|
||||
test_ascii("let x = '\u{FEFF}'", "let x = \"\\uFEFF\";\n");
|
||||
|
||||
// There should still be a space before "extends"
|
||||
// testASCII(t, "class 𐀀 extends π {}", "class \\u{10000} extends \\u03C0 {\n}\n");
|
||||
// testASCII(t, "(class 𐀀 extends π {})", "(class \\u{10000} extends \\u03C0 {\n});\n");
|
||||
// test_minifyASCII(t, "class 𐀀 extends π {}", "class \\u{10000} extends \\u03C0{}");
|
||||
// test_minifyASCII(t, "(class 𐀀 extends π {})", "(class \\u{10000} extends \\u03C0{});");
|
||||
test_ascii("class 𐀀 extends π {}", "class \\u{10000} extends \\u03C0 {}\n");
|
||||
test_ascii("(class 𐀀 extends π {})", "(class \\u{10000} extends \\u03C0 {});\n");
|
||||
test_minify_ascii("class 𐀀 extends π {}", "class \\u{10000} extends \\u03C0{}");
|
||||
test_minify_ascii("(class 𐀀 extends π {})", "(class \\u{10000} extends \\u03C0{});");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -973,13 +972,13 @@ fn test_jsx() {
|
||||
test("<a b={<>{c}</>}/>", "<a b={<>{c}</>} />;\n");
|
||||
|
||||
// These can't be escaped because JSX lacks a syntax for escapes
|
||||
// testJSXASCII(t, "<π/>", "<π />;\n");
|
||||
// testJSXASCII(t, "<π.𐀀/>", "<π.𐀀 />;\n");
|
||||
// testJSXASCII(t, "<𐀀.π/>", "<𐀀.π />;\n");
|
||||
// testJSXASCII(t, "<π>x</π>", "<π>x</π>;\n");
|
||||
// testJSXASCII(t, "<𐀀>x</𐀀>", "<𐀀>x</𐀀>;\n");
|
||||
// testJSXASCII(t, "<a π/>", "<a π />;\n");
|
||||
// testJSXASCII(t, "<a 𐀀/>", "<a 𐀀 />;\n");
|
||||
test_ascii("<π/>", "<π />;\n");
|
||||
test_ascii("<π.𐀀/>", "<π.𐀀 />;\n");
|
||||
test_ascii("<𐀀.π/>", "<𐀀.π />;\n");
|
||||
test_ascii("<π>x</π>", "<π>x</π>;\n");
|
||||
test_ascii("<𐀀>x</𐀀>", "<𐀀>x</𐀀>;\n");
|
||||
test_ascii("<a π/>", "<a π />;\n");
|
||||
test_ascii("<a 𐀀/>", "<a 𐀀 />;\n");
|
||||
|
||||
// JSX text is deliberately not printed as ASCII when JSX preservation is
|
||||
// enabled. This is because:
|
||||
@@ -989,10 +988,10 @@ fn test_jsx() {
|
||||
// c) People do very weird/custom things with JSX that "preserve" shouldn't break
|
||||
//
|
||||
// See also: https://github.com/evanw/esbuild/issues/3605
|
||||
// testJSXASCII(t, "<a b='π'/>", "<a b='π' />;\n");
|
||||
// testJSXASCII(t, "<a b='𐀀'/>", "<a b='𐀀' />;\n");
|
||||
// testJSXASCII(t, "<a>π</a>", "<a>π</a>;\n");
|
||||
// testJSXASCII(t, "<a>𐀀</a>", "<a>𐀀</a>;\n");
|
||||
test_ascii("<a b='π'/>", "<a b=\"π\" />;\n");
|
||||
test_ascii("<a b='𐀀'/>", "<a b=\"𐀀\" />;\n");
|
||||
test_ascii("<a>π</a>", "<a>π</a>;\n");
|
||||
test_ascii("<a>𐀀</a>", "<a>𐀀</a>;\n");
|
||||
|
||||
// testJSXMinify(t, "<a b c={x,y} d='true'/>", "<a b c={(x,y)}d='true'/>;");
|
||||
// testJSXMinify(t, "<a><b/><c/></a>", "<a><b/><c/></a>;");
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#![expect(clippy::missing_panics_doc, clippy::literal_string_with_formatting_args)]
|
||||
pub mod ascii_only;
|
||||
pub mod comments;
|
||||
pub mod esbuild;
|
||||
pub mod js;
|
||||
|
||||
@@ -24,6 +24,20 @@ pub fn test(source_text: &str, expected: &str) {
|
||||
test_options(source_text, expected, default_options());
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
pub fn test_ascii(source_text: &str, expected: &str) {
|
||||
test_options(source_text, expected, CodegenOptions { ascii_only: true, ..default_options() });
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
pub fn test_minify_ascii(source_text: &str, expected: &str) {
|
||||
test_options(
|
||||
source_text,
|
||||
expected,
|
||||
CodegenOptions { ascii_only: true, minify: true, ..CodegenOptions::default() },
|
||||
);
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
pub fn test_same(source_text: &str) {
|
||||
test(source_text, source_text);
|
||||
|
||||
Vendored
+18
@@ -8,6 +8,24 @@ export interface CodegenOptions {
|
||||
* @default true
|
||||
*/
|
||||
removeWhitespace?: boolean
|
||||
/**
|
||||
* Escape non-ASCII characters in string literals, untagged template literals, regular
|
||||
* expression literals and identifier names.
|
||||
*
|
||||
* Uses `\uXXXX` for characters up to U+FFFF and `\u{...}` for higher code points.
|
||||
* Regular expressions use escaped UTF-16 surrogate pairs for higher code points instead;
|
||||
* escaping changes the observable `RegExp.prototype.source` value.
|
||||
*
|
||||
* Code point escapes (`\u{...}`) require ES2015 or later; this option does not provide
|
||||
* ES5-compatible output.
|
||||
*
|
||||
* Non-ASCII characters are left unescaped in tagged template quasis (whose raw text is
|
||||
* observable), JSX names and text, JSX attribute strings, hashbangs and preserved comments.
|
||||
* JavaScript expressions inside tagged templates and JSX are escaped normally.
|
||||
*
|
||||
* @default false
|
||||
*/
|
||||
asciiOnly?: boolean
|
||||
/**
|
||||
* How to handle legal comments (comments containing `@license`, `@preserve`, or starting with `//!`/`/*!`).
|
||||
*
|
||||
|
||||
@@ -8,6 +8,24 @@ export interface CodegenOptions {
|
||||
* @default true
|
||||
*/
|
||||
removeWhitespace?: boolean
|
||||
/**
|
||||
* Escape non-ASCII characters in string literals, untagged template literals, regular
|
||||
* expression literals and identifier names.
|
||||
*
|
||||
* Uses `\uXXXX` for characters up to U+FFFF and `\u{...}` for higher code points.
|
||||
* Regular expressions use escaped UTF-16 surrogate pairs for higher code points instead;
|
||||
* escaping changes the observable `RegExp.prototype.source` value.
|
||||
*
|
||||
* Code point escapes (`\u{...}`) require ES2015 or later; this option does not provide
|
||||
* ES5-compatible output.
|
||||
*
|
||||
* Non-ASCII characters are left unescaped in tagged template quasis (whose raw text is
|
||||
* observable), JSX names and text, JSX attribute strings, hashbangs and preserved comments.
|
||||
* JavaScript expressions inside tagged templates and JSX are escaped normally.
|
||||
*
|
||||
* @default false
|
||||
*/
|
||||
asciiOnly?: boolean
|
||||
/**
|
||||
* How to handle legal comments (comments containing `@license`, `@preserve`, or starting with `//!`/`/*!`).
|
||||
*
|
||||
|
||||
@@ -391,6 +391,23 @@ pub struct CodegenOptions {
|
||||
/// @default true
|
||||
pub remove_whitespace: Option<bool>,
|
||||
|
||||
/// Escape non-ASCII characters in string literals, untagged template literals, regular
|
||||
/// expression literals and identifier names.
|
||||
///
|
||||
/// Uses `\uXXXX` for characters up to U+FFFF and `\u{...}` for higher code points.
|
||||
/// Regular expressions use escaped UTF-16 surrogate pairs for higher code points instead;
|
||||
/// escaping changes the observable `RegExp.prototype.source` value.
|
||||
///
|
||||
/// Code point escapes (`\u{...}`) require ES2015 or later; this option does not provide
|
||||
/// ES5-compatible output.
|
||||
///
|
||||
/// Non-ASCII characters are left unescaped in tagged template quasis (whose raw text is
|
||||
/// observable), JSX names and text, JSX attribute strings, hashbangs and preserved comments.
|
||||
/// JavaScript expressions inside tagged templates and JSX are escaped normally.
|
||||
///
|
||||
/// @default false
|
||||
pub ascii_only: Option<bool>,
|
||||
|
||||
/// How to handle legal comments (comments containing `@license`, `@preserve`, or starting with `//!`/`/*!`).
|
||||
///
|
||||
/// * `"none"` - Do not preserve any legal comments.
|
||||
@@ -406,7 +423,7 @@ pub struct CodegenOptions {
|
||||
|
||||
impl Default for CodegenOptions {
|
||||
fn default() -> Self {
|
||||
Self { remove_whitespace: Some(true), legal_comments: None }
|
||||
Self { remove_whitespace: Some(true), ascii_only: None, legal_comments: None }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -423,6 +440,7 @@ impl CodegenOptions {
|
||||
// Need to remove all comments.
|
||||
oxc_codegen::CodegenOptions { minify: false, ..oxc_codegen::CodegenOptions::minify() }
|
||||
};
|
||||
opts.ascii_only = self.ascii_only.unwrap_or(false);
|
||||
|
||||
if let Some(legal) = &self.legal_comments {
|
||||
opts.comments.legal = match legal {
|
||||
|
||||
@@ -27,6 +27,36 @@ describe("simple", () => {
|
||||
expect(ret.code).toBe("function foo() {\n\tvar bar;\n\tbar(undefined);\n}\nfoo();\n");
|
||||
});
|
||||
|
||||
it.each([
|
||||
// spellchecker:disable-next-line
|
||||
[true, undefined, "export let caf\\u00E9=`na\\u00EFve \\u2615`;"],
|
||||
// spellchecker:disable-next-line
|
||||
[true, true, "export let caf\\u00E9=`na\\u00EFve \\u2615`;"],
|
||||
// spellchecker:disable-next-line
|
||||
[true, false, 'export let caf\\u00E9 = "na\\u00EFve \\u2615";\n'],
|
||||
[false, true, "export let café=`naïve ☕`;"],
|
||||
[false, false, 'export let café = "naïve ☕";\n'],
|
||||
[undefined, true, "export let café=`naïve ☕`;"],
|
||||
[undefined, false, 'export let café = "naïve ☕";\n'],
|
||||
] as const)(
|
||||
"codegen.asciiOnly=%s, removeWhitespace=%s",
|
||||
(asciiOnly, removeWhitespace, expected) => {
|
||||
const ret = minifySync("test.js", "export let café = 'naïve ☕';", {
|
||||
codegen: { asciiOnly, removeWhitespace },
|
||||
});
|
||||
expect(ret.code).toBe(expected);
|
||||
expect(ret.errors.length).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves Unicode legal comments with codegen.asciiOnly", () => {
|
||||
const ret = minifySync("test.js", "/*! café 😀 */\nexport let x = 'é';", {
|
||||
codegen: { asciiOnly: true, legalComments: "inline" },
|
||||
});
|
||||
expect(ret.code).toBe("/*! café 😀 */\nexport let x=`\\u00E9`;");
|
||||
expect(ret.errors.length).toBe(0);
|
||||
});
|
||||
|
||||
it("defaults to esnext", () => {
|
||||
const code = "try { foo } catch (e) {}";
|
||||
const ret = minifySync("test.js", code);
|
||||
|
||||
Reference in new issue
Block a user