Files
deno/cli/module_loader.rs

2376 lines
78 KiB
Rust

// Copyright 2018-2026 the Deno authors. MIT license.
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::future::Future;
use std::path::Path;
use std::path::PathBuf;
use std::pin::Pin;
use std::rc::Rc;
use std::str;
use std::sync::Arc;
use std::sync::atomic::AtomicU16;
use std::sync::atomic::Ordering;
use std::time::SystemTime;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_cache_dir::file_fetcher::FetchLocalOptions;
use deno_cache_dir::file_fetcher::MemoryFiles as _;
use deno_core::FastString;
use deno_core::ModuleLoadOptions;
use deno_core::ModuleLoadReferrer;
use deno_core::ModuleLoader;
use deno_core::ModuleResolutionError;
use deno_core::ModuleSource;
use deno_core::ModuleSourceCode;
use deno_core::ModuleSpecifier;
use deno_core::ModuleType;
use deno_core::RequestedModuleType;
use deno_core::SourceCodeCacheInfo;
use deno_core::anyhow::Context as _;
use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_core::error::ModuleLoaderError;
use deno_core::futures::StreamExt;
use deno_core::futures::future::FutureExt;
use deno_core::futures::io::BufReader;
use deno_core::futures::stream::FuturesOrdered;
use deno_core::parking_lot::Mutex;
use deno_core::resolve_url;
use deno_core::serde_json;
use deno_error::JsErrorBox;
use deno_error::JsErrorClass;
use deno_graph::GraphKind;
use deno_graph::ModuleGraph;
use deno_graph::WalkOptions;
use deno_lib::loader::as_deno_resolver_requested_module_type;
use deno_lib::loader::loaded_module_source_to_module_source_code;
use deno_lib::loader::module_type_from_media_and_requested_type;
use deno_lib::npm::NpmRegistryReadPermissionChecker;
use deno_lib::util::hash::FastInsecureHasher;
use deno_lib::worker::CreateModuleLoaderResult;
use deno_lib::worker::ModuleLoaderFactory;
use deno_npm_installer::resolution::HasJsExecutionStartedFlagRc;
use deno_path_util::PathToUrlError;
use deno_path_util::resolve_url_or_path;
use deno_resolver::cache::ParsedSourceCache;
use deno_resolver::file_fetcher::FetchOptions;
use deno_resolver::file_fetcher::FetchPermissionsOptionRef;
use deno_resolver::graph::ResolveWithGraphErrorKind;
use deno_resolver::graph::ResolveWithGraphOptions;
use deno_resolver::graph::format_range_with_colors;
use deno_resolver::loader::LoadCodeSourceError;
use deno_resolver::loader::LoadPreparedModuleError;
use deno_resolver::loader::LoadedModule;
use deno_resolver::loader::LoadedModuleOrAsset;
use deno_resolver::loader::MemoryFiles;
use deno_resolver::loader::StrippingTypesNodeModulesError;
use deno_resolver::npm::DenoInNpmPackageChecker;
use deno_resolver::npm::is_synthetic_module;
use deno_runtime::code_cache;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::ops::require::UnableToGetCwdError;
use deno_runtime::deno_permissions::CheckSpecifierKind;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_web::Blob;
use deno_runtime::tokio_util::create_basic_runtime;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageReq;
use eszip::EszipV2;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use node_resolver::analyze::CjsAnalysisSourceProvider;
use node_resolver::errors::PackageJsonLoadError;
use rustc_hash::FxHashSet;
use sys_traits::FsCanonicalize;
use sys_traits::FsMetadata;
use sys_traits::FsMetadataValue;
use sys_traits::FsRead;
use tokio_util::compat::TokioAsyncReadCompatExt;
use crate::args::CliLockfile;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::TsTypeLib;
use crate::args::jsr_url;
use crate::cache::CodeCache;
use crate::file_fetcher::CliFileFetcher;
use crate::graph_container::MainModuleGraphContainer;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::BuildGraphRequest;
use crate::graph_util::BuildGraphWithNpmOptions;
use crate::graph_util::GraphRootsValidOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::npm::CliNpmResolver;
use crate::resolver::CliCjsTracker;
use crate::resolver::CliResolver;
use crate::sys::CliSys;
use crate::type_checker::CheckError;
use crate::type_checker::CheckOptions;
use crate::type_checker::TypeChecker;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
pub type CliEmitter =
deno_resolver::emit::Emitter<DenoInNpmPackageChecker, CliSys>;
pub type CliDenoResolverModuleLoader =
deno_resolver::loader::ModuleLoader<CliSys>;
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum PrepareModuleLoadError {
#[class(inherit)]
#[error(transparent)]
BuildGraphWithNpmResolution(
#[from] crate::graph_util::BuildGraphWithNpmResolutionError,
),
#[class(inherit)]
#[error(transparent)]
Check(#[from] CheckError),
#[class(inherit)]
#[error(transparent)]
LockfileWrite(#[from] deno_resolver::lockfile::LockfileWriteError),
#[class(inherit)]
#[error(transparent)]
Other(#[from] JsErrorBox),
}
pub struct ModuleLoadPreparer {
options: Arc<CliOptions>,
lockfile: Option<Arc<CliLockfile>>,
module_graph_builder: Arc<ModuleGraphBuilder>,
progress_bar: ProgressBar,
type_checker: Arc<TypeChecker>,
}
pub struct PrepareModuleLoadOptions<'a> {
pub is_dynamic: bool,
pub lib: TsTypeLib,
pub permissions: PermissionsContainer,
pub file_permission_api_name: Option<&'static str>,
pub ext_overwrite: Option<&'a String>,
pub allow_unknown_media_types: bool,
pub allow_sloppy_imports_hints_for_unreferenced_roots: bool,
/// Whether to skip validating the graph roots. This is useful
/// for when you want to defer doing this until later (ex. get the
/// graph back, reload some specifiers in it, then do graph validation).
pub skip_graph_roots_validation: bool,
pub file_content_overrides: HashMap<ModuleSpecifier, Arc<[u8]>>,
/// Per-specifier response header overrides (e.g. `content-type`) applied to
/// the graph loader. Used to preserve a blob's media type when its content
/// is supplied out-of-band via `file_content_overrides`.
pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
}
impl ModuleLoadPreparer {
#[allow(clippy::too_many_arguments, reason = "construction")]
pub fn new(
options: Arc<CliOptions>,
lockfile: Option<Arc<CliLockfile>>,
module_graph_builder: Arc<ModuleGraphBuilder>,
progress_bar: ProgressBar,
type_checker: Arc<TypeChecker>,
) -> Self {
Self {
options,
lockfile,
module_graph_builder,
progress_bar,
type_checker,
}
}
/// This method must be called for a module or a static importer of that
/// module before attempting to `load()` it from a `JsRuntime`. It will
/// populate the graph data in memory with the necessary source code, write
/// emits where necessary or report any module graph / type checking errors.
pub async fn prepare_module_load(
&self,
graph: &mut ModuleGraph,
roots: &[ModuleSpecifier],
options: PrepareModuleLoadOptions<'_>,
) -> Result<(), PrepareModuleLoadError> {
log::debug!("Preparing module load.");
let PrepareModuleLoadOptions {
is_dynamic,
lib,
permissions,
file_permission_api_name,
ext_overwrite,
allow_unknown_media_types,
allow_sloppy_imports_hints_for_unreferenced_roots,
skip_graph_roots_validation,
file_content_overrides,
file_header_overrides,
} = options;
let _pb_clear_guard = self.progress_bar.deferred_keep_initialize_alive();
let mut loader = self
.module_graph_builder
.create_graph_loader_with_permissions(
permissions,
file_permission_api_name,
);
if !file_content_overrides.is_empty() {
loader.set_file_content_overrides(file_content_overrides);
}
for (specifier, headers) in file_header_overrides {
loader.insert_file_header_override(specifier, headers);
}
if let Some(ext) = ext_overwrite {
let maybe_content_type = match ext.as_str() {
"ts" => Some("text/typescript"),
"tsx" => Some("text/tsx"),
"js" => Some("text/javascript"),
"jsx" => Some("text/jsx"),
_ => None,
};
if let Some(content_type) = maybe_content_type {
for root in roots {
loader.insert_file_header_override(
root.clone(),
std::collections::HashMap::from([(
"content-type".to_string(),
content_type.to_string(),
)]),
);
}
}
}
log::debug!("Building module graph.");
let has_type_checked = !graph.roots.is_empty();
self
.module_graph_builder
.build_graph_roots_with_npm_resolution(
graph,
roots.to_vec(),
BuildGraphWithNpmOptions {
is_dynamic,
loader: Some(&loader),
npm_caching: self.options.default_npm_caching_strategy(),
},
)
.await?;
if !skip_graph_roots_validation {
self.graph_roots_valid(
graph,
roots,
GraphRootsValidOptions {
allow_unknown_media_types,
allow_unknown_jsr_exports: false,
allow_sloppy_imports_hints_for_unreferenced_roots,
},
)?;
}
drop(_pb_clear_guard);
// type check if necessary
if self.options.type_check_mode().is_true() && !has_type_checked {
self.type_checker.check(
// todo(perf): since this is only done the first time the graph is
// created, we could avoid the clone of the graph here by providing
// the actual graph on the first run and then getting the Arc<ModuleGraph>
// back from the return value.
graph.clone(),
CheckOptions {
build_fast_check_graph: true,
lib,
reload: self.options.reload_flag(),
type_check_mode: self.options.type_check_mode(),
},
)?;
}
// write the lockfile if there is one and do so after type checking
// as type checking might discover `@types/node`
if let Some(lockfile) = &self.lockfile {
lockfile.write_if_changed()?;
}
log::debug!("Prepared module load.");
Ok(())
}
pub async fn reload_specifiers(
&self,
graph: &mut ModuleGraph,
specifiers: Vec<ModuleSpecifier>,
is_dynamic: bool,
permissions: PermissionsContainer,
) -> Result<(), PrepareModuleLoadError> {
log::debug!(
"Reloading modified files: {}",
specifiers
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
.join(", ")
);
let _pb_clear_guard = self.progress_bar.deferred_keep_initialize_alive();
let loader = self
.module_graph_builder
.create_graph_loader_with_permissions(permissions, None);
self
.module_graph_builder
.build_graph_with_npm_resolution(
graph,
BuildGraphRequest::Reload(specifiers),
BuildGraphWithNpmOptions {
is_dynamic,
loader: Some(&loader),
npm_caching: self.options.default_npm_caching_strategy(),
},
)
.await?;
if let Some(lockfile) = &self.lockfile {
lockfile.write_if_changed()?;
}
Ok(())
}
pub fn graph_roots_valid(
&self,
graph: &ModuleGraph,
roots: &[ModuleSpecifier],
options: GraphRootsValidOptions,
) -> Result<(), JsErrorBox> {
self
.module_graph_builder
.graph_roots_valid(graph, roots, options)
}
}
struct SharedCliModuleLoaderState {
graph_kind: GraphKind,
lib_window: TsTypeLib,
lib_worker: TsTypeLib,
initial_cwd: PathBuf,
is_inspecting: bool,
is_repl: bool,
cjs_tracker: Arc<CliCjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<CliEmitter>,
file_fetcher: Arc<CliFileFetcher>,
has_js_execution_started_flag: HasJsExecutionStartedFlagRc,
in_npm_pkg_checker: DenoInNpmPackageChecker,
main_module_graph_container: Arc<MainModuleGraphContainer>,
memory_files: Arc<MemoryFiles>,
module_load_preparer: Arc<ModuleLoadPreparer>,
npm_registry_permission_checker:
Arc<NpmRegistryReadPermissionChecker<CliSys>>,
npm_resolver: CliNpmResolver,
parsed_source_cache: Arc<ParsedSourceCache>,
module_loader: Arc<CliDenoResolverModuleLoader>,
resolver: Arc<CliResolver>,
sys: CliSys,
in_flight_loads_tracker: InFlightModuleLoadsTracker,
maybe_eszip_loader: Option<Arc<EszipModuleLoader>>,
watcher_communicator: Option<Arc<WatcherCommunicator>>,
}
struct InFlightModuleLoadsTracker {
loads_number: Arc<AtomicU16>,
cleanup_task_timeout: u64,
cleanup_task_handle: Arc<Mutex<Option<tokio::task::JoinHandle<()>>>>,
}
impl InFlightModuleLoadsTracker {
pub fn increase(&self) {
self.loads_number.fetch_add(1, Ordering::Relaxed);
if let Some(task) = self.cleanup_task_handle.lock().take() {
task.abort();
}
}
pub fn decrease(&self, parsed_source_cache: &Arc<ParsedSourceCache>) {
let prev = self.loads_number.fetch_sub(1, Ordering::Relaxed);
if prev == 1 {
let parsed_source_cache = parsed_source_cache.clone();
let timeout = self.cleanup_task_timeout;
let task_handle = tokio::spawn(async move {
// We use a timeout here, which is defined to 10s,
// so that in situations when dynamic imports are loaded after the startup,
// we don't need to recompute and analyze multiple modules.
tokio::time::sleep(std::time::Duration::from_millis(timeout)).await;
parsed_source_cache.free_all();
});
let maybe_prev_task =
self.cleanup_task_handle.lock().replace(task_handle);
if let Some(prev_task) = maybe_prev_task {
prev_task.abort();
}
}
}
}
pub struct CliModuleLoaderFactory {
shared: Arc<SharedCliModuleLoaderState>,
}
impl CliModuleLoaderFactory {
#[allow(clippy::too_many_arguments, reason = "construction")]
pub fn new(
options: &CliOptions,
cjs_tracker: Arc<CliCjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<CliEmitter>,
file_fetcher: Arc<CliFileFetcher>,
has_js_execution_started_flag: HasJsExecutionStartedFlagRc,
in_npm_pkg_checker: DenoInNpmPackageChecker,
main_module_graph_container: Arc<MainModuleGraphContainer>,
memory_files: Arc<MemoryFiles>,
module_load_preparer: Arc<ModuleLoadPreparer>,
npm_registry_permission_checker: Arc<
NpmRegistryReadPermissionChecker<CliSys>,
>,
npm_resolver: CliNpmResolver,
parsed_source_cache: Arc<ParsedSourceCache>,
module_loader: Arc<CliDenoResolverModuleLoader>,
resolver: Arc<CliResolver>,
sys: CliSys,
maybe_eszip_loader: Option<Arc<EszipModuleLoader>>,
watcher_communicator: Option<Arc<WatcherCommunicator>>,
) -> Self {
Self {
shared: Arc::new(SharedCliModuleLoaderState {
graph_kind: options.graph_kind(),
lib_window: options.ts_type_lib_window(),
lib_worker: options.ts_type_lib_worker(),
initial_cwd: options.initial_cwd().to_path_buf(),
is_inspecting: options.is_inspecting(),
is_repl: matches!(
options.sub_command(),
DenoSubcommand::Repl(_) | DenoSubcommand::Jupyter(_)
),
cjs_tracker,
code_cache,
emitter,
file_fetcher,
has_js_execution_started_flag,
in_npm_pkg_checker,
main_module_graph_container,
memory_files,
module_load_preparer,
npm_registry_permission_checker,
npm_resolver,
parsed_source_cache,
module_loader,
resolver,
sys,
in_flight_loads_tracker: InFlightModuleLoadsTracker {
loads_number: Arc::new(AtomicU16::new(0)),
cleanup_task_timeout: 10_000,
cleanup_task_handle: Arc::new(Mutex::new(None)),
},
maybe_eszip_loader,
watcher_communicator,
}),
}
}
#[allow(clippy::too_many_arguments, reason = "construction")]
fn create_with_lib<TGraphContainer: ModuleGraphContainer>(
&self,
graph_container: TGraphContainer,
lib: TsTypeLib,
is_worker: bool,
parent_permissions: PermissionsContainer,
permissions: PermissionsContainer,
maybe_main_module_blob: Option<(ModuleSpecifier, Arc<Blob>)>,
) -> CreateModuleLoaderResult {
let hook_registry =
deno_runtime::deno_node::ops::module_hooks::LoaderHookRegistry::default();
let module_loader =
Rc::new(CliModuleLoader(Rc::new(CliModuleLoaderInner {
lib,
is_worker,
parent_permissions,
permissions,
graph_container: graph_container.clone(),
shared: self.shared.clone(),
loaded_files: Default::default(),
hook_registry: hook_registry.clone(),
maybe_main_module_blob,
})));
{
// Capture a weak reference: the module loader stores `hook_registry`,
// and the registry stores this `default_resolve` callback, so capturing
// a strong `Rc` to the loader here would form a reference cycle
// (loader inner -> hook_registry -> default_resolve -> loader inner)
// that leaks the whole module loader — and the module graph, npm
// installer and registry cache it transitively owns — on every
// `--watch` reload (see denoland/deno#35664). The callback is only
// invoked while the loader is alive, so the upgrade always succeeds in
// practice.
let inner = Rc::downgrade(&module_loader.0);
hook_registry.set_default_resolve(Rc::new(
move |specifier: &str, referrer: &str| {
let Some(inner) = inner.upgrade() else {
return Err(JsErrorBox::generic(
"module loader was dropped before its resolve hook was called",
));
};
inner
.inner_resolve(
specifier,
referrer,
deno_core::ResolutionKind::Import,
false,
)
.map(|s| s.to_string())
.map_err(|e| JsErrorBox::generic(e.to_string()))
},
));
}
let node_require_loader = Rc::new(CliNodeRequireLoader {
cjs_tracker: self.shared.cjs_tracker.clone(),
emitter: self.shared.emitter.clone(),
npm_resolver: self.shared.npm_resolver.clone(),
sys: self.shared.sys.clone(),
graph_container,
in_npm_pkg_checker: self.shared.in_npm_pkg_checker.clone(),
memory_files: self.shared.memory_files.clone(),
npm_registry_permission_checker: self
.shared
.npm_registry_permission_checker
.clone(),
});
CreateModuleLoaderResult {
module_loader,
node_require_loader,
hook_registry: Some(hook_registry),
}
}
}
impl ModuleLoaderFactory for CliModuleLoaderFactory {
fn create_for_main(
&self,
root_permissions: PermissionsContainer,
) -> CreateModuleLoaderResult {
self.create_with_lib(
(*self.shared.main_module_graph_container).clone(),
self.shared.lib_window,
/* is worker */ false,
root_permissions.clone(),
root_permissions,
None,
)
}
fn create_for_worker(
&self,
parent_permissions: PermissionsContainer,
permissions: PermissionsContainer,
maybe_main_module_blob: Option<(ModuleSpecifier, Arc<Blob>)>,
) -> CreateModuleLoaderResult {
self.create_with_lib(
// create a fresh module graph for the worker
WorkerModuleGraphContainer::new(Arc::new(ModuleGraph::new(
self.shared.graph_kind,
))),
self.shared.lib_worker,
/* is worker */ true,
parent_permissions,
permissions,
maybe_main_module_blob,
)
}
}
struct ModuleCodeStringSource {
pub code: ModuleSourceCode,
pub found_url: ModuleSpecifier,
pub module_type: ModuleType,
}
struct CliModuleLoaderInner<TGraphContainer: ModuleGraphContainer> {
lib: TsTypeLib,
is_worker: bool,
/// The initial set of permissions used to resolve the static imports in the
/// worker. These are "allow all" for main worker, and parent thread
/// permissions for Web Worker.
parent_permissions: PermissionsContainer,
permissions: PermissionsContainer,
shared: Arc<SharedCliModuleLoaderState>,
graph_container: TGraphContainer,
loaded_files: RefCell<FxHashSet<ModuleSpecifier>>,
hook_registry: deno_runtime::deno_node::ops::module_hooks::LoaderHookRegistry,
/// For blob/object-URL module workers, the captured root blob and its
/// specifier. Captured synchronously at worker construction so that a
/// subsequent `URL.revokeObjectURL` cannot race the worker's module load.
/// Its content and media type are injected into the graph during
/// `prepare_load`.
maybe_main_module_blob: Option<(ModuleSpecifier, Arc<Blob>)>,
}
// Recursive CJS analysis must not prompt for or acquire new read access. This
// provider is constructed with a prompt-disabled snapshot of CLI permissions.
struct PermissionedCjsAnalysisSourceProvider<'a> {
permissions: PermissionsContainer,
npm_registry_permission_checker: &'a NpmRegistryReadPermissionChecker<CliSys>,
sys: &'a CliSys,
}
impl CjsAnalysisSourceProvider for PermissionedCjsAnalysisSourceProvider<'_> {
fn load_source<'a>(
&'a self,
specifier: &ModuleSpecifier,
) -> Option<Cow<'a, str>> {
let path = deno_path_util::url_to_file_path(specifier).ok()?;
let mut permissions = self.permissions.clone();
let path = self
.npm_registry_permission_checker
.ensure_read_permission(&mut permissions, Cow::Owned(path))
.ok()?;
self
.sys
.fs_read_to_string_lossy(path)
.ok()
.map(|source| Cow::Owned(source.into_owned()))
}
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum ResolveReferrerError {
#[class(inherit)]
#[error(transparent)]
UnableToGetCwd(#[from] UnableToGetCwdError),
#[class(inherit)]
#[error(transparent)]
PathToUrl(#[from] PathToUrlError),
#[class(inherit)]
#[error(transparent)]
ModuleResolution(#[from] ModuleResolutionError),
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum CliModuleLoaderError {
#[class(inherit)]
#[error(transparent)]
Fetch(#[from] deno_resolver::file_fetcher::FetchError),
#[class(inherit)]
#[error(transparent)]
LoadCodeSource(#[from] LoadCodeSourceError),
#[class(inherit)]
#[error(transparent)]
LoadPreparedModule(#[from] LoadPreparedModuleError),
#[class(inherit)]
#[error(transparent)]
PathToUrl(#[from] PathToUrlError),
#[class(inherit)]
#[error(transparent)]
ResolveNpmReqRef(#[from] deno_resolver::npm::ResolveNpmReqRefError),
#[class(inherit)]
#[error(transparent)]
ResolveReferrer(#[from] ResolveReferrerError),
}
/// Applies `deno_resolver::patch_react_cves` to JavaScript module source,
/// handling both string and (valid UTF-8) byte representations without
/// allocating when no patch is needed.
fn patch_react_cves_source(
specifier: &ModuleSpecifier,
code: ModuleSourceCode,
) -> ModuleSourceCode {
// Borrow the source as `&str` and compute the patched output (if any) in an
// inner scope so the borrow of `code` ends before we move it below.
let patched: Option<String> = {
let src = match &code {
ModuleSourceCode::String(s) => s.as_str(),
ModuleSourceCode::Bytes(b) => match std::str::from_utf8(b.as_bytes()) {
Ok(s) => s,
// Non UTF-8 source can't contain the ASCII patterns we look for.
Err(_) => return code,
},
};
match deno_resolver::patch_react_cves(
specifier.as_str(),
Cow::Borrowed(src),
) {
Cow::Borrowed(_) => None,
Cow::Owned(s) => Some(s),
}
};
match patched {
Some(s) => ModuleSourceCode::String(s.into()),
None => code,
}
}
fn has_prepared_dynamic_import(
graph: &ModuleGraph,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&str>,
) -> bool {
let Some(referrer) =
maybe_referrer.and_then(|referrer| ModuleSpecifier::parse(referrer).ok())
else {
return false;
};
let Some(referrer) = graph.get(&referrer) else {
return false;
};
let specifier = graph.resolve(specifier);
referrer.dependencies().values().any(|dependency| {
dependency.is_dynamic
&& dependency
.get_code()
.is_some_and(|dependency| graph.resolve(dependency) == specifier)
})
}
impl<TGraphContainer: ModuleGraphContainer>
CliModuleLoaderInner<TGraphContainer>
{
/// Best-effort prepare a specifier through the module graph when ESM
/// hooks are active. Used both for the initial load preparation and for
/// redirects introduced by a load hook calling `nextLoad(newUrl)`.
/// Failures are intentionally swallowed: a user hook may resolve to a
/// virtual URL the graph builder cannot fetch, in which case the load
/// is serviced entirely by hooks.
async fn prepare_hooked_specifier(
&self,
specifier: &ModuleSpecifier,
is_dynamic_import: bool,
) {
let permissions = if is_dynamic_import {
&self.permissions
} else {
&self.parent_permissions
};
let is_dynamic = is_dynamic_import || self.is_worker;
let mut update_permit = self.graph_container.acquire_update_permit().await;
let graph = update_permit.graph_mut();
let _ = self
.shared
.module_load_preparer
.prepare_module_load(
graph,
std::slice::from_ref(specifier),
PrepareModuleLoadOptions {
is_dynamic,
lib: self.lib,
permissions: permissions.clone(),
file_permission_api_name: None,
ext_overwrite: None,
allow_unknown_media_types: false,
allow_sloppy_imports_hints_for_unreferenced_roots: !is_dynamic_import,
skip_graph_roots_validation: true,
file_content_overrides: HashMap::new(),
file_header_overrides: HashMap::new(),
},
)
.await;
graph.prune_types();
update_permit.commit();
}
async fn load_inner(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
requested_module_type: &RequestedModuleType,
) -> Result<ModuleSource, ModuleLoaderError> {
if specifier.path().ends_with(".node") {
return Err(JsErrorBox::type_error(format!(
"Cannot import native Node.js addon \"{specifier}\" via ESM. \
Native `.node` addons can only be loaded through `require()`. \
Use `createRequire()` from `node:module`, e.g.:\n \
import {{ createRequire }} from \"node:module\";\n \
const require = createRequire(import.meta.url);\n \
const addon = require(\"./addon.node\");"
)));
}
let code_source = self
.load_code_source(specifier, maybe_referrer, requested_module_type)
.await
.map_err(JsErrorBox::from_err)?;
self.code_source_to_module_source(specifier, code_source)
}
fn try_load_inner_sync(
&self,
specifier: &ModuleSpecifier,
requested_module_type: &RequestedModuleType,
) -> Option<Result<ModuleSource, ModuleLoaderError>> {
if specifier.path().ends_with(".node") {
return None;
}
let code_source = match self
.try_load_code_source_sync(specifier, requested_module_type)
.map_err(JsErrorBox::from_err)
{
Ok(Some(code_source)) => code_source,
Ok(None) => return None,
Err(err) => return Some(Err(err)),
};
Some(self.code_source_to_module_source(specifier, code_source))
}
fn code_source_to_module_source(
&self,
specifier: &ModuleSpecifier,
code_source: ModuleCodeStringSource,
) -> Result<ModuleSource, ModuleLoaderError> {
let code = if self.shared.is_inspecting
|| code_source.module_type == ModuleType::Wasm
{
// we need the code with the source map in order for
// it to work with --inspect or --inspect-brk
code_source.code
} else {
// v8 is slower when source maps are present, so we strip them
code_without_source_map(code_source.code)
};
// Apply load-time security mitigations for known React Server Components
// CVEs to JavaScript source. Opt in via `DENO_PATCH_REACT_CVE`. See
// `deno_resolver::patch_react_cves`.
let code = if code_source.module_type == ModuleType::JavaScript
&& deno_resolver::is_react_cve_patch_enabled(&self.shared.sys)
{
patch_react_cves_source(specifier, code)
} else {
code
};
let code_cache = if code_source.module_type == ModuleType::JavaScript {
self.shared.code_cache.as_ref().map(|cache| {
let code_hash = FastInsecureHasher::new_deno_versioned()
.write_hashable(&code)
.finish();
let data = cache
.get_sync(specifier, code_cache::CodeCacheType::EsModule, code_hash)
.map(Cow::from)
.inspect(|_| {
// This log line is also used by tests.
log::debug!(
"V8 code cache hit for ES module: {specifier}, [{code_hash:?}]"
);
});
SourceCodeCacheInfo {
hash: code_hash,
data,
}
})
} else {
None
};
Ok(ModuleSource::new_with_redirect(
code_source.module_type,
code,
specifier,
&code_source.found_url,
code_cache,
))
}
fn try_load_code_source_sync(
&self,
specifier: &ModuleSpecifier,
requested_module_type: &RequestedModuleType,
) -> Result<Option<ModuleCodeStringSource>, CliModuleLoaderError> {
if NpmPackageReqReference::from_specifier(specifier).is_ok()
|| self.shared.in_npm_pkg_checker.in_npm_package(specifier)
|| self.hook_registry.load_active.get()
{
return Ok(None);
}
let graph = self.graph_container.graph();
let deno_resolver_requested_module_type =
as_deno_resolver_requested_module_type(requested_module_type);
let Some(prepared_module) =
self.shared.module_loader.try_load_prepared_module_sync(
&graph,
specifier,
&deno_resolver_requested_module_type,
)?
else {
return Ok(None);
};
Ok(Some(self.loaded_module_to_module_code_string_source(
prepared_module,
requested_module_type,
)))
}
async fn load_code_source(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleSpecifier>,
requested_module_type: &RequestedModuleType,
) -> Result<ModuleCodeStringSource, CliModuleLoaderError> {
// this loader maintains npm specifiers in dynamic imports when resolving
// so that they can be properly preloaded, but now we might receive them
// here, so we need to actually resolve them to a file: specifier here
let specifier = if let Ok(reference) =
NpmPackageReqReference::from_specifier(specifier)
{
let referrer = match maybe_referrer {
// if we're here, it means it was importing from a dynamic import
// and so there will be a referrer
Some(r) => Cow::Borrowed(r),
// but the repl may also end up here and it won't have
// a referrer so create a referrer for it here
None => Cow::Owned(self.resolve_referrer("")?),
};
Cow::Owned(
self
.shared
.resolver
.resolve_non_workspace_npm_req_ref_to_file(
&reference,
&referrer,
ResolutionMode::Import,
NodeResolutionKind::Execution,
)?
.into_url()?,
)
} else {
Cow::Borrowed(specifier)
};
let graph = self.graph_container.graph();
let deno_resolver_requested_module_type =
as_deno_resolver_requested_module_type(requested_module_type);
let cjs_analysis_source_provider = PermissionedCjsAnalysisSourceProvider {
permissions: self.permissions.deep_clone_without_prompt(),
npm_registry_permission_checker: &self
.shared
.npm_registry_permission_checker,
sys: &self.shared.sys,
};
match self
.shared
.module_loader
.load(
&graph,
&specifier,
maybe_referrer,
&deno_resolver_requested_module_type,
Some(&cjs_analysis_source_provider),
)
.await?
{
LoadedModuleOrAsset::Module(prepared_module) => {
Ok(self.loaded_module_to_module_code_string_source(
prepared_module,
requested_module_type,
))
}
LoadedModuleOrAsset::ExternalAsset {
specifier,
statically_analyzable,
} => {
Ok(
self
.load_asset(
&specifier,
if statically_analyzable {
CheckSpecifierKind::Static
} else {
// force permissions
CheckSpecifierKind::Dynamic
},
requested_module_type,
)
.await?,
)
}
}
}
fn loaded_module_to_module_code_string_source(
&self,
loaded_module: LoadedModule,
requested_module_type: &RequestedModuleType,
) -> ModuleCodeStringSource {
ModuleCodeStringSource {
code: loaded_module_source_to_module_source_code(loaded_module.source),
found_url: loaded_module.specifier.into_owned(),
module_type: module_type_from_media_and_requested_type(
loaded_module.media_type,
requested_module_type,
),
}
}
async fn load_asset(
&self,
specifier: &ModuleSpecifier,
check_specifier_kind: CheckSpecifierKind,
requested_module_type: &RequestedModuleType,
) -> Result<ModuleCodeStringSource, deno_resolver::file_fetcher::FetchError>
{
let file = self
.shared
.file_fetcher
.fetch_with_options(
specifier,
// Always check against this worker's own permissions, never the
// broader `parent_permissions`. Dynamic `import()` of a text/bytes
// asset (`with { type: "text" | "bytes" }`) returns the file's raw
// contents to the caller and therefore requires read access; routing
// it through `parent_permissions` would let a worker created with
// restricted permissions read files its parent can but it cannot.
// For statically analyzable imports `check_specifier` skips the read
// check entirely, so the chosen container does not matter there.
FetchPermissionsOptionRef::Restricted(
&self.permissions,
check_specifier_kind,
),
FetchOptions {
local: FetchLocalOptions {
include_mtime: false,
},
maybe_auth: None,
maybe_accept: None,
maybe_cache_setting: Some(
&deno_cache_dir::file_fetcher::CacheSetting::Use,
),
},
)
.await?;
let module_type = match requested_module_type {
RequestedModuleType::Text => ModuleType::Text,
RequestedModuleType::Bytes => ModuleType::Bytes,
RequestedModuleType::Other(kind) => ModuleType::Other(kind.clone()),
RequestedModuleType::None => {
match file.resolve_media_type_and_charset().0 {
MediaType::Wasm => ModuleType::Wasm,
_ => ModuleType::JavaScript,
}
}
t => unreachable!("{t}"),
};
Ok(ModuleCodeStringSource {
code: ModuleSourceCode::Bytes(file.source.into()),
found_url: file.url,
module_type,
})
}
async fn maybe_reload_dynamic(
&self,
graph: &ModuleGraph,
specifier: &ModuleSpecifier,
permissions: &PermissionsContainer,
) -> Result<bool, PrepareModuleLoadError> {
let specifiers_to_reload =
self.check_specifiers_to_reload_for_dynamic_import(graph, specifier);
if specifiers_to_reload.is_empty() {
return Ok(false);
}
let mut graph_permit = self.graph_container.acquire_update_permit().await;
let graph = graph_permit.graph_mut();
self
.shared
.module_load_preparer
.reload_specifiers(
graph,
specifiers_to_reload,
/* is dynamic */ true,
permissions.clone(),
)
.await?;
graph_permit.commit();
Ok(true)
}
fn check_specifiers_to_reload_for_dynamic_import(
&self,
graph: &ModuleGraph,
specifier: &ModuleSpecifier,
) -> Vec<ModuleSpecifier> {
let mut specifiers_to_reload = Vec::new();
let mut module_iter = graph.walk(
std::iter::once(specifier),
WalkOptions {
check_js: deno_graph::CheckJsOption::False,
follow_dynamic: false,
kind: GraphKind::CodeOnly,
prefer_fast_check_graph: false,
},
);
while let Some((specifier, module_entry)) = module_iter.next() {
if specifier.scheme() != "file"
|| self.loaded_files.borrow().contains(specifier)
{
module_iter.skip_previous_dependencies(); // no need to analyze this module's dependencies
continue;
}
let should_reload = match module_entry {
deno_graph::ModuleEntryRef::Module(module) => {
self.has_module_changed_on_file_system(specifier, module.mtime())
}
deno_graph::ModuleEntryRef::Err(err) => {
if matches!(
err.as_kind(),
deno_graph::ModuleErrorKind::Missing { .. }
) {
self.mtime_of_specifier(specifier).is_some() // it exists now
} else {
self.has_module_changed_on_file_system(specifier, err.mtime())
}
}
deno_graph::ModuleEntryRef::Redirect(_) => false,
};
if should_reload {
specifiers_to_reload.push(specifier.clone());
}
}
self.loaded_files.borrow_mut().insert(specifier.clone());
specifiers_to_reload
}
fn has_module_changed_on_file_system(
&self,
specifier: &ModuleSpecifier,
mtime: Option<SystemTime>,
) -> bool {
let Some(loaded_mtime) = mtime else {
return false;
};
self
.mtime_of_specifier(specifier)
.map(|mtime| mtime > loaded_mtime)
.unwrap_or(false)
}
fn mtime_of_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Option<SystemTime> {
deno_path_util::url_to_file_path(specifier)
.ok()
.and_then(|path| self.shared.sys.fs_symlink_metadata(&path).ok())
.and_then(|metadata| metadata.modified().ok())
}
#[allow(clippy::result_large_err, reason = "TODO: investigate")]
fn resolve_referrer(
&self,
referrer: &str,
) -> Result<ModuleSpecifier, ResolveReferrerError> {
let referrer = if referrer.is_empty() && self.shared.is_repl {
// FIXME(bartlomieju): this is a hacky way to provide compatibility with REPL
// and `Deno.core.evalContext` API. Ideally we should always have a referrer filled
"./$deno$repl.mts"
} else {
referrer
};
Ok(if deno_path_util::specifier_has_uri_scheme(referrer) {
deno_core::resolve_url(referrer)?
} else if referrer == "." {
// main module, use the initial cwd
deno_path_util::resolve_path(referrer, &self.shared.initial_cwd)?
} else {
// this cwd check is slow, so try to avoid it
#[allow(
clippy::disallowed_methods,
reason = "ok, actually needs the current cwd"
)]
let cwd = std::env::current_dir().map_err(UnableToGetCwdError)?;
deno_path_util::resolve_path(referrer, &cwd)?
})
}
fn inner_resolve(
&self,
raw_specifier: &str,
raw_referrer: &str,
kind: deno_core::ResolutionKind,
is_import_meta: bool,
) -> Result<ModuleSpecifier, ModuleLoaderError> {
fn ensure_not_jsr_non_jsr_remote_import(
specifier: &ModuleSpecifier,
referrer: &ModuleSpecifier,
) -> Result<(), JsErrorBox> {
if referrer.as_str().starts_with(jsr_url().as_str())
&& !specifier.as_str().starts_with(jsr_url().as_str())
&& matches!(specifier.scheme(), "http" | "https")
{
return Err(JsErrorBox::generic(format!(
"Importing {} blocked. JSR packages cannot import non-JSR remote modules for security reasons.",
specifier
)));
}
Ok(())
}
// `deno eval` and piped stdin have no file on disk — their source is held
// in memory under a synthetic `$deno$` specifier that is already fully
// resolved. Neither the import map nor node resolution has anything to
// contribute, and node resolution actively breaks it when the cwd is inside
// an npm package, because it checks the file system for a file that is
// never going to be there.
if matches!(kind, deno_core::ResolutionKind::MainModule)
&& let Ok(url) = ModuleSpecifier::parse(raw_specifier)
&& is_synthetic_module(&url)
{
return Ok(url);
}
// An `npm:` package's bin entry chosen as the main module is
// resolved to a concrete `file:` URL inside the global npm cache before it
// reaches here. deno_core then re-resolves that already-resolved specifier
// as the main module, running it through the import map. An
// `"imports": { "/": "./" }` entry (recommended in the docs for
// project-root absolute imports) normalizes to a `file:///` =>
// `file:///<project>/` mapping whose prefix match rewrites any file URL
// outside the project root (such as the npm cache) to live underneath the
// project directory, breaking the load. Such a package-internal main
// module is not a user source file and must not be remapped, so bypass the
// import map. A user-provided path entry (e.g. `deno run ./thing.ts`) is
// intentionally left to the import map.
if matches!(kind, deno_core::ResolutionKind::MainModule)
&& let Ok(url) = ModuleSpecifier::parse(raw_specifier)
&& url.scheme() == "file"
&& self.shared.in_npm_pkg_checker.in_npm_package(&url)
{
let canonicalized_url = deno_path_util::url_to_file_path(&url)
.ok()
.and_then(|path| self.shared.sys.fs_canonicalize(&path).ok())
.and_then(|path| deno_path_util::url_from_file_path(&path).ok())
.unwrap_or(url);
return Ok(canonicalized_url);
}
let referrer = self
.resolve_referrer(raw_referrer)
.map_err(JsErrorBox::from_err)?;
let graph = self.graph_container.graph();
let result = self.shared.resolver.resolve_with_graph(
graph.as_ref(),
raw_specifier,
&referrer,
deno_graph::Position::zeroed(),
ResolveWithGraphOptions {
mode: ResolutionMode::Import,
kind: NodeResolutionKind::Execution,
// leave npm specifiers as-is for dynamic imports so that
// the loader can properly install them if necessary
maintain_npm_specifiers: matches!(
kind,
deno_core::ResolutionKind::DynamicImport
) && !is_import_meta,
},
);
let specifier = match result {
Ok(specifier) => specifier,
Err(err) => {
if let Some(specifier) = err
.maybe_specifier()
.filter(|_| is_import_meta)
.and_then(|s| s.into_owned().into_url().ok())
{
specifier
} else {
match err.into_kind() {
ResolveWithGraphErrorKind::Resolution(err) => {
// todo(dsherret): why do we have a newline here? Document it.
return Err(JsErrorBox::type_error(format!(
"{}\n",
err.to_string_with_range()
)));
}
err => return Err(JsErrorBox::from_err(err)),
}
}
}
};
// only verify this for an import and not import.meta.resolve
if !is_import_meta {
ensure_not_jsr_non_jsr_remote_import(&specifier, &referrer)?;
}
Ok(specifier)
}
}
#[derive(Clone)]
// todo(dsherret): this double Rc boxing is not ideal
pub struct CliModuleLoader<TGraphContainer: ModuleGraphContainer>(
Rc<CliModuleLoaderInner<TGraphContainer>>,
);
impl<TGraphContainer: ModuleGraphContainer> ModuleLoader
for CliModuleLoader<TGraphContainer>
{
fn resolve(
&self,
specifier: &str,
referrer: &str,
kind: deno_core::ResolutionKind,
) -> deno_core::ModuleResolveResponse {
self.0.inner_resolve(specifier, referrer, kind, false)
}
fn resolve_with_scope(
&self,
scope: &mut deno_core::v8::PinScope,
specifier: &str,
referrer: &str,
kind: deno_core::ResolutionKind,
import_attributes: &std::collections::HashMap<String, String>,
) -> deno_core::ModuleResolveResponse {
// CJS modules generated by Deno import `node:module` internally. Node does
// not expose that implementation detail to resolve hooks, so only bypass
// hooks for CJS referrers. User ESM imports of `node:module` still go
// through hooks.
if specifier == "node:module"
&& let Ok(referrer) = ModuleSpecifier::parse(referrer)
&& self.0.shared.cjs_tracker.get_referrer_kind(&referrer)
== ResolutionMode::Require
{
return self
.0
.inner_resolve(specifier, referrer.as_str(), kind, false);
}
if specifier == "node:module"
&& let Ok(referrer) = ModuleSpecifier::parse(referrer)
{
let media_type = MediaType::from_specifier(&referrer);
if self
.0
.shared
.cjs_tracker
.is_maybe_cjs(&referrer, media_type)
.unwrap_or(false)
{
return self
.0
.inner_resolve(specifier, referrer.as_str(), kind, false);
}
}
let resolved = if let Some(url) = self.0.hook_registry.resolve(
scope,
specifier,
referrer,
import_attributes,
)? {
ModuleSpecifier::parse(&url).map_err(JsErrorBox::from_err)
} else {
self.0.inner_resolve(specifier, referrer, kind, false)
};
// Stash the full import attributes against the resolved URL so the load
// hook (which V8 only hands the `type` attribute) can recover them.
if let Ok(resolved) = &resolved {
self
.0
.hook_registry
.record_resolved_attributes(resolved.as_str(), import_attributes);
}
resolved
}
fn pump_event_loop_during_load(&self) -> bool {
// Load hooks respond through the async bridge, so the event loop must be
// pumped during the recursive load or the load deadlocks.
self.0.hook_registry.load_active.get()
}
fn import_meta_resolve(
&self,
specifier: &str,
referrer: &str,
) -> Result<ModuleSpecifier, ModuleLoaderError> {
self.0.inner_resolve(
specifier,
referrer,
deno_core::ResolutionKind::DynamicImport,
true,
)
}
fn get_host_defined_options<'s>(
&self,
scope: &mut deno_core::v8::PinScope<'s, '_>,
name: &str,
) -> Option<deno_core::v8::Local<'s, deno_core::v8::Data>> {
let name = deno_core::ModuleSpecifier::parse(name).ok()?;
if self.0.shared.in_npm_pkg_checker.in_npm_package(&name) {
Some(create_host_defined_options(scope))
} else {
None
}
}
fn load(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<&ModuleLoadReferrer>,
options: ModuleLoadOptions,
) -> deno_core::ModuleLoadResponse {
let inner = self.0.clone();
if let Some(eszip_loader) = &inner.shared.maybe_eszip_loader {
return eszip_loader.load(specifier);
}
self.0.loaded_files.borrow_mut().insert(specifier.clone());
let specifier = specifier.clone();
let maybe_referrer = maybe_referrer.cloned();
// When load hooks are active, delegate to JS hooks first.
// The hook function itself is synchronous, but this load path has no V8
// scope. The async bridge lets the JS event loop run the hook; blocking
// here waiting for JS would deadlock the same runtime.
// Skip the hook bridge for CJS modules — they go through the
// default loader which invokes the sync load hook chain
// (executeLoadHookChain) in Module._load. Going through both
// the ESM bridge AND the CJS path would call hooks twice.
if self.0.hook_registry.load_active.get()
&& !options.is_synchronous
&& !{
let media_type = deno_media_type::MediaType::from_specifier(&specifier);
self
.0
.shared
.cjs_tracker
.is_maybe_cjs(&specifier, media_type)
.unwrap_or(false)
}
{
let import_attributes = hook_load_import_attributes(
&self.0.hook_registry,
specifier.as_str(),
&options.requested_module_type,
);
let receiver = self
.0
.hook_registry
.push_load(specifier.to_string(), import_attributes);
return deno_core::ModuleLoadResponse::Async(
async move {
let hook_result = match receiver.await {
Ok(r) => r,
Err(_) => {
return Err(JsErrorBox::generic("module load hook cancelled"));
}
};
match hook_result {
Ok((Some(source), format, _effective_url)) => {
let module_type = pick_hook_module_type(
format.as_deref(),
&options.requested_module_type,
);
Ok(deno_core::ModuleSource::new(
module_type,
deno_core::ModuleSourceCode::String(source.into()),
&specifier,
None,
))
}
Ok((None, Some(format), _effective_url))
if format == "builtin" && specifier.scheme() == "node" =>
{
Ok(deno_core::ModuleSource::new(
deno_core::ModuleType::JavaScript,
deno_core::ModuleSourceCode::String("".to_string().into()),
&specifier,
None,
))
}
Ok((None, _, effective_url)) => {
// Fallthrough: hooks didn't intercept. If the user's load hook
// chain delegated via `nextLoad(newUrl)`, fetch source from
// that URL but keep the module's identity at the original
// specifier (matches Node's hook semantics).
let fetch_specifier = effective_url
.as_deref()
.and_then(|u| deno_core::ModuleSpecifier::parse(u).ok())
.filter(|u| u != &specifier);
if let Some(redirect) = fetch_specifier.as_ref() {
// The redirect target wasn't seen by the initial graph
// preparation pass; prepare it now so load_inner can find
// it in the graph.
inner
.prepare_hooked_specifier(redirect, options.is_dynamic_import)
.await;
}
let load_url = fetch_specifier.as_ref().unwrap_or(&specifier);
let source = inner
.load_inner(
load_url,
maybe_referrer.as_ref().map(|r| &r.specifier),
&options.requested_module_type,
)
.await?;
if fetch_specifier.is_some() {
Ok(deno_core::ModuleSource::new(
source.module_type,
source.code,
&specifier,
source.code_cache,
))
} else {
Ok(source)
}
}
Err(err) => Err(JsErrorBox::generic(err)),
}
}
.boxed_local(),
);
}
if options.is_synchronous
&& let Some(result) =
inner.try_load_inner_sync(&specifier, &options.requested_module_type)
{
return deno_core::ModuleLoadResponse::Sync(result);
}
deno_core::ModuleLoadResponse::Async(
async move {
// A `registerHooks` resolve hook may have rewritten this specifier
// to a URL the initial graph preparation never saw (e.g. appended a
// fragment for cache busting), so prepare it on demand.
if inner.hook_registry.resolve_active()
&& !matches!(
options.requested_module_type,
RequestedModuleType::Text | RequestedModuleType::Bytes
)
&& !inner.shared.in_npm_pkg_checker.in_npm_package(&specifier)
&& inner.graph_container.graph().get(&specifier).is_none()
{
inner
.prepare_hooked_specifier(&specifier, options.is_dynamic_import)
.await;
}
inner
.load_inner(
&specifier,
maybe_referrer.as_ref().map(|r| &r.specifier),
&options.requested_module_type,
)
.await
.map_err(|err| {
let Some(referrer) = maybe_referrer else {
return err;
};
let position = deno_graph::Position {
line: referrer.line_number as usize - 1,
character: referrer.column_number as usize - 1,
};
JsErrorBox::new(
err.get_class(),
format!(
"{err}\n at {}",
format_range_with_colors(&deno_graph::Range {
specifier: referrer.specifier,
range: deno_graph::PositionRange {
start: position,
end: position
},
resolution_mode: None
})
),
)
})
}
.boxed_local(),
)
}
fn should_load_synthetic_esm(&self, specifier: &str) -> bool {
self.0.hook_registry.load_active.get() && specifier.starts_with("node:")
}
fn prepare_load(
&self,
specifier: &ModuleSpecifier,
maybe_referrer: Option<String>,
maybe_code: Option<String>,
options: ModuleLoadOptions,
) -> Pin<Box<dyn Future<Output = Result<(), ModuleLoaderError>>>> {
// always call this first unconditionally because it will be
// decremented unconditionally in "finish_load"
self.0.shared.in_flight_loads_tracker.increase();
if matches!(
options.requested_module_type,
RequestedModuleType::Text
| RequestedModuleType::Bytes
| RequestedModuleType::Other(_)
) {
// Text/Bytes imports skip graph preparation, so the file watcher's
// graph reporter never sees them. For dynamic imports, register the
// file directly with the watcher so editing it triggers a reload.
// (Static text/bytes imports are picked up by the initial graph
// build via deno_graph's asset/text edge analysis.)
if options.is_dynamic_import
&& let Some(watcher_communicator) =
self.0.shared.watcher_communicator.as_ref()
&& specifier.scheme() == "file"
&& let Ok(file_path) = specifier.to_file_path()
{
let _ = watcher_communicator.watch_paths(vec![file_path]);
}
return Box::pin(deno_core::futures::future::ready(Ok(())));
}
// When ESM hooks are active, attempt graph preparation best-effort.
// A user hook may have resolved this specifier to a virtual URL the
// graph builder cannot fetch; in that case the load itself will be
// serviced entirely by hooks, so failure here is fine and must not
// abort the load.
if self.0.hook_registry.load_active.get() {
self.0.shared.has_js_execution_started_flag.raise();
let specifier = specifier.clone();
let inner = self.0.clone();
let is_dynamic_import = options.is_dynamic_import;
return Box::pin(async move {
inner
.prepare_hooked_specifier(&specifier, is_dynamic_import)
.await;
Ok(())
});
}
if self.0.shared.in_npm_pkg_checker.in_npm_package(specifier) {
self.0.shared.has_js_execution_started_flag.raise();
return Box::pin(deno_core::futures::future::ready(Ok(())));
}
if self.0.shared.maybe_eszip_loader.is_some() {
self.0.shared.has_js_execution_started_flag.raise();
return Box::pin(deno_core::futures::future::ready(Ok(())));
}
let specifier = specifier.clone();
let inner = self.0.clone();
let is_synchronous = options.is_synchronous;
let future = async move {
let graph_container = &inner.graph_container;
let module_load_preparer = &inner.shared.module_load_preparer;
let permissions = if options.is_dynamic_import {
&inner.permissions
} else {
&inner.parent_permissions
};
let allow_sloppy_imports_hints_for_unreferenced_roots =
if options.is_dynamic_import {
// This doesn't acquire a graph update permit because that will
// clone the graph which is a bit slow.
let mut graph = graph_container.graph();
let has_prepared_dynamic_import = has_prepared_dynamic_import(
&graph,
&specifier,
maybe_referrer.as_deref(),
);
// When the specifier is already in the graph then it means it
// was previously loaded, so we can skip that and only check if
// this part of the graph is valid.
if !graph.roots.is_empty() && graph.get(&specifier).is_some() {
let did_reload = inner
.maybe_reload_dynamic(&graph, &specifier, permissions)
.await
.map_err(JsErrorBox::from_err)?;
if did_reload {
graph = inner.graph_container.graph();
}
log::debug!("Skipping prepare module load.");
// roots are already validated so we can skip those
if did_reload || !graph.roots.contains(&specifier) {
module_load_preparer.graph_roots_valid(
&graph,
&[specifier],
GraphRootsValidOptions {
allow_unknown_media_types: false,
allow_unknown_jsr_exports: false,
allow_sloppy_imports_hints_for_unreferenced_roots:
has_prepared_dynamic_import,
},
)?;
}
return Ok(());
}
has_prepared_dynamic_import
} else {
true
};
let is_dynamic = options.is_dynamic_import || inner.is_worker; // consider workers as dynamic for permissions
let lib = inner.lib;
let mut update_permit = graph_container.acquire_update_permit().await;
// For a blob/object-URL worker root, inject the captured blob's content
// and media type so the graph builds from the synchronously-captured
// bytes even if the object URL has since been revoked. Otherwise fall
// back to any caller-provided source code.
let captured_blob =
inner
.maybe_main_module_blob
.as_ref()
.filter(|(blob_specifier, _)| {
maybe_code.is_none() && *blob_specifier == specifier
});
let (file_overrides, header_overrides) =
if let Some((blob_specifier, blob)) = captured_blob {
let bytes = blob.read_all().await;
(
HashMap::from([(blob_specifier.clone(), Arc::from(bytes))]),
HashMap::from([(
blob_specifier.clone(),
HashMap::from([(
"content-type".to_string(),
blob.media_type.clone(),
)]),
)]),
)
} else {
let file_overrides = maybe_code
.map(|code| {
HashMap::from([(specifier.clone(), Arc::from(code.into_bytes()))])
})
.unwrap_or_default();
(file_overrides, HashMap::new())
};
let specifiers = &[specifier];
{
let graph = update_permit.graph_mut();
module_load_preparer
.prepare_module_load(
graph,
specifiers,
PrepareModuleLoadOptions {
is_dynamic,
lib,
permissions: permissions.clone(),
file_permission_api_name: None,
ext_overwrite: None,
allow_unknown_media_types: false,
allow_sloppy_imports_hints_for_unreferenced_roots,
skip_graph_roots_validation: is_dynamic,
file_content_overrides: file_overrides,
file_header_overrides: header_overrides,
},
)
.await
.map_err(JsErrorBox::from_err)?;
graph.prune_types();
update_permit.commit();
inner.shared.has_js_execution_started_flag.raise();
}
if is_dynamic {
inner
.maybe_reload_dynamic(
&graph_container.graph(),
&specifiers[0],
permissions,
)
.await
.map_err(JsErrorBox::from_err)?;
// always validate the graph roots because we skipped doing it above
module_load_preparer.graph_roots_valid(
&graph_container.graph(),
specifiers,
GraphRootsValidOptions {
allow_unknown_media_types: false,
allow_unknown_jsr_exports: false,
allow_sloppy_imports_hints_for_unreferenced_roots,
},
)?;
}
Ok(())
};
// when is_synchronous is true, the caller is only polling the future
// without a tokio runtime, so we need to run it in a new runtime
if is_synchronous {
let result = run_future_in_new_runtime(future);
Box::pin(std::future::ready(result))
} else {
Box::pin(future)
}
}
fn finish_load(&self) {
self
.0
.shared
.in_flight_loads_tracker
.decrease(&self.0.shared.parsed_source_cache);
}
fn get_code_cache(
&self,
specifier: &ModuleSpecifier,
source: &str,
) -> Option<SourceCodeCacheInfo> {
let cache = self.0.shared.code_cache.as_ref()?;
// Key on the full source contents. V8 only checks the source length when
// validating cached data, so an equal-length edit would otherwise be
// served stale bytecode.
//
// This hashes a `&str` while `code_source_to_module_source` hashes a
// `ModuleSourceCode`, so the two never share an entry. That's fine: this
// path only ever sees residual `ext:`/`node:` specifiers, which never go
// through `load()`.
let source_hash = FastInsecureHasher::new_deno_versioned()
.write_hashable(source)
.finish();
let data = cache
.get_sync(specifier, code_cache::CodeCacheType::EsModule, source_hash)
.inspect(|_| {
// This log line is also used by tests.
log::debug!(
"V8 code cache hit for ES module: {specifier}, [{source_hash}]"
);
})
.map(Cow::from);
Some(SourceCodeCacheInfo {
hash: source_hash,
data,
})
}
fn code_cache_ready(
&self,
specifier: ModuleSpecifier,
source_hash: u64,
code_cache: &[u8],
) -> Pin<Box<dyn Future<Output = ()>>> {
if let Some(cache) = self.0.shared.code_cache.as_ref() {
// This log line is also used by tests.
log::debug!(
"Updating V8 code cache for ES module: {specifier}, [{source_hash:?}]"
);
cache.set_sync(
&specifier,
code_cache::CodeCacheType::EsModule,
source_hash,
code_cache,
);
}
std::future::ready(()).boxed_local()
}
fn get_source_map(&self, file_name: &str) -> Option<Cow<'_, [u8]>> {
let specifier = resolve_url(file_name).ok()?;
match specifier.scheme() {
// we should only be looking for emits for schemes that denote external
// modules, which the disk_cache supports
"wasm" | "file" | "http" | "https" | "data" | "blob" => (),
_ => return None,
}
// Load the prepared module and extract inline source map
let graph = self.0.graph_container.graph();
let source = self
.0
.shared
.module_loader
.load_prepared_module_for_source_map_sync(&graph, &specifier)
.ok()??;
source_map_from_code(source.source.as_bytes()).map(Cow::Owned)
}
fn load_external_source_map(
&self,
source_map_url: &str,
) -> Option<Cow<'_, [u8]>> {
let specifier = resolve_url(source_map_url).ok()?;
if let Ok(Some(file)) = self
.0
.shared
.file_fetcher
.get_cached_source_or_local(&specifier)
{
return Some(Cow::Owned(file.source.to_vec()));
}
None
}
// todo(dsherret): this method is actually only to determine whether
// to show the filename in the stack traces so we should rename it
// to something more clear that reflects that (since we skip checking
// this for non-npm packages)
fn source_map_source_exists(&self, source_url: &str) -> Option<bool> {
let specifier = resolve_url(source_url).ok()?;
// some npm packages rely on the file existing or not to end up in
// the stack trace, so for backwards compat reasons only check this
// for npm packages because we don't want the perf hit otherwise
if self.0.shared.in_npm_pkg_checker.in_npm_package(&specifier)
&& let Ok(path) = deno_path_util::url_to_file_path(&specifier)
{
return Some(path.is_file());
}
Some(true)
}
fn get_source_mapped_source_line(
&self,
file_name: &str,
line_number: usize,
) -> Option<String> {
let specifier = resolve_url(file_name).ok()?;
let graph = self.0.graph_container.graph();
let code = match graph.get(&specifier) {
Some(deno_graph::Module::Js(module)) => &module.source.text,
Some(deno_graph::Module::Json(module)) => &module.source.text,
Some(
deno_graph::Module::Wasm(_)
| deno_graph::Module::Npm(_)
| deno_graph::Module::Node(_)
| deno_graph::Module::External(_),
) => {
return None;
}
None => {
// Not in graph, try to read from file system (for source-mapped original files)
if let Ok(Some(file)) = self
.0
.shared
.file_fetcher
.get_cached_source_or_local(&specifier)
{
return extract_source_line(
&String::from_utf8_lossy(&file.source),
line_number,
);
} else {
return None;
}
}
};
extract_source_line(code, line_number)
}
}
/// Build the `importAttributes` object handed to a `module.registerHooks()`
/// load hook. V8 only surfaces the `type` attribute at load time, so recover
/// the full `with { ... }` clause recorded during resolution and make sure
/// `type` is present for the common JSON/text/bytes case.
fn hook_load_import_attributes(
registry: &deno_runtime::deno_node::ops::module_hooks::LoaderHookRegistry,
specifier: &str,
requested_module_type: &deno_core::RequestedModuleType,
) -> std::collections::HashMap<String, String> {
let mut attrs = registry.take_resolved_attributes(specifier);
if let Some(ty) = requested_module_type.as_str() {
attrs
.entry("type".to_string())
.or_insert_with(|| ty.to_string());
}
attrs
}
/// Pick the `ModuleType` for source returned by a `module.registerHooks()`
/// load hook. Honors a hook-supplied `format` first (Node's hook contract),
/// then falls back to the importer's `with { type: "..." }` attribute, so
/// e.g. `import x from "./x.json" with { type: "json" }` still parses as
/// JSON when the hook passes the source straight through.
fn pick_hook_module_type(
format: Option<&str>,
requested: &deno_core::RequestedModuleType,
) -> deno_core::ModuleType {
if let Some(format) = format {
match format {
"json" => return deno_core::ModuleType::Json,
"text" => return deno_core::ModuleType::Text,
"bytes" => return deno_core::ModuleType::Bytes,
"wasm" => return deno_core::ModuleType::Wasm,
_ => {}
}
}
match requested {
deno_core::RequestedModuleType::Json => deno_core::ModuleType::Json,
deno_core::RequestedModuleType::Text => deno_core::ModuleType::Text,
deno_core::RequestedModuleType::Bytes => deno_core::ModuleType::Bytes,
_ => deno_core::ModuleType::JavaScript,
}
}
/// Extracts a specific line from source code text.
fn extract_source_line(text: &str, line_number: usize) -> Option<String> {
// Do NOT use .lines(): it skips the terminating empty line.
// (due to internally using_terminator() instead of .split())
match text.split('\n').nth(line_number) {
Some(line) => Some(line.to_string()),
None => Some(format!(
"{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)",
crate::colors::yellow("Warning"),
line_number + 1,
)),
}
}
/// Holds the `ModuleGraph` in workers.
#[derive(Clone)]
struct WorkerModuleGraphContainer {
// Allow only one request to update the graph data at a time,
// but allow other requests to read from it at any time even
// while another request is updating the data.
update_queue: Rc<deno_core::unsync::TaskQueue>,
inner: Rc<RefCell<Arc<ModuleGraph>>>,
}
impl WorkerModuleGraphContainer {
pub fn new(module_graph: Arc<ModuleGraph>) -> Self {
Self {
update_queue: Default::default(),
inner: Rc::new(RefCell::new(module_graph)),
}
}
}
impl ModuleGraphContainer for WorkerModuleGraphContainer {
async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit {
let permit = self.update_queue.acquire().await;
WorkerModuleGraphUpdatePermit {
permit,
inner: self.inner.clone(),
graph: (**self.inner.borrow()).clone(),
}
}
fn graph(&self) -> Arc<ModuleGraph> {
self.inner.borrow().clone()
}
}
struct WorkerModuleGraphUpdatePermit {
permit: deno_core::unsync::TaskQueuePermit,
inner: Rc<RefCell<Arc<ModuleGraph>>>,
graph: ModuleGraph,
}
impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit {
fn graph_mut(&mut self) -> &mut ModuleGraph {
&mut self.graph
}
fn commit(self) {
*self.inner.borrow_mut() = Arc::new(self.graph);
drop(self.permit); // explicit drop for clarity
}
}
#[derive(Debug)]
struct CliNodeRequireLoader<TGraphContainer: ModuleGraphContainer> {
cjs_tracker: Arc<CliCjsTracker>,
emitter: Arc<CliEmitter>,
memory_files: Arc<MemoryFiles>,
npm_resolver: CliNpmResolver,
sys: CliSys,
graph_container: TGraphContainer,
in_npm_pkg_checker: DenoInNpmPackageChecker,
npm_registry_permission_checker:
Arc<NpmRegistryReadPermissionChecker<CliSys>>,
}
impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader
for CliNodeRequireLoader<TGraphContainer>
{
fn ensure_read_permission<'a>(
&self,
permissions: &mut PermissionsContainer,
path: Cow<'a, Path>,
) -> Result<Cow<'a, Path>, JsErrorBox> {
if let Ok(url) = deno_path_util::url_from_file_path(&path) {
// allow reading if it's in the module graph
if self.graph_container.graph().get(&url).is_some() {
return Ok(path);
}
}
self
.npm_registry_permission_checker
.ensure_read_permission(permissions, path)
.map_err(JsErrorBox::from_err)
}
fn load_text_file_lossy(
&self,
path: &Path,
) -> Result<FastString, JsErrorBox> {
// todo(dsherret): use the preloaded module from the graph if available?
let media_type = MediaType::from_path(path);
let text_result = self.sys.fs_read_to_string_lossy(path);
let text = match text_result {
Ok(text) => text,
Err(err) => {
// only bother on error for performance reasons
if let Some(file) = deno_path_util::url_from_file_path(path)
.ok()
.and_then(|s| self.memory_files.get(&s))
{
Cow::Owned(String::from_utf8_lossy(&file.source).into_owned())
} else {
return Err(JsErrorBox::from_err(err));
}
}
};
if media_type.is_emittable() {
let specifier = deno_path_util::url_from_file_path(path)
.map_err(JsErrorBox::from_err)?;
if self.in_npm_pkg_checker.in_npm_package(&specifier) {
return Err(JsErrorBox::from_err(StrippingTypesNodeModulesError {
specifier,
}));
}
let text = self
.emitter
.maybe_emit_source_sync(
&specifier,
media_type,
// this is probably not super accurate due to require esm, but probably ok.
// If we find this causes a lot of churn in the emit cache then we should
// investigate how we can make this better
ModuleKind::Cjs,
&text.into(),
)
.map_err(JsErrorBox::from_err)?;
Ok(text.into())
} else {
Ok(match text {
Cow::Borrowed(s) => FastString::from_static(s),
Cow::Owned(s) => s.into(),
})
}
}
fn is_maybe_cjs(
&self,
specifier: &ModuleSpecifier,
) -> Result<bool, PackageJsonLoadError> {
let media_type = MediaType::from_specifier(specifier);
self.cjs_tracker.is_maybe_cjs(specifier, media_type)
}
fn is_maybe_cjs_from_require(
&self,
specifier: &ModuleSpecifier,
) -> Result<bool, PackageJsonLoadError> {
let media_type = MediaType::from_specifier(specifier);
self
.cjs_tracker
.is_maybe_cjs_from_require(specifier, media_type)
}
fn resolve_require_node_module_paths(&self, from: &Path) -> Vec<String> {
let is_global_resolver_and_from_in_global_cache = self
.npm_resolver
.as_managed()
.filter(|r| r.root_node_modules_path().is_none())
.map(|r| r.global_cache_root_path())
.filter(|global_cache_path| from.starts_with(global_cache_path))
.is_some();
if is_global_resolver_and_from_in_global_cache {
Vec::new()
} else {
deno_runtime::deno_node::default_resolve_require_node_module_paths(from)
}
}
fn resolve_package_folder_from_name(
&self,
package_name: &str,
) -> Option<PathBuf> {
// Only meaningful in global-cache mode: when a local node_modules
// directory exists, byonm-style walking handles the lookup.
let managed = self
.npm_resolver
.as_managed()
.filter(|r| r.root_node_modules_path().is_none())?;
let req = PackageReq::from_str(package_name).ok()?;
managed.resolve_pkg_folder_from_deno_module_req(&req).ok()
}
}
#[derive(Debug, Default)]
pub struct EszipModuleLoader {
files: HashMap<ModuleSpecifier, Arc<[u8]>>,
}
impl EszipModuleLoader {
pub async fn create(script: &str, cwd: &Path) -> Result<Self, AnyError> {
// entrypoint#path1,path2,...
let (_entrypoint, files) = script
.split_once("#")
.with_context(|| "eszip: invalid script string")?;
// TODO: handle paths that contain ','
let files = files.split(",").collect::<Vec<_>>();
let mut loaded_eszips = FuturesOrdered::new();
for path in files {
let file = tokio::fs::File::open(path).await?;
let eszip = BufReader::new(file.compat());
let path = path.to_string();
loaded_eszips.push_back(async move {
let (eszip, loader) = EszipV2::parse(eszip)
.await
.with_context(|| format!("Error parsing eszip header at {}", path))?;
loader
.await
.with_context(|| format!("Error loading eszip at {}", path))?;
Ok(eszip)
});
}
// At this point all eszips are fully loaded
let loaded_eszips: Vec<Result<EszipV2, AnyError>> =
loaded_eszips.collect::<Vec<_>>().await;
let mut loader = Self::default();
for loaded_eszip_result in loaded_eszips {
let loaded_eszip = loaded_eszip_result?;
let specifiers = loaded_eszip.specifiers();
loader.files.reserve(specifiers.len());
for specifier in specifiers {
let module = loaded_eszip.get_module(&specifier).unwrap();
let source = module.take_source().await.unwrap();
let resolved_specifier = resolve_url_or_path(&specifier, cwd)?;
let prev = loader.files.insert(resolved_specifier, source);
assert!(prev.is_none());
}
}
Ok(loader)
}
pub fn load_import_map_value(
&self,
specifier: &ModuleSpecifier,
) -> Result<serde_json::Value, AnyError> {
match self.files.get(specifier) {
Some(bytes) => Ok(serde_json::from_slice(bytes.as_ref())?),
None => bail!("Import map not found in eszip: {}", specifier),
}
}
fn load(&self, specifier: &ModuleSpecifier) -> deno_core::ModuleLoadResponse {
match self.files.get(specifier) {
Some(source) => {
let module_source = ModuleSource::new(
ModuleType::JavaScript,
ModuleSourceCode::Bytes(deno_core::ModuleCodeBytes::Arc(
source.clone(),
)),
specifier,
None,
);
deno_core::ModuleLoadResponse::Sync(Ok(module_source))
}
None => deno_core::ModuleLoadResponse::Sync(Err(JsErrorBox::generic(
"Module not found",
))),
}
}
}
/// Runs the future to completion in a new tokio runtime.
///
/// This is used for synchronous module loading (require ESM) where deno_core is blocking
/// the main thread and polling the future. In this case, tasks in the current runtime
/// aren't being processed, so we can get around this by creating a new runtime.
fn run_future_in_new_runtime<F, R>(future: F) -> R
where
F: Future<Output = R> + 'static,
R: Send + 'static,
{
// SAFETY: few points here...
// 1. The future is moved entirely to the new thread before any execution
// 2. The future executes exclusively on that single thread
// 3. The original thread waits (joins) until completion
// 4. No concurrent access to the future's captured state occurs
// 5. HOWEVER: Any thread local state won't work properly... which could
// possibly cause issues.
let masked = unsafe { deno_core::unsync::MaskFutureAsSend::new(future) };
std::thread::scope(|s| {
s.spawn(move || {
let rt = create_basic_runtime();
rt.block_on(masked).into_inner()
})
.join()
.unwrap()
})
}
#[cfg(test)]
mod tests {
use deno_graph::ast::ParsedSourceStore;
use super::*;
#[tokio::test]
async fn test_inflight_module_loads_tracker() {
let tracker = InFlightModuleLoadsTracker {
loads_number: Default::default(),
cleanup_task_timeout: 10,
cleanup_task_handle: Default::default(),
};
let specifier = ModuleSpecifier::parse("file:///a.js").unwrap();
let source = "const a = 'hello';";
let parsed_source_cache = Arc::new(ParsedSourceCache::default());
let parsed_source = parsed_source_cache
.remove_or_parse_module(&specifier, MediaType::JavaScript, source.into())
.unwrap();
parsed_source_cache.set_parsed_source(specifier, parsed_source);
assert_eq!(parsed_source_cache.len(), 1);
assert!(tracker.cleanup_task_handle.lock().is_none());
tracker.increase();
tracker.increase();
assert!(tracker.cleanup_task_handle.lock().is_none());
tracker.decrease(&parsed_source_cache);
assert!(tracker.cleanup_task_handle.lock().is_none());
tracker.decrease(&parsed_source_cache);
assert!(tracker.cleanup_task_handle.lock().is_some());
assert_eq!(parsed_source_cache.len(), 1);
tracker.increase();
assert!(tracker.cleanup_task_handle.lock().is_none());
assert_eq!(parsed_source_cache.len(), 1);
tracker.decrease(&parsed_source_cache);
// Rather long timeout, but to make sure CI is not flaking on it.
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
assert_eq!(parsed_source_cache.len(), 0);
}
}