deno/cli/http_util.rs

// Copyright 2018-2026 the Deno authors. MIT license.

use std::io::BufReader;
use std::path::Path;
use std::sync::Arc;

use boxed_error::Boxed;
use dashmap::DashMap;
use deno_cache_dir::file_fetcher::RedirectHeaderParseError;
use deno_core::error::AnyError;
use deno_core::futures::StreamExt;
use deno_core::serde;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_error::JsError;
use deno_error::JsErrorBox;
use deno_lib::version::DENO_VERSION_INFO;
use deno_runtime::deno_fetch;
use deno_runtime::deno_fetch::CreateHttpClientOptions;
use deno_runtime::deno_fetch::ResBody;
use deno_runtime::deno_fetch::create_http_client;
use deno_runtime::deno_fetch::dns::ResolvedDenyCheckKind;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_runtime::deno_tls::TlsKey;
use deno_runtime::deno_tls::load_certs;
use deno_runtime::deno_tls::load_private_keys;
use http::HeaderMap;
use http::StatusCode;
use http::header::CONTENT_ENCODING;
use http::header::CONTENT_LENGTH;
use http::header::HeaderName;
use http::header::HeaderValue;
use http_body_util::BodyExt;
use once_cell::sync::OnceCell;
use thiserror::Error;

use crate::util::progress_bar::UpdateGuard;

#[derive(Debug, Error)]
pub enum SendError {
  #[error(transparent)]
  Send(#[from] deno_fetch::ClientSendError),
  #[error(transparent)]
  InvalidUri(#[from] http::uri::InvalidUri),
}

pub struct HttpClientProvider {
  options: CreateHttpClientOptions,
  root_cert_store_provider: Option<Arc<dyn RootCertStoreProvider>>,
  client: OnceCell<deno_fetch::Client>,
  cert_clients: DashMap<(String, String), deno_fetch::Client>,
}

impl std::fmt::Debug for HttpClientProvider {
  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    f.debug_struct("HttpClient")
      .field("options", &self.options)
      .finish()
  }
}

impl HttpClientProvider {
  pub fn new(
    root_cert_store_provider: Option<Arc<dyn RootCertStoreProvider>>,
    unsafely_ignore_certificate_errors: Option<Vec<String>>,
  ) -> Self {
    Self {
      options: CreateHttpClientOptions {
        unsafely_ignore_certificate_errors,
        ..Default::default()
      },
      root_cert_store_provider,
      client: OnceCell::new(),
      cert_clients: DashMap::new(),
    }
  }

  /// Makes every connection this provider's clients open re-check the address
  /// it resolved to against the `--deny-import` list.
  ///
  /// The import permission check runs on the specifier's hostname before any
  /// DNS lookup, so on its own it cannot see that a name resolves to a denied
  /// address. Used by the module fetcher, whose requests are the ones
  /// `--deny-import` governs.
  pub fn with_import_permissions(
    mut self,
    permissions: PermissionsContainer,
  ) -> Self {
    self.options.permissions = Some(permissions);
    self.options.resolved_deny_check_kind = ResolvedDenyCheckKind::Import;
    self
  }

  pub fn get_or_create(&self) -> Result<HttpClient, JsErrorBox> {
    let client = self.client.get_or_try_init(|| {
      create_http_client(
        DENO_VERSION_INFO.user_agent,
        CreateHttpClientOptions {
          root_cert_store: match &self.root_cert_store_provider {
            Some(provider) => Some(provider.get_or_try_init()?.clone()),
            None => None,
          },
          ..self.options.clone()
        },
      )
      .map_err(JsErrorBox::from_err)
    })?;
    Ok(HttpClient::new(client.clone()))
  }

  pub fn get_or_create_with_client_cert(
    &self,
    certfile: &Path,
    keyfile: &Path,
  ) -> Result<HttpClient, JsErrorBox> {
    let cache_key = (
      certfile.to_string_lossy().into_owned(),
      keyfile.to_string_lossy().into_owned(),
    );
    if let Some(client) = self.cert_clients.get(&cache_key) {
      return Ok(HttpClient::new(client.clone()));
    }
    let cert_data = std::fs::read(certfile).map_err(|e| {
      JsErrorBox::generic(format!(
        "Failed to read npmrc certfile '{}': {e}",
        certfile.display()
      ))
    })?;
    let key_data = std::fs::read(keyfile).map_err(|e| {
      JsErrorBox::generic(format!(
        "Failed to read npmrc keyfile '{}': {e}",
        keyfile.display()
      ))
    })?;
    let certs =
      load_certs(&mut BufReader::new(cert_data.as_slice())).map_err(|e| {
        JsErrorBox::generic(format!(
          "Failed to parse npmrc certfile '{}': {e}",
          certfile.display()
        ))
      })?;
    let mut keys = load_private_keys(&key_data).map_err(|e| {
      JsErrorBox::generic(format!(
        "Failed to parse npmrc keyfile '{}': {e}",
        keyfile.display()
      ))
    })?;
    if keys.is_empty() {
      return Err(JsErrorBox::generic(format!(
        "No private keys found in npmrc keyfile '{}'",
        keyfile.display()
      )));
    }
    let tls_key = TlsKey(certs, keys.remove(0));
    let client = create_http_client(
      DENO_VERSION_INFO.user_agent,
      CreateHttpClientOptions {
        root_cert_store: match &self.root_cert_store_provider {
          Some(provider) => Some(provider.get_or_try_init()?.clone()),
          None => None,
        },
        client_cert_chain_and_key: Some(tls_key),
        ..self.options.clone()
      },
    )
    .map_err(JsErrorBox::from_err)?;
    self.cert_clients.insert(cache_key, client.clone());
    Ok(HttpClient::new(client))
  }
}

#[derive(Debug, Error, JsError)]
#[class(type)]
#[error("Bad response: {:?}{}", .status_code, .response_text.as_ref().map(|s| format!("\n\n{}", s)).unwrap_or_else(String::new))]
pub struct BadResponseError {
  pub status_code: StatusCode,
  pub response_text: Option<String>,
}

#[derive(Debug, Boxed, JsError)]
pub struct DownloadError(pub Box<DownloadErrorKind>);

#[derive(Debug, Error, JsError)]
pub enum DownloadErrorKind {
  #[class(inherit)]
  #[error(transparent)]
  Fetch(deno_fetch::ClientSendError),
  #[class(inherit)]
  #[error(transparent)]
  UrlParse(#[from] deno_core::url::ParseError),
  #[class(generic)]
  #[error(transparent)]
  HttpParse(#[from] http::Error),
  #[class(inherit)]
  #[error(transparent)]
  Json(#[from] serde_json::Error),
  #[class(generic)]
  #[error(transparent)]
  ToStr(#[from] http::header::ToStrError),
  #[class(inherit)]
  #[error(transparent)]
  RedirectHeaderParse(RedirectHeaderParseError),
  #[class(type)]
  #[error("Too many redirects.")]
  TooManyRedirects,
  #[class(inherit)]
  #[error(transparent)]
  BadResponse(#[from] BadResponseError),
  #[class("Http")]
  #[error("Not Found.")]
  NotFound,
  #[class("Http")]
  #[error("Received unhandled Not Modified response.")]
  UnhandledNotModified,
  #[class(inherit)]
  #[error(transparent)]
  Other(JsErrorBox),
}

#[derive(Debug)]
pub enum HttpClientResponse {
  Success {
    headers: HeaderMap<HeaderValue>,
    body: Vec<u8>,
  },
  NotFound,
  NotModified,
}

impl HttpClientResponse {
  pub fn into_bytes(self) -> Result<Vec<u8>, DownloadError> {
    match self {
      Self::Success { body, .. } => Ok(body),
      Self::NotFound => Err(DownloadErrorKind::NotFound.into_box()),
      Self::NotModified => {
        Err(DownloadErrorKind::UnhandledNotModified.into_box())
      }
    }
  }

  pub fn into_maybe_bytes(self) -> Result<Option<Vec<u8>>, DownloadError> {
    match self {
      Self::Success { body, .. } => Ok(Some(body)),
      Self::NotFound => Ok(None),
      Self::NotModified => {
        Err(DownloadErrorKind::UnhandledNotModified.into_box())
      }
    }
  }
}

#[derive(Debug)]
pub struct HttpClient {
  client: deno_fetch::Client,
}

impl HttpClient {
  // DO NOT make this public. You should always be creating one of these from
  // the HttpClientProvider
  fn new(client: deno_fetch::Client) -> Self {
    Self { client }
  }

  pub fn get(&self, url: Url) -> Result<RequestBuilder, http::Error> {
    let body = deno_fetch::ReqBody::empty();
    let mut req = http::Request::new(body);
    *req.uri_mut() = url.as_str().parse()?;
    Ok(RequestBuilder {
      client: self.client.clone(),
      req,
    })
  }

  pub fn post(
    &self,
    url: Url,
    body: deno_fetch::ReqBody,
  ) -> Result<RequestBuilder, http::Error> {
    let mut req = http::Request::new(body);
    *req.method_mut() = http::Method::POST;
    *req.uri_mut() = url.as_str().parse()?;
    Ok(RequestBuilder {
      client: self.client.clone(),
      req,
    })
  }

  pub fn post_json<S>(
    &self,
    url: Url,
    ser: &S,
  ) -> Result<RequestBuilder, DownloadError>
  where
    S: serde::Serialize,
  {
    let json = deno_core::serde_json::to_vec(ser)?;
    let body = deno_fetch::ReqBody::full(json.into());
    let builder = self.post(url, body)?;
    Ok(builder.header(
      http::header::CONTENT_TYPE,
      "application/json".parse().map_err(http::Error::from)?,
    ))
  }

  pub async fn send(
    &self,
    url: &Url,
    headers: HeaderMap,
  ) -> Result<http::Response<ResBody>, SendError> {
    let body = deno_fetch::ReqBody::empty();
    let mut request = http::Request::new(body);
    *request.uri_mut() = http::Uri::try_from(url.as_str())?;
    *request.headers_mut() = headers;

    self
      .client
      .clone()
      .send(request)
      .await
      .map_err(SendError::Send)
  }

  pub async fn download_text(&self, url: Url) -> Result<String, AnyError> {
    let bytes = self.download(url).await?;
    Ok(String::from_utf8(bytes)?)
  }

  pub async fn download(&self, url: Url) -> Result<Vec<u8>, DownloadError> {
    let response = self
      .download_inner(url, &Default::default(), None, true)
      .await?;
    response.into_bytes()
  }

  pub async fn download_with_progress_and_retries(
    &self,
    url: Url,
    headers: &HeaderMap,
    progress_guard: &UpdateGuard,
  ) -> Result<HttpClientResponse, DownloadError> {
    crate::util::retry::retry(
      || self.download_inner(url.clone(), headers, Some(progress_guard), true),
      |e| {
        matches!(
          e.as_kind(),
          DownloadErrorKind::BadResponse(_) | DownloadErrorKind::Fetch(_)
        )
      },
    )
    .await
  }

  /// Like `download_with_progress_and_retries`, but bypasses the transparent
  /// decompression middleware. The response body will contain raw bytes
  /// (potentially gzip-compressed). The caller should check the
  /// Content-Encoding header and decompress if needed.
  pub async fn download_with_progress_and_retries_no_decompress(
    &self,
    url: Url,
    headers: &HeaderMap,
    progress_guard: &UpdateGuard,
  ) -> Result<HttpClientResponse, DownloadError> {
    crate::util::retry::retry(
      || self.download_inner(url.clone(), headers, Some(progress_guard), false),
      |e| {
        matches!(
          e.as_kind(),
          DownloadErrorKind::BadResponse(_) | DownloadErrorKind::Fetch(_)
        )
      },
    )
    .await
  }

  pub async fn get_redirected_url(
    &self,
    url: Url,
    headers: &HeaderMap<HeaderValue>,
  ) -> Result<Url, AnyError> {
    let (_, url) = self.get_redirected_response(url, headers, true).await?;
    Ok(url)
  }

  async fn download_inner(
    &self,
    url: Url,
    headers: &HeaderMap<HeaderValue>,
    progress_guard: Option<&UpdateGuard>,
    should_decompress: bool,
  ) -> Result<HttpClientResponse, DownloadError> {
    let (response, _) = self
      .get_redirected_response(url, headers, should_decompress)
      .await?;

    if response.status() == 404 {
      return Ok(HttpClientResponse::NotFound);
    } else if response.status() == 304 {
      return Ok(HttpClientResponse::NotModified);
    } else if !response.status().is_success() {
      let status = response.status();
      let maybe_response_text = body_to_string(response).await.ok();
      return Err(
        DownloadErrorKind::BadResponse(BadResponseError {
          status_code: status,
          response_text: maybe_response_text
            .map(|s| s.trim().to_string())
            .filter(|s| !s.is_empty()),
        })
        .into_box(),
      );
    }

    get_response_body_with_progress(response, progress_guard)
      .await
      .map(|(headers, body)| HttpClientResponse::Success { headers, body })
      .map_err(|err| DownloadErrorKind::Other(err).into_box())
  }

  async fn get_redirected_response(
    &self,
    mut url: Url,
    headers: &HeaderMap<HeaderValue>,
    should_decompress: bool,
  ) -> Result<(http::Response<deno_fetch::ResBody>, Url), DownloadError> {
    let mut req = self.get(url.clone())?.build();
    *req.headers_mut() = headers.clone();
    let mut response = if should_decompress {
      self.client.clone().send(req).await
    } else {
      self.client.clone().send_no_decompress(req).await
    }
    .map_err(|e| DownloadErrorKind::Fetch(e).into_box())?;
    let status = response.status();
    if status.is_redirection() && status != http::StatusCode::NOT_MODIFIED {
      let mut redirect_headers = headers.clone();
      for _ in 0..5 {
        let new_url = resolve_redirect_from_response(&url, &response)?;
        let mut req = self.get(new_url.clone())?.build();

        if should_strip_auth_on_redirect(&url, &new_url) {
          redirect_headers.remove(http::header::AUTHORIZATION);
        }
        *req.headers_mut() = redirect_headers.clone();

        let new_response = if should_decompress {
          self.client.clone().send(req).await
        } else {
          self.client.clone().send_no_decompress(req).await
        }
        .map_err(|e| DownloadErrorKind::Fetch(e).into_box())?;
        let status = new_response.status();
        if status.is_redirection() {
          response = new_response;
          url = new_url;
        } else {
          return Ok((new_response, new_url));
        }
      }
      Err(DownloadErrorKind::TooManyRedirects.into_box())
    } else {
      Ok((response, url))
    }
  }
}

/// Returns true if auth headers should be stripped when redirecting from
/// `original` to `new_url`. Auth is stripped when the host or port changes,
/// or on a scheme downgrade (https -> http) to avoid leaking credentials
/// over plaintext. Same-host scheme upgrades (http -> https) are allowed.
fn should_strip_auth_on_redirect(original: &Url, new_url: &Url) -> bool {
  new_url.host() != original.host()
    || new_url.port() != original.port()
    || (original.scheme() == "https" && new_url.scheme() == "http")
}

pub async fn get_response_body_with_progress(
  response: http::Response<deno_fetch::ResBody>,
  progress_guard: Option<&UpdateGuard>,
) -> Result<(HeaderMap, Vec<u8>), JsErrorBox> {
  use http_body::Body as _;
  if let Some(progress_guard) = progress_guard {
    let mut total_size = response.body().size_hint().exact();
    if total_size.is_none() {
      // `Content-Length` describes the encoded body, so it can't be used as
      // the progress total when the body was transparently decompressed.
      total_size = response
        .headers()
        .get(CONTENT_LENGTH)
        .filter(|_| !response.headers().contains_key(CONTENT_ENCODING))
        .and_then(|val| val.to_str().ok())
        .and_then(|s| s.parse::<u64>().ok());
    }
    if let Some(total_size) = total_size {
      progress_guard.set_total_size(total_size);
      let mut current_size = 0;
      let mut data = Vec::with_capacity(total_size as usize);
      let (parts, body) = response.into_parts();
      let mut stream = body.into_data_stream();
      while let Some(item) = stream.next().await {
        let bytes = item?;
        current_size += bytes.len() as u64;
        progress_guard.set_position(current_size);
        data.extend_from_slice(&bytes);
      }
      return Ok((parts.headers, data));
    }
  }

  let (parts, body) = response.into_parts();
  let bytes = body.collect().await?.to_bytes();
  Ok((parts.headers, bytes.into()))
}

fn resolve_redirect_from_response<B>(
  request_url: &Url,
  response: &http::Response<B>,
) -> Result<Url, DownloadError> {
  debug_assert!(response.status().is_redirection());
  deno_cache_dir::file_fetcher::resolve_redirect_from_headers(
    request_url,
    response.headers(),
  )
  .map_err(|err| DownloadErrorKind::RedirectHeaderParse(*err).into_box())
}

pub async fn body_to_string<B>(body: B) -> Result<String, AnyError>
where
  B: http_body::Body,
  AnyError: From<B::Error>,
{
  let bytes = body.collect().await?.to_bytes();
  let s = std::str::from_utf8(&bytes)?;
  Ok(s.into())
}

pub async fn body_to_json<B, D>(body: B) -> Result<D, AnyError>
where
  B: http_body::Body,
  AnyError: From<B::Error>,
  D: serde::de::DeserializeOwned,
{
  let bytes = body.collect().await?.to_bytes();
  let val = deno_core::serde_json::from_slice(&bytes)?;
  Ok(val)
}

pub struct RequestBuilder {
  client: deno_fetch::Client,
  req: http::Request<deno_fetch::ReqBody>,
}

impl RequestBuilder {
  pub fn header(mut self, name: HeaderName, value: HeaderValue) -> Self {
    self.req.headers_mut().append(name, value);
    self
  }

  pub async fn send(
    self,
  ) -> Result<http::Response<deno_fetch::ResBody>, AnyError> {
    self.client.send(self.req).await.map_err(Into::into)
  }

  pub fn build(self) -> http::Request<deno_fetch::ReqBody> {
    self.req
  }
}

#[allow(clippy::print_stdout, reason = "test code")]
#[allow(clippy::print_stderr, reason = "test code")]
#[cfg(test)]
mod test {
  use std::collections::HashSet;
  use std::hash::RandomState;

  use deno_runtime::deno_tls::rustls::RootCertStore;
  use tokio::io::AsyncReadExt;
  use tokio::io::AsyncWriteExt;
  use tokio::net::TcpListener;

  use super::*;

  #[tokio::test]
  async fn test_http_client_download_redirect() {
    let _http_server_guard = test_util::http_server();
    let client = HttpClientProvider::new(None, None).get_or_create().unwrap();

    // make a request to the redirect server
    let text = client
      .download_text(
        Url::parse("http://localhost:4546/subdir/redirects/redirect1.js")
          .unwrap(),
      )
      .await
      .unwrap();
    assert_eq!(text, "export const redirect = 1;\n");

    // now make one to the infinite redirects server
    let err = client
      .download_text(
        Url::parse("http://localhost:4549/subdir/redirects/redirect1.js")
          .unwrap(),
      )
      .await
      .err()
      .unwrap();
    assert_eq!(err.to_string(), "Too many redirects.");
  }

  async fn read_request(
    stream: &mut tokio::net::TcpStream,
  ) -> (String, Option<String>) {
    let mut request = Vec::new();
    let mut buffer = [0; 1024];
    loop {
      let count = stream.read(&mut buffer).await.unwrap();
      assert_ne!(count, 0, "connection closed before request headers");
      request.extend_from_slice(&buffer[..count]);
      if request.windows(4).any(|window| window == b"\r\n\r\n") {
        break;
      }
    }

    let request = String::from_utf8(request).unwrap();
    let path = request
      .lines()
      .next()
      .and_then(|line| line.split_whitespace().nth(1))
      .unwrap()
      .to_string();
    let authorization = request.lines().find_map(|line| {
      let (name, value) = line.split_once(':')?;
      name
        .eq_ignore_ascii_case("authorization")
        .then(|| value.trim().to_string())
    });
    (path, authorization)
  }

  #[tokio::test]
  async fn test_http_client_redirect_does_not_restore_auth() {
    let source_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let source_addr = source_listener.local_addr().unwrap();
    let destination_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let destination_addr = destination_listener.local_addr().unwrap();

    let source_server = tokio::spawn(async move {
      let requests = [
        ("/start", format!("http://{source_addr}/same")),
        ("/same", format!("http://{destination_addr}/first")),
      ];
      let mut auth_headers = Vec::new();
      for (expected_path, location) in requests {
        let (mut stream, _) = source_listener.accept().await.unwrap();
        let (path, authorization) = read_request(&mut stream).await;
        assert_eq!(path, expected_path);
        auth_headers.push(authorization);
        stream
          .write_all(
            format!(
              "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
            )
            .as_bytes(),
          )
          .await
          .unwrap();
      }
      auth_headers
    });

    let destination_server = tokio::spawn(async move {
      let mut auth_headers = Vec::new();
      for expected_path in ["/first", "/last"] {
        let (mut stream, _) = destination_listener.accept().await.unwrap();
        let (path, authorization) = read_request(&mut stream).await;
        assert_eq!(path, expected_path);
        auth_headers.push(authorization);
        let response = if path == "/first" {
          format!(
            "HTTP/1.1 302 Found\r\nLocation: http://{destination_addr}/last\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
          )
        } else {
          "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok"
            .to_string()
        };
        stream.write_all(response.as_bytes()).await.unwrap();
      }
      auth_headers
    });

    let client = HttpClientProvider::new(None, None).get_or_create().unwrap();
    let mut headers = HeaderMap::new();
    headers.insert(
      http::header::AUTHORIZATION,
      HeaderValue::from_static("Bearer test-token"),
    );
    let (response, url) = client
      .get_redirected_response(
        Url::parse(&format!("http://{source_addr}/start")).unwrap(),
        &headers,
        true,
      )
      .await
      .unwrap();

    assert_eq!(response.status(), StatusCode::OK);
    assert_eq!(url.as_str(), format!("http://{destination_addr}/last"));
    assert_eq!(
      source_server.await.unwrap(),
      [
        Some("Bearer test-token".to_string()),
        Some("Bearer test-token".to_string())
      ]
    );
    assert_eq!(destination_server.await.unwrap(), [None, None]);
  }

  #[tokio::test]
  async fn test_fetch_with_cafile_string() {
    let _http_server_guard = test_util::http_server();
    let url = Url::parse("https://localhost:5545/assets/fixture.json").unwrap();

    let client = HttpClient::new(
      create_http_client(
        DENO_VERSION_INFO.user_agent,
        CreateHttpClientOptions {
          ca_certs: vec![
            std::fs::read(test_util::testdata_path().join("tls/RootCA.pem"))
              .unwrap(),
          ],
          ..Default::default()
        },
      )
      .unwrap(),
    );
    let response = client.send(&url, Default::default()).await.unwrap();
    assert!(response.status().is_success());
    let (parts, body) = response.into_parts();
    let headers = parts.headers;
    let body = body.collect().await.unwrap().to_bytes();
    assert!(!body.is_empty());
    assert_eq!(headers.get("content-type").unwrap(), "application/json");
    assert_eq!(headers.get("etag"), None);
    assert_eq!(headers.get("x-typescript-types"), None);
  }

  static PUBLIC_HTTPS_URLS: &[&str] = &[
    "https://deno.com/",
    "https://example.com/",
    "https://github.com/",
    "https://www.w3.org/",
  ];

  /// This test depends on external servers, so we need to be careful to avoid mistaking an offline machine with a
  /// test failure.
  #[tokio::test]
  async fn test_fetch_with_default_certificate_store() {
    let urls: HashSet<_, RandomState> =
      HashSet::from_iter(PUBLIC_HTTPS_URLS.iter());

    // Rely on the randomization of hashset iteration
    for url in urls {
      // Relies on external http server with a valid mozilla root CA cert.
      let url = Url::parse(url).unwrap();
      eprintln!("Attempting to fetch {url}...");

      let client = HttpClient::new(
        create_http_client(
          DENO_VERSION_INFO.user_agent,
          CreateHttpClientOptions::default(),
        )
        .unwrap(),
      );

      let result = client.send(&url, Default::default()).await;
      match result {
        Ok(response) if response.status().is_success() => {
          return; // success
        }
        _ => {
          // keep going
        }
      }
    }

    // Use 1.1.1.1 and 8.8.8.8 as our last-ditch internet check
    if std::net::TcpStream::connect("8.8.8.8:80").is_err()
      && std::net::TcpStream::connect("1.1.1.1:80").is_err()
    {
      return;
    }

    panic!(
      "None of the expected public URLs were available but internet appears to be available"
    );
  }

  #[tokio::test]
  async fn test_fetch_with_empty_certificate_store() {
    let root_cert_store = RootCertStore::empty();
    let urls: HashSet<_, RandomState> =
      HashSet::from_iter(PUBLIC_HTTPS_URLS.iter());

    // Rely on the randomization of hashset iteration
    let url = urls.into_iter().next().unwrap();
    // Relies on external http server with a valid mozilla root CA cert.
    let url = Url::parse(url).unwrap();
    eprintln!("Attempting to fetch {url}...");

    let client = HttpClient::new(
      create_http_client(
        DENO_VERSION_INFO.user_agent,
        CreateHttpClientOptions {
          root_cert_store: Some(root_cert_store),
          ..Default::default()
        },
      )
      .unwrap(),
    );

    let result = client.send(&url, HeaderMap::new()).await;
    assert!(result.is_err() || !result.unwrap().status().is_success());
  }

  #[tokio::test]
  async fn test_fetch_with_cafile_gzip() {
    let _http_server_guard = test_util::http_server();
    let url =
      Url::parse("https://localhost:5545/run/import_compression/gziped")
        .unwrap();
    let client = HttpClient::new(
      create_http_client(
        DENO_VERSION_INFO.user_agent,
        CreateHttpClientOptions {
          ca_certs: vec![
            std::fs::read(
              test_util::testdata_path()
                .join("tls/RootCA.pem")
                .to_string(),
            )
            .unwrap(),
          ],
          ..Default::default()
        },
      )
      .unwrap(),
    );
    let response = client.send(&url, Default::default()).await.unwrap();
    assert!(response.status().is_success());
    let (parts, body) = response.into_parts();
    let headers = parts.headers;
    let body = body.collect().await.unwrap().to_bytes().to_vec();
    assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
    assert_eq!(
      headers.get("content-type").unwrap(),
      "application/javascript"
    );
    assert_eq!(headers.get("etag"), None);
    assert_eq!(headers.get("x-typescript-types"), None);
  }

  #[tokio::test]
  async fn test_fetch_with_cafile_with_etag() {
    let _http_server_guard = test_util::http_server();
    let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap();
    let client = HttpClient::new(
      create_http_client(
        DENO_VERSION_INFO.user_agent,
        CreateHttpClientOptions {
          ca_certs: vec![
            std::fs::read(
              test_util::testdata_path()
                .join("tls/RootCA.pem")
                .to_string(),
            )
            .unwrap(),
          ],
          ..Default::default()
        },
      )
      .unwrap(),
    );
    let response = client.send(&url, Default::default()).await.unwrap();
    assert!(response.status().is_success());
    let (parts, body) = response.into_parts();
    let headers = parts.headers;
    let body = body.collect().await.unwrap().to_bytes().to_vec();
    assert!(!body.is_empty());
    assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
    assert_eq!(
      headers.get("content-type").unwrap(),
      "application/typescript"
    );
    assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e");
    assert_eq!(headers.get("x-typescript-types"), None);

    let mut headers = HeaderMap::new();
    headers.insert("If-None-Match", "33a64df551425fcc55e".parse().unwrap());
    let res = client.send(&url, headers).await.unwrap();
    assert_eq!(res.status(), StatusCode::NOT_MODIFIED);
  }

  #[tokio::test]
  async fn test_fetch_with_cafile_brotli() {
    let _http_server_guard = test_util::http_server();
    let url =
      Url::parse("https://localhost:5545/run/import_compression/brotli")
        .unwrap();
    let client = HttpClient::new(
      create_http_client(
        DENO_VERSION_INFO.user_agent,
        CreateHttpClientOptions {
          ca_certs: vec![
            std::fs::read(
              test_util::testdata_path()
                .join("tls/RootCA.pem")
                .to_string(),
            )
            .unwrap(),
          ],
          ..Default::default()
        },
      )
      .unwrap(),
    );
    let response = client.send(&url, Default::default()).await.unwrap();
    assert!(response.status().is_success());
    let (parts, body) = response.into_parts();
    let headers = parts.headers;
    let body = body.collect().await.unwrap().to_bytes().to_vec();
    assert!(!body.is_empty());
    assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
    assert_eq!(
      headers.get("content-type").unwrap(),
      "application/javascript"
    );
    assert_eq!(headers.get("etag"), None);
    assert_eq!(headers.get("x-typescript-types"), None);
  }

  #[test]
  fn test_should_strip_auth_on_redirect() {
    // http -> https same host: safe upgrade, retain auth
    assert!(!should_strip_auth_on_redirect(
      &Url::parse("http://npm.pkg.github.com/package").unwrap(),
      &Url::parse("https://npm.pkg.github.com/package").unwrap(),
    ));

    // same origin: retain auth
    assert!(!should_strip_auth_on_redirect(
      &Url::parse("https://registry.example.com/a").unwrap(),
      &Url::parse("https://registry.example.com/b").unwrap(),
    ));

    // different host: strip auth
    assert!(should_strip_auth_on_redirect(
      &Url::parse("https://registry.example.com/package").unwrap(),
      &Url::parse("https://other.example.com/package").unwrap(),
    ));

    // different port: strip auth
    assert!(should_strip_auth_on_redirect(
      &Url::parse("https://registry.example.com:8080/package").unwrap(),
      &Url::parse("https://registry.example.com:9090/package").unwrap(),
    ));

    // https -> http same host: scheme downgrade, strip auth
    assert!(should_strip_auth_on_redirect(
      &Url::parse("https://npm.pkg.github.com/package").unwrap(),
      &Url::parse("http://npm.pkg.github.com/package").unwrap(),
    ));
  }
}