diff --git a/crates/next-core/src/next_config.rs b/crates/next-core/src/next_config.rs index 08626ad108bf61..143684adf1d8b0 100644 --- a/crates/next-core/src/next_config.rs +++ b/crates/next-core/src/next_config.rs @@ -9,7 +9,7 @@ use turbo_tasks::{ trace::TraceRawVcs, }; use turbo_tasks_env::{EnvMap, ProcessEnv}; -use turbo_tasks_fetch::ReqwestClientConfig; +use turbo_tasks_fetch::FetchClient; use turbo_tasks_fs::FileSystemPath; use turbopack::module_options::{ ConditionItem, ConditionPath, LoaderRuleItem, OptionWebpackRules, @@ -1725,10 +1725,7 @@ impl NextConfig { } #[turbo_tasks::function] - pub async fn reqwest_client_config( - &self, - env: Vc>, - ) -> Result> { + pub async fn fetch_client(&self, env: Vc>) -> Result> { // Support both an env var and the experimental flag to provide more flexibility to // developers on locked down systems, depending on if they want to configure this on a // per-system or per-project basis. @@ -1742,7 +1739,7 @@ impl NextConfig { }) .or(self.experimental.turbopack_use_system_tls_certs) .unwrap_or(false); - Ok(ReqwestClientConfig { + Ok(FetchClient { tls_built_in_webpki_certs: !use_system_tls_certs, tls_built_in_native_certs: use_system_tls_certs, } diff --git a/crates/next-core/src/next_font/google/mod.rs b/crates/next-core/src/next_font/google/mod.rs index cf59798e14526f..d3473453ff60a2 100644 --- a/crates/next-core/src/next_font/google/mod.rs +++ b/crates/next-core/src/next_font/google/mod.rs @@ -9,7 +9,7 @@ use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{Completion, FxIndexMap, ResolvedVc, Vc}; use turbo_tasks_bytes::stream::SingleValue; use turbo_tasks_env::{CommandLineProcessEnv, ProcessEnv}; -use turbo_tasks_fetch::{HttpResponseBody, ReqwestClientConfig, fetch}; +use turbo_tasks_fetch::{FetchClient, HttpResponseBody}; use turbo_tasks_fs::{ DiskFileSystem, File, FileContent, FileSystem, FileSystemPath, json::parse_json_with_source_context, @@ -182,7 +182,7 @@ pub struct NextFontGoogleCssModuleReplacer { project_path: FileSystemPath, execution_context: ResolvedVc, next_mode: ResolvedVc, - reqwest_client_config: ResolvedVc, + fetch_client: ResolvedVc, } #[turbo_tasks::value_impl] @@ -192,13 +192,13 @@ impl NextFontGoogleCssModuleReplacer { project_path: FileSystemPath, execution_context: ResolvedVc, next_mode: ResolvedVc, - reqwest_client_config: ResolvedVc, + fetch_client: ResolvedVc, ) -> Vc { Self::cell(NextFontGoogleCssModuleReplacer { project_path, execution_context, next_mode, - reqwest_client_config, + fetch_client, }) } @@ -233,9 +233,9 @@ impl NextFontGoogleCssModuleReplacer { .map_or_else( || { fetch_real_stylesheet( + *self.fetch_client, stylesheet_url.clone(), css_virtual_path.clone(), - *self.reqwest_client_config, ) .boxed() }, @@ -375,19 +375,16 @@ struct NextFontGoogleFontFileOptions { #[turbo_tasks::value(shared)] pub struct NextFontGoogleFontFileReplacer { project_path: FileSystemPath, - reqwest_client_config: ResolvedVc, + fetch_client: ResolvedVc, } #[turbo_tasks::value_impl] impl NextFontGoogleFontFileReplacer { #[turbo_tasks::function] - pub fn new( - project_path: FileSystemPath, - reqwest_client_config: ResolvedVc, - ) -> Vc { + pub fn new(project_path: FileSystemPath, fetch_client: ResolvedVc) -> Vc { Self::cell(NextFontGoogleFontFileReplacer { project_path, - reqwest_client_config, + fetch_client, }) } } @@ -443,12 +440,9 @@ impl ImportMappingReplacement for NextFontGoogleFontFileReplacer { // doesn't seem ideal to download the font into a string, but probably doesn't // really matter either. - let Some(font) = fetch_from_google_fonts( - url.into(), - font_virtual_path.clone(), - *self.reqwest_client_config, - ) - .await? + let Some(font) = + fetch_from_google_fonts(*self.fetch_client, url.into(), font_virtual_path.clone()) + .await? else { return Ok(ImportMapResult::Result(ResolveResult::unresolvable()).cell()); }; @@ -670,27 +664,23 @@ fn font_file_options_from_query_map(query: &RcStr) -> Result, stylesheet_url: RcStr, css_virtual_path: FileSystemPath, - reqwest_client_config: Vc, ) -> Result>> { - let body = - fetch_from_google_fonts(stylesheet_url, css_virtual_path, reqwest_client_config).await?; + let body = fetch_from_google_fonts(fetch_client, stylesheet_url, css_virtual_path).await?; Ok(body.map(|body| body.to_string())) } async fn fetch_from_google_fonts( + fetch_client: Vc, url: RcStr, virtual_path: FileSystemPath, - reqwest_client_config: Vc, ) -> Result>> { - let result = fetch( - url, - Some(rcstr!(USER_AGENT_FOR_GOOGLE_FONTS)), - reqwest_client_config, - ) - .await?; + let result = fetch_client + .fetch(url, Some(rcstr!(USER_AGENT_FOR_GOOGLE_FONTS))) + .await?; Ok(match *result { Ok(r) => Some(*r.await?.body), diff --git a/crates/next-core/src/next_import_map.rs b/crates/next-core/src/next_import_map.rs index 83e9a623a6ceab..63178258f988dd 100644 --- a/crates/next-core/src/next_import_map.rs +++ b/crates/next-core/src/next_import_map.rs @@ -1022,7 +1022,7 @@ async fn insert_next_shared_aliases( next_font_google_replacer_mapping, ); - let reqwest_client_config = next_config.reqwest_client_config(execution_context.env()); + let fetch_client = next_config.fetch_client(execution_context.env()); import_map.insert_alias( AliasPattern::exact("@vercel/turbopack-next/internal/font/google/cssmodule.module.css"), ImportMapping::Dynamic(ResolvedVc::upcast( @@ -1030,7 +1030,7 @@ async fn insert_next_shared_aliases( project_path.clone(), execution_context, next_mode, - reqwest_client_config, + fetch_client, ) .to_resolved() .await?, @@ -1041,7 +1041,7 @@ async fn insert_next_shared_aliases( import_map.insert_alias( AliasPattern::exact(GOOGLE_FONTS_INTERNAL_PREFIX), ImportMapping::Dynamic(ResolvedVc::upcast( - NextFontGoogleFontFileReplacer::new(project_path.clone(), reqwest_client_config) + NextFontGoogleFontFileReplacer::new(project_path.clone(), fetch_client) .to_resolved() .await?, )) diff --git a/lerna.json b/lerna.json index c80f060354ff70..5867045e06b971 100644 --- a/lerna.json +++ b/lerna.json @@ -16,5 +16,5 @@ "registry": "https://registry.npmjs.org/" } }, - "version": "15.4.2-canary.15" + "version": "15.4.2-canary.16" } diff --git a/packages/create-next-app/package.json b/packages/create-next-app/package.json index cfab3190c8b3c4..c88cb5c40bbdbc 100644 --- a/packages/create-next-app/package.json +++ b/packages/create-next-app/package.json @@ -1,6 +1,6 @@ { "name": "create-next-app", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "keywords": [ "react", "next", diff --git a/packages/eslint-config-next/package.json b/packages/eslint-config-next/package.json index 143d1b316136f9..ef444cd490a9d6 100644 --- a/packages/eslint-config-next/package.json +++ b/packages/eslint-config-next/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-next", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "description": "ESLint configuration used by Next.js.", "main": "index.js", "license": "MIT", @@ -10,7 +10,7 @@ }, "homepage": "https://nextjs.org/docs/app/api-reference/config/eslint", "dependencies": { - "@next/eslint-plugin-next": "15.4.2-canary.15", + "@next/eslint-plugin-next": "15.4.2-canary.16", "@rushstack/eslint-patch": "^1.10.3", "@typescript-eslint/eslint-plugin": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", "@typescript-eslint/parser": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", diff --git a/packages/eslint-plugin-internal/package.json b/packages/eslint-plugin-internal/package.json index 0bb57fdeb72b03..c7cf1964eb5710 100644 --- a/packages/eslint-plugin-internal/package.json +++ b/packages/eslint-plugin-internal/package.json @@ -1,7 +1,7 @@ { "name": "@next/eslint-plugin-internal", "private": true, - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "description": "ESLint plugin for working on Next.js.", "exports": { ".": "./src/eslint-plugin-internal.js" diff --git a/packages/eslint-plugin-next/package.json b/packages/eslint-plugin-next/package.json index 87cdab5209eab4..4681c711e7fc91 100644 --- a/packages/eslint-plugin-next/package.json +++ b/packages/eslint-plugin-next/package.json @@ -1,6 +1,6 @@ { "name": "@next/eslint-plugin-next", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "description": "ESLint plugin for Next.js.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/font/package.json b/packages/font/package.json index fe8c202074a301..2f4521a4f22f9e 100644 --- a/packages/font/package.json +++ b/packages/font/package.json @@ -1,7 +1,7 @@ { "name": "@next/font", "private": true, - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "repository": { "url": "vercel/next.js", "directory": "packages/font" diff --git a/packages/next-bundle-analyzer/package.json b/packages/next-bundle-analyzer/package.json index 12088f1ec46a38..f9408a209c2454 100644 --- a/packages/next-bundle-analyzer/package.json +++ b/packages/next-bundle-analyzer/package.json @@ -1,6 +1,6 @@ { "name": "@next/bundle-analyzer", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "main": "index.js", "types": "index.d.ts", "license": "MIT", diff --git a/packages/next-codemod/package.json b/packages/next-codemod/package.json index d4350875fd5c5e..0658f165c6ad59 100644 --- a/packages/next-codemod/package.json +++ b/packages/next-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@next/codemod", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "license": "MIT", "repository": { "type": "git", diff --git a/packages/next-env/package.json b/packages/next-env/package.json index 08b9e0a754dbed..7b2b610c3caae4 100644 --- a/packages/next-env/package.json +++ b/packages/next-env/package.json @@ -1,6 +1,6 @@ { "name": "@next/env", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "keywords": [ "react", "next", diff --git a/packages/next-mdx/package.json b/packages/next-mdx/package.json index a76f4b5890438e..611f499ad12396 100644 --- a/packages/next-mdx/package.json +++ b/packages/next-mdx/package.json @@ -1,6 +1,6 @@ { "name": "@next/mdx", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "main": "index.js", "license": "MIT", "repository": { diff --git a/packages/next-plugin-storybook/package.json b/packages/next-plugin-storybook/package.json index b41af01f97c782..2313663a0d78a0 100644 --- a/packages/next-plugin-storybook/package.json +++ b/packages/next-plugin-storybook/package.json @@ -1,6 +1,6 @@ { "name": "@next/plugin-storybook", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "repository": { "url": "vercel/next.js", "directory": "packages/next-plugin-storybook" diff --git a/packages/next-polyfill-module/package.json b/packages/next-polyfill-module/package.json index 4636d2e4e71ce4..e8f0e6cc0a772a 100644 --- a/packages/next-polyfill-module/package.json +++ b/packages/next-polyfill-module/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-module", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "description": "A standard library polyfill for ES Modules supporting browsers (Edge 16+, Firefox 60+, Chrome 61+, Safari 10.1+)", "main": "dist/polyfill-module.js", "license": "MIT", diff --git a/packages/next-polyfill-nomodule/package.json b/packages/next-polyfill-nomodule/package.json index 3ecf45b1f2f575..8363d896667770 100644 --- a/packages/next-polyfill-nomodule/package.json +++ b/packages/next-polyfill-nomodule/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-nomodule", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "description": "A polyfill for non-dead, nomodule browsers.", "main": "dist/polyfill-nomodule.js", "license": "MIT", diff --git a/packages/next-rspack/package.json b/packages/next-rspack/package.json index e13631ab0ac543..bce6382337a156 100644 --- a/packages/next-rspack/package.json +++ b/packages/next-rspack/package.json @@ -1,6 +1,6 @@ { "name": "next-rspack", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "repository": { "url": "vercel/next.js", "directory": "packages/next-rspack" diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index 7b39bfd37a21ad..9833ff9aeb1340 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -1,6 +1,6 @@ { "name": "@next/swc", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "private": true, "files": [ "native/" diff --git a/packages/next/package.json b/packages/next/package.json index 8a99a6f8011ac7..caff6005644abe 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -1,6 +1,6 @@ { "name": "next", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "description": "The React Framework", "main": "./dist/server/next.js", "license": "MIT", @@ -100,7 +100,7 @@ ] }, "dependencies": { - "@next/env": "15.4.2-canary.15", + "@next/env": "15.4.2-canary.16", "@swc/helpers": "0.5.15", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", @@ -168,11 +168,11 @@ "@modelcontextprotocol/sdk": "1.15.1", "@mswjs/interceptors": "0.23.0", "@napi-rs/triples": "1.2.0", - "@next/font": "15.4.2-canary.15", - "@next/polyfill-module": "15.4.2-canary.15", - "@next/polyfill-nomodule": "15.4.2-canary.15", - "@next/react-refresh-utils": "15.4.2-canary.15", - "@next/swc": "15.4.2-canary.15", + "@next/font": "15.4.2-canary.16", + "@next/polyfill-module": "15.4.2-canary.16", + "@next/polyfill-nomodule": "15.4.2-canary.16", + "@next/react-refresh-utils": "15.4.2-canary.16", + "@next/swc": "15.4.2-canary.16", "@opentelemetry/api": "1.6.0", "@playwright/test": "1.51.1", "@rspack/core": "1.4.5", diff --git a/packages/react-refresh-utils/package.json b/packages/react-refresh-utils/package.json index 71f42b95eedae0..6a922f84fc5c3f 100644 --- a/packages/react-refresh-utils/package.json +++ b/packages/react-refresh-utils/package.json @@ -1,6 +1,6 @@ { "name": "@next/react-refresh-utils", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "description": "An experimental package providing utilities for React Refresh.", "repository": { "url": "vercel/next.js", diff --git a/packages/third-parties/package.json b/packages/third-parties/package.json index 579e252147a8d1..9461a666575a3d 100644 --- a/packages/third-parties/package.json +++ b/packages/third-parties/package.json @@ -1,6 +1,6 @@ { "name": "@next/third-parties", - "version": "15.4.2-canary.15", + "version": "15.4.2-canary.16", "repository": { "url": "vercel/next.js", "directory": "packages/third-parties" @@ -26,7 +26,7 @@ "third-party-capital": "1.0.20" }, "devDependencies": { - "next": "15.4.2-canary.15", + "next": "15.4.2-canary.16", "outdent": "0.8.0", "prettier": "2.5.1", "typescript": "5.8.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 252f6bc59049fc..c2b3ccd1b38ec8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -854,7 +854,7 @@ importers: packages/eslint-config-next: dependencies: '@next/eslint-plugin-next': - specifier: 15.4.2-canary.15 + specifier: 15.4.2-canary.16 version: link:../eslint-plugin-next '@rushstack/eslint-patch': specifier: ^1.10.3 @@ -924,7 +924,7 @@ importers: packages/next: dependencies: '@next/env': - specifier: 15.4.2-canary.15 + specifier: 15.4.2-canary.16 version: link:../next-env '@swc/helpers': specifier: 0.5.15 @@ -1049,19 +1049,19 @@ importers: specifier: 1.2.0 version: 1.2.0 '@next/font': - specifier: 15.4.2-canary.15 + specifier: 15.4.2-canary.16 version: link:../font '@next/polyfill-module': - specifier: 15.4.2-canary.15 + specifier: 15.4.2-canary.16 version: link:../next-polyfill-module '@next/polyfill-nomodule': - specifier: 15.4.2-canary.15 + specifier: 15.4.2-canary.16 version: link:../next-polyfill-nomodule '@next/react-refresh-utils': - specifier: 15.4.2-canary.15 + specifier: 15.4.2-canary.16 version: link:../react-refresh-utils '@next/swc': - specifier: 15.4.2-canary.15 + specifier: 15.4.2-canary.16 version: link:../next-swc '@opentelemetry/api': specifier: 1.6.0 @@ -1761,7 +1761,7 @@ importers: version: 1.0.20 devDependencies: next: - specifier: 15.4.2-canary.15 + specifier: 15.4.2-canary.16 version: link:../next outdent: specifier: 0.8.0 diff --git a/turbopack/crates/turbo-tasks-fetch/Cargo.toml b/turbopack/crates/turbo-tasks-fetch/Cargo.toml index 2113a046d64c06..dc1f6ddcd084bc 100644 --- a/turbopack/crates/turbo-tasks-fetch/Cargo.toml +++ b/turbopack/crates/turbo-tasks-fetch/Cargo.toml @@ -20,7 +20,7 @@ workspace = true [target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies] reqwest = { workspace = true, features = ["native-tls"] } -# If you modify this cfg, make sure you update `ReqwestClientConfig::try_build`. +# If you modify this cfg, make sure you update `FetchClient::try_build_uncached_reqwest_client`. [target.'cfg(not(any(all(target_os = "windows", target_arch = "aarch64"), target_arch="wasm32")))'.dependencies] reqwest = { workspace = true, features = ["rustls-tls-webpki-roots", "rustls-tls-native-roots"] } diff --git a/turbopack/crates/turbo-tasks-fetch/src/client.rs b/turbopack/crates/turbo-tasks-fetch/src/client.rs new file mode 100644 index 00000000000000..296c58ef8cd605 --- /dev/null +++ b/turbopack/crates/turbo-tasks-fetch/src/client.rs @@ -0,0 +1,156 @@ +use std::{hash::Hash, sync::LazyLock}; + +use anyhow::Result; +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use turbo_rcstr::RcStr; +use turbo_tasks::{ + NonLocalValue, ReadRef, Vc, duration_span, mark_session_dependent, trace::TraceRawVcs, +}; + +use crate::{FetchError, FetchResult, HttpResponse, HttpResponseBody}; + +const MAX_CLIENTS: usize = 16; +static CLIENT_CACHE: LazyLock, reqwest::Client>> = + LazyLock::new(|| Cache::new(MAX_CLIENTS)); + +#[derive(Hash, PartialEq, Eq, Serialize, Deserialize, NonLocalValue, Debug, TraceRawVcs)] +pub enum ProxyConfig { + Http(RcStr), + Https(RcStr), +} + +/// Represents the configuration needed to construct a [`reqwest::Client`]. +/// +/// This is used to cache clients keyed by their configuration, so the configuration should contain +/// as few fields as possible and change infrequently. +/// +/// This is needed because [`reqwest::ClientBuilder`] does not implement the required traits. This +/// factory cannot be a closure because closures do not implement `Eq` or `Hash`. +#[turbo_tasks::value(shared)] +#[derive(Hash)] +pub struct FetchClient { + /// Whether to load embedded webpki root certs with rustls. Default is true. + /// + /// Ignored for: + /// - Windows on ARM, which uses `native-tls` instead of `rustls-tls`. + /// - Ignored for WASM targets, which use the runtime's TLS implementation. + pub tls_built_in_webpki_certs: bool, + /// Whether to load native root certs using the `rustls-native-certs` crate. This may make + /// reqwest client initialization slower, so it's not used by default. + /// + /// Ignored for: + /// - Windows on ARM, which uses `native-tls` instead of `rustls-tls`. + /// - Ignored for WASM targets, which use the runtime's TLS implementation. + pub tls_built_in_native_certs: bool, +} + +impl Default for FetchClient { + fn default() -> Self { + Self { + tls_built_in_webpki_certs: true, + tls_built_in_native_certs: false, + } + } +} + +impl FetchClient { + /// Returns a cached instance of `reqwest::Client` it exists, otherwise constructs a new one. + /// + /// The cache is bound in size to prevent accidental blowups or leaks. However, in practice, + /// very few clients should be created, likely only when the bundler configuration changes. + /// + /// Client construction is largely deterministic, aside from changes to system TLS + /// configuration. + /// + /// The reqwest client fails to construct if the TLS backend cannot be initialized, or the + /// resolver cannot load the system configuration. These failures should be treated as + /// cached for some amount of time, but ultimately transient (e.g. using + /// [`turbo_tasks::mark_session_dependent`]). + pub fn try_get_cached_reqwest_client( + self: ReadRef, + ) -> reqwest::Result { + CLIENT_CACHE.get_or_insert_with(&self, { + let this = ReadRef::clone(&self); + move || this.try_build_uncached_reqwest_client() + }) + } + + fn try_build_uncached_reqwest_client(&self) -> reqwest::Result { + let client_builder = reqwest::Client::builder(); + + // make sure this cfg matches the one in `Cargo.toml`! + #[cfg(not(any( + all(target_os = "windows", target_arch = "aarch64"), + target_arch = "wasm32" + )))] + let client_builder = client_builder + .tls_built_in_root_certs(false) + .tls_built_in_webpki_certs(self.tls_built_in_webpki_certs) + .tls_built_in_native_certs(self.tls_built_in_native_certs); + + client_builder.build() + } +} + +#[turbo_tasks::value_impl] +impl FetchClient { + #[turbo_tasks::function(network)] + pub async fn fetch( + self: Vc, + url: RcStr, + user_agent: Option, + ) -> Result> { + let url_ref = &*url; + let this = self.await?; + let response_result: reqwest::Result = async move { + let reqwest_client = this.try_get_cached_reqwest_client()?; + + let mut builder = reqwest_client.get(url_ref); + if let Some(user_agent) = user_agent { + builder = builder.header("User-Agent", user_agent.as_str()); + } + + let response = { + let _span = duration_span!("fetch request", url = url_ref); + builder.send().await + } + .and_then(|r| r.error_for_status())?; + + let status = response.status().as_u16(); + + let body = { + let _span = duration_span!("fetch response", url = url_ref); + response.bytes().await? + } + .to_vec(); + + Ok(HttpResponse { + status, + body: HttpResponseBody(body).resolved_cell(), + }) + } + .await; + + match response_result { + Ok(resp) => Ok(Vc::cell(Ok(resp.resolved_cell()))), + Err(err) => { + // the client failed to construct or the HTTP request failed + mark_session_dependent(); + Ok(Vc::cell(Err( + FetchError::from_reqwest_error(&err, &url).resolved_cell() + ))) + } + } + } +} + +#[doc(hidden)] +pub fn __test_only_reqwest_client_cache_clear() { + CLIENT_CACHE.clear() +} + +#[doc(hidden)] +pub fn __test_only_reqwest_client_cache_len() -> usize { + CLIENT_CACHE.len() +} diff --git a/turbopack/crates/turbo-tasks-fetch/src/error.rs b/turbopack/crates/turbo-tasks-fetch/src/error.rs new file mode 100644 index 00000000000000..03ab2b901ad69a --- /dev/null +++ b/turbopack/crates/turbo-tasks-fetch/src/error.rs @@ -0,0 +1,119 @@ +use anyhow::Result; +use turbo_rcstr::{RcStr, rcstr}; +use turbo_tasks::{ResolvedVc, Vc}; +use turbo_tasks_fs::FileSystemPath; +use turbopack_core::issue::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString}; + +#[derive(Debug)] +#[turbo_tasks::value(shared)] +pub enum FetchErrorKind { + Connect, + Timeout, + Status(u16), + Other, +} + +#[turbo_tasks::value(shared)] +pub struct FetchError { + pub url: ResolvedVc, + pub kind: ResolvedVc, + pub detail: ResolvedVc, +} + +impl FetchError { + pub(crate) fn from_reqwest_error(error: &reqwest::Error, url: &str) -> FetchError { + let kind = if error.is_connect() { + FetchErrorKind::Connect + } else if error.is_timeout() { + FetchErrorKind::Timeout + } else if let Some(status) = error.status() { + FetchErrorKind::Status(status.as_u16()) + } else { + FetchErrorKind::Other + }; + + FetchError { + detail: StyledString::Text(error.to_string().into()).resolved_cell(), + url: ResolvedVc::cell(url.into()), + kind: kind.resolved_cell(), + } + } +} + +#[turbo_tasks::value_impl] +impl FetchError { + #[turbo_tasks::function] + pub fn to_issue( + &self, + severity: IssueSeverity, + issue_context: FileSystemPath, + ) -> Vc { + FetchIssue { + issue_context, + severity, + url: self.url, + kind: self.kind, + detail: self.detail, + } + .cell() + } +} + +#[turbo_tasks::value(shared)] +pub struct FetchIssue { + pub issue_context: FileSystemPath, + pub severity: IssueSeverity, + pub url: ResolvedVc, + pub kind: ResolvedVc, + pub detail: ResolvedVc, +} + +#[turbo_tasks::value_impl] +impl Issue for FetchIssue { + #[turbo_tasks::function] + fn file_path(&self) -> Vc { + self.issue_context.clone().cell() + } + + fn severity(&self) -> IssueSeverity { + self.severity + } + + #[turbo_tasks::function] + fn title(&self) -> Vc { + StyledString::Text(rcstr!("Error while requesting resource")).cell() + } + + #[turbo_tasks::function] + fn stage(&self) -> Vc { + IssueStage::Load.into() + } + + #[turbo_tasks::function] + async fn description(&self) -> Result> { + let url = &*self.url.await?; + let kind = &*self.kind.await?; + + Ok(Vc::cell(Some( + StyledString::Text(match kind { + FetchErrorKind::Connect => { + format!("There was an issue establishing a connection while requesting {url}.") + .into() + } + FetchErrorKind::Status(status) => { + format!("Received response with status {status} when requesting {url}").into() + } + FetchErrorKind::Timeout => { + format!("Connection timed out when requesting {url}").into() + } + FetchErrorKind::Other => format!("There was an issue requesting {url}").into(), + }) + .resolved_cell(), + ))) + } + + #[turbo_tasks::function] + fn detail(&self) -> Vc { + Vc::cell(Some(self.detail)) + } +} diff --git a/turbopack/crates/turbo-tasks-fetch/src/lib.rs b/turbopack/crates/turbo-tasks-fetch/src/lib.rs index 259d3f005f45d8..3595055a4d44ff 100644 --- a/turbopack/crates/turbo-tasks-fetch/src/lib.rs +++ b/turbopack/crates/turbo-tasks-fetch/src/lib.rs @@ -2,18 +2,17 @@ #![feature(arbitrary_self_types)] #![feature(arbitrary_self_types_pointers)] -mod reqwest_client_cache; - -use anyhow::Result; -use turbo_rcstr::{RcStr, rcstr}; -use turbo_tasks::{ResolvedVc, Vc, duration_span, mark_session_dependent}; -use turbo_tasks_fs::FileSystemPath; -use turbopack_core::issue::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString}; - -use crate::reqwest_client_cache::try_get_cached_reqwest_client; -pub use crate::reqwest_client_cache::{ - __test_only_reqwest_client_cache_clear, __test_only_reqwest_client_cache_len, ProxyConfig, - ReqwestClientConfig, +mod client; +mod error; +mod response; + +pub use crate::{ + client::{ + __test_only_reqwest_client_cache_clear, __test_only_reqwest_client_cache_len, FetchClient, + ProxyConfig, + }, + error::{FetchError, FetchErrorKind, FetchIssue}, + response::{FetchResult, HttpResponse, HttpResponseBody}, }; pub fn register() { @@ -22,189 +21,3 @@ pub fn register() { turbopack_core::register(); include!(concat!(env!("OUT_DIR"), "/register.rs")); } - -#[turbo_tasks::value(transparent)] -pub struct FetchResult(Result, ResolvedVc>); - -#[turbo_tasks::value(shared)] -#[derive(Debug)] -pub struct HttpResponse { - pub status: u16, - pub body: ResolvedVc, -} - -#[turbo_tasks::value(shared)] -#[derive(Debug)] -pub struct HttpResponseBody(pub Vec); - -#[turbo_tasks::value_impl] -impl HttpResponseBody { - #[turbo_tasks::function] - pub async fn to_string(self: Vc) -> Result> { - let this = &*self.await?; - Ok(Vc::cell(std::str::from_utf8(&this.0)?.into())) - } -} - -#[turbo_tasks::function(network)] -pub async fn fetch( - url: RcStr, - user_agent: Option, - client_config: Vc, -) -> Result> { - let url_ref = &*url; - let client_config = client_config.await?; - let response_result: reqwest::Result = async move { - let client = try_get_cached_reqwest_client(client_config)?; - - let mut builder = client.get(url_ref); - if let Some(user_agent) = user_agent { - builder = builder.header("User-Agent", user_agent.as_str()); - } - - let response = { - let _span = duration_span!("fetch request", url = url_ref); - builder.send().await - } - .and_then(|r| r.error_for_status())?; - - let status = response.status().as_u16(); - - let body = { - let _span = duration_span!("fetch response", url = url_ref); - response.bytes().await? - } - .to_vec(); - - Ok(HttpResponse { - status, - body: HttpResponseBody(body).resolved_cell(), - }) - } - .await; - - match response_result { - Ok(resp) => Ok(Vc::cell(Ok(resp.resolved_cell()))), - Err(err) => { - // the client failed to construct or the HTTP request failed - mark_session_dependent(); - Ok(Vc::cell(Err( - FetchError::from_reqwest_error(&err, &url).resolved_cell() - ))) - } - } -} - -#[derive(Debug)] -#[turbo_tasks::value(shared)] -pub enum FetchErrorKind { - Connect, - Timeout, - Status(u16), - Other, -} - -#[turbo_tasks::value(shared)] -pub struct FetchError { - pub url: ResolvedVc, - pub kind: ResolvedVc, - pub detail: ResolvedVc, -} - -impl FetchError { - fn from_reqwest_error(error: &reqwest::Error, url: &str) -> FetchError { - let kind = if error.is_connect() { - FetchErrorKind::Connect - } else if error.is_timeout() { - FetchErrorKind::Timeout - } else if let Some(status) = error.status() { - FetchErrorKind::Status(status.as_u16()) - } else { - FetchErrorKind::Other - }; - - FetchError { - detail: StyledString::Text(error.to_string().into()).resolved_cell(), - url: ResolvedVc::cell(url.into()), - kind: kind.resolved_cell(), - } - } -} - -#[turbo_tasks::value_impl] -impl FetchError { - #[turbo_tasks::function] - pub fn to_issue( - &self, - severity: IssueSeverity, - issue_context: FileSystemPath, - ) -> Vc { - FetchIssue { - issue_context, - severity, - url: self.url, - kind: self.kind, - detail: self.detail, - } - .cell() - } -} - -#[turbo_tasks::value(shared)] -pub struct FetchIssue { - pub issue_context: FileSystemPath, - pub severity: IssueSeverity, - pub url: ResolvedVc, - pub kind: ResolvedVc, - pub detail: ResolvedVc, -} - -#[turbo_tasks::value_impl] -impl Issue for FetchIssue { - #[turbo_tasks::function] - fn file_path(&self) -> Vc { - self.issue_context.clone().cell() - } - - fn severity(&self) -> IssueSeverity { - self.severity - } - - #[turbo_tasks::function] - fn title(&self) -> Vc { - StyledString::Text(rcstr!("Error while requesting resource")).cell() - } - - #[turbo_tasks::function] - fn stage(&self) -> Vc { - IssueStage::Load.into() - } - - #[turbo_tasks::function] - async fn description(&self) -> Result> { - let url = &*self.url.await?; - let kind = &*self.kind.await?; - - Ok(Vc::cell(Some( - StyledString::Text(match kind { - FetchErrorKind::Connect => { - format!("There was an issue establishing a connection while requesting {url}.") - .into() - } - FetchErrorKind::Status(status) => { - format!("Received response with status {status} when requesting {url}").into() - } - FetchErrorKind::Timeout => { - format!("Connection timed out when requesting {url}").into() - } - FetchErrorKind::Other => format!("There was an issue requesting {url}").into(), - }) - .resolved_cell(), - ))) - } - - #[turbo_tasks::function] - fn detail(&self) -> Vc { - Vc::cell(Some(self.detail)) - } -} diff --git a/turbopack/crates/turbo-tasks-fetch/src/reqwest_client_cache.rs b/turbopack/crates/turbo-tasks-fetch/src/reqwest_client_cache.rs deleted file mode 100644 index af39f3acdd76e6..00000000000000 --- a/turbopack/crates/turbo-tasks-fetch/src/reqwest_client_cache.rs +++ /dev/null @@ -1,97 +0,0 @@ -use std::{hash::Hash, sync::LazyLock}; - -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use turbo_rcstr::RcStr; -use turbo_tasks::{NonLocalValue, ReadRef, trace::TraceRawVcs}; - -const MAX_CLIENTS: usize = 16; -static CLIENT_CACHE: LazyLock, reqwest::Client>> = - LazyLock::new(|| Cache::new(MAX_CLIENTS)); - -#[derive(Hash, PartialEq, Eq, Serialize, Deserialize, NonLocalValue, Debug, TraceRawVcs)] -pub enum ProxyConfig { - Http(RcStr), - Https(RcStr), -} - -/// Represents the configuration needed to construct a [`reqwest::Client`]. -/// -/// This is used to cache clients keyed by their configuration, so the configuration should contain -/// as few fields as possible and change infrequently. -/// -/// This is needed because [`reqwest::ClientBuilder`] does not implement the required traits. This -/// factory cannot be a closure because closures do not implement `Eq` or `Hash`. -#[turbo_tasks::value(shared)] -#[derive(Hash)] -pub struct ReqwestClientConfig { - /// Whether to load embedded webpki root certs with rustls. Default is true. - /// - /// Ignored for: - /// - Windows on ARM, which uses `native-tls` instead of `rustls-tls`. - /// - Ignored for WASM targets, which use the runtime's TLS implementation. - pub tls_built_in_webpki_certs: bool, - /// Whether to load native root certs using the `rustls-native-certs` crate. This may make - /// reqwest client initialization slower, so it's not used by default. - /// - /// Ignored for: - /// - Windows on ARM, which uses `native-tls` instead of `rustls-tls`. - /// - Ignored for WASM targets, which use the runtime's TLS implementation. - pub tls_built_in_native_certs: bool, -} - -impl Default for ReqwestClientConfig { - fn default() -> Self { - Self { - tls_built_in_webpki_certs: true, - tls_built_in_native_certs: false, - } - } -} - -impl ReqwestClientConfig { - fn try_build(&self) -> reqwest::Result { - let client_builder = reqwest::Client::builder(); - - // make sure this cfg matches the one in `Cargo.toml`! - #[cfg(not(any( - all(target_os = "windows", target_arch = "aarch64"), - target_arch = "wasm32" - )))] - let client_builder = client_builder - .tls_built_in_root_certs(false) - .tls_built_in_webpki_certs(self.tls_built_in_webpki_certs) - .tls_built_in_native_certs(self.tls_built_in_native_certs); - - client_builder.build() - } -} - -/// Given a config, returns a cached instance if it exists, otherwise constructs a new one. -/// -/// The cache is bound in size to prevent accidental blowups or leaks. However, in practice, very -/// few clients should be created, likely only when the bundler configuration changes. -/// -/// Client construction is largely deterministic, aside from changes to system TLS configuration. -/// -/// The reqwest client fails to construct if the TLS backend cannot be initialized, or the resolver -/// cannot load the system configuration. These failures should be treated as cached for some amount -/// of time, but ultimately transient (e.g. using [`turbo_tasks::mark_session_dependent`]). -pub fn try_get_cached_reqwest_client( - config: ReadRef, -) -> reqwest::Result { - CLIENT_CACHE.get_or_insert_with(&config, { - let config = ReadRef::clone(&config); - move || config.try_build() - }) -} - -#[doc(hidden)] -pub fn __test_only_reqwest_client_cache_clear() { - CLIENT_CACHE.clear() -} - -#[doc(hidden)] -pub fn __test_only_reqwest_client_cache_len() -> usize { - CLIENT_CACHE.len() -} diff --git a/turbopack/crates/turbo-tasks-fetch/src/response.rs b/turbopack/crates/turbo-tasks-fetch/src/response.rs new file mode 100644 index 00000000000000..9df08b24998c5e --- /dev/null +++ b/turbopack/crates/turbo-tasks-fetch/src/response.rs @@ -0,0 +1,28 @@ +use anyhow::Result; +use turbo_rcstr::RcStr; +use turbo_tasks::{ResolvedVc, Vc}; + +use crate::FetchError; + +#[turbo_tasks::value(transparent)] +pub struct FetchResult(Result, ResolvedVc>); + +#[turbo_tasks::value(shared)] +#[derive(Debug)] +pub struct HttpResponse { + pub status: u16, + pub body: ResolvedVc, +} + +#[turbo_tasks::value(shared)] +#[derive(Debug)] +pub struct HttpResponseBody(pub Vec); + +#[turbo_tasks::value_impl] +impl HttpResponseBody { + #[turbo_tasks::function] + pub async fn to_string(self: Vc) -> Result> { + let this = &*self.await?; + Ok(Vc::cell(std::str::from_utf8(&this.0)?.into())) + } +} diff --git a/turbopack/crates/turbo-tasks-fetch/tests/fetch.rs b/turbopack/crates/turbo-tasks-fetch/tests/fetch.rs index 31fd898a14aa4b..eec957a84c4c88 100644 --- a/turbopack/crates/turbo-tasks-fetch/tests/fetch.rs +++ b/turbopack/crates/turbo-tasks-fetch/tests/fetch.rs @@ -5,8 +5,8 @@ use tokio::sync::Mutex as TokioMutex; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::Vc; use turbo_tasks_fetch::{ - __test_only_reqwest_client_cache_clear, __test_only_reqwest_client_cache_len, FetchErrorKind, - ReqwestClientConfig, fetch, + __test_only_reqwest_client_cache_clear, __test_only_reqwest_client_cache_len, FetchClient, + FetchErrorKind, }; use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath}; use turbo_tasks_testing::{Registration, register, run}; @@ -29,15 +29,15 @@ async fn basic_get() { .create_async() .await; - let config_vc = ReqwestClientConfig::default().cell(); - let response = &*fetch( - RcStr::from(format!("{}/foo.woff", server.url())), - /* user_agent */ None, - config_vc, - ) - .await? - .unwrap() - .await?; + let client_vc = FetchClient::default().cell(); + let response = &*client_vc + .fetch( + RcStr::from(format!("{}/foo.woff", server.url())), + /* user_agent */ None, + ) + .await? + .unwrap() + .await?; resource_mock.assert_async().await; @@ -63,15 +63,15 @@ async fn sends_user_agent() { eprintln!("{}", server.url()); - let config_vc = ReqwestClientConfig::default().cell(); - let response = &*fetch( - RcStr::from(format!("{}/foo.woff", server.url())), - Some(rcstr!("mock-user-agent")), - config_vc, - ) - .await? - .unwrap() - .await?; + let client_vc = FetchClient::default().cell(); + let response = &*client_vc + .fetch( + RcStr::from(format!("{}/foo.woff", server.url())), + Some(rcstr!("mock-user-agent")), + ) + .await? + .unwrap() + .await?; resource_mock.assert_async().await; @@ -98,8 +98,9 @@ async fn invalidation_does_not_invalidate() { .await; let url = RcStr::from(format!("{}/foo.woff", server.url())); - let config_vc = ReqwestClientConfig::default().cell(); - let response = &*fetch(url.clone(), /* user_agent */ None, config_vc) + let client_vc = FetchClient::default().cell(); + let response = &*client_vc + .fetch(url.clone(), /* user_agent */ None) .await? .unwrap() .await?; @@ -109,7 +110,8 @@ async fn invalidation_does_not_invalidate() { assert_eq!(response.status, 200); assert_eq!(*response.body.to_string().await?, "responsebody"); - let second_response = &*fetch(url.clone(), /* user_agent */ None, config_vc) + let second_response = &*client_vc + .fetch(url.clone(), /* user_agent */ None) .await? .unwrap() .await?; @@ -136,8 +138,8 @@ async fn errors_on_failed_connection() { // `ECONNREFUSED`. // Other values (e.g. domain name, reserved IP address block) may result in long timeouts. let url = rcstr!("http://127.0.0.1:0/foo.woff"); - let config_vc = ReqwestClientConfig::default().cell(); - let response_vc = fetch(url.clone(), None, config_vc); + let client_vc = FetchClient::default().cell(); + let response_vc = client_vc.fetch(url.clone(), None); let err_vc = &*response_vc.await?.unwrap_err(); let err = err_vc.await?; @@ -171,8 +173,8 @@ async fn errors_on_404() { .await; let url = RcStr::from(server.url()); - let config_vc = ReqwestClientConfig::default().cell(); - let response_vc = fetch(url.clone(), None, config_vc); + let client_vc = FetchClient::default().cell(); + let response_vc = client_vc.fetch(url.clone(), None); let err_vc = &*response_vc.await?.unwrap_err(); let err = err_vc.await?; @@ -197,7 +199,7 @@ async fn errors_on_404() { #[tokio::test] async fn client_cache() { // a simple fetch that should always succeed - async fn simple_fetch(path: &str, config: ReqwestClientConfig) -> anyhow::Result<()> { + async fn simple_fetch(path: &str, client: FetchClient) -> anyhow::Result<()> { let mut server = mockito::Server::new_async().await; let _resource_mock = server .mock("GET", &*format!("/{path}")) @@ -206,7 +208,11 @@ async fn client_cache() { .await; let url = RcStr::from(format!("{}/{}", server.url(), path)); - let response = match &*fetch(url.clone(), /* user_agent */ None, config.cell()).await? { + let response = match &*client + .cell() + .fetch(url.clone(), /* user_agent */ None) + .await? + { Ok(resp) => resp.await?, Err(_err) => { anyhow::bail!("fetch error") @@ -227,7 +233,7 @@ async fn client_cache() { simple_fetch( "/foo", - ReqwestClientConfig { + FetchClient { tls_built_in_native_certs: false, ..Default::default() }, @@ -239,7 +245,7 @@ async fn client_cache() { // the client is reused if the config is the same (by equality) simple_fetch( "/bar", - ReqwestClientConfig { + FetchClient { tls_built_in_native_certs: false, ..Default::default() }, @@ -251,7 +257,7 @@ async fn client_cache() { // the client is recreated if the config is different simple_fetch( "/bar", - ReqwestClientConfig { + FetchClient { tls_built_in_native_certs: true, ..Default::default() },