diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs index 0364c664ba512..aec78e653041f 100644 --- a/src/bootstrap/src/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -179,6 +179,16 @@ fn main() { } } + // Here we pass additional paths that essentially act as a sysroot. + // These are used to load rustc crates (e.g. `extern crate rustc_ast;`) + // for rustc_private tools, so that we do not have to copy them into the + // actual sysroot of the compiler that builds the tool. + if let Ok(dirs) = env::var("RUSTC_ADDITIONAL_SYSROOT_PATHS") { + for dir in dirs.split(",") { + cmd.arg(format!("-L{dir}")); + } + } + // Force all crates compiled by this compiler to (a) be unstable and (b) // allow the `rustc_private` feature to link to other unstable crates // also in the sysroot. We also do this for host crates, since those diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs index f0acb7f71411a..cce066c63f526 100644 --- a/src/bootstrap/src/core/build_steps/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -1,5 +1,8 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. +use std::fs; +use std::path::PathBuf; + use crate::core::build_steps::compile::{ add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make, }; @@ -9,11 +12,11 @@ use crate::core::build_steps::tool::{ prepare_tool_cargo, }; use crate::core::builder::{ - self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description, + self, Alias, Builder, Cargo, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description, }; use crate::core::config::TargetSelection; use crate::utils::build_stamp::{self, BuildStamp}; -use crate::{Compiler, Mode, Subcommand}; +use crate::{Compiler, Mode, Subcommand, t}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Std { @@ -64,7 +67,8 @@ impl Step for Std { let crates = std_crates_for_run_make(&run); run.builder.ensure(Std { - build_compiler: prepare_compiler_for_check(run.builder, run.target, Mode::Std), + build_compiler: prepare_compiler_for_check(run.builder, run.target, Mode::Std) + .build_compiler, target: run.target, crates, }); @@ -145,8 +149,65 @@ impl Step for Std { } } -/// Checks rustc using `build_compiler` and copies the built -/// .rmeta files into the sysroot of `build_compiler`. +/// Represents a proof that rustc was checked. +/// Contains directories that contain .rmeta files generated by checking rustc for a specific +/// target. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct RustcRmetaSysroot { + host_dir: PathBuf, + target_dir: PathBuf, +} + +impl RustcRmetaSysroot { + /// Configure the given cargo invocation so that the compiled crate will be able to use + /// rustc rmeta artifacts that were previously generated. + fn configure_cargo(&self, cargo: &mut Cargo) { + cargo.env( + "RUSTC_ADDITIONAL_SYSROOT_PATHS", + format!("{},{}", self.host_dir.display(), self.target_dir.display()), + ); + } +} + +/// Checks rustc using the given `build_compiler` for the given `target`, and produces +/// a sysroot in the build directory that stores the generated .rmeta files. +/// +/// This step exists so that we can separate the generated .rmeta artifacts into a separate +/// directory. Before, we used to copy them into the sysroot of `build_compiler`, but that +/// "pollutes" its sysroot, which is problematic especially for external stage0 rustc. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct PrepareRustcRmetaSysroot { + build_compiler: Compiler, + target: TargetSelection, +} + +impl Step for PrepareRustcRmetaSysroot { + type Output = RustcRmetaSysroot; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + // Check rustc + let stamp = builder.ensure(Rustc::new(builder, self.build_compiler, self.target)); + + // Copy the generated rmeta artifacts to a separate directory + let dir = builder + .out + .join(self.build_compiler.host) + .join(format!("stage{}-rustc-check-artifacts", self.build_compiler.stage + 1)); + let host_dir = dir.join("host"); + let target_dir = dir.join(self.target); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + add_to_sysroot(builder, &target_dir, &host_dir, &stamp); + + RustcRmetaSysroot { host_dir, target_dir } + } +} + +/// Checks rustc using `build_compiler`. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Rustc { /// Compiler that will check this rustc. @@ -172,7 +233,7 @@ impl Rustc { } impl Step for Rustc { - type Output = (); + type Output = BuildStamp; const ONLY_HOSTS: bool = true; const DEFAULT: bool = true; @@ -184,7 +245,8 @@ impl Step for Rustc { let crates = run.make_run_crates(Alias::Compiler); run.builder.ensure(Rustc { target: run.target, - build_compiler: prepare_compiler_for_check(run.builder, run.target, Mode::Rustc), + build_compiler: prepare_compiler_for_check(run.builder, run.target, Mode::Rustc) + .build_compiler, crates, }); } @@ -196,7 +258,7 @@ impl Step for Rustc { /// created will also be linked into the sysroot directory. /// /// If we check a stage 2 compiler, we will have to first build a stage 1 compiler to check it. - fn run(self, builder: &Builder<'_>) { + fn run(self, builder: &Builder<'_>) -> Self::Output { let build_compiler = self.build_compiler; let target = self.target; @@ -238,9 +300,7 @@ impl Step for Rustc { run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); - let libdir = builder.sysroot_target_libdir(build_compiler, target); - let hostdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); - add_to_sysroot(builder, &libdir, &hostdir, &stamp); + stamp } fn metadata(&self) -> Option { @@ -248,15 +308,25 @@ impl Step for Rustc { } } +/// Represents a compiler that can check something. +/// It optionally includes .rmeta artifacts from rustc that was already checked using +/// `build_compiler`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct CompilerForCheck { + build_compiler: Compiler, + rustc_rmeta_sysroot: Option, +} + /// Prepares a compiler that will check something with the given `mode`. fn prepare_compiler_for_check( builder: &Builder<'_>, target: TargetSelection, mode: Mode, -) -> Compiler { +) -> CompilerForCheck { let host = builder.host_target; - match mode { + let mut rmeta_sysroot = None; + let build_compiler = match mode { Mode::ToolBootstrap => builder.compiler(0, host), Mode::ToolTarget => get_tool_target_compiler(builder, ToolTargetBuildMode::Build(target)), Mode::ToolStd => { @@ -264,27 +334,28 @@ fn prepare_compiler_for_check( // When --compile-time-deps is passed, we can't use any rustc // other than the bootstrap compiler. Luckily build scripts and // proc macros for tools are unlikely to need nightly. - return builder.compiler(0, host); + builder.compiler(0, host) + } else { + // These tools require the local standard library to be checked + let build_compiler = builder.compiler(builder.top_stage, host); + + // We need to build the host stdlib to check the tool itself. + // We need to build the target stdlib so that the tool can link to it. + builder.std(build_compiler, host); + // We could only check this library in theory, but `check::Std` doesn't copy rmetas + // into `build_compiler`'s sysroot to avoid clashes with `.rlibs`, so we build it + // instead. + builder.std(build_compiler, target); + build_compiler } - - // These tools require the local standard library to be checked - let build_compiler = builder.compiler(builder.top_stage, host); - - // We need to build the host stdlib to check the tool itself. - // We need to build the target stdlib so that the tool can link to it. - builder.std(build_compiler, host); - // We could only check this library in theory, but `check::Std` doesn't copy rmetas - // into `build_compiler`'s sysroot to avoid clashes with `.rlibs`, so we build it - // instead. - builder.std(build_compiler, target); - build_compiler } Mode::ToolRustc | Mode::Codegen => { // FIXME: this is a hack, see description of Mode::Rustc below let stage = if host == target { builder.top_stage - 1 } else { builder.top_stage }; // When checking tool stage N, we check it with compiler stage N-1 let build_compiler = builder.compiler(stage, host); - builder.ensure(Rustc::new(builder, build_compiler, target)); + rmeta_sysroot = + Some(builder.ensure(PrepareRustcRmetaSysroot { build_compiler, target })); build_compiler } Mode::Rustc => { @@ -304,15 +375,17 @@ fn prepare_compiler_for_check( // stage 0 stdlib is used to compile build scripts and proc macros. builder.compiler(builder.top_stage, host) } - } + }; + CompilerForCheck { build_compiler, rustc_rmeta_sysroot: rmeta_sysroot } } /// Checks a single codegen backend. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CodegenBackend { - pub build_compiler: Compiler, - pub target: TargetSelection, - pub backend: &'static str, + build_compiler: Compiler, + rmeta_sysroot: RustcRmetaSysroot, + target: TargetSelection, + backend: &'static str, } impl Step for CodegenBackend { @@ -326,9 +399,17 @@ impl Step for CodegenBackend { fn make_run(run: RunConfig<'_>) { // FIXME: only check the backend(s) that were actually selected in run.paths - let build_compiler = prepare_compiler_for_check(run.builder, run.target, Mode::Codegen); + let CompilerForCheck { build_compiler, rustc_rmeta_sysroot: rmeta_sysroot } = + prepare_compiler_for_check(run.builder, run.target, Mode::Codegen); + let rmeta_sysroot = + rmeta_sysroot.expect("Rustc rmeta sysroot missing for checking codegen"); for &backend in &["cranelift", "gcc"] { - run.builder.ensure(CodegenBackend { build_compiler, target: run.target, backend }); + run.builder.ensure(CodegenBackend { + build_compiler, + rmeta_sysroot: rmeta_sysroot.clone(), + target: run.target, + backend, + }); } } @@ -351,6 +432,7 @@ impl Step for CodegenBackend { target, builder.kind, ); + self.rmeta_sysroot.configure_cargo(&mut cargo); cargo .arg("--manifest-path") @@ -370,140 +452,26 @@ impl Step for CodegenBackend { } } -/// Checks Rust analyzer that links to .rmetas from a checked rustc. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RustAnalyzer { - pub build_compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/rust-analyzer").default_condition( - builder - .config - .tools - .as_ref() - .is_none_or(|tools| tools.iter().any(|tool| tool == "rust-analyzer")), - ) - } - - fn make_run(run: RunConfig<'_>) { - let build_compiler = prepare_compiler_for_check(run.builder, run.target, Mode::ToolRustc); - run.builder.ensure(RustAnalyzer { build_compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let build_compiler = self.build_compiler; - let target = self.target; - - let mut cargo = prepare_tool_cargo( - builder, - build_compiler, - Mode::ToolRustc, - target, - builder.kind, - "src/tools/rust-analyzer", - SourceType::InTree, - &["in-rust-tree".to_owned()], - ); - - cargo.allow_features(crate::core::build_steps::tool::RustAnalyzer::ALLOW_FEATURES); - - cargo.arg("--bins"); - cargo.arg("--tests"); - cargo.arg("--benches"); - - // Cargo's output path in a given stage, compiled by a particular - // compiler for the specified target. - let stamp = BuildStamp::new(&builder.cargo_out(build_compiler, Mode::ToolRustc, target)) - .with_prefix("rust-analyzer-check"); - - let _guard = builder.msg_check("rust-analyzer artifacts", target, None); - run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); - } - - fn metadata(&self) -> Option { - Some(StepMetadata::check("rust-analyzer", self.target).built_by(self.build_compiler)) - } -} - -/// Compiletest is implicitly "checked" when it gets built in order to run tests, -/// so this is mainly for people working on compiletest to run locally. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Compiletest { - pub target: TargetSelection, -} - -impl Step for Compiletest { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/compiletest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Compiletest { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let mode = if builder.config.compiletest_use_stage0_libtest { - Mode::ToolBootstrap - } else { - Mode::ToolStd - }; - let build_compiler = prepare_compiler_for_check(builder, self.target, mode); - - let mut cargo = prepare_tool_cargo( - builder, - build_compiler, - mode, - self.target, - builder.kind, - "src/tools/compiletest", - SourceType::InTree, - &[], - ); - - cargo.allow_features(COMPILETEST_ALLOW_FEATURES); - - cargo.arg("--all-targets"); - - let stamp = BuildStamp::new(&builder.cargo_out(build_compiler, mode, self.target)) - .with_prefix("compiletest-check"); - - let _guard = builder.msg_check("compiletest artifacts", self.target, None); - run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); - } - - fn metadata(&self) -> Option { - Some(StepMetadata::check("compiletest", self.target)) - } -} - macro_rules! tool_check_step { ( $name:ident { // The part of this path after the final '/' is also used as a display name. path: $path:literal $(, alt_path: $alt_path:literal )* - , mode: $mode:path + // Closure that returns `Mode` based on the passed `&Builder<'_>` + , mode: $mode:expr + // Subset of nightly features that are allowed to be used when checking $(, allow_features: $allow_features:expr )? + // Features that should be enabled when checking + $(, enable_features: [$($enable_features:expr),*] )? $(, default: $default:literal )? $( , )? } ) => { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct $name { - pub build_compiler: Compiler, - pub target: TargetSelection, + compiler: CompilerForCheck, + target: TargetSelection, } impl Step for $name { @@ -518,29 +486,34 @@ macro_rules! tool_check_step { fn make_run(run: RunConfig<'_>) { let target = run.target; - let build_compiler = prepare_compiler_for_check(run.builder, target, $mode); + let builder = run.builder; + let mode = $mode(builder); + + let compiler = prepare_compiler_for_check(run.builder, target, mode); // It doesn't make sense to cross-check bootstrap tools - if $mode == Mode::ToolBootstrap && target != run.builder.host_target { + if mode == Mode::ToolBootstrap && target != run.builder.host_target { println!("WARNING: not checking bootstrap tool {} for target {target} as it is a bootstrap (host-only) tool", stringify!($path)); return; }; - run.builder.ensure($name { target, build_compiler }); + run.builder.ensure($name { target, compiler }); } fn run(self, builder: &Builder<'_>) { - let Self { target, build_compiler } = self; + let Self { target, compiler } = self; let allow_features = { let mut _value = ""; $( _value = $allow_features; )? _value }; - run_tool_check_step(builder, build_compiler, target, $path, $mode, allow_features); + let extra_features: &[&str] = &[$($($enable_features),*)?]; + let mode = $mode(builder); + run_tool_check_step(builder, compiler, target, $path, mode, allow_features, extra_features); } fn metadata(&self) -> Option { - Some(StepMetadata::check(stringify!($name), self.target).built_by(self.build_compiler)) + Some(StepMetadata::check(stringify!($name), self.target).built_by(self.compiler.build_compiler)) } } } @@ -549,14 +522,18 @@ macro_rules! tool_check_step { /// Used by the implementation of `Step::run` in `tool_check_step!`. fn run_tool_check_step( builder: &Builder<'_>, - build_compiler: Compiler, + compiler: CompilerForCheck, target: TargetSelection, path: &str, mode: Mode, allow_features: &str, + extra_features: &[&str], ) { let display_name = path.rsplit('/').next().unwrap(); + let CompilerForCheck { build_compiler, rustc_rmeta_sysroot: rmeta_sysroot } = compiler; + + let extra_features = extra_features.iter().map(|f| f.to_string()).collect::>(); let mut cargo = prepare_tool_cargo( builder, build_compiler, @@ -569,12 +546,24 @@ fn run_tool_check_step( // steps should probably be marked non-default so that the default // checks aren't affected by toolstate being broken. SourceType::InTree, - &[], + &extra_features, ); cargo.allow_features(allow_features); + if mode == Mode::ToolRustc { + let rmeta_sysroot = + rmeta_sysroot.expect("rustc rmeta sysroot missing for a ToolRustc tool"); + rmeta_sysroot.configure_cargo(&mut cargo); + } - // FIXME: check bootstrap doesn't currently work with --all-targets - cargo.arg("--all-targets"); + // FIXME: check bootstrap doesn't currently work when multiple targets are checked + // FIXME: rust-analyzer does not work with --all-targets + if display_name == "rust-analyzer" { + cargo.arg("--bins"); + cargo.arg("--tests"); + cargo.arg("--benches"); + } else { + cargo.arg("--all-targets"); + } let stamp = BuildStamp::new(&builder.cargo_out(build_compiler, mode, target)) .with_prefix(&format!("{display_name}-check")); @@ -593,43 +582,66 @@ fn run_tool_check_step( tool_check_step!(Rustdoc { path: "src/tools/rustdoc", alt_path: "src/librustdoc", - mode: Mode::ToolRustc + mode: |_builder| Mode::ToolRustc }); // Clippy, miri and Rustfmt are hybrids. They are external tools, but use a git subtree instead // of a submodule. Since the SourceType only drives the deny-warnings // behavior, treat it as in-tree so that any new warnings in clippy will be // rejected. -tool_check_step!(Clippy { path: "src/tools/clippy", mode: Mode::ToolRustc }); -tool_check_step!(Miri { path: "src/tools/miri", mode: Mode::ToolRustc }); -tool_check_step!(CargoMiri { path: "src/tools/miri/cargo-miri", mode: Mode::ToolRustc }); -tool_check_step!(Rustfmt { path: "src/tools/rustfmt", mode: Mode::ToolRustc }); +tool_check_step!(Clippy { path: "src/tools/clippy", mode: |_builder| Mode::ToolRustc }); +tool_check_step!(Miri { path: "src/tools/miri", mode: |_builder| Mode::ToolRustc }); +tool_check_step!(CargoMiri { path: "src/tools/miri/cargo-miri", mode: |_builder| Mode::ToolRustc }); +tool_check_step!(Rustfmt { path: "src/tools/rustfmt", mode: |_builder| Mode::ToolRustc }); +tool_check_step!(RustAnalyzer { + path: "src/tools/rust-analyzer", + mode: |_builder| Mode::ToolRustc, + allow_features: tool::RustAnalyzer::ALLOW_FEATURES, + enable_features: ["in-rust-tree"], +}); tool_check_step!(MiroptTestTools { path: "src/tools/miropt-test-tools", - mode: Mode::ToolBootstrap + mode: |_builder| Mode::ToolBootstrap }); // We want to test the local std tool_check_step!(TestFloatParse { path: "src/tools/test-float-parse", - mode: Mode::ToolStd, + mode: |_builder| Mode::ToolStd, allow_features: tool::TestFloatParse::ALLOW_FEATURES }); tool_check_step!(FeaturesStatusDump { path: "src/tools/features-status-dump", - mode: Mode::ToolBootstrap + mode: |_builder| Mode::ToolBootstrap }); -tool_check_step!(Bootstrap { path: "src/bootstrap", mode: Mode::ToolBootstrap, default: false }); +tool_check_step!(Bootstrap { + path: "src/bootstrap", + mode: |_builder| Mode::ToolBootstrap, + default: false +}); // `run-make-support` will be built as part of suitable run-make compiletest test steps, but support // check to make it easier to work on. tool_check_step!(RunMakeSupport { path: "src/tools/run-make-support", - mode: Mode::ToolBootstrap, + mode: |_builder| Mode::ToolBootstrap, default: false }); tool_check_step!(CoverageDump { path: "src/tools/coverage-dump", - mode: Mode::ToolBootstrap, + mode: |_builder| Mode::ToolBootstrap, default: false }); + +// Compiletest is implicitly "checked" when it gets built in order to run tests, +// so this is mainly for people working on compiletest to run locally. +tool_check_step!(Compiletest { + path: "src/tools/compiletest", + mode: |builder: &Builder<'_>| if builder.config.compiletest_use_stage0_libtest { + Mode::ToolBootstrap + } else { + Mode::ToolStd + }, + allow_features: COMPILETEST_ALLOW_FEATURES, + default: false, +}); diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index e60a115b19d68..c025a8b15d039 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -1382,7 +1382,7 @@ mod snapshot { [check] rustc 1 -> Miri 2 [check] rustc 1 -> CargoMiri 2 [check] rustc 1 -> Rustfmt 2 - [check] rustc 1 -> rust-analyzer 2 + [check] rustc 1 -> RustAnalyzer 2 [check] rustc 1 -> TestFloatParse 2 [check] rustc 1 -> std 1 "); @@ -1530,7 +1530,7 @@ mod snapshot { insta::assert_snapshot!( ctx.config("check") .path("compiletest") - .render_steps(), @"[check] compiletest "); + .render_steps(), @"[check] rustc 0 -> Compiletest 1 "); } #[test] @@ -1544,7 +1544,7 @@ mod snapshot { [build] llvm [build] rustc 0 -> rustc 1 [build] rustc 1 -> std 1 - [check] compiletest + [check] rustc 1 -> Compiletest 2 "); } @@ -1569,7 +1569,7 @@ mod snapshot { .path("rust-analyzer") .render_steps(), @r" [check] rustc 0 -> rustc 1 - [check] rustc 0 -> rust-analyzer 1 + [check] rustc 0 -> RustAnalyzer 1 "); }