Skip to content

Consolidate staging for rustc_private tools #144303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 22 commits into
base: master
Choose a base branch
from
Open

Consolidate staging for rustc_private tools #144303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
b6f12d7
Rename extended rustc tool macros
Kobzol Jun 18, 2025
65c3597
Add metadata to `Cargo` and `RustAnalyzer` tools
Kobzol Jul 22, 2025
57a6c93
Cleanup `ensure_if_default` to not require `Option` output
Kobzol Jul 22, 2025
8fed3fb
Add step metadata to `RustAnalyzerProcMacroSrv`
Kobzol Jul 22, 2025
2f3bfff
Rename `Builder::rustdoc` to `Builder::rustdoc_for_compiler`
Kobzol Jul 22, 2025
6d562b2
Implement `RustcPrivateCompilers` to unify building of `rustc_private…
Kobzol Jul 22, 2025
b944144
Add step metadata and a few tests for `Doc` steps
Kobzol Jul 22, 2025
2c7581f
Refactor `Rustdoc`
Kobzol Jul 22, 2025
91d40d2
Fix `ToolRustc` build with `download-rustc`
Kobzol Jul 22, 2025
7a2c4d3
Add step metadata and a simple test for codegen backends
Kobzol Jul 22, 2025
06855be
Port codegen backends to `RustcPrivateCompilers`
Kobzol Jul 22, 2025
c3da07b
Rename `link_compiler` to `target_compiler`
Kobzol Jul 22, 2025
0ff8ff3
Appease Clippy
Kobzol Jul 22, 2025
316b5d3
Add basic Cargo snapshot test
Kobzol Jul 21, 2025
21d7a54
Make `Cargo` a `ToolTarget` tool
Kobzol Jul 21, 2025
021f2ff
Update `CargoTest`
Kobzol Jul 21, 2025
3e6aa81
Add snapshot tests for `test cargo` and update Cargo snapshot tests
Kobzol Jul 21, 2025
528e7dc
Make build compiler explicit in `dist::Cargo`
Kobzol Jul 28, 2025
1d1efed
Fix `x test cargo`
Kobzol Jul 29, 2025
47da014
Make `x test cargo` stage N test rustc stage N
Kobzol Jul 31, 2025
c46f42d
Clarify comments on Cargo (self-)test steps
Kobzol Aug 1, 2025
ac28b5b
Fix splitting dylib paths
Kobzol Aug 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 35 additions & 28 deletions src/bootstrap/src/core/build_steps/compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use serde_derive::Deserialize;
use tracing::{instrument, span};

use crate::core::build_steps::gcc::{Gcc, add_cg_gcc_cargo_flags};
use crate::core::build_steps::tool::{SourceType, copy_lld_artifacts};
use crate::core::build_steps::tool::{RustcPrivateCompilers, SourceType, copy_lld_artifacts};
use crate::core::build_steps::{dist, llvm};
use crate::core::builder;
use crate::core::builder::{
Expand Down Expand Up @@ -1131,7 +1131,7 @@ impl Step for Rustc {
cargo.env("RUSTC_BOLT_LINK_FLAGS", "1");
}

let _guard = builder.msg_sysroot_tool(
let _guard = builder.msg_rustc_tool(
Kind::Build,
build_compiler.stage,
format_args!("compiler artifacts{}", crate_description(&self.crates)),
Expand Down Expand Up @@ -1544,9 +1544,8 @@ impl Step for RustcLink {

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct CodegenBackend {
pub target: TargetSelection,
pub compiler: Compiler,
pub backend: CodegenBackendKind,
compilers: RustcPrivateCompilers,
backend: CodegenBackendKind,
}

fn needs_codegen_config(run: &RunConfig<'_>) -> bool {
Expand Down Expand Up @@ -1610,8 +1609,11 @@ impl Step for CodegenBackend {
}

run.builder.ensure(CodegenBackend {
target: run.target,
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
compilers: RustcPrivateCompilers::new(
run.builder,
run.builder.top_stage,
run.target,
),
backend: backend.clone(),
});
}
Expand All @@ -1624,20 +1626,17 @@ impl Step for CodegenBackend {
name = "CodegenBackend::run",
skip_all,
fields(
compiler = ?self.compiler,
target = ?self.target,
backend = ?self.target,
compilers = ?self.compilers,
backend = ?self.backend,
),
),
)]
fn run(self, builder: &Builder<'_>) {
let compiler = self.compiler;
let target = self.target;
let backend = self.backend;
let target = self.compilers.target();
let build_compiler = self.compilers.build_compiler();

builder.ensure(Rustc::new(compiler, target));

if builder.config.keep_stage.contains(&compiler.stage) {
if builder.config.keep_stage.contains(&build_compiler.stage) {
trace!("`keep-stage` requested");
builder.info(
"WARNING: Using a potentially old codegen backend. \
Expand All @@ -1648,17 +1647,11 @@ impl Step for CodegenBackend {
return;
}

let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
if compiler_to_use != compiler {
builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend });
return;
}

let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
let out_dir = builder.cargo_out(build_compiler, Mode::Codegen, target);

let mut cargo = builder::Cargo::new(
builder,
compiler,
build_compiler,
Mode::Codegen,
SourceType::InTree,
target,
Expand All @@ -1679,8 +1672,13 @@ impl Step for CodegenBackend {

let tmp_stamp = BuildStamp::new(&out_dir).with_prefix("tmp");

let _guard =
builder.msg_build(compiler, format_args!("codegen backend {}", backend.name()), target);
let _guard = builder.msg_rustc_tool(
Kind::Build,
build_compiler.stage,
format_args!("codegen backend {}", backend.name()),
build_compiler.host,
target,
);
let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false);
if builder.config.dry_run() {
return;
Expand All @@ -1700,10 +1698,17 @@ impl Step for CodegenBackend {
f.display()
);
}
let stamp = build_stamp::codegen_backend_stamp(builder, compiler, target, &backend);
let stamp = build_stamp::codegen_backend_stamp(builder, build_compiler, target, &backend);
let codegen_backend = codegen_backend.to_str().unwrap();
t!(stamp.add_stamp(codegen_backend).write());
}

fn metadata(&self) -> Option<StepMetadata> {
Some(
StepMetadata::build(&self.backend.crate_name(), self.compilers.target())
.built_by(self.compilers.build_compiler()),
)
}
}

/// Creates the `codegen-backends` folder for a compiler that's about to be
Expand Down Expand Up @@ -2190,8 +2195,10 @@ impl Step for Assemble {
continue;
}
builder.ensure(CodegenBackend {
compiler: build_compiler,
target: target_compiler.host,
compilers: RustcPrivateCompilers::from_build_and_target_compiler(
build_compiler,
target_compiler,
),
backend: backend.clone(),
});
}
Expand Down
99 changes: 46 additions & 53 deletions src/bootstrap/src/core/build_steps/dist.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use object::read::archive::ArchiveFile;
use tracing::instrument;

use crate::core::build_steps::doc::DocumentationFormat;
use crate::core::build_steps::tool::{self, Tool};
use crate::core::build_steps::tool::{self, RustcPrivateCompilers, Tool};
use crate::core::build_steps::vendor::{VENDOR_DIR, Vendor};
use crate::core::build_steps::{compile, llvm};
use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata};
Expand Down Expand Up @@ -425,19 +425,20 @@ impl Step for Rustc {
.as_ref()
.is_none_or(|tools| tools.iter().any(|tool| tool == "rustdoc"))
{
let rustdoc = builder.rustdoc(compiler);
let rustdoc = builder.rustdoc_for_compiler(compiler);
builder.install(&rustdoc, &image.join("bin"), FileType::Executable);
}

let ra_proc_macro_srv_compiler =
builder.compiler_for(compiler.stage, builder.config.host_target, compiler.host);
builder.ensure(compile::Rustc::new(ra_proc_macro_srv_compiler, compiler.host));
let compilers = RustcPrivateCompilers::from_build_compiler(
builder,
ra_proc_macro_srv_compiler,
compiler.host,
);

if let Some(ra_proc_macro_srv) = builder.ensure_if_default(
tool::RustAnalyzerProcMacroSrv {
compiler: ra_proc_macro_srv_compiler,
target: compiler.host,
},
tool::RustAnalyzerProcMacroSrv::from_compilers(compilers),
builder.kind,
) {
let dst = image.join("libexec");
Expand Down Expand Up @@ -1172,7 +1173,7 @@ impl Step for PlainSourceTarball {

#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
pub struct Cargo {
pub compiler: Compiler,
pub build_compiler: Compiler,
pub target: TargetSelection,
}

Expand All @@ -1188,7 +1189,7 @@ impl Step for Cargo {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Cargo {
compiler: run.builder.compiler_for(
build_compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.host_target,
run.target,
Expand All @@ -1198,12 +1199,10 @@ impl Step for Cargo {
}

fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let build_compiler = self.build_compiler;
let target = self.target;

builder.ensure(compile::Rustc::new(compiler, target));

let cargo = builder.ensure(tool::Cargo { compiler, target });
let cargo = builder.ensure(tool::Cargo::from_build_compiler(build_compiler, target));
let src = builder.src.join("src/tools/cargo");
let etc = src.join("src/etc");

Expand All @@ -1228,7 +1227,7 @@ impl Step for Cargo {

#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
pub struct RustAnalyzer {
pub compiler: Compiler,
pub build_compiler: Compiler,
pub target: TargetSelection,
}

Expand All @@ -1244,7 +1243,7 @@ impl Step for RustAnalyzer {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(RustAnalyzer {
compiler: run.builder.compiler_for(
build_compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.host_target,
run.target,
Expand All @@ -1254,12 +1253,11 @@ impl Step for RustAnalyzer {
}

fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;
let compilers =
RustcPrivateCompilers::from_build_compiler(builder, self.build_compiler, self.target);

builder.ensure(compile::Rustc::new(compiler, target));

let rust_analyzer = builder.ensure(tool::RustAnalyzer { compiler, target });
let rust_analyzer = builder.ensure(tool::RustAnalyzer::from_compilers(compilers));

let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
tarball.set_overlay(OverlayKind::RustAnalyzer);
Expand All @@ -1270,9 +1268,9 @@ impl Step for RustAnalyzer {
}
}

#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct Clippy {
pub compiler: Compiler,
pub build_compiler: Compiler,
pub target: TargetSelection,
}

Expand All @@ -1288,7 +1286,7 @@ impl Step for Clippy {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Clippy {
compiler: run.builder.compiler_for(
build_compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.host_target,
run.target,
Expand All @@ -1298,16 +1296,15 @@ impl Step for Clippy {
}

fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;

builder.ensure(compile::Rustc::new(compiler, target));
let compilers =
RustcPrivateCompilers::from_build_compiler(builder, self.build_compiler, target);

// Prepare the image directory
// We expect clippy to build, because we've exited this step above if tool
// state for clippy isn't testing.
let clippy = builder.ensure(tool::Clippy { compiler, target });
let cargoclippy = builder.ensure(tool::CargoClippy { compiler, target });
let clippy = builder.ensure(tool::Clippy::from_compilers(compilers));
let cargoclippy = builder.ensure(tool::CargoClippy::from_compilers(compilers));

let mut tarball = Tarball::new(builder, "clippy", &target.triple);
tarball.set_overlay(OverlayKind::Clippy);
Expand All @@ -1319,9 +1316,9 @@ impl Step for Clippy {
}
}

#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct Miri {
pub compiler: Compiler,
pub build_compiler: Compiler,
pub target: TargetSelection,
}

Expand All @@ -1337,7 +1334,7 @@ impl Step for Miri {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Miri {
compiler: run.builder.compiler_for(
build_compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.host_target,
run.target,
Expand All @@ -1354,15 +1351,12 @@ impl Step for Miri {
return None;
}

let compiler = self.compiler;
let target = self.target;

builder.ensure(compile::Rustc::new(compiler, target));
let compilers =
RustcPrivateCompilers::from_build_compiler(builder, self.build_compiler, self.target);
let miri = builder.ensure(tool::Miri::from_compilers(compilers));
let cargomiri = builder.ensure(tool::CargoMiri::from_compilers(compilers));

let miri = builder.ensure(tool::Miri { compiler, target });
let cargomiri = builder.ensure(tool::CargoMiri { compiler, target });

let mut tarball = Tarball::new(builder, "miri", &target.triple);
let mut tarball = Tarball::new(builder, "miri", &self.target.triple);
tarball.set_overlay(OverlayKind::Miri);
tarball.is_preview(true);
tarball.add_file(&miri.tool_path, "bin", FileType::Executable);
Expand Down Expand Up @@ -1466,9 +1460,9 @@ impl Step for CodegenBackend {
}
}

#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct Rustfmt {
pub compiler: Compiler,
pub build_compiler: Compiler,
pub target: TargetSelection,
}

Expand All @@ -1484,7 +1478,7 @@ impl Step for Rustfmt {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Rustfmt {
compiler: run.builder.compiler_for(
build_compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.host_target,
run.target,
Expand All @@ -1494,14 +1488,13 @@ impl Step for Rustfmt {
}

fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;
let compilers =
RustcPrivateCompilers::from_build_compiler(builder, self.build_compiler, self.target);

builder.ensure(compile::Rustc::new(compiler, target));
let rustfmt = builder.ensure(tool::Rustfmt::from_compilers(compilers));
let cargofmt = builder.ensure(tool::Cargofmt::from_compilers(compilers));

let rustfmt = builder.ensure(tool::Rustfmt { compiler, target });
let cargofmt = builder.ensure(tool::Cargofmt { compiler, target });
let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
let mut tarball = Tarball::new(builder, "rustfmt", &self.target.triple);
tarball.set_overlay(OverlayKind::Rustfmt);
tarball.is_preview(true);
tarball.add_file(&rustfmt.tool_path, "bin", FileType::Executable);
Expand Down Expand Up @@ -1548,7 +1541,7 @@ impl Step for Extended {
let mut built_tools = HashSet::new();
macro_rules! add_component {
($name:expr => $step:expr) => {
if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
if let Some(Some(tarball)) = builder.ensure_if_default($step, Kind::Dist) {
tarballs.push(tarball);
built_tools.insert($name);
}
Expand All @@ -1568,12 +1561,12 @@ impl Step for Extended {

add_component!("rust-docs" => Docs { host: target });
add_component!("rust-json-docs" => JsonDocs { host: target });
add_component!("cargo" => Cargo { compiler, target });
add_component!("rustfmt" => Rustfmt { compiler, target });
add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
add_component!("cargo" => Cargo { build_compiler: compiler, target });
add_component!("rustfmt" => Rustfmt { build_compiler: compiler, target });
add_component!("rust-analyzer" => RustAnalyzer { build_compiler: compiler, target });
add_component!("llvm-components" => LlvmTools { target });
add_component!("clippy" => Clippy { compiler, target });
add_component!("miri" => Miri { compiler, target });
add_component!("clippy" => Clippy { build_compiler: compiler, target });
add_component!("miri" => Miri { build_compiler: compiler, target });
add_component!("analysis" => Analysis { compiler, target });
add_component!("rustc-codegen-cranelift" => CodegenBackend {
compiler: builder.compiler(stage, target),
Expand Down
Loading
Loading