Skip to content

Commit f4b36c8

Browse files
rust-bors[bot]rust-bors[bot]
authored
Auto merge of #144483 - scottmcm:early-inline, r=<try>
EXPERIMENT: Use the force-inline pass for more stuff
2 parents 383b9c4 + a6adf2d commit f4b36c8

23 files changed

+256
-54
lines changed

‎compiler/rustc_attr_parsing/src/attributes/inline.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,3 +93,16 @@ impl<S: Stage> SingleAttributeParser<S> for RustcForceInlineParser {
9393
))
9494
}
9595
}
96+
97+
pub(crate) struct RustcEarlyInlineParser;
98+
99+
impl<S: Stage> SingleAttributeParser<S> for RustcEarlyInlineParser {
100+
const PATH: &'static [Symbol] = &[sym::rustc_early_inline];
101+
const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
102+
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
103+
const TEMPLATE: AttributeTemplate = template!(Word);
104+
105+
fn convert(cx: &mut AcceptContext<'_, '_, S>, _args: &ArgParser<'_>) -> Option<AttributeKind> {
106+
Some(AttributeKind::Inline(InlineAttr::Early, cx.attr_span))
107+
}
108+
}

‎compiler/rustc_attr_parsing/src/context.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use crate::attributes::codegen_attrs::{
2323
use crate::attributes::confusables::ConfusablesParser;
2424
use crate::attributes::deprecation::DeprecationParser;
2525
use crate::attributes::dummy::DummyParser;
26-
use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
26+
use crate::attributes::inline::{InlineParser, RustcEarlyInlineParser, RustcForceInlineParser};
2727
use crate::attributes::link_attrs::{
2828
ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser,
2929
LinkSectionParser, StdInternalSymbolParser,
@@ -169,6 +169,7 @@ attribute_parsers!(
169169
Single<PathAttributeParser>,
170170
Single<ProcMacroDeriveParser>,
171171
Single<RustcBuiltinMacroParser>,
172+
Single<RustcEarlyInlineParser>,
172173
Single<RustcForceInlineParser>,
173174
Single<RustcLayoutScalarValidRangeEnd>,
174175
Single<RustcLayoutScalarValidRangeStart>,

‎compiler/rustc_codegen_gcc/src/attributes.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ fn inline_attr<'gcc, 'tcx>(
6767
Some(FnAttribute::AlwaysInline)
6868
}
6969
}
70-
InlineAttr::Hint => Some(FnAttribute::Inline),
70+
InlineAttr::Hint | InlineAttr::Early => Some(FnAttribute::Inline),
7171
InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
7272
InlineAttr::Never => {
7373
if cx.sess().target.arch != "amdgpu" {

‎compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
5252
return Some(AttributeKind::NoInline.create_attr(cx.llcx));
5353
}
5454
match inline {
55-
InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
55+
InlineAttr::Hint | InlineAttr::Early => {
56+
Some(AttributeKind::InlineHint.create_attr(cx.llcx))
57+
}
5658
InlineAttr::Always | InlineAttr::Force { .. } => {
5759
Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
5860
}

‎compiler/rustc_codegen_ssa/src/codegen_attrs.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ fn check_result(
473473

474474
// warn that inline has no effect when no_sanitize is present
475475
if !codegen_fn_attrs.no_sanitize.is_empty()
476-
&& codegen_fn_attrs.inline.always()
476+
&& codegen_fn_attrs.inline.always_in_codegen()
477477
&& let (Some(no_sanitize_span), Some(inline_span)) =
478478
(interesting_spans.no_sanitize, interesting_spans.inline)
479479
{

‎compiler/rustc_feature/src/builtin_attrs.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1123,6 +1123,11 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
11231123
rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
11241124
"`#[rustc_no_mir_inline]` prevents the MIR inliner from inlining a function while not affecting codegen"
11251125
),
1126+
rustc_attr!(
1127+
rustc_early_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
1128+
"`#[rustc_early_inline]` inlines non-generic calls to trivial methods even in debug, \
1129+
while still allowing them to be codegen'd for generic calls"
1130+
),
11261131
rustc_attr!(
11271132
rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
11281133
"`#[rustc_force_inline]` forces a free function to be inlined"

‎compiler/rustc_hir/src/attrs/data_structures.rs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,20 @@ pub enum InlineAttr {
2424
attr_span: Span,
2525
reason: Option<Symbol>,
2626
},
27+
/// `#[rustc_early_inline]` will always inline calls to a known impl in MIR.
28+
///
29+
/// You can think of this as either
30+
/// - Force, but without the "do not codegen as a function ever" restriction.
31+
/// - Always, but only for MIR.
32+
Early,
2733
}
2834

2935
impl InlineAttr {
30-
pub fn always(&self) -> bool {
36+
pub fn always_in_codegen(&self) -> bool {
3137
match self {
32-
InlineAttr::Always | InlineAttr::Force { .. } => true,
33-
InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
38+
InlineAttr::Always => true,
39+
InlineAttr::None | InlineAttr::Hint | InlineAttr::Early | InlineAttr::Never => false,
40+
InlineAttr::Force { .. } => panic!("Shouldn't be codegen'ing {self:?}"),
3441
}
3542
}
3643
}
@@ -342,7 +349,7 @@ pub enum AttributeKind {
342349
reason: Option<Symbol>,
343350
},
344351

345-
/// Represents `#[inline]` and `#[rustc_force_inline]`.
352+
/// Represents `#[inline]` and `#[rustc_force_inline]` and `#[rustc_early_inline]`.
346353
Inline(InlineAttr, Span),
347354

348355
/// Represents `#[link_name]`.

‎compiler/rustc_middle/src/mir/mono.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ impl<'tcx> MonoItem<'tcx> {
205205

206206
// To ensure that #[inline(always)] can be inlined as much as possible, especially in unoptimized
207207
// builds, we always select LocalCopy.
208-
if codegen_fn_attrs.inline.always() {
208+
if codegen_fn_attrs.inline.always_in_codegen() {
209209
return InstantiationMode::LocalCopy;
210210
}
211211

‎compiler/rustc_mir_transform/src/cross_crate_inline.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,10 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
4646
// #[inline(never)] to force code generation.
4747
match codegen_fn_attrs.inline {
4848
InlineAttr::Never => return false,
49-
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
50-
_ => {}
49+
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Early | InlineAttr::Force { .. } => {
50+
return true;
51+
}
52+
InlineAttr::None => {}
5153
}
5254

5355
// If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until

‎compiler/rustc_mir_transform/src/inline.rs

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ pub struct ForceInline;
7777

7878
impl ForceInline {
7979
pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
80-
matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
80+
matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. } | InlineAttr::Early)
8181
}
8282
}
8383

@@ -195,7 +195,7 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
195195
&self,
196196
callee_attrs: &CodegenFnAttrs,
197197
) -> Result<(), &'static str> {
198-
debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
198+
debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. } | InlineAttr::Early);
199199
Ok(())
200200
}
201201

@@ -247,23 +247,26 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
247247

248248
fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
249249
let tcx = self.tcx();
250-
let InlineAttr::Force { attr_span, reason: justification } =
251-
tcx.codegen_fn_attrs(callsite.callee.def_id()).inline
252-
else {
253-
bug!("called on item without required inlining");
254-
};
255-
256-
let call_span = callsite.source_info.span;
257-
tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
258-
call_span,
259-
attr_span,
260-
caller_span: tcx.def_span(self.def_id),
261-
caller: tcx.def_path_str(self.def_id),
262-
callee_span: tcx.def_span(callsite.callee.def_id()),
263-
callee: tcx.def_path_str(callsite.callee.def_id()),
264-
reason,
265-
justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
266-
});
250+
match tcx.codegen_fn_attrs(callsite.callee.def_id()).inline {
251+
InlineAttr::Early => {
252+
// Ok, we don't actually mind if this fails.
253+
}
254+
InlineAttr::Force { attr_span, reason: justification } => {
255+
let call_span = callsite.source_info.span;
256+
tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
257+
call_span,
258+
attr_span,
259+
caller_span: tcx.def_span(self.def_id),
260+
caller: tcx.def_path_str(self.def_id),
261+
callee_span: tcx.def_span(callsite.callee.def_id()),
262+
callee: tcx.def_path_str(callsite.callee.def_id()),
263+
reason,
264+
justification: justification
265+
.map(|sym| crate::errors::ForceInlineJustification { sym }),
266+
});
267+
}
268+
_ => bug!("called on item without required inlining"),
269+
}
267270
}
268271
}
269272

0 commit comments

Comments
 (0)