EXPERIMENT: Use the force-inline pass for more stuff

compiler/rustc_attr_parsing/src/attributes/inline.rs

@@ -93,3 +93,16 @@ impl<S: Stage> SingleAttributeParser<S> for RustcForceInlineParser {
         ))
     }
 }
+
+pub(crate) struct RustcEarlyInlineParser;
+
+impl<S: Stage> SingleAttributeParser<S> for RustcEarlyInlineParser {
+    const PATH: &'static [Symbol] = &[sym::rustc_early_inline];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
+    const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
+    const TEMPLATE: AttributeTemplate = template!(Word);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, _args: &ArgParser<'_>) -> Option<AttributeKind> {
+        Some(AttributeKind::Inline(InlineAttr::Early, cx.attr_span))
+    }
+}

compiler/rustc_attr_parsing/src/context.rs

@@ -23,7 +23,7 @@ use crate::attributes::codegen_attrs::{
 use crate::attributes::confusables::ConfusablesParser;
 use crate::attributes::deprecation::DeprecationParser;
 use crate::attributes::dummy::DummyParser;
-use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
+use crate::attributes::inline::{InlineParser, RustcEarlyInlineParser, RustcForceInlineParser};
 use crate::attributes::link_attrs::{
     ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser,
     LinkSectionParser, StdInternalSymbolParser,
@@ -169,6 +169,7 @@ attribute_parsers!(
         Single<PathAttributeParser>,
         Single<ProcMacroDeriveParser>,
         Single<RustcBuiltinMacroParser>,
+        Single<RustcEarlyInlineParser>,
         Single<RustcForceInlineParser>,
         Single<RustcLayoutScalarValidRangeEnd>,
         Single<RustcLayoutScalarValidRangeStart>,

compiler/rustc_codegen_gcc/src/attributes.rs

@@ -67,7 +67,7 @@ fn inline_attr<'gcc, 'tcx>(
                 Some(FnAttribute::AlwaysInline)
             }
         }
-        InlineAttr::Hint => Some(FnAttribute::Inline),
+        InlineAttr::Hint | InlineAttr::Early => Some(FnAttribute::Inline),
         InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
         InlineAttr::Never => {
             if cx.sess().target.arch != "amdgpu" {

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -52,7 +52,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
         return Some(AttributeKind::NoInline.create_attr(cx.llcx));
     }
     match inline {
-        InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
+        InlineAttr::Hint | InlineAttr::Early => {
+            Some(AttributeKind::InlineHint.create_attr(cx.llcx))
+        }
         InlineAttr::Always | InlineAttr::Force { .. } => {
             Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
         }

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -473,7 +473,7 @@ fn check_result(
 
     // warn that inline has no effect when no_sanitize is present
     if !codegen_fn_attrs.no_sanitize.is_empty()
-        && codegen_fn_attrs.inline.always()
+        && codegen_fn_attrs.inline.always_in_codegen()
         && let (Some(no_sanitize_span), Some(inline_span)) =
             (interesting_spans.no_sanitize, interesting_spans.inline)
     {

compiler/rustc_feature/src/builtin_attrs.rs

@@ -1123,6 +1123,11 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_no_mir_inline]` prevents the MIR inliner from inlining a function while not affecting codegen"
     ),
+    rustc_attr!(
+        rustc_early_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
+        "`#[rustc_early_inline]` inlines non-generic calls to trivial methods even in debug, \
+         while still allowing them to be codegen'd for generic calls"
+    ),
     rustc_attr!(
         rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_force_inline]` forces a free function to be inlined"

compiler/rustc_hir/src/attrs/data_structures.rs

@@ -24,13 +24,20 @@ pub enum InlineAttr {
         attr_span: Span,
         reason: Option<Symbol>,
     },
+    /// `#[rustc_early_inline]` will always inline calls to a known impl in MIR.
+    ///
+    /// You can think of this as either
+    /// - Force, but without the "do not codegen as a function ever" restriction.
+    /// - Always, but only for MIR.
+    Early,
 }
 
 impl InlineAttr {
-    pub fn always(&self) -> bool {
+    pub fn always_in_codegen(&self) -> bool {
         match self {
-            InlineAttr::Always | InlineAttr::Force { .. } => true,
-            InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
+            InlineAttr::Always => true,
+            InlineAttr::None | InlineAttr::Hint | InlineAttr::Early | InlineAttr::Never => false,
+            InlineAttr::Force { .. } => panic!("Shouldn't be codegen'ing {self:?}"),
         }
     }
 }
@@ -342,7 +349,7 @@ pub enum AttributeKind {
         reason: Option<Symbol>,
     },
 
-    /// Represents `#[inline]` and `#[rustc_force_inline]`.
+    /// Represents `#[inline]` and `#[rustc_force_inline]` and `#[rustc_early_inline]`.
     Inline(InlineAttr, Span),
 
     /// Represents `#[link_name]`.

compiler/rustc_middle/src/mir/mono.rs

@@ -205,7 +205,7 @@ impl<'tcx> MonoItem<'tcx> {
 
         // To ensure that #[inline(always)] can be inlined as much as possible, especially in unoptimized
         // builds, we always select LocalCopy.
-        if codegen_fn_attrs.inline.always() {
+        if codegen_fn_attrs.inline.always_in_codegen() {
             return InstantiationMode::LocalCopy;
         }
 

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -46,8 +46,10 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
-        _ => {}
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Early | InlineAttr::Force { .. } => {
+            return true;
+        }
+        InlineAttr::None => {}
     }
 
     // If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until

compiler/rustc_mir_transform/src/inline.rs

@@ -77,7 +77,7 @@ pub struct ForceInline;
 
 impl ForceInline {
     pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
-        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
+        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. } | InlineAttr::Early)
     }
 }
 
@@ -195,7 +195,7 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
         &self,
         callee_attrs: &CodegenFnAttrs,
     ) -> Result<(), &'static str> {
-        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
+        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. } | InlineAttr::Early);
         Ok(())
     }
 
@@ -247,23 +247,26 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
 
     fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
         let tcx = self.tcx();
-        let InlineAttr::Force { attr_span, reason: justification } =
-            tcx.codegen_fn_attrs(callsite.callee.def_id()).inline
-        else {
-            bug!("called on item without required inlining");
-        };
-
-        let call_span = callsite.source_info.span;
-        tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
-            call_span,
-            attr_span,
-            caller_span: tcx.def_span(self.def_id),
-            caller: tcx.def_path_str(self.def_id),
-            callee_span: tcx.def_span(callsite.callee.def_id()),
-            callee: tcx.def_path_str(callsite.callee.def_id()),
-            reason,
-            justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
-        });
+        match tcx.codegen_fn_attrs(callsite.callee.def_id()).inline {
+            InlineAttr::Early => {
+                // Ok, we don't actually mind if this fails.
+            }
+            InlineAttr::Force { attr_span, reason: justification } => {
+                let call_span = callsite.source_info.span;
+                tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
+                    call_span,
+                    attr_span,
+                    caller_span: tcx.def_span(self.def_id),
+                    caller: tcx.def_path_str(self.def_id),
+                    callee_span: tcx.def_span(callsite.callee.def_id()),
+                    callee: tcx.def_path_str(callsite.callee.def_id()),
+                    reason,
+                    justification: justification
+                        .map(|sym| crate::errors::ForceInlineJustification { sym }),
+                });
+            }
+            _ => bug!("called on item without required inlining"),
+        }
     }
 }
 

compiler/rustc_span/src/symbol.rs

@@ -1848,6 +1848,7 @@ symbols! {
         rustc_dump_predicates,
         rustc_dump_user_args,
         rustc_dump_vtable,
+        rustc_early_inline,
         rustc_effective_visibility,
         rustc_evaluate_where_clauses,
         rustc_expected_cgu_reuse,

library/alloc/src/boxed.rs

@@ -252,7 +252,7 @@ impl<T> Box<T> {
     /// let five = Box::new(5);
     /// ```
     #[cfg(not(no_global_oom_handling))]
-    #[inline(always)]
+    #[rustc_early_inline]
     #[stable(feature = "rust1", since = "1.0.0")]
     #[must_use]
     #[rustc_diagnostic_item = "box_new"]
@@ -317,7 +317,7 @@ impl<T> Box<T> {
     #[cfg(not(no_global_oom_handling))]
     #[stable(feature = "pin", since = "1.33.0")]
     #[must_use]
-    #[inline(always)]
+    #[rustc_early_inline]
     pub fn pin(x: T) -> Pin<Box<T>> {
         Box::new(x).into()
     }
@@ -594,7 +594,7 @@ impl<T, A: Allocator> Box<T, A> {
     #[cfg(not(no_global_oom_handling))]
     #[unstable(feature = "allocator_api", issue = "32838")]
     #[must_use]
-    #[inline(always)]
+    #[rustc_early_inline]
     pub fn pin_in(x: T, alloc: A) -> Pin<Self>
     where
         A: 'static + Allocator,

library/alloc/src/collections/binary_heap/mod.rs

@@ -875,7 +875,7 @@ impl<T: Ord, A: Allocator> BinaryHeap<T, A> {
 
         let tail_len = self.len() - start;
 
-        #[inline(always)]
+        #[rustc_early_inline]
         fn log2_fast(x: usize) -> usize {
             (usize::BITS - x.leading_zeros() - 1) as usize
         }

library/alloc/src/collections/linked_list.rs

@@ -1507,7 +1507,7 @@ impl<'a, T, A: Allocator> Cursor<'a, T, A> {
 
     /// Provides a reference to the cursor's parent list.
     #[must_use]
-    #[inline(always)]
+    #[rustc_early_inline]
     #[unstable(feature = "linked_list_cursors", issue = "58533")]
     pub fn as_list(&self) -> &'a LinkedList<T, A> {
         self.list
@@ -1629,7 +1629,7 @@ impl<'a, T, A: Allocator> CursorMut<'a, T, A> {
     /// `CursorMut`, which means it cannot outlive the `CursorMut` and that the
     /// `CursorMut` is frozen for the lifetime of the reference.
     #[must_use]
-    #[inline(always)]
+    #[rustc_early_inline]
     #[unstable(feature = "linked_list_cursors", issue = "58533")]
     pub fn as_list(&self) -> &LinkedList<T, A> {
         self.list

library/alloc/src/ffi/c_str.rs

@@ -269,7 +269,7 @@ impl CString {
         }
 
         // Specialization for avoiding reallocation
-        #[inline(always)] // Without that it is not inlined into specializations
+        #[rustc_early_inline] // Without that it is not inlined into specializations
         fn spec_new_impl_bytes(bytes: &[u8]) -> Result<CString, NulError> {
             // We cannot have such large slice that we would overflow here
             // but using `checked_add` allows LLVM to assume that capacity never overflows

library/alloc/src/rc.rs

@@ -351,7 +351,7 @@ impl<T: ?Sized> Rc<T> {
 }
 
 impl<T: ?Sized, A: Allocator> Rc<T, A> {
-    #[inline(always)]
+    #[rustc_early_inline]
     fn inner(&self) -> &RcInner<T> {
         // This unsafety is ok because while this Rc is alive we're guaranteed
         // that the inner pointer is valid.
@@ -2251,7 +2251,7 @@ impl<T: Copy> RcFromSlice<T> for Rc<[T]> {
 impl<T: ?Sized, A: Allocator> Deref for Rc<T, A> {
     type Target = T;
 
-    #[inline(always)]
+    #[rustc_early_inline]
     fn deref(&self) -> &T {
         &self.inner().value
     }

library/alloc/src/task.rs

@@ -133,18 +133,18 @@ impl<W: Wake + Send + Sync + 'static> From<Arc<W>> for RawWaker {
 // trait dispatch - instead both impls call this function directly and
 // explicitly.
 #[cfg(target_has_atomic = "ptr")]
-#[inline(always)]
+#[rustc_early_inline]
 fn raw_waker<W: Wake + Send + Sync + 'static>(waker: Arc<W>) -> RawWaker {
     // Increment the reference count of the arc to clone it.
     //
-    // The #[inline(always)] is to ensure that raw_waker and clone_waker are
+    // The #[rustc_early_inline] is to ensure that raw_waker and clone_waker are
     // always generated in the same code generation unit as one another, and
     // therefore that the structurally identical const-promoted RawWakerVTable
     // within both functions is deduplicated at LLVM IR code generation time.
     // This allows optimizing Waker::will_wake to a single pointer comparison of
     // the vtable pointers, rather than comparing all four function pointers
     // within the vtables.
-    #[inline(always)]
+    #[rustc_early_inline]
     unsafe fn clone_waker<W: Wake + Send + Sync + 'static>(waker: *const ()) -> RawWaker {
         unsafe { Arc::increment_strong_count(waker as *const W) };
         RawWaker::new(
@@ -311,13 +311,13 @@ impl<W: LocalWake + 'static> From<Rc<W>> for RawWaker {
 // the safety of `From<Rc<W>> for Waker` does not depend on the correct
 // trait dispatch - instead both impls call this function directly and
 // explicitly.
-#[inline(always)]
+#[rustc_early_inline]
 fn local_raw_waker<W: LocalWake + 'static>(waker: Rc<W>) -> RawWaker {
     // Increment the reference count of the Rc to clone it.
     //
     // Refer to the comment on raw_waker's clone_waker regarding why this is
     // always inline.
-    #[inline(always)]
+    #[rustc_early_inline]
     unsafe fn clone_waker<W: LocalWake + 'static>(waker: *const ()) -> RawWaker {
         unsafe { Rc::increment_strong_count(waker as *const W) };
         RawWaker::new(

library/alloctests/tests/sort/known_good_stable_sort.rs

@@ -19,12 +19,12 @@ use std::ptr;
 /// interior mutability will be observable. Same is true if `T: Ord` panics.
 ///
 /// Panics if allocating the auxiliary memory fails.
-#[inline(always)]
+#[rustc_early_inline]
 pub fn sort<T: Ord>(v: &mut [T]) {
     stable_sort(v, |a, b| a.lt(b))
 }
 
-#[inline(always)]
+#[rustc_early_inline]
 fn stable_sort<T, F: FnMut(&T, &T) -> bool>(v: &mut [T], mut is_less: F) {
     if size_of::<T>() == 0 {
         return;
@@ -65,7 +65,7 @@ unsafe fn mergesort_main<T, F: FnMut(&T, &T) -> bool>(v: &mut [T], is_less: &mut
 /// no run detection, etc.
 ///
 /// Buffer as pointed to by `scratch` must have space for `v.len()` writes. And must not alias `v`.
-#[inline(always)]
+#[rustc_early_inline]
 unsafe fn mergesort_core<T, F: FnMut(&T, &T) -> bool>(
     v: &mut [T],
     scratch_ptr: *mut T,
@@ -96,7 +96,7 @@ unsafe fn mergesort_core<T, F: FnMut(&T, &T) -> bool>(
 ///
 /// SAFETY: The caller must ensure that `scratch_ptr` is valid for `v.len()` writes. And that mid is
 /// in-bounds.
-#[inline(always)]
+#[rustc_early_inline]
 unsafe fn merge<T, F>(v: &mut [T], scratch_ptr: *mut T, is_less: &mut F, mid: usize)
 where
     F: FnMut(&T, &T) -> bool,

library/core/src/alloc/layout.rs

@@ -83,7 +83,7 @@ impl Layout {
         true
     }
 
-    #[inline(always)]
+    #[rustc_early_inline]
     const fn max_size_for_align(align: Alignment) -> usize {
         // (power-of-two implies align != 0.)