compiler-builtins: plumb LSE support for aarch64 on linux/gnu

Add dynamic support for aarch64 LSE atomic ops on linux/gnu targets
when optimized-compiler-builtins is not enabled.

The resulting asm should exactly match that of LLVM's compiler-rt
builtins, though the symbol naming for the support function and
global does not.

Author: Paul Murphy

library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs

@@ -6,9 +6,6 @@
 //! which is supported on the current CPU.
 //! See <https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10#:~:text=out%20of%20line%20atomics> for more discussion.
 //!
-//! Currently we only support LL/SC, because LSE requires `getauxval` from libc in order to do runtime detection.
-//! Use the `compiler-rt` intrinsics if you want LSE support.
-//!
 //! Ported from `aarch64/lse.S` in LLVM's compiler-rt.
 //!
 //! Generate functions for each of the following symbols:
@@ -24,7 +21,38 @@
 //! We do something similar, but with macro arguments.
 #![cfg_attr(feature = "c", allow(unused_macros))] // avoid putting the macros into a submodule
 
-// We don't do runtime dispatch so we don't have to worry about the `__aarch64_have_lse_atomics` global ctor.
+/// non-zero if the host supports LSE atomics.
+#[cfg(not(feature = "c"))]
+static mut HAVE_LSE_ATOMICS: u8 = 0;
+
+/// This calls into libc's getauxval to check hwcap bits for LSE support. Note, at this time
+/// outline-atomics are only enabled on gnu targets.
+#[cfg(all(not(feature = "c"), target_env = "gnu"))]
+extern "C" fn aarch64_rust_init_lse_atomics() {
+    // The most straightforward path to querying for LSE support is the host's libc.
+    // We can't use the libc crate here, we are a dependency.
+    unsafe extern "C" {
+        fn getauxval(num: core::ffi::c_ulong) -> core::ffi::c_ulong;
+    }
+
+    const AT_HWCAP: core::ffi::c_ulong = 16;
+    const HWCAP_ATOMICS: core::ffi::c_ulong = 0x100;
+
+    unsafe {
+        let hwcap = getauxval(AT_HWCAP);
+        if hwcap & HWCAP_ATOMICS != 0 {
+            HAVE_LSE_ATOMICS = 1;
+        }
+    }
+}
+
+/// The entry into startup array for LSE initialization. This should be placed within
+/// the init array similar to compiler-rt. However, if it isn't called very early, that
+/// should also be OK as the fallback atomic operations always work.
+#[used]
+#[unsafe(link_section = ".init_array.90")]
+#[cfg(all(not(feature = "c"), target_env = "gnu"))]
+static RUST_LSE_INIT: extern "C" fn() = aarch64_rust_init_lse_atomics;
 
 /// Translate a byte size to a Rust type.
 #[rustfmt::skip]
@@ -126,6 +154,42 @@ macro_rules! stxp {
     };
 }
 
+// Check if LSE intrinsic can be used, and jump to label if not.
+macro_rules! jmp_if_no_lse {
+    ($label:literal) => {
+        concat!(
+            ".arch_extension lse; ",
+            ".macro  load sym;",
+            "adrp    x16, \\sym; ",
+            "ldrb    w16, [x16, :lo12:\\sym]; ",
+            ".endm;",
+            "load    sym={};",
+            "cbz     w16, ",
+            $label,
+            ";"
+        )
+    };
+}
+
+// Translate memory ordering to the LSE suffix
+#[rustfmt::skip]
+macro_rules! lse_mem_sfx {
+    (Relaxed) => { "" };
+    (Acquire) => { "a" };
+    (Release) => { "l" };
+    (AcqRel) => { "al" };
+}
+
+// Generate the aarch64 LSE operation for memory ordering and width
+macro_rules! lse {
+    ($op:literal, $order:ident, 16) => {
+        concat!($op, "p", lse_mem_sfx!($order))
+    };
+    ($op:literal, $order:ident, $bytes:tt) => {
+        concat!($op, lse_mem_sfx!($order), size!($bytes))
+    };
+}
+
 /// See <https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicI8.html#method.compare_and_swap>.
 macro_rules! compare_and_swap {
     ($ordering:ident, $bytes:tt, $name:ident) => {
@@ -137,6 +201,11 @@ macro_rules! compare_and_swap {
             ) -> int_ty!($bytes) {
                 // We can't use `AtomicI8::compare_and_swap`; we *are* compare_and_swap.
                 core::arch::naked_asm! {
+                    jmp_if_no_lse!("8f"),
+                    // CAS    s(0), s(1), [x2]
+                    concat!(lse!("cas", $ordering, $bytes), " ", reg!($bytes, 0), ", ", reg!($bytes, 1), ", [x2]"),
+                    "ret",
+                    "8:",
                     // UXT s(tmp0), s(0)
                     concat!(uxt!($bytes), " ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
                     "0:",
@@ -150,6 +219,7 @@ macro_rules! compare_and_swap {
                     "cbnz   w17, 0b",
                     "1:",
                     "ret",
+                    sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
                 }
             }
         }
@@ -166,6 +236,11 @@ macro_rules! compare_and_swap_i128 {
                 expected: i128, desired: i128, ptr: *mut i128
             ) -> i128 {
                 core::arch::naked_asm! {
+                    jmp_if_no_lse!("8f"),
+                    // CASP   x0, x1, x2, x3, [x4]
+                    concat!(lse!("cas", $ordering, 16), " x0, x1, x2, x3, [x4]"),
+                    "ret",
+                    "8:",
                     "mov    x16, x0",
                     "mov    x17, x1",
                     "0:",
@@ -179,6 +254,7 @@ macro_rules! compare_and_swap_i128 {
                     "cbnz   w15, 0b",
                     "1:",
                     "ret",
+                    sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
                 }
             }
         }
@@ -195,6 +271,11 @@ macro_rules! swap {
                 left: int_ty!($bytes), right_ptr: *mut int_ty!($bytes)
             ) -> int_ty!($bytes) {
                 core::arch::naked_asm! {
+                    jmp_if_no_lse!("8f"),
+                    // SWP    s(0), s(0), [x1]
+                    concat!(lse!("swp", $ordering, $bytes), " ", reg!($bytes, 0), ", ", reg!($bytes, 0), ", [x1]"),
+                    "ret",
+                    "8:",
                     // mov    s(tmp0), s(0)
                     concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
                     "0:",
@@ -204,6 +285,7 @@ macro_rules! swap {
                     concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"),
                     "cbnz   w17, 0b",
                     "ret",
+                    sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
                 }
             }
         }
@@ -212,14 +294,19 @@ macro_rules! swap {
 
 /// See (e.g.) <https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicI8.html#method.fetch_add>.
 macro_rules! fetch_op {
-    ($ordering:ident, $bytes:tt, $name:ident, $op:literal) => {
+    ($ordering:ident, $bytes:tt, $name:ident, $op:literal, $lse_op:literal) => {
         intrinsics! {
             #[maybe_use_optimized_c_shim]
             #[unsafe(naked)]
             pub unsafe extern "C" fn $name (
                 val: int_ty!($bytes), ptr: *mut int_ty!($bytes)
             ) -> int_ty!($bytes) {
                 core::arch::naked_asm! {
+                    jmp_if_no_lse!("8f"),
+                    // LSEOP  s(0), s(0), [x1]
+                    concat!(lse!($lse_op, $ordering, $bytes), " ", reg!($bytes, 0), ", ", reg!($bytes, 0), ", [x1]"),
+                    "ret",
+                    "8:",
                     // mov    s(tmp0), s(0)
                     concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)),
                     "0:",
@@ -231,6 +318,7 @@ macro_rules! fetch_op {
                     concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"),
                     "cbnz  w15, 0b",
                     "ret",
+                    sym crate::aarch64_linux::HAVE_LSE_ATOMICS,
                 }
             }
         }
@@ -240,25 +328,25 @@ macro_rules! fetch_op {
 // We need a single macro to pass to `foreach_ldadd`.
 macro_rules! add {
     ($ordering:ident, $bytes:tt, $name:ident) => {
-        fetch_op! { $ordering, $bytes, $name, "add" }
+        fetch_op! { $ordering, $bytes, $name, "add", "ldadd" }
     };
 }
 
 macro_rules! and {
     ($ordering:ident, $bytes:tt, $name:ident) => {
-        fetch_op! { $ordering, $bytes, $name, "bic" }
+        fetch_op! { $ordering, $bytes, $name, "bic", "ldclr" }
     };
 }
 
 macro_rules! xor {
     ($ordering:ident, $bytes:tt, $name:ident) => {
-        fetch_op! { $ordering, $bytes, $name, "eor" }
+        fetch_op! { $ordering, $bytes, $name, "eor", "ldeor" }
     };
 }
 
 macro_rules! or {
     ($ordering:ident, $bytes:tt, $name:ident) => {
-        fetch_op! { $ordering, $bytes, $name, "orr" }
+        fetch_op! { $ordering, $bytes, $name, "orr", "ldset" }
     };
 }