riscv64: Support 128-bit atomics (Zacas extension)

.github/.cspell/project-dictionary.txt

@@ -3,6 +3,7 @@ adde
 alcgr
 algr
 allnoconfig
+amocas
 aosp
 aqrl
 armasm
@@ -31,6 +32,7 @@ cpsid
 cpsie
 CPSR
 cpuid
+cpus
 cputable
 csel
 cset
@@ -192,4 +194,5 @@ xsave
 xsub
 zaamo
 zabha
+zacas
 Zhaoxin

.github/workflows/ci.yml

@@ -259,6 +259,8 @@ jobs:
             target: riscv64gc-unknown-linux-gnu
           - rust: '1.59' # LLVM 13
             target: riscv64gc-unknown-linux-gnu
+          - rust: '1.73' # LLVM 17 (oldest version we can use experimental-zacas on this target)
+            target: riscv64gc-unknown-linux-gnu
           - rust: stable
             target: riscv64gc-unknown-linux-gnu
           - rust: nightly
@@ -386,6 +388,21 @@ jobs:
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-cpu=pwr8
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-cpu=pwr8
         if: startsWith(matrix.target, 'powerpc64-')
+      # riscv64 +zabha
+      - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+zabha
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+zabha
+          QEMU_CPU: max
+        # TODO: cranelift doesn't support cfg(target_feature): /~https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400
+        if: startsWith(matrix.target, 'riscv64') && !contains(matrix.flags, 'codegen-backend=cranelift')
+      # riscv64 +experimental-zacas
+      - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+experimental-zacas
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+experimental-zacas
+        # TODO: cranelift doesn't support cfg(target_feature): /~https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400
+        if: startsWith(matrix.target, 'riscv64') && !contains(matrix.flags, 'codegen-backend=cranelift')
       # s390x z196 (arch9)
       - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
         env:

build.rs

@@ -47,7 +47,7 @@ fn main() {
 
     if version.minor >= 80 {
         println!(
-            r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","zabha","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
+            r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","zabha","experimental-zacas","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
         );
 
         // Custom cfgs set by build script. Not public API.
@@ -58,7 +58,7 @@ fn main() {
         // TODO: handle multi-line target_feature_fallback
         // grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
         println!(
-            r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo","zabha"))"#
+            r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","experimental-zacas","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo","zabha"))"#
         );
     }
 
@@ -308,16 +308,26 @@ fn main() {
             }
         }
         "riscv32" | "riscv64" => {
-            // As of rustc 1.80, target_feature "zaamo"/"zabha" is not available on rustc side:
+            // As of rustc 1.80, target_feature "zaamo"/"zabha"/"zacas" is not available on rustc side:
             // /~https://github.com/rust-lang/rust/blob/1.80.0/compiler/rustc_target/src/target_features.rs#L273
-            // zabha implies zaamo in GCC, but do not in LLVM (but enabling it without zaamo is not allowed).
-            // /~https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/TargetParser/RISCVISAInfo.cpp#L776-L778
-            // /~https://github.com/gcc-mirror/gcc/blob/08693e29ec186fd7941d0b73d4d466388971fe2f/gcc/config/riscv/arch-canonicalize#L45
+            // zabha and zacas imply zaamo in GCC, but do not in LLVM (but enabling them without zaamo is not allowed).
+            // /~https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/TargetParser/RISCVISAInfo.cpp#L772-L778
+            // /~https://github.com/gcc-mirror/gcc/blob/08693e29ec186fd7941d0b73d4d466388971fe2f/gcc/config/riscv/arch-canonicalize#L45-L46
             if version.llvm >= 19 {
                 // amo*.{b,h}
-                // available since 19 /~https://github.com/llvm/llvm-project/commit/89f87c387627150d342722b79c78cea2311cddf7 / /~https://github.com/llvm/llvm-project/commit/6b7444964a8d028989beee554a1f5c61d16a1cac
+                // available since LLVM 19 /~https://github.com/llvm/llvm-project/commit/89f87c387627150d342722b79c78cea2311cddf7 / /~https://github.com/llvm/llvm-project/commit/6b7444964a8d028989beee554a1f5c61d16a1cac
                 target_feature_fallback("zabha", false);
             }
+            if version.llvm == 19 {
+                // amocas.{w,d,q} (and amocas.{b,h} if zabha is also available)
+                // available as experimental since LLVM 17 /~https://github.com/llvm/llvm-project/commit/29f630a1ddcbb03caa31b5002f0cbc105ff3a869
+                // attempted to make non-experimental in LLVM 19 /~https://github.com/llvm/llvm-project/commit/95aab69c109adf29e183090c25dc95c773215746
+                // but reverted in /~https://github.com/llvm/llvm-project/commit/70e7d26e560173c8b9db4c75ab4a3004cd5f021a
+                // check == 19 instead of range 17..=19 because it is more experimental in LLVM 17/18.
+                // check == 19 instead of >= 19 because "experimental-zacas" feature
+                // may no longer exist when it is marked as non-experimental in LLVM 20.
+                target_feature_fallback("experimental-zacas", false);
+            }
             // amo*.{w,d}
             target_feature_fallback("zaamo", false);
         }

src/cfgs.rs

@@ -241,6 +241,35 @@ mod atomic_64_macros {
                 ),
             ),
         ),
+        all(
+            target_arch = "riscv64",
+            not(portable_atomic_no_asm),
+            any(
+                target_feature = "experimental-zacas",
+                portable_atomic_target_feature = "experimental-zacas",
+                // TODO(riscv64)
+                // all(
+                //     feature = "fallback",
+                //     not(portable_atomic_no_outline_atomics),
+                //     any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+                //     any(
+                //         all(
+                //             target_os = "linux",
+                //             any(
+                //                 target_env = "gnu",
+                //                 all(
+                //                     any(target_env = "musl", target_env = "ohos"),
+                //                     not(target_feature = "crt-static"),
+                //                 ),
+                //                 portable_atomic_outline_atomics,
+                //             ),
+                //         ),
+                //         target_os = "android",
+                //     ),
+                //     not(any(miri, portable_atomic_sanitize_thread)),
+                // ),
+            ),
+        ),
         all(
             target_arch = "powerpc64",
             portable_atomic_unstable_asm_experimental_arch,
@@ -331,6 +360,35 @@ mod atomic_128_macros {
                 ),
             ),
         ),
+        all(
+            target_arch = "riscv64",
+            not(portable_atomic_no_asm),
+            any(
+                target_feature = "experimental-zacas",
+                portable_atomic_target_feature = "experimental-zacas",
+                // TODO(riscv64)
+                // all(
+                //     feature = "fallback",
+                //     not(portable_atomic_no_outline_atomics),
+                //     any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+                //     any(
+                //         all(
+                //             target_os = "linux",
+                //             any(
+                //                 target_env = "gnu",
+                //                 all(
+                //                     any(target_env = "musl", target_env = "ohos"),
+                //                     not(target_feature = "crt-static"),
+                //                 ),
+                //                 portable_atomic_outline_atomics,
+                //             ),
+                //         ),
+                //         target_os = "android",
+                //     ),
+                //     not(any(miri, portable_atomic_sanitize_thread)),
+                // ),
+            ),
+        ),
         all(
             target_arch = "powerpc64",
             portable_atomic_unstable_asm_experimental_arch,

src/imp/atomic128/README.md

@@ -10,6 +10,7 @@ Here is the table of targets that support 128-bit atomics and the instructions u
 | ----------- | ---- | ----- | --- | --- | ---- |
 | x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel, AMD, or Zhaoxin CPU with AVX. <br> Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only. <br> Requires rustc 1.59+ |
 | aarch64 | ldxp/stxp or casp or ldp/ldiapp | ldxp/stxp or casp or stp/stilp/swpp | ldxp/stxp or casp | ldxp/stxp or casp/swpp/ldclrp/ldsetp | casp requires lse target feature, ldp/stp requires lse2 target feature, ldiapp/stilp requires lse2 and rcpc3 target features, swpp/ldclrp/ldsetp requires lse128 target feature. <br> Both compile-time and run-time detection are supported. <br> Requires rustc 1.59+ |
+| riscv64 | amocas.q | amocas.q | amocas.q | amocas.q | Experimental. Requires experimental-zacas target feature. Currently compile-time detection only due to LLVM marking it as experimental. <br> Requires 1.82+ (LLVM 19+) |
 | powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Requires target-cpu pwr8+ (powerpc64le is pwr8 by default). Both compile-time and run-time detection are supported (run-time detection is currently disabled by default). <br> Requires nightly |
 | s390x | lpq | stpq | cdsg | cdsg | Requires nightly |
 
@@ -19,7 +20,7 @@ See [aarch64.rs](aarch64.rs) module-level comments for more details on the instr
 
 ## Comparison with core::intrinsics::atomic_\* (core::sync::atomic::Atomic{I,U}128)
 
-This directory has target-specific implementations with inline assembly ([aarch64.rs](aarch64.rs), [x86_64.rs](x86_64.rs), [powerpc64.rs](powerpc64.rs), [s390x.rs](s390x.rs)) and an implementation without inline assembly ([intrinsics.rs](intrinsics.rs)). The latter currently always needs nightly compilers and is only used for Miri and ThreadSanitizer, which do not support inline assembly.
+This directory has target-specific implementations with inline assembly ([aarch64.rs](aarch64.rs), [x86_64.rs](x86_64.rs), [powerpc64.rs](powerpc64.rs), [riscv64.rs](riscv64.rs), [s390x.rs](s390x.rs)) and an implementation without inline assembly ([intrinsics.rs](intrinsics.rs)). The latter currently always needs nightly compilers and is only used for Miri and ThreadSanitizer, which do not support inline assembly.
 
 Implementations with inline assembly generate assemblies almost equivalent to the `core::intrinsics::atomic_*` (used in `core::sync::atomic::Atomic{I,U}128`) for many operations, but some operations may or may not generate more efficient code. For example:
 
@@ -47,6 +48,7 @@ Here is the table of targets that support run-time CPU feature detection and the
 | aarch64     | macos/ios/tvos/watchos/visionos | sysctlbyname    | all      | Currently only used in tests (see detect/aarch64_apple.rs). |
 | aarch64     | windows              | IsProcessorFeaturePresent | lse | Enabled by default |
 | aarch64     | fuchsia              | zx_system_get_features | lse | Enabled by default |
+| riscv64     | linux                | riscv_hwprobe   | all      | Currently only used in tests due to LLVM marking zacas as experimental |
 | powerpc64   | linux                | getauxval       | all      | Disabled by default |
 | powerpc64   | freebsd              | elf_aux_info    | all      | Disabled by default |
 | powerpc64   | openbsd              | elf_aux_info    | all      | Disabled by default |

src/imp/atomic128/macros.rs

@@ -261,7 +261,12 @@ macro_rules! atomic128 {
     };
 }
 
-#[cfg(any(target_arch = "powerpc64", target_arch = "s390x", target_arch = "x86_64"))]
+#[cfg(any(
+    target_arch = "powerpc64",
+    target_arch = "riscv64",
+    target_arch = "s390x",
+    target_arch = "x86_64",
+))]
 #[allow(unused_macros)] // also used by intrinsics.rs
 macro_rules! atomic_rmw_by_atomic_update {
     () => {