diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt
index 1cdef37b3..59de5b0d6 100644
--- a/.github/.cspell/project-dictionary.txt
+++ b/.github/.cspell/project-dictionary.txt
@@ -3,6 +3,7 @@ adde
alcgr
algr
allnoconfig
+amocas
aosp
aqrl
armasm
@@ -183,4 +184,5 @@ xsave
xsub
zaamo
zabha
+zacas
Zhaoxin
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 95576f37d..0d3402330 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -232,6 +232,8 @@ jobs:
target: riscv32gc-unknown-linux-gnu
- rust: '1.59'
target: riscv64gc-unknown-linux-gnu
+ - rust: '1.73' # LLVM 17 (oldest version we can use experimental-zacas on this target)
+ target: riscv64gc-unknown-linux-gnu
- rust: stable
target: riscv64gc-unknown-linux-gnu
- rust: nightly
@@ -355,6 +357,13 @@ jobs:
RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-cpu=pwr8
RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-cpu=pwr8
if: startsWith(matrix.target, 'powerpc64-')
+ # riscv64 +experimental-zacas
+ - run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
+ env:
+ RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+experimental-zacas
+ RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+experimental-zacas
+ # TODO: cranelift doesn't support cfg(target_feature): /~https://github.com/rust-lang/rustc_codegen_cranelift/issues/1400
+ if: startsWith(matrix.target, 'riscv64') && !contains(matrix.flags, 'codegen-backend=cranelift')
# s390x z196 (arch9)
- run: tools/test.sh -vv --tests ${TARGET:-} ${BUILD_STD:-} ${RELEASE:-}
env:
diff --git a/build.rs b/build.rs
index e91503684..3a34b2589 100644
--- a/build.rs
+++ b/build.rs
@@ -47,7 +47,7 @@ fn main() {
if version.minor >= 80 {
println!(
- r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","zabha","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
+ r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","zabha","experimental-zacas","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
);
// Custom cfgs set by build script. Not public API.
@@ -58,7 +58,7 @@ fn main() {
// TODO: handle multi-line target_feature_fallback
// grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
println!(
- r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo","zabha"))"#
+ r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","experimental-zacas","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo","zabha"))"#
);
}
@@ -311,15 +311,22 @@ fn main() {
}
}
"riscv32" | "riscv64" => {
- // As of rustc 1.80, target_feature "zaamo"/"zabha" is not available on rustc side:
+ // As of rustc 1.80, target_feature "zaamo"/"zabha"/"zacas" is not available on rustc side:
// /~https://github.com/rust-lang/rust/blob/1.80.0/compiler/rustc_target/src/target_features.rs#L273
- // zabha implies zaamo in GCC, but do not in LLVM (but enabling it without zaamo is not allowed).
- // /~https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0-rc3/llvm/lib/TargetParser/RISCVISAInfo.cpp#L776-L778
- // /~https://github.com/gcc-mirror/gcc/blob/08693e29ec186fd7941d0b73d4d466388971fe2f/gcc/config/riscv/arch-canonicalize#L45
- if version.llvm >= 19 {
- // amo*.{b,h}
- // available since 19 /~https://github.com/llvm/llvm-project/commit/89f87c387627150d342722b79c78cea2311cddf7 / /~https://github.com/llvm/llvm-project/commit/6b7444964a8d028989beee554a1f5c61d16a1cac
- target_feature_fallback("zabha", false);
+ // zacas and zabha imply zaamo in GCC, but do not in LLVM (but enabling them without zaamo is not allowed).
+ // /~https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0-rc3/llvm/lib/TargetParser/RISCVISAInfo.cpp#L772-L778
+ // /~https://github.com/gcc-mirror/gcc/blob/08693e29ec186fd7941d0b73d4d466388971fe2f/gcc/config/riscv/arch-canonicalize#L45-L46
+ if version.llvm >= 17 {
+ // amocas.{w,d,q} (amocas.{b,h} if zabha is also available)
+ // available as experimental since 17 /~https://github.com/llvm/llvm-project/commit/29f630a1ddcbb03caa31b5002f0cbc105ff3a869
+ // attempted to make non-experimental in 19 /~https://github.com/llvm/llvm-project/commit/95aab69c109adf29e183090c25dc95c773215746
+ // but reverted in /~https://github.com/llvm/llvm-project/commit/70e7d26e560173c8b9db4c75ab4a3004cd5f021a
+ target_feature_fallback("experimental-zacas", false);
+ if version.llvm >= 19 {
+ // amo*.{b,h}
+ // available since 19 /~https://github.com/llvm/llvm-project/commit/89f87c387627150d342722b79c78cea2311cddf7 / /~https://github.com/llvm/llvm-project/commit/6b7444964a8d028989beee554a1f5c61d16a1cac
+ target_feature_fallback("zabha", false);
+ }
}
// amo*.{w,d}
target_feature_fallback("zaamo", false);
diff --git a/src/cfgs.rs b/src/cfgs.rs
index afb72503e..b90ce262c 100644
--- a/src/cfgs.rs
+++ b/src/cfgs.rs
@@ -241,6 +241,35 @@ mod atomic_64_macros {
),
),
),
+ all(
+ target_arch = "riscv64",
+ not(portable_atomic_no_asm),
+ any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas",
+ // TODO(riscv64)
+ // all(
+ // feature = "fallback",
+ // not(portable_atomic_no_outline_atomics),
+ // any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+ // any(
+ // all(
+ // target_os = "linux",
+ // any(
+ // target_env = "gnu",
+ // all(
+ // any(target_env = "musl", target_env = "ohos"),
+ // not(target_feature = "crt-static"),
+ // ),
+ // portable_atomic_outline_atomics,
+ // ),
+ // ),
+ // target_os = "android",
+ // ),
+ // not(any(miri, portable_atomic_sanitize_thread)),
+ // ),
+ ),
+ ),
all(
target_arch = "powerpc64",
portable_atomic_unstable_asm_experimental_arch,
@@ -331,6 +360,35 @@ mod atomic_128_macros {
),
),
),
+ all(
+ target_arch = "riscv64",
+ not(portable_atomic_no_asm),
+ any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas",
+ // TODO(riscv64)
+ // all(
+ // feature = "fallback",
+ // not(portable_atomic_no_outline_atomics),
+ // any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+ // any(
+ // all(
+ // target_os = "linux",
+ // any(
+ // target_env = "gnu",
+ // all(
+ // any(target_env = "musl", target_env = "ohos"),
+ // not(target_feature = "crt-static"),
+ // ),
+ // portable_atomic_outline_atomics,
+ // ),
+ // ),
+ // target_os = "android",
+ // ),
+ // not(any(miri, portable_atomic_sanitize_thread)),
+ // ),
+ ),
+ ),
all(
target_arch = "powerpc64",
portable_atomic_unstable_asm_experimental_arch,
diff --git a/src/imp/atomic128/README.md b/src/imp/atomic128/README.md
index 1d53e8ee4..efe911f5e 100644
--- a/src/imp/atomic128/README.md
+++ b/src/imp/atomic128/README.md
@@ -8,6 +8,7 @@ Here is the table of targets that support 128-bit atomics and the instructions u
| ----------- | ---- | ----- | --- | --- | ---- |
| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel, AMD, or Zhaoxin CPU with AVX.
Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.
Requires rustc 1.59+ |
| aarch64 | ldxp/stxp or casp or ldp/ldiapp | ldxp/stxp or casp or stp/stilp/swpp | ldxp/stxp or casp | ldxp/stxp or casp/swpp/ldclrp/ldsetp | casp requires lse target feature, ldp/stp requires lse2 target feature, ldiapp/stilp requires lse2 and rcpc3 target features, swpp/ldclrp/ldsetp requires lse128 target feature.
Both compile-time and run-time detection are supported.
Requires rustc 1.59+ |
+| riscv64 | amocas.q | amocas.q | amocas.q | amocas.q | Requires experimental-zacas target feature. Currently compile-time detection only due to LLVM marking it as experimental.
Requires 1.73+ (LLVM 17+) |
| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Requires target-cpu pwr8+ (powerpc64le is pwr8 by default). Both compile-time and run-time detection are supported (run-time detection is currently disabled by default).
Requires nightly |
| s390x | lpq | stpq | cdsg | cdsg | Requires nightly |
@@ -17,7 +18,7 @@ See [aarch64.rs](aarch64.rs) module-level comments for more details on the instr
## Comparison with core::intrinsics::atomic_\* (core::sync::atomic::Atomic{I,U}128)
-This directory has target-specific implementations with inline assembly ([aarch64.rs](aarch64.rs), [x86_64.rs](x86_64.rs), [powerpc64.rs](powerpc64.rs), [s390x.rs](s390x.rs)) and an implementation without inline assembly ([intrinsics.rs](intrinsics.rs)). The latter currently always needs nightly compilers and is only used for Miri and ThreadSanitizer, which do not support inline assembly.
+This directory has target-specific implementations with inline assembly ([aarch64.rs](aarch64.rs), [x86_64.rs](x86_64.rs), [powerpc64.rs](powerpc64.rs), [riscv64.rs](riscv64.rs), [s390x.rs](s390x.rs)) and an implementation without inline assembly ([intrinsics.rs](intrinsics.rs)). The latter currently always needs nightly compilers and is only used for Miri and ThreadSanitizer, which do not support inline assembly.
Implementations with inline assembly generate assemblies almost equivalent to the `core::intrinsics::atomic_*` (used in `core::sync::atomic::Atomic{I,U}128`) for many operations, but some operations may or may not generate more efficient code. For example:
@@ -45,6 +46,7 @@ Here is the table of targets that support run-time CPU feature detection and the
| aarch64 | macos | sysctl | all | Currently only used in tests because FEAT_LSE and FEAT_LSE2 are always available at compile-time. |
| aarch64 | windows | IsProcessorFeaturePresent | lse | Enabled by default |
| aarch64 | fuchsia | zx_system_get_features | lse | Enabled by default |
+| riscv64 | linux | riscv_hwprobe | all | Currently only used in tests due to LLVM marking zacas as experimental |
| powerpc64 | linux | getauxval | all | Disabled by default |
| powerpc64 | freebsd | elf_aux_info | all | Disabled by default |
| powerpc64 | openbsd | elf_aux_info | all | Disabled by default |
diff --git a/src/imp/atomic128/detect/common.rs b/src/imp/atomic128/detect/common.rs
index 3e680226b..087ce822d 100644
--- a/src/imp/atomic128/detect/common.rs
+++ b/src/imp/atomic128/detect/common.rs
@@ -109,6 +109,12 @@ flags! {
HAS_VMOVDQA_ATOMIC(2, has_vmovdqa_atomic, "vmovdqa-atomic", any(/* always false */)),
}
+#[cfg(target_arch = "riscv64")]
+flags! {
+ // amocas.{w,d,q}
+ HAS_ZACAS(1, has_zacas, "zacas", any(target_feature = "experimental-zacas", portable_atomic_target_feature = "experimental-zacas")),
+}
+
#[cfg(target_arch = "powerpc64")]
flags! {
// lqarx and stqcx.
@@ -116,7 +122,7 @@ flags! {
}
// core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47
-#[cfg(any(target_arch = "aarch64", target_arch = "powerpc64"))]
+#[cfg(any(target_arch = "aarch64", target_arch = "powerpc64", target_arch = "riscv64"))]
#[cfg(not(windows))]
#[allow(dead_code, non_camel_case_types)]
mod c_types {
@@ -164,6 +170,9 @@ mod c_types {
let _: c_long = 0 as std::os::raw::c_long;
let _: c_ulong = 0 as std::os::raw::c_ulong;
let _: c_size_t = 0 as libc::size_t; // std::os::raw::c_size_t is unstable
+ #[cfg(not(
+ all(target_arch = "riscv64", target_os = "android"), // TODO: /~https://github.com/rust-lang/rust/issues/129945
+ ))]
let _: c_char = 0 as std::os::raw::c_char;
let _: c_char = 0 as sys::c_char;
};
@@ -310,6 +319,16 @@ mod tests_common {
assert!(!detect().test(CpuInfo::HAS_VMOVDQA_ATOMIC));
}
}
+ #[cfg(target_arch = "riscv64")]
+ #[test]
+ #[cfg_attr(portable_atomic_test_outline_atomics_detect_false, ignore)]
+ fn test_detect() {
+ if detect().has_zacas() {
+ assert!(detect().test(CpuInfo::HAS_ZACAS));
+ } else {
+ assert!(!detect().test(CpuInfo::HAS_ZACAS));
+ }
+ }
#[cfg(target_arch = "powerpc64")]
#[test]
#[cfg_attr(portable_atomic_test_outline_atomics_detect_false, ignore)]
diff --git a/src/imp/atomic128/detect/riscv64_linux.rs b/src/imp/atomic128/detect/riscv64_linux.rs
new file mode 100644
index 000000000..24a56d04b
--- /dev/null
+++ b/src/imp/atomic128/detect/riscv64_linux.rs
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+/*
+Run-time CPU feature detection on RISC-V Linux/Android by using riscv_hwprobe.
+
+On RISC-V, detection using auxv only supports single-letter extensions.
+
+Refs:
+- /~https://github.com/torvalds/linux/blob/v6.10/Documentation/arch/riscv/hwprobe.rst
+- /~https://github.com/golang/sys/commit/3283fc3f6160baf63bec24fbaa24e094e9ff6daf
+*/
+
+include!("common.rs");
+
+// core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47
+#[allow(non_camel_case_types, non_upper_case_globals)]
+mod ffi {
+ pub(crate) use super::c_types::c_long;
+
+ // /~https://github.com/torvalds/linux/blob/v6.10/arch/riscv/include/uapi/asm/hwprobe.h
+ #[derive(Copy, Clone)]
+ #[repr(C)]
+ pub(crate) struct riscv_hwprobe {
+ pub(crate) key: i64,
+ pub(crate) value: u64,
+ }
+
+ pub(crate) const __NR_riscv_hwprobe: c_long = 258;
+
+ // /~https://github.com/torvalds/linux/blob/v6.10/arch/riscv/include/uapi/asm/hwprobe.h
+ pub(crate) const RISCV_HWPROBE_KEY_IMA_EXT_0: i64 = 4;
+ // Linux 6.8+
+ // /~https://github.com/torvalds/linux/commit/154a3706122978eeb34d8223d49285ed4f3c61fa
+ pub(crate) const RISCV_HWPROBE_EXT_ZACAS: u64 = 1 << 34;
+
+ extern "C" {
+ // https://man7.org/linux/man-pages/man2/syscall.2.html
+ pub(crate) fn syscall(number: c_long, ...) -> c_long;
+ }
+}
+
+// syscall returns an unsupported error if riscv_hwprobe is not supported,
+// so we can safely use this function on older versions of Linux.
+fn riscv_hwprobe(out: &mut ffi::riscv_hwprobe, flags: usize) -> bool {
+ // SAFETY: We've passed the valid pointer and length.
+ unsafe {
+ ffi::syscall(
+ ffi::__NR_riscv_hwprobe,
+ out as *mut ffi::riscv_hwprobe,
+ 1_usize,
+ 0_usize,
+ 0_usize,
+ flags,
+ 0_usize,
+ ) == 0
+ }
+}
+
+#[cold]
+fn _detect(info: &mut CpuInfo) {
+ let mut out = ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 };
+ if riscv_hwprobe(&mut out, 0) {
+ if out.key != -1 {
+ let value = out.value;
+ if value & ffi::RISCV_HWPROBE_EXT_ZACAS != 0 {
+ info.set(CpuInfo::HAS_ZACAS);
+ }
+ }
+ }
+}
+
+#[allow(
+ clippy::alloc_instead_of_core,
+ clippy::std_instead_of_alloc,
+ clippy::std_instead_of_core,
+ clippy::undocumented_unsafe_blocks,
+ clippy::wildcard_imports
+)]
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ // Static assertions for FFI bindings.
+ // This checks that FFI bindings defined in this crate, FFI bindings defined
+ // in libc, and FFI bindings generated for the platform's latest header file
+ // using bindgen have compatible signatures (or the same values if constants).
+ // Since this is static assertion, we can detect problems with
+ // `cargo check --tests --target ` run in CI (via TESTS=1 build.sh)
+ // without actually running tests on these platforms.
+ // See also tools/codegen/src/ffi.rs.
+ // TODO(codegen): auto-generate this test
+ #[allow(
+ clippy::cast_possible_wrap,
+ clippy::cast_sign_loss,
+ clippy::no_effect_underscore_binding
+ )]
+ const _: fn() = || {
+ use test_helper::sys;
+ // TODO: syscall,riscv_hwprobe
+ // static_assert!(ffi::__NR_riscv_hwprobe == libc::__NR_riscv_hwprobe); // libc doesn't have this
+ static_assert!(ffi::__NR_riscv_hwprobe == sys::__NR_riscv_hwprobe as ffi::c_long);
+ // static_assert!(ffi::RISCV_HWPROBE_KEY_IMA_EXT_0 == libc::RISCV_HWPROBE_KEY_IMA_EXT_0); // libc doesn't have this
+ static_assert!(ffi::RISCV_HWPROBE_KEY_IMA_EXT_0 == sys::RISCV_HWPROBE_KEY_IMA_EXT_0 as i64);
+ // static_assert!(ffi::RISCV_HWPROBE_EXT_ZACAS == libc::RISCV_HWPROBE_EXT_ZACAS); // libc doesn't have this
+ static_assert!(ffi::RISCV_HWPROBE_EXT_ZACAS == sys::RISCV_HWPROBE_EXT_ZACAS);
+ };
+}
diff --git a/src/imp/atomic128/macros.rs b/src/imp/atomic128/macros.rs
index 251120f20..dd1f34e9e 100644
--- a/src/imp/atomic128/macros.rs
+++ b/src/imp/atomic128/macros.rs
@@ -264,7 +264,12 @@ macro_rules! atomic128 {
};
}
-#[cfg(any(target_arch = "powerpc64", target_arch = "s390x", target_arch = "x86_64"))]
+#[cfg(any(
+ target_arch = "powerpc64",
+ target_arch = "riscv64",
+ target_arch = "s390x",
+ target_arch = "x86_64",
+))]
#[allow(unused_macros)] // also used by intrinsics.rs
macro_rules! atomic_rmw_by_atomic_update {
() => {
diff --git a/src/imp/atomic128/riscv64.rs b/src/imp/atomic128/riscv64.rs
new file mode 100644
index 000000000..920fedeb9
--- /dev/null
+++ b/src/imp/atomic128/riscv64.rs
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+/*
+Atomic{I,U}128 implementation on riscv64 using amocas.q (DWCAS).
+
+Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use
+this module and use intrinsics.rs instead.
+
+Refs:
+- RISC-V Instruction Set Manual
+ /~https://github.com/riscv/riscv-isa-manual/tree/riscv-isa-release-8b9dc50-2024-08-30
+ "Zacas" Extension for Atomic Compare-and-Swap (CAS) Instructions
+ /~https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/zacas.adoc
+- RISC-V Atomics ABI Specification
+ /~https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20240829-13bfa9f54634cb60d86b9b333e109f077805b4b3/riscv-atomic.adoc
+
+Generated asm:
+- riscv64 (+experimental-zacas) https://godbolt.org/z/crdhjKPdq
+*/
+
+// TODO: 64-bit atomic using amocas.d for riscv32
+
+include!("macros.rs");
+
+// TODO
+// #[cfg(not(any(target_feature = "experimental-zacas", portable_atomic_target_feature = "experimental-zacas")))]
+// #[path = "../fallback/outline_atomics.rs"]
+// mod fallback;
+
+// On musl with static linking, it seems that libc is not always available.
+// See detect/auxv.rs for more.
+#[cfg(test)] // TODO
+#[cfg(not(portable_atomic_no_outline_atomics))]
+#[cfg(any(test, portable_atomic_outline_atomics))] // TODO(riscv64): currently disabled by default
+#[cfg(any(
+ test,
+ not(any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas"
+ ))
+))]
+#[cfg(any(
+ all(
+ target_os = "linux",
+ any(
+ target_env = "gnu",
+ all(any(target_env = "musl", target_env = "ohos"), not(target_feature = "crt-static")),
+ portable_atomic_outline_atomics,
+ ),
+ ),
+ target_os = "android",
+))]
+#[path = "detect/riscv64_linux.rs"]
+mod detect;
+
+use core::{arch::asm, sync::atomic::Ordering};
+
+use crate::utils::{Pair, U128};
+
+// /~https://github.com/riscv-non-isa/riscv-asm-manual/blob/ad0de8c004e29c9a7ac33cfd054f4d4f9392f2fb/src/asm-manual.adoc#arch
+#[cfg(any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas"
+))]
+macro_rules! start_zacas {
+ () => {
+ // zacas available, no-op
+ ""
+ };
+}
+#[cfg(any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas"
+))]
+macro_rules! end_zacas {
+ () => {
+ // zacas available, no-op
+ ""
+ };
+}
+#[cfg(not(any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas"
+)))]
+macro_rules! start_zacas {
+ () => {
+ ".option push\n.option arch, +experimental-zacas"
+ };
+}
+#[cfg(not(any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas"
+)))]
+macro_rules! end_zacas {
+ () => {
+ ".option pop"
+ };
+}
+
+macro_rules! atomic_rmw_amocas_order {
+ ($op:ident, $order:ident) => {
+ atomic_rmw_amocas_order!($op, $order, failure = $order)
+ };
+ ($op:ident, $order:ident, failure = $failure:ident) => {
+ match $order {
+ Ordering::Relaxed => $op!("", ""),
+ Ordering::Acquire => $op!("", ".aq"),
+ Ordering::Release => $op!("", ".rl"),
+ Ordering::AcqRel => $op!("", ".aqrl"),
+ Ordering::SeqCst if $failure == Ordering::SeqCst => $op!("fence rw,rw", ".aqrl"),
+ Ordering::SeqCst => $op!("", ".aqrl"),
+ _ => unreachable!(),
+ }
+ };
+}
+
+#[inline]
+unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
+ debug_assert!(src as usize % 16 == 0);
+
+ // SAFETY: the caller must uphold the safety contract.
+ unsafe {
+ let (out_lo, out_hi);
+ macro_rules! load {
+ ($fence:tt, $asm_order:tt) => {
+ asm!(
+ start_zacas!(),
+ $fence,
+ concat!("amocas.q", $asm_order, " a2, a2, 0({src})"),
+ end_zacas!(),
+ src = in(reg) ptr_reg!(src),
+ inout("a2") 0_u64 => out_lo,
+ inout("a3") 0_u64 => out_hi,
+ options(nostack, preserves_flags),
+ )
+ };
+ }
+ atomic_rmw_amocas_order!(load, order);
+ U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
+ }
+}
+
+#[inline]
+unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
+ // SAFETY: the caller must uphold the safety contract.
+ unsafe {
+ atomic_swap(dst, val, order);
+ }
+}
+
+#[inline]
+unsafe fn atomic_compare_exchange(
+ dst: *mut u128,
+ old: u128,
+ new: u128,
+ success: Ordering,
+ failure: Ordering,
+) -> Result {
+ debug_assert!(dst as usize % 16 == 0);
+ let order = crate::utils::upgrade_success_ordering(success, failure);
+
+ // SAFETY: the caller must uphold the safety contract.
+ let prev = unsafe {
+ let old = U128 { whole: old };
+ let new = U128 { whole: new };
+ let (prev_lo, prev_hi);
+ macro_rules! cmpxchg {
+ ($fence:tt, $asm_order:tt) => {
+ asm!(
+ start_zacas!(),
+ $fence,
+ concat!("amocas.q", $asm_order, " a4, a2, 0({dst})"),
+ end_zacas!(),
+ dst = in(reg) ptr_reg!(dst),
+ // must be allocated to even/odd register pair
+ inout("a4") old.pair.lo => prev_lo,
+ inout("a5") old.pair.hi => prev_hi,
+ // must be allocated to even/odd register pair
+ in("a2") new.pair.lo,
+ in("a3") new.pair.hi,
+ options(nostack, preserves_flags),
+ )
+ };
+ }
+ atomic_rmw_amocas_order!(cmpxchg, order, failure = failure);
+ U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole
+ };
+ if prev == old {
+ Ok(prev)
+ } else {
+ Err(prev)
+ }
+}
+
+// amocas is always strong.
+use atomic_compare_exchange as atomic_compare_exchange_weak;
+
+// 128-bit atomic load by two 64-bit atomic loads.
+#[inline]
+unsafe fn byte_wise_atomic_load(src: *const u128) -> u128 {
+ // SAFETY: the caller must uphold the safety contract.
+ unsafe {
+ let (out_lo, out_hi);
+ asm!(
+ "ld {out_lo}, ({src})",
+ "ld {out_hi}, 8({src})",
+ src = in(reg) src,
+ out_lo = out(reg) out_lo,
+ out_hi = out(reg) out_hi,
+ options(pure, nostack, preserves_flags, readonly),
+ );
+ U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole
+ }
+}
+
+#[inline(always)]
+unsafe fn atomic_update(dst: *mut u128, order: Ordering, mut f: F) -> u128
+where
+ F: FnMut(u128) -> u128,
+{
+ // SAFETY: the caller must uphold the safety contract.
+ unsafe {
+ let mut prev = byte_wise_atomic_load(dst);
+ loop {
+ let next = f(prev);
+ match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) {
+ Ok(x) => return x,
+ Err(x) => prev = x,
+ }
+ }
+ }
+}
+
+atomic_rmw_by_atomic_update!();
+
+#[inline]
+fn is_lock_free() -> bool {
+ #[cfg(any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas"
+ ))]
+ {
+ // zacas is available at compile-time.
+ true
+ }
+ #[cfg(not(any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas"
+ )))]
+ {
+ detect::detect().has_zacas()
+ }
+}
+const IS_ALWAYS_LOCK_FREE: bool = cfg!(any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas"
+));
+
+atomic128!(AtomicI128, i128, atomic_max, atomic_min);
+atomic128!(AtomicU128, u128, atomic_umax, atomic_umin);
+
+#[allow(clippy::undocumented_unsafe_blocks, clippy::wildcard_imports)]
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ test_atomic_int!(i128);
+ test_atomic_int!(u128);
+
+ // load/store/swap implementation is not affected by signedness, so it is
+ // enough to test only unsigned types.
+ stress_test!(u128);
+}
diff --git a/src/imp/mod.rs b/src/imp/mod.rs
index 593ea978c..ed3521a89 100644
--- a/src/imp/mod.rs
+++ b/src/imp/mod.rs
@@ -64,6 +64,41 @@ mod aarch64;
#[cfg_attr(not(any(miri, portable_atomic_sanitize_thread)), path = "atomic128/x86_64.rs")]
mod x86_64;
+// riscv64 128-bit atomics
+#[cfg(all(
+ target_arch = "riscv64",
+ not(portable_atomic_no_asm),
+ any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas",
+ // TODO(riscv64)
+ // all(
+ // feature = "fallback",
+ // not(portable_atomic_no_outline_atomics),
+ // any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+ // any(
+ // all(
+ // target_os = "linux",
+ // any(
+ // target_env = "gnu",
+ // all(
+ // any(target_env = "musl", target_env = "ohos"),
+ // not(target_feature = "crt-static"),
+ // ),
+ // portable_atomic_outline_atomics,
+ // ),
+ // ),
+ // target_os = "android",
+ // ),
+ // not(any(miri, portable_atomic_sanitize_thread)),
+ // ),
+ ),
+))]
+// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
+#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "atomic128/intrinsics.rs")]
+#[cfg_attr(not(any(miri, portable_atomic_sanitize_thread)), path = "atomic128/riscv64.rs")]
+mod riscv64;
+
// powerpc64 128-bit atomics
#[cfg(all(
target_arch = "powerpc64",
@@ -178,6 +213,14 @@ mod x86;
any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
),
+ all(
+ target_arch = "riscv64",
+ not(portable_atomic_no_asm),
+ any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas",
+ ),
+ ),
all(
target_arch = "powerpc64",
portable_atomic_unstable_asm_experimental_arch,
@@ -354,6 +397,35 @@ items! {
),
),
),
+ all(
+ target_arch = "riscv64",
+ not(portable_atomic_no_asm),
+ any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas",
+ // TODO(riscv64)
+ // all(
+ // feature = "fallback",
+ // not(portable_atomic_no_outline_atomics),
+ // any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+ // any(
+ // all(
+ // target_os = "linux",
+ // any(
+ // target_env = "gnu",
+ // all(
+ // any(target_env = "musl", target_env = "ohos"),
+ // not(target_feature = "crt-static"),
+ // ),
+ // portable_atomic_outline_atomics,
+ // ),
+ // ),
+ // target_os = "android",
+ // ),
+ // not(any(miri, portable_atomic_sanitize_thread)),
+ // ),
+ ),
+ ),
all(
target_arch = "powerpc64",
portable_atomic_unstable_asm_experimental_arch,
@@ -427,6 +499,37 @@ pub(crate) use self::aarch64::{AtomicI128, AtomicU128};
),
))]
pub(crate) use self::x86_64::{AtomicI128, AtomicU128};
+// riscv64 & zacas
+#[cfg(all(
+ target_arch = "riscv64",
+ not(portable_atomic_no_asm),
+ any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas",
+ // TODO(riscv64)
+ // all(
+ // feature = "fallback",
+ // not(portable_atomic_no_outline_atomics),
+ // any(test, portable_atomic_outline_atomics), // TODO(riscv64): currently disabled by default
+ // any(
+ // all(
+ // target_os = "linux",
+ // any(
+ // target_env = "gnu",
+ // all(
+ // any(target_env = "musl", target_env = "ohos"),
+ // not(target_feature = "crt-static"),
+ // ),
+ // portable_atomic_outline_atomics,
+ // ),
+ // ),
+ // target_os = "android",
+ // ),
+ // not(any(miri, portable_atomic_sanitize_thread)),
+ // ),
+ ),
+))]
+pub(crate) use self::riscv64::{AtomicI128, AtomicU128};
// powerpc64 & (pwr8 | outline-atomics)
#[cfg(all(
target_arch = "powerpc64",
diff --git a/src/imp/riscv.rs b/src/imp/riscv.rs
index c8b55f85f..3129fd1e8 100644
--- a/src/imp/riscv.rs
+++ b/src/imp/riscv.rs
@@ -26,6 +26,8 @@ Generated asm:
- riscv32imac https://godbolt.org/z/aG9157dhW
*/
+// TODO: Zacas
+
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::{cell::UnsafeCell, sync::atomic::Ordering};
diff --git a/src/lib.rs b/src/lib.rs
index 64b000782..39208d215 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -270,14 +270,24 @@ RUSTFLAGS="--cfg portable_atomic_no_outline_atomics" cargo ...
// fallback and causing memory ordering problems to be missed by these checkers.
#![cfg_attr(
all(
- any(target_arch = "aarch64", target_arch = "powerpc64", target_arch = "s390x"),
+ any(
+ target_arch = "aarch64",
+ target_arch = "powerpc64",
+ target_arch = "riscv64",
+ target_arch = "s390x",
+ ),
any(miri, portable_atomic_sanitize_thread),
),
allow(internal_features)
)]
#![cfg_attr(
all(
- any(target_arch = "aarch64", target_arch = "powerpc64", target_arch = "s390x"),
+ any(
+ target_arch = "aarch64",
+ target_arch = "powerpc64",
+ target_arch = "riscv64",
+ target_arch = "s390x",
+ ),
any(miri, portable_atomic_sanitize_thread),
),
feature(core_intrinsics)
diff --git a/src/tests/mod.rs b/src/tests/mod.rs
index 1042a9ae8..2f9514a83 100644
--- a/src/tests/mod.rs
+++ b/src/tests/mod.rs
@@ -117,6 +117,14 @@ fn test_is_lock_free() {
target_arch = "x86_64",
any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
),
+ all(
+ target_arch = "riscv64",
+ not(portable_atomic_no_asm),
+ any(
+ target_feature = "experimental-zacas",
+ portable_atomic_target_feature = "experimental-zacas",
+ ),
+ ),
all(
target_arch = "powerpc64",
portable_atomic_unstable_asm_experimental_arch,
diff --git a/src/utils.rs b/src/utils.rs
index 2ad488224..7dda05a2c 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -344,6 +344,7 @@ pub(crate) fn zero_extend64_ptr(v: *mut ()) -> core::mem::MaybeUninit {
#[cfg(any(
target_arch = "aarch64",
target_arch = "powerpc64",
+ target_arch = "riscv64",
target_arch = "s390x",
target_arch = "x86_64",
))]
diff --git a/tools/build.sh b/tools/build.sh
index 1c44dad39..7383ab27a 100755
--- a/tools/build.sh
+++ b/tools/build.sh
@@ -591,6 +591,11 @@ build() {
RUSTFLAGS="${target_rustflags} -C target-cpu=pwr7" \
x_cargo "${args[@]}" "$@"
;;
+ riscv64*)
+ CARGO_TARGET_DIR="${target_dir}/zacas" \
+ RUSTFLAGS="${target_rustflags} -C target-feature=experimental-zacas" \
+ x_cargo "${args[@]}" "$@"
+ ;;
s390x*)
CARGO_TARGET_DIR="${target_dir}/z196" \
RUSTFLAGS="${target_rustflags} -C target-cpu=z196" \