1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Pre-defined atomic types 4 5 use crate::prelude::*; 6 7 // Ensure size and alignment requirements are checked. 8 static_assert!(size_of::<bool>() == size_of::<i8>()); 9 static_assert!(align_of::<bool>() == align_of::<i8>()); 10 11 // SAFETY: `bool` has the same size and alignment as `i8`, and Rust guarantees that `bool` has 12 // only two valid bit patterns: 0 (false) and 1 (true). Those are valid `i8` values, so `bool` is 13 // round-trip transmutable to `i8`. 14 unsafe impl super::AtomicType for bool { 15 type Repr = i8; 16 } 17 18 // SAFETY: `i8` has the same size and alignment with itself, and is round-trip transmutable to 19 // itself. 20 unsafe impl super::AtomicType for i8 { 21 type Repr = i8; 22 } 23 24 // SAFETY: `i16` has the same size and alignment with itself, and is round-trip transmutable to 25 // itself. 26 unsafe impl super::AtomicType for i16 { 27 type Repr = i16; 28 } 29 30 // SAFETY: 31 // 32 // - `*mut T` has the same size and alignment with `*const c_void`, and is round-trip 33 // transmutable to `*const c_void`. 34 // - `*mut T` is safe to transfer between execution contexts. See the safety requirement of 35 // [`AtomicType`]. 36 unsafe impl<T: Sized> super::AtomicType for *mut T { 37 type Repr = *const c_void; 38 } 39 40 // SAFETY: 41 // 42 // - `*const T` has the same size and alignment with `*const c_void`, and is round-trip 43 // transmutable to `*const c_void`. 44 // - `*const T` is safe to transfer between execution contexts. See the safety requirement of 45 // [`AtomicType`]. 46 unsafe impl<T: Sized> super::AtomicType for *const T { 47 type Repr = *const c_void; 48 } 49 50 // SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to 51 // itself. 52 unsafe impl super::AtomicType for i32 { 53 type Repr = i32; 54 } 55 56 // SAFETY: The wrapping add result of two `i32`s is a valid `i32`. 57 unsafe impl super::AtomicAdd<i32> for i32 { 58 fn rhs_into_delta(rhs: i32) -> i32 { 59 rhs 60 } 61 } 62 63 // SAFETY: `i64` has the same size and alignment with itself, and is round-trip transmutable to 64 // itself. 65 unsafe impl super::AtomicType for i64 { 66 type Repr = i64; 67 } 68 69 // SAFETY: The wrapping add result of two `i64`s is a valid `i64`. 70 unsafe impl super::AtomicAdd<i64> for i64 { 71 fn rhs_into_delta(rhs: i64) -> i64 { 72 rhs 73 } 74 } 75 76 // Defines an internal type that always maps to the integer type which has the same size alignment 77 // as `isize` and `usize`, and `isize` and `usize` are always bi-directional transmutable to 78 // `isize_atomic_repr`, which also always implements `AtomicImpl`. 79 #[allow(non_camel_case_types)] 80 #[cfg(not(testlib))] 81 #[cfg(not(CONFIG_64BIT))] 82 type isize_atomic_repr = i32; 83 #[allow(non_camel_case_types)] 84 #[cfg(not(testlib))] 85 #[cfg(CONFIG_64BIT)] 86 type isize_atomic_repr = i64; 87 88 #[allow(non_camel_case_types)] 89 #[cfg(testlib)] 90 #[cfg(target_pointer_width = "32")] 91 type isize_atomic_repr = i32; 92 #[allow(non_camel_case_types)] 93 #[cfg(testlib)] 94 #[cfg(target_pointer_width = "64")] 95 type isize_atomic_repr = i64; 96 97 // Ensure size and alignment requirements are checked. 98 static_assert!(size_of::<isize>() == size_of::<isize_atomic_repr>()); 99 static_assert!(align_of::<isize>() == align_of::<isize_atomic_repr>()); 100 static_assert!(size_of::<usize>() == size_of::<isize_atomic_repr>()); 101 static_assert!(align_of::<usize>() == align_of::<isize_atomic_repr>()); 102 103 // SAFETY: `isize` has the same size and alignment with `isize_atomic_repr`, and is round-trip 104 // transmutable to `isize_atomic_repr`. 105 unsafe impl super::AtomicType for isize { 106 type Repr = isize_atomic_repr; 107 } 108 109 // SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`. 110 unsafe impl super::AtomicAdd<isize> for isize { 111 fn rhs_into_delta(rhs: isize) -> isize_atomic_repr { 112 rhs as isize_atomic_repr 113 } 114 } 115 116 // SAFETY: `u32` and `i32` has the same size and alignment, and `u32` is round-trip transmutable to 117 // `i32`. 118 unsafe impl super::AtomicType for u32 { 119 type Repr = i32; 120 } 121 122 // SAFETY: The wrapping add result of two `i32`s is a valid `u32`. 123 unsafe impl super::AtomicAdd<u32> for u32 { 124 fn rhs_into_delta(rhs: u32) -> i32 { 125 rhs as i32 126 } 127 } 128 129 // SAFETY: `u64` and `i64` has the same size and alignment, and `u64` is round-trip transmutable to 130 // `i64`. 131 unsafe impl super::AtomicType for u64 { 132 type Repr = i64; 133 } 134 135 // SAFETY: The wrapping add result of two `i64`s is a valid `u64`. 136 unsafe impl super::AtomicAdd<u64> for u64 { 137 fn rhs_into_delta(rhs: u64) -> i64 { 138 rhs as i64 139 } 140 } 141 142 // SAFETY: `usize` has the same size and alignment with `isize_atomic_repr`, and is round-trip 143 // transmutable to `isize_atomic_repr`. 144 unsafe impl super::AtomicType for usize { 145 type Repr = isize_atomic_repr; 146 } 147 148 // SAFETY: The wrapping add result of two `isize_atomic_repr`s is a valid `usize`. 149 unsafe impl super::AtomicAdd<usize> for usize { 150 fn rhs_into_delta(rhs: usize) -> isize_atomic_repr { 151 rhs as isize_atomic_repr 152 } 153 } 154 155 #[cfg(CONFIG_RUST_ATOMICS_KUNIT_TEST)] 156 #[macros::kunit_tests(rust_atomics)] 157 mod tests { 158 use super::super::*; 159 160 // Call $fn($val) with each $type of $val. 161 macro_rules! for_each_type { 162 ($val:literal in [$($type:ty),*] $fn:expr) => { 163 $({ 164 let v: $type = $val; 165 166 $fn(v); 167 })* 168 } 169 } 170 171 #[test] 172 fn atomic_basic_tests() { 173 for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 174 let x = Atomic::new(v); 175 176 assert_eq!(v, x.load(Relaxed)); 177 }); 178 179 for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 180 let x = Atomic::new(v); 181 let ptr = x.as_ptr(); 182 183 // SAFETY: `ptr` is a valid pointer and no concurrent access. 184 assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) }); 185 }); 186 } 187 188 #[test] 189 fn atomic_acquire_release_tests() { 190 for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 191 let x = Atomic::new(0); 192 193 x.store(v, Release); 194 assert_eq!(v, x.load(Acquire)); 195 }); 196 197 for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 198 let x = Atomic::new(0); 199 let ptr = x.as_ptr(); 200 201 // SAFETY: `ptr` is a valid pointer and no concurrent access. 202 unsafe { atomic_store(ptr, v, Release) }; 203 204 // SAFETY: `ptr` is a valid pointer and no concurrent access. 205 assert_eq!(v, unsafe { atomic_load(ptr, Acquire) }); 206 }); 207 } 208 209 #[test] 210 fn atomic_xchg_tests() { 211 for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 212 let x = Atomic::new(v); 213 214 let old = v; 215 let new = v + 1; 216 217 assert_eq!(old, x.xchg(new, Full)); 218 assert_eq!(new, x.load(Relaxed)); 219 }); 220 221 for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 222 let x = Atomic::new(v); 223 let ptr = x.as_ptr(); 224 225 let old = v; 226 let new = v + 1; 227 228 // SAFETY: `ptr` is a valid pointer and no concurrent access. 229 assert_eq!(old, unsafe { xchg(ptr, new, Full) }); 230 assert_eq!(new, x.load(Relaxed)); 231 }); 232 } 233 234 #[test] 235 fn atomic_cmpxchg_tests() { 236 for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 237 let x = Atomic::new(v); 238 239 let old = v; 240 let new = v + 1; 241 242 assert_eq!(Err(old), x.cmpxchg(new, new, Full)); 243 assert_eq!(old, x.load(Relaxed)); 244 assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed)); 245 assert_eq!(new, x.load(Relaxed)); 246 }); 247 248 for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { 249 let x = Atomic::new(v); 250 let ptr = x.as_ptr(); 251 252 let old = v; 253 let new = v + 1; 254 255 // SAFETY: `ptr` is a valid pointer and no concurrent access. 256 assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) }); 257 assert_eq!(old, x.load(Relaxed)); 258 // SAFETY: `ptr` is a valid pointer and no concurrent access. 259 assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) }); 260 assert_eq!(new, x.load(Relaxed)); 261 }); 262 } 263 264 #[test] 265 fn atomic_arithmetic_tests() { 266 for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { 267 let x = Atomic::new(v); 268 269 assert_eq!(v, x.fetch_add(12, Full)); 270 assert_eq!(v + 12, x.load(Relaxed)); 271 272 x.add(13, Relaxed); 273 274 assert_eq!(v + 25, x.load(Relaxed)); 275 }); 276 } 277 278 #[test] 279 fn atomic_bool_tests() { 280 let x = Atomic::new(false); 281 282 assert_eq!(false, x.load(Relaxed)); 283 x.store(true, Relaxed); 284 assert_eq!(true, x.load(Relaxed)); 285 286 assert_eq!(true, x.xchg(false, Relaxed)); 287 assert_eq!(false, x.load(Relaxed)); 288 289 assert_eq!(Err(false), x.cmpxchg(true, true, Relaxed)); 290 assert_eq!(false, x.load(Relaxed)); 291 assert_eq!(Ok(false), x.cmpxchg(false, true, Full)); 292 } 293 294 #[test] 295 fn atomic_ptr_tests() { 296 let mut v = 42; 297 let mut u = 43; 298 let x = Atomic::new(&raw mut v); 299 300 assert_eq!(x.load(Acquire), &raw mut v); 301 assert_eq!(x.cmpxchg(&raw mut u, &raw mut u, Relaxed), Err(&raw mut v)); 302 assert_eq!(x.cmpxchg(&raw mut v, &raw mut u, Relaxed), Ok(&raw mut v)); 303 assert_eq!(x.load(Relaxed), &raw mut u); 304 305 let x = Atomic::new(&raw const v); 306 307 assert_eq!(x.load(Acquire), &raw const v); 308 assert_eq!( 309 x.cmpxchg(&raw const u, &raw const u, Relaxed), 310 Err(&raw const v) 311 ); 312 assert_eq!( 313 x.cmpxchg(&raw const v, &raw const u, Relaxed), 314 Ok(&raw const v) 315 ); 316 assert_eq!(x.load(Relaxed), &raw const u); 317 } 318 319 #[test] 320 fn atomic_flag_tests() { 321 let mut flag = AtomicFlag::new(false); 322 323 assert_eq!(false, flag.load(Relaxed)); 324 325 *flag.get_mut() = true; 326 assert_eq!(true, flag.load(Relaxed)); 327 328 assert_eq!(true, flag.xchg(false, Relaxed)); 329 assert_eq!(false, flag.load(Relaxed)); 330 331 *flag.get_mut() = true; 332 assert_eq!(Ok(true), flag.cmpxchg(true, false, Full)); 333 assert_eq!(false, flag.load(Relaxed)); 334 } 335 } 336