1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Atomic primitives. 4 //! 5 //! These primitives have the same semantics as their C counterparts: and the precise definitions of 6 //! semantics can be found at [`LKMM`]. Note that Linux Kernel Memory (Consistency) Model is the 7 //! only model for Rust code in kernel, and Rust's own atomics should be avoided. 8 //! 9 //! # Data races 10 //! 11 //! [`LKMM`] atomics have different rules regarding data races: 12 //! 13 //! - A normal write from C side is treated as an atomic write if 14 //! CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=y. 15 //! - Mixed-size atomic accesses don't cause data races. 16 //! 17 //! [`LKMM`]: srctree/tools/memory-model/ 18 19 mod internal; 20 pub mod ordering; 21 mod predefine; 22 23 pub use internal::AtomicImpl; 24 pub use ordering::{Acquire, Full, Relaxed, Release}; 25 26 pub(crate) use internal::{AtomicArithmeticOps, AtomicBasicOps, AtomicExchangeOps}; 27 28 use crate::build_error; 29 use internal::AtomicRepr; 30 use ordering::OrderingType; 31 32 /// A memory location which can be safely modified from multiple execution contexts. 33 /// 34 /// This has the same size, alignment and bit validity as the underlying type `T`. And it disables 35 /// niche optimization for the same reason as [`UnsafeCell`]. 36 /// 37 /// The atomic operations are implemented in a way that is fully compatible with the [Linux Kernel 38 /// Memory (Consistency) Model][LKMM], hence they should be modeled as the corresponding 39 /// [`LKMM`][LKMM] atomic primitives. With the help of [`Atomic::from_ptr()`] and 40 /// [`Atomic::as_ptr()`], this provides a way to interact with [C-side atomic operations] 41 /// (including those without the `atomic` prefix, e.g. `READ_ONCE()`, `WRITE_ONCE()`, 42 /// `smp_load_acquire()` and `smp_store_release()`). 43 /// 44 /// # Invariants 45 /// 46 /// `self.0` is a valid `T`. 47 /// 48 /// [`UnsafeCell`]: core::cell::UnsafeCell 49 /// [LKMM]: srctree/tools/memory-model/ 50 /// [C-side atomic operations]: srctree/Documentation/atomic_t.txt 51 #[repr(transparent)] 52 pub struct Atomic<T: AtomicType>(AtomicRepr<T::Repr>); 53 54 // SAFETY: `Atomic<T>` is safe to transfer between execution contexts because of the safety 55 // requirement of `AtomicType`. 56 unsafe impl<T: AtomicType> Send for Atomic<T> {} 57 58 // SAFETY: `Atomic<T>` is safe to share among execution contexts because all accesses are atomic. 59 unsafe impl<T: AtomicType> Sync for Atomic<T> {} 60 61 /// Types that support basic atomic operations. 62 /// 63 /// # Round-trip transmutability 64 /// 65 /// `T` is round-trip transmutable to `U` if and only if both of these properties hold: 66 /// 67 /// - Any valid bit pattern for `T` is also a valid bit pattern for `U`. 68 /// - Transmuting (e.g. using [`transmute()`]) a value of type `T` to `U` and then to `T` again 69 /// yields a value that is in all aspects equivalent to the original value. 70 /// 71 /// # Safety 72 /// 73 /// - [`Self`] must have the same size and alignment as [`Self::Repr`]. 74 /// - [`Self`] must be [round-trip transmutable] to [`Self::Repr`]. 75 /// - [`Self`] must be safe to transfer between execution contexts, if it's [`Send`], this is 76 /// automatically satisfied. The exception is pointer types that are even though marked as 77 /// `!Send` (e.g. raw pointers and [`NonNull<T>`]) but requiring `unsafe` to do anything 78 /// meaningful on them. This is because transferring pointer values between execution contexts is 79 /// safe as long as the actual `unsafe` dereferencing is justified. 80 /// 81 /// Note that this is more relaxed than requiring the bi-directional transmutability (i.e. 82 /// [`transmute()`] is always sound between `U` and `T`) because of the support for atomic 83 /// variables over unit-only enums, see [Examples]. 84 /// 85 /// # Limitations 86 /// 87 /// Because C primitives are used to implement the atomic operations, and a C function requires a 88 /// valid object of a type to operate on (i.e. no `MaybeUninit<_>`), hence at the Rust <-> C 89 /// surface, only types with all the bits initialized can be passed. As a result, types like `(u8, 90 /// u16)` (padding bytes are uninitialized) are currently not supported. 91 /// 92 /// # Examples 93 /// 94 /// A unit-only enum that implements [`AtomicType`]: 95 /// 96 /// ``` 97 /// use kernel::sync::atomic::{AtomicType, Atomic, Relaxed}; 98 /// 99 /// #[derive(Clone, Copy, PartialEq, Eq)] 100 /// #[repr(i32)] 101 /// enum State { 102 /// Uninit = 0, 103 /// Working = 1, 104 /// Done = 2, 105 /// }; 106 /// 107 /// // SAFETY: `State` and `i32` has the same size and alignment, and it's round-trip 108 /// // transmutable to `i32`. 109 /// unsafe impl AtomicType for State { 110 /// type Repr = i32; 111 /// } 112 /// 113 /// let s = Atomic::new(State::Uninit); 114 /// 115 /// assert_eq!(State::Uninit, s.load(Relaxed)); 116 /// ``` 117 /// [`transmute()`]: core::mem::transmute 118 /// [round-trip transmutable]: AtomicType#round-trip-transmutability 119 /// [Examples]: AtomicType#examples 120 /// [`NonNull<T>`]: core::ptr::NonNull 121 pub unsafe trait AtomicType: Sized + Copy { 122 /// The backing atomic implementation type. 123 type Repr: AtomicImpl; 124 } 125 126 /// Types that support atomic add operations. 127 /// 128 /// # Safety 129 /// 130 // TODO: Properly defines `wrapping_add` in the following comment. 131 /// `wrapping_add` any value of type `Self::Repr::Delta` obtained by [`Self::rhs_into_delta()`] to 132 /// any value of type `Self::Repr` obtained through transmuting a value of type `Self` to must 133 /// yield a value with a bit pattern also valid for `Self`. 134 pub unsafe trait AtomicAdd<Rhs = Self>: AtomicType { 135 /// Converts `Rhs` into the `Delta` type of the atomic implementation. 136 fn rhs_into_delta(rhs: Rhs) -> <Self::Repr as AtomicImpl>::Delta; 137 } 138 139 #[inline(always)] 140 const fn into_repr<T: AtomicType>(v: T) -> T::Repr { 141 // SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to 142 // `T::Repr`, therefore the transmute operation is sound. 143 unsafe { core::mem::transmute_copy(&v) } 144 } 145 146 /// # Safety 147 /// 148 /// `r` must be a valid bit pattern of `T`. 149 #[inline(always)] 150 const unsafe fn from_repr<T: AtomicType>(r: T::Repr) -> T { 151 // SAFETY: Per the safety requirement of the function, the transmute operation is sound. 152 unsafe { core::mem::transmute_copy(&r) } 153 } 154 155 impl<T: AtomicType> Atomic<T> { 156 /// Creates a new atomic `T`. 157 pub const fn new(v: T) -> Self { 158 // INVARIANT: Per the safety requirement of `AtomicType`, `into_repr(v)` is a valid `T`. 159 Self(AtomicRepr::new(into_repr(v))) 160 } 161 162 /// Creates a reference to an atomic `T` from a pointer of `T`. 163 /// 164 /// This usually is used when communicating with C side or manipulating a C struct, see 165 /// examples below. 166 /// 167 /// # Safety 168 /// 169 /// - `ptr` is aligned to `align_of::<T>()`. 170 /// - `ptr` is valid for reads and writes for `'a`. 171 /// - For the duration of `'a`, other accesses to `*ptr` must not cause data races (defined 172 /// by [`LKMM`]) against atomic operations on the returned reference. Note that if all other 173 /// accesses are atomic, then this safety requirement is trivially fulfilled. 174 /// 175 /// [`LKMM`]: srctree/tools/memory-model 176 /// 177 /// # Examples 178 /// 179 /// Using [`Atomic::from_ptr()`] combined with [`Atomic::load()`] or [`Atomic::store()`] can 180 /// achieve the same functionality as `READ_ONCE()`/`smp_load_acquire()` or 181 /// `WRITE_ONCE()`/`smp_store_release()` in C side: 182 /// 183 /// ``` 184 /// # use kernel::types::Opaque; 185 /// use kernel::sync::atomic::{Atomic, Relaxed, Release}; 186 /// 187 /// // Assume there is a C struct `foo`. 188 /// mod cbindings { 189 /// #[repr(C)] 190 /// pub(crate) struct foo { 191 /// pub(crate) a: i32, 192 /// pub(crate) b: i32 193 /// } 194 /// } 195 /// 196 /// let tmp = Opaque::new(cbindings::foo { a: 1, b: 2 }); 197 /// 198 /// // struct foo *foo_ptr = ..; 199 /// let foo_ptr = tmp.get(); 200 /// 201 /// // SAFETY: `foo_ptr` is valid, and `.a` is in bounds. 202 /// let foo_a_ptr = unsafe { &raw mut (*foo_ptr).a }; 203 /// 204 /// // a = READ_ONCE(foo_ptr->a); 205 /// // 206 /// // SAFETY: `foo_a_ptr` is valid for read, and all other accesses on it is atomic, so no 207 /// // data race. 208 /// let a = unsafe { Atomic::from_ptr(foo_a_ptr) }.load(Relaxed); 209 /// # assert_eq!(a, 1); 210 /// 211 /// // smp_store_release(&foo_ptr->a, 2); 212 /// // 213 /// // SAFETY: `foo_a_ptr` is valid for writes, and all other accesses on it is atomic, so 214 /// // no data race. 215 /// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release); 216 /// ``` 217 pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self { 218 // CAST: `T` and `Atomic<T>` have the same size, alignment and bit validity. 219 // SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will 220 // live long enough. It's safe to return a `&Atomic<T>` because function safety requirement 221 // guarantees other accesses won't cause data races. 222 unsafe { &*ptr.cast::<Self>() } 223 } 224 225 /// Returns a pointer to the underlying atomic `T`. 226 /// 227 /// Note that use of the return pointer must not cause data races defined by [`LKMM`]. 228 /// 229 /// # Guarantees 230 /// 231 /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::<T>()`]). 232 /// 233 /// [`LKMM`]: srctree/tools/memory-model 234 /// [`align_of::<T>()`]: core::mem::align_of 235 pub const fn as_ptr(&self) -> *mut T { 236 // GUARANTEE: Per the function guarantee of `AtomicRepr::as_ptr()`, the `self.0.as_ptr()` 237 // must be a valid and properly aligned pointer for `T::Repr`, and per the safety guarantee 238 // of `AtomicType`, it's a valid and properly aligned pointer of `T`. 239 self.0.as_ptr().cast() 240 } 241 242 /// Returns a mutable reference to the underlying atomic `T`. 243 /// 244 /// This is safe because the mutable reference of the atomic `T` guarantees exclusive access. 245 /// 246 /// # Examples 247 /// 248 /// ``` 249 /// use kernel::sync::atomic::{Atomic, Relaxed}; 250 /// 251 /// let mut atomic_val = Atomic::new(0u32); 252 /// let val_mut = atomic_val.get_mut(); 253 /// *val_mut = 101; 254 /// assert_eq!(101, atomic_val.load(Relaxed)); 255 /// ``` 256 pub fn get_mut(&mut self) -> &mut T { 257 // CAST: `T` and `T::Repr` has the same size and alignment per the safety requirement of 258 // `AtomicType`, and per the type invariants `self.0` is a valid `T`, therefore the casting 259 // result is a valid pointer of `T`. 260 // SAFETY: The pointer is valid per the CAST comment above, and the mutable reference 261 // guarantees exclusive access. 262 unsafe { &mut *self.0.as_ptr().cast() } 263 } 264 } 265 266 impl<T: AtomicType> Atomic<T> 267 where 268 T::Repr: AtomicBasicOps, 269 { 270 /// Loads the value from the atomic `T`. 271 /// 272 /// # Examples 273 /// 274 /// ``` 275 /// use kernel::sync::atomic::{Atomic, Relaxed}; 276 /// 277 /// let x = Atomic::new(42i32); 278 /// 279 /// assert_eq!(42, x.load(Relaxed)); 280 /// 281 /// let x = Atomic::new(42i64); 282 /// 283 /// assert_eq!(42, x.load(Relaxed)); 284 /// ``` 285 #[doc(alias("atomic_read", "atomic64_read"))] 286 #[inline(always)] 287 pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, _: Ordering) -> T { 288 let v = { 289 match Ordering::TYPE { 290 OrderingType::Relaxed => T::Repr::atomic_read(&self.0), 291 OrderingType::Acquire => T::Repr::atomic_read_acquire(&self.0), 292 _ => build_error!("Wrong ordering"), 293 } 294 }; 295 296 // SAFETY: `v` comes from reading `self.0`, which is a valid `T` per the type invariants. 297 unsafe { from_repr(v) } 298 } 299 300 /// Stores a value to the atomic `T`. 301 /// 302 /// # Examples 303 /// 304 /// ``` 305 /// use kernel::sync::atomic::{Atomic, Relaxed}; 306 /// 307 /// let x = Atomic::new(42i32); 308 /// 309 /// assert_eq!(42, x.load(Relaxed)); 310 /// 311 /// x.store(43, Relaxed); 312 /// 313 /// assert_eq!(43, x.load(Relaxed)); 314 /// ``` 315 #[doc(alias("atomic_set", "atomic64_set"))] 316 #[inline(always)] 317 pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: T, _: Ordering) { 318 let v = into_repr(v); 319 320 // INVARIANT: `v` is a valid `T`, and is stored to `self.0` by `atomic_set*()`. 321 match Ordering::TYPE { 322 OrderingType::Relaxed => T::Repr::atomic_set(&self.0, v), 323 OrderingType::Release => T::Repr::atomic_set_release(&self.0, v), 324 _ => build_error!("Wrong ordering"), 325 } 326 } 327 } 328 329 impl<T: AtomicType + core::fmt::Debug> core::fmt::Debug for Atomic<T> 330 where 331 T::Repr: AtomicBasicOps, 332 { 333 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { 334 core::fmt::Debug::fmt(&self.load(Relaxed), f) 335 } 336 } 337 338 impl<T: AtomicType> Atomic<T> 339 where 340 T::Repr: AtomicExchangeOps, 341 { 342 /// Atomic exchange. 343 /// 344 /// Atomically updates `*self` to `v` and returns the old value of `*self`. 345 /// 346 /// # Examples 347 /// 348 /// ``` 349 /// use kernel::sync::atomic::{Atomic, Acquire, Relaxed}; 350 /// 351 /// let x = Atomic::new(42); 352 /// 353 /// assert_eq!(42, x.xchg(52, Acquire)); 354 /// assert_eq!(52, x.load(Relaxed)); 355 /// ``` 356 #[doc(alias("atomic_xchg", "atomic64_xchg", "swap"))] 357 #[inline(always)] 358 pub fn xchg<Ordering: ordering::Ordering>(&self, v: T, _: Ordering) -> T { 359 let v = into_repr(v); 360 361 // INVARIANT: `self.0` is a valid `T` after `atomic_xchg*()` because `v` is transmutable to 362 // `T`. 363 let ret = { 364 match Ordering::TYPE { 365 OrderingType::Full => T::Repr::atomic_xchg(&self.0, v), 366 OrderingType::Acquire => T::Repr::atomic_xchg_acquire(&self.0, v), 367 OrderingType::Release => T::Repr::atomic_xchg_release(&self.0, v), 368 OrderingType::Relaxed => T::Repr::atomic_xchg_relaxed(&self.0, v), 369 } 370 }; 371 372 // SAFETY: `ret` comes from reading `*self`, which is a valid `T` per type invariants. 373 unsafe { from_repr(ret) } 374 } 375 376 /// Atomic compare and exchange. 377 /// 378 /// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not 379 /// modified. 380 /// 381 /// Compare: The comparison is done via the byte level comparison between `*self` and `old`. 382 /// 383 /// Ordering: When succeeds, provides the corresponding ordering as the `Ordering` type 384 /// parameter indicates, and a failed one doesn't provide any ordering, the load part of a 385 /// failed cmpxchg is a [`Relaxed`] load. 386 /// 387 /// Returns `Ok(value)` if cmpxchg succeeds, and `value` is guaranteed to be equal to `old`, 388 /// otherwise returns `Err(value)`, and `value` is the current value of `*self`. 389 /// 390 /// # Examples 391 /// 392 /// ``` 393 /// use kernel::sync::atomic::{Atomic, Full, Relaxed}; 394 /// 395 /// let x = Atomic::new(42); 396 /// 397 /// // Checks whether cmpxchg succeeded. 398 /// let success = x.cmpxchg(52, 64, Relaxed).is_ok(); 399 /// # assert!(!success); 400 /// 401 /// // Checks whether cmpxchg failed. 402 /// let failure = x.cmpxchg(52, 64, Relaxed).is_err(); 403 /// # assert!(failure); 404 /// 405 /// // Uses the old value if failed, probably re-try cmpxchg. 406 /// match x.cmpxchg(52, 64, Relaxed) { 407 /// Ok(_) => { }, 408 /// Err(old) => { 409 /// // do something with `old`. 410 /// # assert_eq!(old, 42); 411 /// } 412 /// } 413 /// 414 /// // Uses the latest value regardlessly, same as atomic_cmpxchg() in C. 415 /// let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old); 416 /// # assert_eq!(42, latest); 417 /// assert_eq!(64, x.load(Relaxed)); 418 /// ``` 419 /// 420 /// [`Relaxed`]: ordering::Relaxed 421 #[doc(alias( 422 "atomic_cmpxchg", 423 "atomic64_cmpxchg", 424 "atomic_try_cmpxchg", 425 "atomic64_try_cmpxchg", 426 "compare_exchange" 427 ))] 428 #[inline(always)] 429 pub fn cmpxchg<Ordering: ordering::Ordering>( 430 &self, 431 mut old: T, 432 new: T, 433 o: Ordering, 434 ) -> Result<T, T> { 435 // Note on code generation: 436 // 437 // try_cmpxchg() is used to implement cmpxchg(), and if the helper functions are inlined, 438 // the compiler is able to figure out that branch is not needed if the users don't care 439 // about whether the operation succeeds or not. One exception is on x86, due to commit 440 // 44fe84459faf ("locking/atomic: Fix atomic_try_cmpxchg() semantics"), the 441 // atomic_try_cmpxchg() on x86 has a branch even if the caller doesn't care about the 442 // success of cmpxchg and only wants to use the old value. For example, for code like: 443 // 444 // let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(|old| old); 445 // 446 // It will still generate code: 447 // 448 // movl $0x40, %ecx 449 // movl $0x34, %eax 450 // lock 451 // cmpxchgl %ecx, 0x4(%rsp) 452 // jne 1f 453 // 2: 454 // ... 455 // 1: movl %eax, %ecx 456 // jmp 2b 457 // 458 // This might be "fixed" by introducing a try_cmpxchg_exclusive() that knows the "*old" 459 // location in the C function is always safe to write. 460 if self.try_cmpxchg(&mut old, new, o) { 461 Ok(old) 462 } else { 463 Err(old) 464 } 465 } 466 467 /// Atomic compare and exchange and returns whether the operation succeeds. 468 /// 469 /// If `*self` == `old`, atomically updates `*self` to `new`. Otherwise, `*self` is not 470 /// modified, `*old` is updated to the current value of `*self`. 471 /// 472 /// "Compare" and "Ordering" part are the same as [`Atomic::cmpxchg()`]. 473 /// 474 /// Returns `true` means the cmpxchg succeeds otherwise returns `false`. 475 #[inline(always)] 476 fn try_cmpxchg<Ordering: ordering::Ordering>(&self, old: &mut T, new: T, _: Ordering) -> bool { 477 let mut tmp = into_repr(*old); 478 let new = into_repr(new); 479 480 // INVARIANT: `self.0` is a valid `T` after `atomic_try_cmpxchg*()` because `new` is 481 // transmutable to `T`. 482 let ret = { 483 match Ordering::TYPE { 484 OrderingType::Full => T::Repr::atomic_try_cmpxchg(&self.0, &mut tmp, new), 485 OrderingType::Acquire => { 486 T::Repr::atomic_try_cmpxchg_acquire(&self.0, &mut tmp, new) 487 } 488 OrderingType::Release => { 489 T::Repr::atomic_try_cmpxchg_release(&self.0, &mut tmp, new) 490 } 491 OrderingType::Relaxed => { 492 T::Repr::atomic_try_cmpxchg_relaxed(&self.0, &mut tmp, new) 493 } 494 } 495 }; 496 497 // SAFETY: `tmp` comes from reading `*self`, which is a valid `T` per type invariants. 498 *old = unsafe { from_repr(tmp) }; 499 500 ret 501 } 502 } 503 504 impl<T: AtomicType> Atomic<T> 505 where 506 T::Repr: AtomicArithmeticOps, 507 { 508 /// Atomic add. 509 /// 510 /// Atomically updates `*self` to `(*self).wrapping_add(v)`. 511 /// 512 /// # Examples 513 /// 514 /// ``` 515 /// use kernel::sync::atomic::{Atomic, Relaxed}; 516 /// 517 /// let x = Atomic::new(42); 518 /// 519 /// assert_eq!(42, x.load(Relaxed)); 520 /// 521 /// x.add(12, Relaxed); 522 /// 523 /// assert_eq!(54, x.load(Relaxed)); 524 /// ``` 525 #[inline(always)] 526 pub fn add<Rhs>(&self, v: Rhs, _: ordering::Relaxed) 527 where 528 T: AtomicAdd<Rhs>, 529 { 530 let v = T::rhs_into_delta(v); 531 532 // INVARIANT: `self.0` is a valid `T` after `atomic_add()` due to safety requirement of 533 // `AtomicAdd`. 534 T::Repr::atomic_add(&self.0, v); 535 } 536 537 /// Atomic fetch and add. 538 /// 539 /// Atomically updates `*self` to `(*self).wrapping_add(v)`, and returns the value of `*self` 540 /// before the update. 541 /// 542 /// # Examples 543 /// 544 /// ``` 545 /// use kernel::sync::atomic::{Atomic, Acquire, Full, Relaxed}; 546 /// 547 /// let x = Atomic::new(42); 548 /// 549 /// assert_eq!(42, x.load(Relaxed)); 550 /// 551 /// assert_eq!(54, { x.fetch_add(12, Acquire); x.load(Relaxed) }); 552 /// 553 /// let x = Atomic::new(42); 554 /// 555 /// assert_eq!(42, x.load(Relaxed)); 556 /// 557 /// assert_eq!(54, { x.fetch_add(12, Full); x.load(Relaxed) } ); 558 /// ``` 559 #[inline(always)] 560 pub fn fetch_add<Rhs, Ordering: ordering::Ordering>(&self, v: Rhs, _: Ordering) -> T 561 where 562 T: AtomicAdd<Rhs>, 563 { 564 let v = T::rhs_into_delta(v); 565 566 // INVARIANT: `self.0` is a valid `T` after `atomic_fetch_add*()` due to safety requirement 567 // of `AtomicAdd`. 568 let ret = { 569 match Ordering::TYPE { 570 OrderingType::Full => T::Repr::atomic_fetch_add(&self.0, v), 571 OrderingType::Acquire => T::Repr::atomic_fetch_add_acquire(&self.0, v), 572 OrderingType::Release => T::Repr::atomic_fetch_add_release(&self.0, v), 573 OrderingType::Relaxed => T::Repr::atomic_fetch_add_relaxed(&self.0, v), 574 } 575 }; 576 577 // SAFETY: `ret` comes from reading `self.0`, which is a valid `T` per type invariants. 578 unsafe { from_repr(ret) } 579 } 580 } 581 582 #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] 583 #[repr(C)] 584 #[derive(Clone, Copy)] 585 struct Flag { 586 bool_field: bool, 587 } 588 589 /// # Invariants 590 /// 591 /// `padding` must be all zeroes. 592 #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 593 #[repr(C, align(4))] 594 #[derive(Clone, Copy)] 595 struct Flag { 596 #[cfg(target_endian = "big")] 597 padding: [u8; 3], 598 bool_field: bool, 599 #[cfg(target_endian = "little")] 600 padding: [u8; 3], 601 } 602 603 impl Flag { 604 #[inline(always)] 605 const fn new(b: bool) -> Self { 606 // INVARIANT: `padding` is all zeroes. 607 Self { 608 bool_field: b, 609 #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 610 padding: [0; 3], 611 } 612 } 613 } 614 615 // SAFETY: `Flag` and `Repr` have the same size and alignment, and `Flag` is round-trip 616 // transmutable to the selected representation (`i8` or `i32`). 617 unsafe impl AtomicType for Flag { 618 #[cfg(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64))] 619 type Repr = i8; 620 #[cfg(not(any(CONFIG_X86_64, CONFIG_UML, CONFIG_ARM, CONFIG_ARM64)))] 621 type Repr = i32; 622 } 623 624 /// An atomic flag type intended to be backed by performance-optimal integer type. 625 /// 626 /// The backing integer type is an implementation detail; it may vary by architecture and change 627 /// in the future. 628 /// 629 /// [`AtomicFlag`] is generally preferable to [`Atomic<bool>`] when you need read-modify-write 630 /// (RMW) operations (e.g. [`Atomic::xchg()`]/[`Atomic::cmpxchg()`]) or when [`Atomic<bool>`] does 631 /// not save memory due to padding. On some architectures that do not support byte-sized atomic 632 /// RMW operations, RMW operations on [`Atomic<bool>`] are slower. 633 /// 634 /// If you only use [`Atomic::load()`]/[`Atomic::store()`], [`Atomic<bool>`] is fine. 635 /// 636 /// # Examples 637 /// 638 /// ``` 639 /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; 640 /// 641 /// let flag = AtomicFlag::new(false); 642 /// assert_eq!(false, flag.load(Relaxed)); 643 /// flag.store(true, Relaxed); 644 /// assert_eq!(true, flag.load(Relaxed)); 645 /// ``` 646 pub struct AtomicFlag(Atomic<Flag>); 647 648 impl AtomicFlag { 649 /// Creates a new atomic flag. 650 #[inline(always)] 651 pub const fn new(b: bool) -> Self { 652 Self(Atomic::new(Flag::new(b))) 653 } 654 655 /// Returns a mutable reference to the underlying flag as a [`bool`]. 656 /// 657 /// This is safe because the mutable reference of the atomic flag guarantees exclusive access. 658 /// 659 /// # Examples 660 /// 661 /// ``` 662 /// use kernel::sync::atomic::{AtomicFlag, Relaxed}; 663 /// 664 /// let mut atomic_flag = AtomicFlag::new(false); 665 /// assert_eq!(false, atomic_flag.load(Relaxed)); 666 /// *atomic_flag.get_mut() = true; 667 /// assert_eq!(true, atomic_flag.load(Relaxed)); 668 /// ``` 669 #[inline(always)] 670 pub fn get_mut(&mut self) -> &mut bool { 671 &mut self.0.get_mut().bool_field 672 } 673 674 /// Loads the value from the atomic flag. 675 #[inline(always)] 676 pub fn load<Ordering: ordering::AcquireOrRelaxed>(&self, o: Ordering) -> bool { 677 self.0.load(o).bool_field 678 } 679 680 /// Stores a value to the atomic flag. 681 #[inline(always)] 682 pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: bool, o: Ordering) { 683 self.0.store(Flag::new(v), o); 684 } 685 686 /// Stores a value to the atomic flag and returns the previous value. 687 #[inline(always)] 688 pub fn xchg<Ordering: ordering::Ordering>(&self, new: bool, o: Ordering) -> bool { 689 self.0.xchg(Flag::new(new), o).bool_field 690 } 691 692 /// Store a value to the atomic flag if the current value is equal to `old`. 693 #[inline(always)] 694 pub fn cmpxchg<Ordering: ordering::Ordering>( 695 &self, 696 old: bool, 697 new: bool, 698 o: Ordering, 699 ) -> Result<bool, bool> { 700 match self.0.cmpxchg(Flag::new(old), Flag::new(new), o) { 701 Ok(_) => Ok(old), 702 Err(f) => Err(f.bool_field), 703 } 704 } 705 } 706