1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Memory barriers. 4 //! 5 //! These primitives have the same semantics as their C counterparts: and the precise definitions 6 //! of semantics can be found at [`LKMM`]. 7 //! 8 //! [`LKMM`]: srctree/tools/memory-model/ 9 10 #![expect(private_bounds, reason = "sealed implementation")] 11 12 /// Memory barrier orderings. 13 /// 14 /// The semantics of these orderings follows the [`LKMM`] definitions and rules. 15 /// 16 /// - [`Read`] provides ordering between preceding load operations and succeeding load operations. 17 /// - [`Write`] provides ordering between preceding store operations and succeeding store 18 /// operations. 19 /// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory 20 /// accesses. 21 /// 22 /// [`LKMM`]: srctree/tools/memory-model/ 23 pub mod ordering { 24 pub use crate::sync::atomic::ordering::Full; 25 26 /// The annotation type for read-read barrier ordering. 27 pub struct Read; 28 29 /// The annotation type for write-write barrier ordering. 30 pub struct Write; 31 } 32 33 pub use ordering::{ 34 Full, 35 Read, 36 Write, // 37 }; 38 39 struct Smp; 40 struct Dma; 41 42 /// A compiler barrier. 43 /// 44 /// A barrier that prevents compiler from reordering memory accesses across the barrier. 45 #[inline(always)] 46 pub(crate) fn barrier() { 47 // By default, Rust inline asms are treated as being able to access any memory or flags, hence 48 // it suffices as a compiler barrier. 49 // 50 // SAFETY: An empty asm block. 51 unsafe { core::arch::asm!("") }; 52 } 53 54 trait MemoryBarrier<Flavour = ()> { 55 fn run(); 56 } 57 58 macro_rules! define_barrier { 59 ($([$flavour:ident])? $ordering:ident, $binding:ident) => { 60 impl MemoryBarrier$(<$flavour>)? for $ordering { 61 #[inline] 62 fn run() { 63 // SAFETY: barrier methods are safe to call. 64 unsafe { bindings::$binding() }; 65 } 66 } 67 }; 68 } 69 70 define_barrier!(Full, mb); 71 define_barrier!(Read, rmb); 72 define_barrier!(Write, wmb); 73 define_barrier!([Dma] Full, dma_mb); 74 define_barrier!([Dma] Read, dma_rmb); 75 define_barrier!([Dma] Write, dma_wmb); 76 define_barrier!([Smp] Full, smp_mb); 77 define_barrier!([Smp] Read, smp_rmb); 78 define_barrier!([Smp] Write, smp_wmb); 79 80 /// Memory barrier. 81 /// 82 /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. 83 /// 84 /// The specific forms of reordering can be specified using the parameter. 85 /// - `mb(Read)` provides a read-read barrier. 86 /// - `mb(Write)` provides a write-write barrier. 87 /// - `mb(Full)` provides a full barrier. 88 /// 89 /// # Examples 90 /// 91 /// ``` 92 /// # use kernel::sync::barrier::*; 93 /// mb(Read); 94 /// mb(Write); 95 /// mb(Full); 96 /// ``` 97 #[inline] 98 #[doc(alias = "rmb")] 99 #[doc(alias = "wmb")] 100 pub fn mb<T: MemoryBarrier>(_: T) { 101 T::run() 102 } 103 104 /// Memory barrier between CPUs. 105 /// 106 /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. 107 /// Does not prevent re-ordering with respect to other bus-mastering devices. 108 /// 109 /// See [`mb`] for usage. 110 #[inline] 111 #[doc(alias = "smp_rmb")] 112 #[doc(alias = "smp_wmb")] 113 pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) { 114 if cfg!(CONFIG_SMP) { 115 T::run() 116 } else { 117 barrier() 118 } 119 } 120 121 /// Memory barrier between local CPU and bus-mastering devices. 122 /// 123 /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. 124 /// Does not prevent re-ordering with respect to other CPUs. 125 /// 126 /// See [`mb`] for usage. 127 #[inline] 128 #[doc(alias = "dma_rmb")] 129 #[doc(alias = "dma_wmb")] 130 pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) { 131 T::run() 132 } 133