1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Implementation of the kernel's memory allocation infrastructure. 4 5 pub mod allocator; 6 pub mod kbox; 7 pub mod kvec; 8 pub mod layout; 9 10 pub use self::kbox::Box; 11 pub use self::kbox::KBox; 12 pub use self::kbox::KVBox; 13 pub use self::kbox::VBox; 14 15 pub use self::kvec::IntoIter; 16 pub use self::kvec::KVVec; 17 pub use self::kvec::KVec; 18 pub use self::kvec::VVec; 19 pub use self::kvec::Vec; 20 21 /// Indicates an allocation error. 22 #[derive(Copy, Clone, PartialEq, Eq, Debug)] 23 pub struct AllocError; 24 25 use crate::prelude::*; 26 27 use core::{ 28 alloc::Layout, 29 ptr::NonNull, // 30 }; 31 32 /// Flags to be used when allocating memory. 33 /// 34 /// They can be combined with the operators `|`, `&`, and `!`. 35 /// 36 /// Values can be used from the [`flags`] module. 37 #[derive(Clone, Copy, PartialEq)] 38 pub struct Flags(u32); 39 40 impl Flags { 41 /// Get the raw representation of this flag. 42 pub(crate) fn as_raw(self) -> u32 { 43 self.0 44 } 45 46 /// Check whether `flags` is contained in `self`. 47 pub fn contains(self, flags: Flags) -> bool { 48 (self & flags) == flags 49 } 50 } 51 52 impl core::ops::BitOr for Flags { 53 type Output = Self; 54 fn bitor(self, rhs: Self) -> Self::Output { 55 Self(self.0 | rhs.0) 56 } 57 } 58 59 impl core::ops::BitAnd for Flags { 60 type Output = Self; 61 fn bitand(self, rhs: Self) -> Self::Output { 62 Self(self.0 & rhs.0) 63 } 64 } 65 66 impl core::ops::Not for Flags { 67 type Output = Self; 68 fn not(self) -> Self::Output { 69 Self(!self.0) 70 } 71 } 72 73 /// Allocation flags. 74 /// 75 /// These are meant to be used in functions that can allocate memory. 76 pub mod flags { 77 use super::Flags; 78 79 /// Zeroes out the allocated memory. 80 /// 81 /// This is normally or'd with other flags. 82 pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO); 83 84 /// Allow the allocation to be in high memory. 85 /// 86 /// Allocations in high memory may not be mapped into the kernel's address space, so this can't 87 /// be used with `kmalloc` and other similar methods. 88 /// 89 /// This is normally or'd with other flags. 90 pub const __GFP_HIGHMEM: Flags = Flags(bindings::__GFP_HIGHMEM); 91 92 /// Users can not sleep and need the allocation to succeed. 93 /// 94 /// A lower watermark is applied to allow access to "atomic reserves". The current 95 /// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g. 96 /// `raw_spin_lock`). The same applies to [`GFP_NOWAIT`]. 97 pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC); 98 99 /// Typical for kernel-internal allocations. The caller requires `ZONE_NORMAL` or a lower zone 100 /// for direct access but can direct reclaim. 101 pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL); 102 103 /// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg. 104 pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT); 105 106 /// For kernel allocations that should not stall for direct reclaim, start physical IO or 107 /// use any filesystem callback. It is very likely to fail to allocate memory, even for very 108 /// small allocations. 109 pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); 110 111 /// Suppresses allocation failure reports. 112 /// 113 /// This is normally or'd with other flags. 114 pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN); 115 } 116 117 /// Non Uniform Memory Access (NUMA) node identifier. 118 #[derive(Clone, Copy, PartialEq)] 119 pub struct NumaNode(i32); 120 121 impl NumaNode { 122 /// Create a new NUMA node identifier (non-negative integer). 123 /// 124 /// Returns [`EINVAL`] if a negative id or an id exceeding [`bindings::MAX_NUMNODES`] is 125 /// specified. 126 pub fn new(node: i32) -> Result<Self> { 127 // MAX_NUMNODES never exceeds 2**10 because NODES_SHIFT is 0..10. 128 if node < 0 || node >= bindings::MAX_NUMNODES as i32 { 129 return Err(EINVAL); 130 } 131 Ok(Self(node)) 132 } 133 134 /// Returns the raw NUMA node identifier as an `i32`. 135 #[inline] 136 pub fn id(&self) -> i32 { 137 self.0 138 } 139 } 140 141 /// Specify necessary constant to pass the information to Allocator that the caller doesn't care 142 /// about the NUMA node to allocate memory from. 143 impl NumaNode { 144 /// No node preference. 145 pub const NO_NODE: NumaNode = NumaNode(bindings::NUMA_NO_NODE); 146 } 147 148 /// The kernel's [`Allocator`] trait. 149 /// 150 /// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described 151 /// via [`Layout`]. 152 /// 153 /// [`Allocator`] is designed to be implemented as a ZST; [`Allocator`] functions do not operate on 154 /// an object instance. 155 /// 156 /// In order to be able to support `#[derive(CoercePointee)]` later on, we need to avoid a design 157 /// that requires an `Allocator` to be instantiated, hence its functions must not contain any kind 158 /// of `self` parameter. 159 /// 160 /// # Safety 161 /// 162 /// - A memory allocation returned from an allocator must remain valid until it is explicitly freed. 163 /// 164 /// - Any pointer to a valid memory allocation must be valid to be passed to any other [`Allocator`] 165 /// function of the same type. 166 /// 167 /// - Implementers must ensure that all trait functions abide by the guarantees documented in the 168 /// `# Guarantees` sections. 169 pub unsafe trait Allocator { 170 /// The minimum alignment satisfied by all allocations from this allocator. 171 /// 172 /// # Guarantees 173 /// 174 /// Any pointer allocated by this allocator is guaranteed to be aligned to `MIN_ALIGN` even if 175 /// the requested layout has a smaller alignment. 176 const MIN_ALIGN: usize; 177 178 /// Allocate memory based on `layout`, `flags` and `nid`. 179 /// 180 /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout 181 /// constraints (i.e. minimum size and alignment as specified by `layout`). 182 /// 183 /// This function is equivalent to `realloc` when called with `None`. 184 /// 185 /// # Guarantees 186 /// 187 /// When the return value is `Ok(ptr)`, then `ptr` is 188 /// - valid for reads and writes for `layout.size()` bytes, until it is passed to 189 /// [`Allocator::free`] or [`Allocator::realloc`], 190 /// - aligned to `layout.align()`, 191 /// 192 /// Additionally, `Flags` are honored as documented in 193 /// <https://docs.kernel.org/core-api/mm-api.html#mm-api-gfp-flags>. 194 fn alloc(layout: Layout, flags: Flags, nid: NumaNode) -> Result<NonNull<[u8]>, AllocError> { 195 // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a 196 // new memory allocation. 197 unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, nid) } 198 } 199 200 /// Re-allocate an existing memory allocation to satisfy the requested `layout` and 201 /// a specific NUMA node request to allocate the memory for. 202 /// 203 /// Systems employing a Non Uniform Memory Access (NUMA) architecture contain collections of 204 /// hardware resources including processors, memory, and I/O buses, that comprise what is 205 /// commonly known as a NUMA node. 206 /// 207 /// `nid` stands for NUMA id, i. e. NUMA node identifier, which is a non-negative integer 208 /// if a node needs to be specified, or [`NumaNode::NO_NODE`] if the caller doesn't care. 209 /// 210 /// If the requested size is zero, `realloc` behaves equivalent to `free`. 211 /// 212 /// If the requested size is larger than the size of the existing allocation, a successful call 213 /// to `realloc` guarantees that the new or grown buffer has at least `Layout::size` bytes, but 214 /// may also be larger. 215 /// 216 /// If the requested size is smaller than the size of the existing allocation, `realloc` may or 217 /// may not shrink the buffer; this is implementation specific to the allocator. 218 /// 219 /// On allocation failure, the existing buffer, if any, remains valid. 220 /// 221 /// The buffer is represented as `NonNull<[u8]>`. 222 /// 223 /// # Safety 224 /// 225 /// - If `ptr == Some(p)`, then `p` must point to an existing and valid memory allocation 226 /// created by this [`Allocator`]; if `old_layout` is zero-sized `p` does not need to be a 227 /// pointer returned by this [`Allocator`]. 228 /// - `ptr` is allowed to be `None`; in this case a new memory allocation is created and 229 /// `old_layout` is ignored. 230 /// - `old_layout` must match the `Layout` the allocation has been created with. 231 /// 232 /// # Guarantees 233 /// 234 /// This function has the same guarantees as [`Allocator::alloc`]. When `ptr == Some(p)`, then 235 /// it additionally guarantees that: 236 /// - the contents of the memory pointed to by `p` are preserved up to the lesser of the new 237 /// and old size, i.e. `ret_ptr[0..min(layout.size(), old_layout.size())] == 238 /// p[0..min(layout.size(), old_layout.size())]`. 239 /// - when the return value is `Err(AllocError)`, then `ptr` is still valid. 240 unsafe fn realloc( 241 ptr: Option<NonNull<u8>>, 242 layout: Layout, 243 old_layout: Layout, 244 flags: Flags, 245 nid: NumaNode, 246 ) -> Result<NonNull<[u8]>, AllocError>; 247 248 /// Free an existing memory allocation. 249 /// 250 /// # Safety 251 /// 252 /// - `ptr` must point to an existing and valid memory allocation created by this [`Allocator`]; 253 /// if `old_layout` is zero-sized `p` does not need to be a pointer returned by this 254 /// [`Allocator`]. 255 /// - `layout` must match the `Layout` the allocation has been created with. 256 /// - The memory allocation at `ptr` must never again be read from or written to. 257 unsafe fn free(ptr: NonNull<u8>, layout: Layout) { 258 // SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this 259 // allocator. We are passing a `Layout` with the smallest possible alignment, so it is 260 // smaller than or equal to the alignment previously used with this allocation. 261 let _ = unsafe { 262 Self::realloc( 263 Some(ptr), 264 Layout::new::<()>(), 265 layout, 266 Flags(0), 267 NumaNode::NO_NODE, 268 ) 269 }; 270 } 271 } 272 273 /// Returns a properly aligned dangling pointer from the given `layout`. 274 pub(crate) fn dangling_from_layout(layout: Layout) -> NonNull<u8> { 275 let ptr = layout.align() as *mut u8; 276 277 // SAFETY: `layout.align()` (and hence `ptr`) is guaranteed to be non-zero. 278 unsafe { NonNull::new_unchecked(ptr) } 279 } 280