1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Allocator support. 4 //! 5 //! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" 6 //! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the 7 //! typical application of the different kernel allocators. 8 //! 9 //! Reference: <https://docs.kernel.org/core-api/memory-allocation.html> 10 11 use super::Flags; 12 use core::alloc::Layout; 13 use core::ptr; 14 use core::ptr::NonNull; 15 16 use crate::alloc::{AllocError, Allocator, NumaNode}; 17 use crate::bindings; 18 19 /// The contiguous kernel allocator. 20 /// 21 /// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also 22 /// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. 23 /// 24 /// For more details see [self]. 25 pub struct Kmalloc; 26 27 /// The virtually contiguous kernel allocator. 28 /// 29 /// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel 30 /// virtual space. It is typically used for large allocations. The memory allocated with this 31 /// allocator is not physically contiguous. 32 /// 33 /// For more details see [self]. 34 pub struct Vmalloc; 35 36 /// The kvmalloc kernel allocator. 37 /// 38 /// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon 39 /// failure. This allocator is typically used when the size for the requested allocation is not 40 /// known and may exceed the capabilities of `Kmalloc`. 41 /// 42 /// For more details see [self]. 43 pub struct KVmalloc; 44 45 /// # Invariants 46 /// 47 /// One of the following: `krealloc_node_align`, `vrealloc_node_align`, `kvrealloc_node_align`. 48 struct ReallocFunc( 49 unsafe extern "C" fn( 50 *const crate::ffi::c_void, 51 usize, 52 crate::ffi::c_ulong, 53 u32, 54 crate::ffi::c_int, 55 ) -> *mut crate::ffi::c_void, 56 ); 57 58 impl ReallocFunc { 59 // INVARIANT: `krealloc_node_align` satisfies the type invariants. 60 const KREALLOC: Self = Self(bindings::krealloc_node_align); 61 62 // INVARIANT: `vrealloc_node_align` satisfies the type invariants. 63 const VREALLOC: Self = Self(bindings::vrealloc_node_align); 64 65 // INVARIANT: `kvrealloc_node_align` satisfies the type invariants. 66 const KVREALLOC: Self = Self(bindings::kvrealloc_node_align); 67 68 /// # Safety 69 /// 70 /// This method has the same safety requirements as [`Allocator::realloc`]. 71 /// 72 /// # Guarantees 73 /// 74 /// This method has the same guarantees as `Allocator::realloc`. Additionally 75 /// - it accepts any pointer to a valid memory allocation allocated by this function. 76 /// - memory allocated by this function remains valid until it is passed to this function. 77 #[inline] 78 unsafe fn call( 79 &self, 80 ptr: Option<NonNull<u8>>, 81 layout: Layout, 82 old_layout: Layout, 83 flags: Flags, 84 nid: NumaNode, 85 ) -> Result<NonNull<[u8]>, AllocError> { 86 let size = layout.size(); 87 let ptr = match ptr { 88 Some(ptr) => { 89 if old_layout.size() == 0 { 90 ptr::null() 91 } else { 92 ptr.as_ptr() 93 } 94 } 95 None => ptr::null(), 96 }; 97 98 // SAFETY: 99 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that 100 // `ptr` is NULL or valid. 101 // - `ptr` is either NULL or valid by the safety requirements of this function. 102 // 103 // GUARANTEE: 104 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. 105 // - Those functions provide the guarantees of this function. 106 let raw_ptr = unsafe { 107 // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. 108 self.0(ptr.cast(), size, layout.align(), flags.0, nid.0).cast() 109 }; 110 111 let ptr = if size == 0 { 112 crate::alloc::dangling_from_layout(layout) 113 } else { 114 NonNull::new(raw_ptr).ok_or(AllocError)? 115 }; 116 117 Ok(NonNull::slice_from_raw_parts(ptr, size)) 118 } 119 } 120 121 impl Kmalloc { 122 /// Returns a [`Layout`] that makes [`Kmalloc`] fulfill the requested size and alignment of 123 /// `layout`. 124 pub fn aligned_layout(layout: Layout) -> Layout { 125 // Note that `layout.size()` (after padding) is guaranteed to be a multiple of 126 // `layout.align()` which together with the slab guarantees means that `Kmalloc` will return 127 // a properly aligned object (see comments in `kmalloc()` for more information). 128 layout.pad_to_align() 129 } 130 } 131 132 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 133 // - memory remains valid until it is explicitly freed, 134 // - passing a pointer to a valid memory allocation is OK, 135 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 136 unsafe impl Allocator for Kmalloc { 137 #[inline] 138 unsafe fn realloc( 139 ptr: Option<NonNull<u8>>, 140 layout: Layout, 141 old_layout: Layout, 142 flags: Flags, 143 nid: NumaNode, 144 ) -> Result<NonNull<[u8]>, AllocError> { 145 let layout = Kmalloc::aligned_layout(layout); 146 147 // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. 148 unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) } 149 } 150 } 151 152 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 153 // - memory remains valid until it is explicitly freed, 154 // - passing a pointer to a valid memory allocation is OK, 155 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 156 unsafe impl Allocator for Vmalloc { 157 #[inline] 158 unsafe fn realloc( 159 ptr: Option<NonNull<u8>>, 160 layout: Layout, 161 old_layout: Layout, 162 flags: Flags, 163 nid: NumaNode, 164 ) -> Result<NonNull<[u8]>, AllocError> { 165 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 166 // allocated with this `Allocator`. 167 unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) } 168 } 169 } 170 171 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 172 // - memory remains valid until it is explicitly freed, 173 // - passing a pointer to a valid memory allocation is OK, 174 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 175 unsafe impl Allocator for KVmalloc { 176 #[inline] 177 unsafe fn realloc( 178 ptr: Option<NonNull<u8>>, 179 layout: Layout, 180 old_layout: Layout, 181 flags: Flags, 182 nid: NumaNode, 183 ) -> Result<NonNull<[u8]>, AllocError> { 184 // `KVmalloc` may use the `Kmalloc` backend, hence we have to enforce a `Kmalloc` 185 // compatible layout. 186 let layout = Kmalloc::aligned_layout(layout); 187 188 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 189 // allocated with this `Allocator`. 190 unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) } 191 } 192 } 193 194 #[macros::kunit_tests(rust_allocator)] 195 mod tests { 196 use super::*; 197 use core::mem::MaybeUninit; 198 use kernel::prelude::*; 199 200 #[test] 201 fn test_alignment() -> Result { 202 const TEST_SIZE: usize = 1024; 203 const TEST_LARGE_ALIGN_SIZE: usize = kernel::page::PAGE_SIZE * 4; 204 205 // These two structs are used to test allocating aligned memory. 206 // they don't need to be accessed, so they're marked as dead_code. 207 #[expect(dead_code)] 208 #[repr(align(128))] 209 struct Blob([u8; TEST_SIZE]); 210 #[expect(dead_code)] 211 #[repr(align(8192))] 212 struct LargeAlignBlob([u8; TEST_LARGE_ALIGN_SIZE]); 213 214 struct TestAlign<T, A: Allocator>(Box<MaybeUninit<T>, A>); 215 impl<T, A: Allocator> TestAlign<T, A> { 216 fn new() -> Result<Self> { 217 Ok(Self(Box::<_, A>::new_uninit(GFP_KERNEL)?)) 218 } 219 220 fn is_aligned_to(&self, align: usize) -> bool { 221 assert!(align.is_power_of_two()); 222 223 let addr = self.0.as_ptr() as usize; 224 addr & (align - 1) == 0 225 } 226 } 227 228 let ta = TestAlign::<Blob, Kmalloc>::new()?; 229 assert!(ta.is_aligned_to(128)); 230 231 let ta = TestAlign::<LargeAlignBlob, Kmalloc>::new()?; 232 assert!(ta.is_aligned_to(8192)); 233 234 let ta = TestAlign::<Blob, Vmalloc>::new()?; 235 assert!(ta.is_aligned_to(128)); 236 237 let ta = TestAlign::<LargeAlignBlob, Vmalloc>::new()?; 238 assert!(ta.is_aligned_to(8192)); 239 240 let ta = TestAlign::<Blob, KVmalloc>::new()?; 241 assert!(ta.is_aligned_to(128)); 242 243 let ta = TestAlign::<LargeAlignBlob, KVmalloc>::new()?; 244 assert!(ta.is_aligned_to(8192)); 245 246 Ok(()) 247 } 248 } 249