1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Allocator support. 4 //! 5 //! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" 6 //! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the 7 //! typical application of the different kernel allocators. 8 //! 9 //! Reference: <https://docs.kernel.org/core-api/memory-allocation.html> 10 11 use super::Flags; 12 use core::alloc::Layout; 13 use core::ptr; 14 use core::ptr::NonNull; 15 16 use crate::alloc::{AllocError, Allocator}; 17 use crate::bindings; 18 use crate::pr_warn; 19 20 const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN; 21 22 /// The contiguous kernel allocator. 23 /// 24 /// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also 25 /// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. 26 /// 27 /// For more details see [self]. 28 pub struct Kmalloc; 29 30 /// The virtually contiguous kernel allocator. 31 /// 32 /// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel 33 /// virtual space. It is typically used for large allocations. The memory allocated with this 34 /// allocator is not physically contiguous. 35 /// 36 /// For more details see [self]. 37 pub struct Vmalloc; 38 39 /// The kvmalloc kernel allocator. 40 /// 41 /// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon 42 /// failure. This allocator is typically used when the size for the requested allocation is not 43 /// known and may exceed the capabilities of `Kmalloc`. 44 /// 45 /// For more details see [self]. 46 pub struct KVmalloc; 47 48 /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. 49 fn aligned_size(new_layout: Layout) -> usize { 50 // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. 51 let layout = new_layout.pad_to_align(); 52 53 // Note that `layout.size()` (after padding) is guaranteed to be a multiple of `layout.align()` 54 // which together with the slab guarantees means the `krealloc` will return a properly aligned 55 // object (see comments in `kmalloc()` for more information). 56 layout.size() 57 } 58 59 /// # Invariants 60 /// 61 /// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. 62 struct ReallocFunc( 63 unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void, 64 ); 65 66 impl ReallocFunc { 67 // INVARIANT: `krealloc` satisfies the type invariants. 68 const KREALLOC: Self = Self(bindings::krealloc); 69 70 // INVARIANT: `vrealloc` satisfies the type invariants. 71 const VREALLOC: Self = Self(bindings::vrealloc); 72 73 // INVARIANT: `kvrealloc` satisfies the type invariants. 74 const KVREALLOC: Self = Self(bindings::kvrealloc); 75 76 /// # Safety 77 /// 78 /// This method has the same safety requirements as [`Allocator::realloc`]. 79 /// 80 /// # Guarantees 81 /// 82 /// This method has the same guarantees as `Allocator::realloc`. Additionally 83 /// - it accepts any pointer to a valid memory allocation allocated by this function. 84 /// - memory allocated by this function remains valid until it is passed to this function. 85 #[inline] 86 unsafe fn call( 87 &self, 88 ptr: Option<NonNull<u8>>, 89 layout: Layout, 90 old_layout: Layout, 91 flags: Flags, 92 ) -> Result<NonNull<[u8]>, AllocError> { 93 let size = aligned_size(layout); 94 let ptr = match ptr { 95 Some(ptr) => { 96 if old_layout.size() == 0 { 97 ptr::null() 98 } else { 99 ptr.as_ptr() 100 } 101 } 102 None => ptr::null(), 103 }; 104 105 // SAFETY: 106 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that 107 // `ptr` is NULL or valid. 108 // - `ptr` is either NULL or valid by the safety requirements of this function. 109 // 110 // GUARANTEE: 111 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. 112 // - Those functions provide the guarantees of this function. 113 let raw_ptr = unsafe { 114 // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. 115 self.0(ptr.cast(), size, flags.0).cast() 116 }; 117 118 let ptr = if size == 0 { 119 crate::alloc::dangling_from_layout(layout) 120 } else { 121 NonNull::new(raw_ptr).ok_or(AllocError)? 122 }; 123 124 Ok(NonNull::slice_from_raw_parts(ptr, size)) 125 } 126 } 127 128 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 129 // - memory remains valid until it is explicitly freed, 130 // - passing a pointer to a valid memory allocation is OK, 131 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 132 unsafe impl Allocator for Kmalloc { 133 const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; 134 135 #[inline] 136 unsafe fn realloc( 137 ptr: Option<NonNull<u8>>, 138 layout: Layout, 139 old_layout: Layout, 140 flags: Flags, 141 ) -> Result<NonNull<[u8]>, AllocError> { 142 // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. 143 unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } 144 } 145 } 146 147 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 148 // - memory remains valid until it is explicitly freed, 149 // - passing a pointer to a valid memory allocation is OK, 150 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 151 unsafe impl Allocator for Vmalloc { 152 const MIN_ALIGN: usize = kernel::page::PAGE_SIZE; 153 154 #[inline] 155 unsafe fn realloc( 156 ptr: Option<NonNull<u8>>, 157 layout: Layout, 158 old_layout: Layout, 159 flags: Flags, 160 ) -> Result<NonNull<[u8]>, AllocError> { 161 // TODO: Support alignments larger than PAGE_SIZE. 162 if layout.align() > bindings::PAGE_SIZE { 163 pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); 164 return Err(AllocError); 165 } 166 167 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 168 // allocated with this `Allocator`. 169 unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } 170 } 171 } 172 173 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 174 // - memory remains valid until it is explicitly freed, 175 // - passing a pointer to a valid memory allocation is OK, 176 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 177 unsafe impl Allocator for KVmalloc { 178 const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN; 179 180 #[inline] 181 unsafe fn realloc( 182 ptr: Option<NonNull<u8>>, 183 layout: Layout, 184 old_layout: Layout, 185 flags: Flags, 186 ) -> Result<NonNull<[u8]>, AllocError> { 187 // TODO: Support alignments larger than PAGE_SIZE. 188 if layout.align() > bindings::PAGE_SIZE { 189 pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n"); 190 return Err(AllocError); 191 } 192 193 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 194 // allocated with this `Allocator`. 195 unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } 196 } 197 } 198