1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Allocator support. 4 //! 5 //! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide" 6 //! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the 7 //! typical application of the different kernel allocators. 8 //! 9 //! Reference: <https://docs.kernel.org/core-api/memory-allocation.html> 10 11 use super::{flags::*, Flags}; 12 use core::alloc::{GlobalAlloc, Layout}; 13 use core::ptr; 14 use core::ptr::NonNull; 15 16 use crate::alloc::{AllocError, Allocator}; 17 use crate::bindings; 18 use crate::pr_warn; 19 20 /// The contiguous kernel allocator. 21 /// 22 /// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also 23 /// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific. 24 /// 25 /// For more details see [self]. 26 pub struct Kmalloc; 27 28 /// The virtually contiguous kernel allocator. 29 /// 30 /// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel 31 /// virtual space. It is typically used for large allocations. The memory allocated with this 32 /// allocator is not physically contiguous. 33 /// 34 /// For more details see [self]. 35 pub struct Vmalloc; 36 37 /// The kvmalloc kernel allocator. 38 /// 39 /// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon 40 /// failure. This allocator is typically used when the size for the requested allocation is not 41 /// known and may exceed the capabilities of `Kmalloc`. 42 /// 43 /// For more details see [self]. 44 pub struct KVmalloc; 45 46 /// Returns a proper size to alloc a new object aligned to `new_layout`'s alignment. 47 fn aligned_size(new_layout: Layout) -> usize { 48 // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. 49 let layout = new_layout.pad_to_align(); 50 51 // Note that `layout.size()` (after padding) is guaranteed to be a multiple of `layout.align()` 52 // which together with the slab guarantees means the `krealloc` will return a properly aligned 53 // object (see comments in `kmalloc()` for more information). 54 layout.size() 55 } 56 57 /// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. 58 /// 59 /// # Safety 60 /// 61 /// - `ptr` can be either null or a pointer which has been allocated by this allocator. 62 /// - `new_layout` must have a non-zero size. 63 pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 { 64 let size = aligned_size(new_layout); 65 66 // SAFETY: 67 // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the 68 // function safety requirement. 69 // - `size` is greater than 0 since it's from `layout.size()` (which cannot be zero according 70 // to the function safety requirement) 71 unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } 72 } 73 74 /// # Invariants 75 /// 76 /// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. 77 struct ReallocFunc( 78 unsafe extern "C" fn(*const core::ffi::c_void, usize, u32) -> *mut core::ffi::c_void, 79 ); 80 81 impl ReallocFunc { 82 // INVARIANT: `krealloc` satisfies the type invariants. 83 const KREALLOC: Self = Self(bindings::krealloc); 84 85 // INVARIANT: `vrealloc` satisfies the type invariants. 86 const VREALLOC: Self = Self(bindings::vrealloc); 87 88 // INVARIANT: `kvrealloc` satisfies the type invariants. 89 const KVREALLOC: Self = Self(bindings::kvrealloc); 90 91 /// # Safety 92 /// 93 /// This method has the same safety requirements as [`Allocator::realloc`]. 94 /// 95 /// # Guarantees 96 /// 97 /// This method has the same guarantees as `Allocator::realloc`. Additionally 98 /// - it accepts any pointer to a valid memory allocation allocated by this function. 99 /// - memory allocated by this function remains valid until it is passed to this function. 100 unsafe fn call( 101 &self, 102 ptr: Option<NonNull<u8>>, 103 layout: Layout, 104 old_layout: Layout, 105 flags: Flags, 106 ) -> Result<NonNull<[u8]>, AllocError> { 107 let size = aligned_size(layout); 108 let ptr = match ptr { 109 Some(ptr) => { 110 if old_layout.size() == 0 { 111 ptr::null() 112 } else { 113 ptr.as_ptr() 114 } 115 } 116 None => ptr::null(), 117 }; 118 119 // SAFETY: 120 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that 121 // `ptr` is NULL or valid. 122 // - `ptr` is either NULL or valid by the safety requirements of this function. 123 // 124 // GUARANTEE: 125 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`. 126 // - Those functions provide the guarantees of this function. 127 let raw_ptr = unsafe { 128 // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. 129 self.0(ptr.cast(), size, flags.0).cast() 130 }; 131 132 let ptr = if size == 0 { 133 crate::alloc::dangling_from_layout(layout) 134 } else { 135 NonNull::new(raw_ptr).ok_or(AllocError)? 136 }; 137 138 Ok(NonNull::slice_from_raw_parts(ptr, size)) 139 } 140 } 141 142 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 143 // - memory remains valid until it is explicitly freed, 144 // - passing a pointer to a valid memory allocation is OK, 145 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 146 unsafe impl Allocator for Kmalloc { 147 #[inline] 148 unsafe fn realloc( 149 ptr: Option<NonNull<u8>>, 150 layout: Layout, 151 old_layout: Layout, 152 flags: Flags, 153 ) -> Result<NonNull<[u8]>, AllocError> { 154 // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. 155 unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } 156 } 157 } 158 159 // SAFETY: TODO. 160 unsafe impl GlobalAlloc for Kmalloc { 161 unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 162 // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety 163 // requirement. 164 unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) } 165 } 166 167 unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { 168 // SAFETY: TODO. 169 unsafe { 170 bindings::kfree(ptr as *const core::ffi::c_void); 171 } 172 } 173 174 unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { 175 // SAFETY: 176 // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not 177 // overflow `isize` by the function safety requirement. 178 // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two). 179 let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; 180 181 // SAFETY: 182 // - `ptr` is either null or a pointer allocated by this allocator by the function safety 183 // requirement. 184 // - the size of `layout` is not zero because `new_size` is not zero by the function safety 185 // requirement. 186 unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) } 187 } 188 189 unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { 190 // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety 191 // requirement. 192 unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) } 193 } 194 } 195 196 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 197 // - memory remains valid until it is explicitly freed, 198 // - passing a pointer to a valid memory allocation is OK, 199 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 200 unsafe impl Allocator for Vmalloc { 201 #[inline] 202 unsafe fn realloc( 203 ptr: Option<NonNull<u8>>, 204 layout: Layout, 205 old_layout: Layout, 206 flags: Flags, 207 ) -> Result<NonNull<[u8]>, AllocError> { 208 // TODO: Support alignments larger than PAGE_SIZE. 209 if layout.align() > bindings::PAGE_SIZE { 210 pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); 211 return Err(AllocError); 212 } 213 214 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 215 // allocated with this `Allocator`. 216 unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } 217 } 218 } 219 220 // SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that 221 // - memory remains valid until it is explicitly freed, 222 // - passing a pointer to a valid memory allocation is OK, 223 // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. 224 unsafe impl Allocator for KVmalloc { 225 #[inline] 226 unsafe fn realloc( 227 ptr: Option<NonNull<u8>>, 228 layout: Layout, 229 old_layout: Layout, 230 flags: Flags, 231 ) -> Result<NonNull<[u8]>, AllocError> { 232 // TODO: Support alignments larger than PAGE_SIZE. 233 if layout.align() > bindings::PAGE_SIZE { 234 pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n"); 235 return Err(AllocError); 236 } 237 238 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously 239 // allocated with this `Allocator`. 240 unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } 241 } 242 } 243 244 #[global_allocator] 245 static ALLOCATOR: Kmalloc = Kmalloc; 246 247 // See <https://github.com/rust-lang/rust/pull/86844>. 248 #[no_mangle] 249 static __rust_no_alloc_shim_is_unstable: u8 = 0; 250