1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Kernel page allocation and management. 4 5 use crate::{ 6 alloc::{ 7 AllocError, 8 Flags, // 9 }, 10 bindings, 11 error::{ 12 code::*, 13 Result, // 14 }, 15 uaccess::UserSliceReader, // 16 }; 17 use core::{ 18 marker::PhantomData, 19 mem::ManuallyDrop, 20 ops::Deref, 21 ptr::{ 22 self, 23 NonNull, // 24 }, // 25 }; 26 27 /// A bitwise shift for the page size. 28 pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize; 29 30 /// The number of bytes in a page. 31 pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; 32 33 /// A bitmask that gives the page containing a given address. 34 pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); 35 36 /// Rounds up to the next multiple of [`PAGE_SIZE`]. 37 /// 38 /// Returns [`None`] on integer overflow. 39 /// 40 /// # Examples 41 /// 42 /// ``` 43 /// use kernel::page::{ 44 /// page_align, 45 /// PAGE_SIZE, 46 /// }; 47 /// 48 /// // Requested address is already aligned. 49 /// assert_eq!(page_align(0x0), Some(0x0)); 50 /// assert_eq!(page_align(PAGE_SIZE), Some(PAGE_SIZE)); 51 /// 52 /// // Requested address needs alignment up. 53 /// assert_eq!(page_align(0x1), Some(PAGE_SIZE)); 54 /// assert_eq!(page_align(PAGE_SIZE + 1), Some(2 * PAGE_SIZE)); 55 /// 56 /// // Requested address causes overflow (returns `None`). 57 /// let overflow_addr = usize::MAX - (PAGE_SIZE / 2); 58 /// assert_eq!(page_align(overflow_addr), None); 59 /// ``` 60 #[inline(always)] 61 pub const fn page_align(addr: usize) -> Option<usize> { 62 let Some(sum) = addr.checked_add(PAGE_SIZE - 1) else { 63 return None; 64 }; 65 Some(sum & PAGE_MASK) 66 } 67 68 /// Representation of a non-owning reference to a [`Page`]. 69 /// 70 /// This type provides a borrowed version of a [`Page`] that is owned by some other entity, e.g. a 71 /// [`Vmalloc`] allocation such as [`VBox`]. 72 /// 73 /// # Example 74 /// 75 /// ``` 76 /// # use kernel::{bindings, prelude::*}; 77 /// use kernel::page::{BorrowedPage, Page, PAGE_SIZE}; 78 /// # use core::{mem::MaybeUninit, ptr, ptr::NonNull }; 79 /// 80 /// fn borrow_page<'a>(vbox: &'a mut VBox<MaybeUninit<[u8; PAGE_SIZE]>>) -> BorrowedPage<'a> { 81 /// let ptr = ptr::from_ref(&**vbox); 82 /// 83 /// // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory. 84 /// let page = unsafe { bindings::vmalloc_to_page(ptr.cast()) }; 85 /// 86 /// // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid 87 /// // pointer to `Vmalloc` memory. 88 /// let page = unsafe { NonNull::new_unchecked(page) }; 89 /// 90 /// // SAFETY: 91 /// // - `self.0` is a valid pointer to a `struct page`. 92 /// // - `self.0` is valid for the entire lifetime of `self`. 93 /// unsafe { BorrowedPage::from_raw(page) } 94 /// } 95 /// 96 /// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; 97 /// let page = borrow_page(&mut vbox); 98 /// 99 /// // SAFETY: There is no concurrent read or write to this page. 100 /// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; 101 /// # Ok::<(), Error>(()) 102 /// ``` 103 /// 104 /// # Invariants 105 /// 106 /// The borrowed underlying pointer to a `struct page` is valid for the entire lifetime `'a`. 107 /// 108 /// [`VBox`]: kernel::alloc::VBox 109 /// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc 110 pub struct BorrowedPage<'a>(ManuallyDrop<Page>, PhantomData<&'a Page>); 111 112 impl<'a> BorrowedPage<'a> { 113 /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. 114 /// 115 /// # Safety 116 /// 117 /// - `ptr` must point to a valid `bindings::page`. 118 /// - `ptr` must remain valid for the entire lifetime `'a`. 119 pub unsafe fn from_raw(ptr: NonNull<bindings::page>) -> Self { 120 let page = Page { page: ptr }; 121 122 // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime 123 // `'a`. 124 Self(ManuallyDrop::new(page), PhantomData) 125 } 126 } 127 128 impl<'a> Deref for BorrowedPage<'a> { 129 type Target = Page; 130 131 fn deref(&self) -> &Self::Target { 132 &self.0 133 } 134 } 135 136 /// Trait to be implemented by types which provide an [`Iterator`] implementation of 137 /// [`BorrowedPage`] items, such as [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). 138 pub trait AsPageIter { 139 /// The [`Iterator`] type, e.g. [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). 140 type Iter<'a>: Iterator<Item = BorrowedPage<'a>> 141 where 142 Self: 'a; 143 144 /// Returns an [`Iterator`] of [`BorrowedPage`] items over all pages owned by `self`. 145 fn page_iter(&mut self) -> Self::Iter<'_>; 146 } 147 148 /// A pointer to a page that owns the page allocation. 149 /// 150 /// # Invariants 151 /// 152 /// The pointer is valid, and has ownership over the page. 153 pub struct Page { 154 page: NonNull<bindings::page>, 155 } 156 157 // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across 158 // threads is safe. 159 unsafe impl Send for Page {} 160 161 // SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing 162 // them concurrently is safe. 163 unsafe impl Sync for Page {} 164 165 impl Page { 166 /// Allocates a new page. 167 /// 168 /// # Examples 169 /// 170 /// Allocate memory for a page. 171 /// 172 /// ``` 173 /// use kernel::page::Page; 174 /// 175 /// let page = Page::alloc_page(GFP_KERNEL)?; 176 /// # Ok::<(), kernel::alloc::AllocError>(()) 177 /// ``` 178 /// 179 /// Allocate memory for a page and zero its contents. 180 /// 181 /// ``` 182 /// use kernel::page::Page; 183 /// 184 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; 185 /// # Ok::<(), kernel::alloc::AllocError>(()) 186 /// ``` 187 #[inline] 188 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> { 189 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it 190 // is always safe to call this method. 191 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; 192 let page = NonNull::new(page).ok_or(AllocError)?; 193 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly 194 // allocated page. We transfer that ownership to the new `Page` object. 195 Ok(Self { page }) 196 } 197 198 /// Returns a raw pointer to the page. 199 pub fn as_ptr(&self) -> *mut bindings::page { 200 self.page.as_ptr() 201 } 202 203 /// Get the node id containing this page. 204 pub fn nid(&self) -> i32 { 205 // SAFETY: Always safe to call with a valid page. 206 unsafe { bindings::page_to_nid(self.as_ptr()) } 207 } 208 209 /// Runs a piece of code with this page mapped to an address. 210 /// 211 /// The page is unmapped when this call returns. 212 /// 213 /// # Using the raw pointer 214 /// 215 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 216 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might 217 /// only be mapped on the current thread, and when that is the case, dereferencing it on other 218 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't 219 /// cause data races, the memory may be uninitialized, and so on. 220 /// 221 /// If multiple threads map the same page at the same time, then they may reference with 222 /// different addresses. However, even if the addresses are different, the underlying memory is 223 /// still the same for these purposes (e.g., it's still a data race if they both write to the 224 /// same underlying byte at the same time). 225 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T { 226 // SAFETY: `page` is valid due to the type invariants on `Page`. 227 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; 228 229 let res = f(mapped_addr.cast()); 230 231 // This unmaps the page mapped above. 232 // 233 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner 234 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. 235 // 236 // In other words, if this call to `kunmap_local` happens when a different page should be 237 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the 238 // call just above in `with_page_mapped` that made that possible. In this case, it is the 239 // unsafe block that wraps that other call that is incorrect. 240 unsafe { bindings::kunmap_local(mapped_addr) }; 241 242 res 243 } 244 245 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. 246 /// 247 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the 248 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on 249 /// this task, as this method uses a local mapping. 250 /// 251 /// If `off` and `len` refers to a region outside of this page, then this method returns 252 /// [`EINVAL`] and does not call `f`. 253 /// 254 /// # Using the raw pointer 255 /// 256 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 257 /// `len` bytes and for the duration in which the closure is called. The pointer might only be 258 /// mapped on the current thread, and when that is the case, dereferencing it on other threads 259 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause 260 /// data races, the memory may be uninitialized, and so on. 261 /// 262 /// If multiple threads map the same page at the same time, then they may reference with 263 /// different addresses. However, even if the addresses are different, the underlying memory is 264 /// still the same for these purposes (e.g., it's still a data race if they both write to the 265 /// same underlying byte at the same time). 266 fn with_pointer_into_page<T>( 267 &self, 268 off: usize, 269 len: usize, 270 f: impl FnOnce(*mut u8) -> Result<T>, 271 ) -> Result<T> { 272 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE; 273 274 if bounds_ok { 275 self.with_page_mapped(move |page_addr| { 276 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will 277 // result in a pointer that is in bounds or one off the end of the page. 278 f(unsafe { page_addr.add(off) }) 279 }) 280 } else { 281 Err(EINVAL) 282 } 283 } 284 285 /// Maps the page and reads from it into the given buffer. 286 /// 287 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 288 /// outside of the page, then this call returns [`EINVAL`]. 289 /// 290 /// # Safety 291 /// 292 /// * Callers must ensure that `dst` is valid for writing `len` bytes. 293 /// * Callers must ensure that this call does not race with a write to the same page that 294 /// overlaps with this read. 295 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { 296 self.with_pointer_into_page(offset, len, move |src| { 297 // SAFETY: If `with_pointer_into_page` calls into this closure, then 298 // it has performed a bounds check and guarantees that `src` is 299 // valid for `len` bytes. 300 // 301 // There caller guarantees that there is no data race. 302 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 303 Ok(()) 304 }) 305 } 306 307 /// Maps the page and writes into it from the given buffer. 308 /// 309 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 310 /// outside of the page, then this call returns [`EINVAL`]. 311 /// 312 /// # Safety 313 /// 314 /// * Callers must ensure that `src` is valid for reading `len` bytes. 315 /// * Callers must ensure that this call does not race with a read or write to the same page 316 /// that overlaps with this write. 317 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { 318 self.with_pointer_into_page(offset, len, move |dst| { 319 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 320 // bounds check and guarantees that `dst` is valid for `len` bytes. 321 // 322 // There caller guarantees that there is no data race. 323 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 324 Ok(()) 325 }) 326 } 327 328 /// Maps the page and zeroes the given slice. 329 /// 330 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 331 /// outside of the page, then this call returns [`EINVAL`]. 332 /// 333 /// # Safety 334 /// 335 /// Callers must ensure that this call does not race with a read or write to the same page that 336 /// overlaps with this write. 337 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { 338 self.with_pointer_into_page(offset, len, move |dst| { 339 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 340 // bounds check and guarantees that `dst` is valid for `len` bytes. 341 // 342 // There caller guarantees that there is no data race. 343 unsafe { ptr::write_bytes(dst, 0u8, len) }; 344 Ok(()) 345 }) 346 } 347 348 /// Copies data from userspace into this page. 349 /// 350 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 351 /// outside of the page, then this call returns [`EINVAL`]. 352 /// 353 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. 354 /// However, they are not allowed on the page you are copying into. 355 /// 356 /// # Safety 357 /// 358 /// Callers must ensure that this call does not race with a read or write to the same page that 359 /// overlaps with this write. 360 pub unsafe fn copy_from_user_slice_raw( 361 &self, 362 reader: &mut UserSliceReader, 363 offset: usize, 364 len: usize, 365 ) -> Result { 366 self.with_pointer_into_page(offset, len, move |dst| { 367 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 368 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have 369 // exclusive access to the slice since the caller guarantees that there are no races. 370 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) 371 }) 372 } 373 } 374 375 impl Drop for Page { 376 #[inline] 377 fn drop(&mut self) { 378 // SAFETY: By the type invariants, we have ownership of the page and can free it. 379 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; 380 } 381 } 382