1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Kernel page allocation and management. 4 5 use crate::{ 6 alloc::{AllocError, Flags}, 7 bindings, 8 error::code::*, 9 error::Result, 10 uaccess::UserSliceReader, 11 }; 12 use core::{ 13 marker::PhantomData, 14 mem::ManuallyDrop, 15 ops::Deref, 16 ptr::{self, NonNull}, 17 }; 18 19 /// A bitwise shift for the page size. 20 pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize; 21 22 /// The number of bytes in a page. 23 pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; 24 25 /// A bitmask that gives the page containing a given address. 26 pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); 27 28 /// Round up the given number to the next multiple of [`PAGE_SIZE`]. 29 /// 30 /// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a 31 /// [`usize`]. 32 pub const fn page_align(addr: usize) -> usize { 33 // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong 34 // cases. 35 (addr + (PAGE_SIZE - 1)) & PAGE_MASK 36 } 37 38 /// Representation of a non-owning reference to a [`Page`]. 39 /// 40 /// This type provides a borrowed version of a [`Page`] that is owned by some other entity, e.g. a 41 /// [`Vmalloc`] allocation such as [`VBox`]. 42 /// 43 /// # Example 44 /// 45 /// ``` 46 /// # use kernel::{bindings, prelude::*}; 47 /// use kernel::page::{BorrowedPage, Page, PAGE_SIZE}; 48 /// # use core::{mem::MaybeUninit, ptr, ptr::NonNull }; 49 /// 50 /// fn borrow_page<'a>(vbox: &'a mut VBox<MaybeUninit<[u8; PAGE_SIZE]>>) -> BorrowedPage<'a> { 51 /// let ptr = ptr::from_ref(&**vbox); 52 /// 53 /// // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory. 54 /// let page = unsafe { bindings::vmalloc_to_page(ptr.cast()) }; 55 /// 56 /// // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid 57 /// // pointer to `Vmalloc` memory. 58 /// let page = unsafe { NonNull::new_unchecked(page) }; 59 /// 60 /// // SAFETY: 61 /// // - `self.0` is a valid pointer to a `struct page`. 62 /// // - `self.0` is valid for the entire lifetime of `self`. 63 /// unsafe { BorrowedPage::from_raw(page) } 64 /// } 65 /// 66 /// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; 67 /// let page = borrow_page(&mut vbox); 68 /// 69 /// // SAFETY: There is no concurrent read or write to this page. 70 /// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; 71 /// # Ok::<(), Error>(()) 72 /// ``` 73 /// 74 /// # Invariants 75 /// 76 /// The borrowed underlying pointer to a `struct page` is valid for the entire lifetime `'a`. 77 /// 78 /// [`VBox`]: kernel::alloc::VBox 79 /// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc 80 pub struct BorrowedPage<'a>(ManuallyDrop<Page>, PhantomData<&'a Page>); 81 82 impl<'a> BorrowedPage<'a> { 83 /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. 84 /// 85 /// # Safety 86 /// 87 /// - `ptr` must point to a valid `bindings::page`. 88 /// - `ptr` must remain valid for the entire lifetime `'a`. 89 pub unsafe fn from_raw(ptr: NonNull<bindings::page>) -> Self { 90 let page = Page { page: ptr }; 91 92 // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime 93 // `'a`. 94 Self(ManuallyDrop::new(page), PhantomData) 95 } 96 } 97 98 impl<'a> Deref for BorrowedPage<'a> { 99 type Target = Page; 100 101 fn deref(&self) -> &Self::Target { 102 &self.0 103 } 104 } 105 106 /// Trait to be implemented by types which provide an [`Iterator`] implementation of 107 /// [`BorrowedPage`] items, such as [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). 108 pub trait AsPageIter { 109 /// The [`Iterator`] type, e.g. [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). 110 type Iter<'a>: Iterator<Item = BorrowedPage<'a>> 111 where 112 Self: 'a; 113 114 /// Returns an [`Iterator`] of [`BorrowedPage`] items over all pages owned by `self`. 115 fn page_iter(&mut self) -> Self::Iter<'_>; 116 } 117 118 /// A pointer to a page that owns the page allocation. 119 /// 120 /// # Invariants 121 /// 122 /// The pointer is valid, and has ownership over the page. 123 pub struct Page { 124 page: NonNull<bindings::page>, 125 } 126 127 // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across 128 // threads is safe. 129 unsafe impl Send for Page {} 130 131 // SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing 132 // them concurrently is safe. 133 unsafe impl Sync for Page {} 134 135 impl Page { 136 /// Allocates a new page. 137 /// 138 /// # Examples 139 /// 140 /// Allocate memory for a page. 141 /// 142 /// ``` 143 /// use kernel::page::Page; 144 /// 145 /// let page = Page::alloc_page(GFP_KERNEL)?; 146 /// # Ok::<(), kernel::alloc::AllocError>(()) 147 /// ``` 148 /// 149 /// Allocate memory for a page and zero its contents. 150 /// 151 /// ``` 152 /// use kernel::page::Page; 153 /// 154 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; 155 /// # Ok::<(), kernel::alloc::AllocError>(()) 156 /// ``` 157 #[inline] 158 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> { 159 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it 160 // is always safe to call this method. 161 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; 162 let page = NonNull::new(page).ok_or(AllocError)?; 163 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly 164 // allocated page. We transfer that ownership to the new `Page` object. 165 Ok(Self { page }) 166 } 167 168 /// Returns a raw pointer to the page. 169 pub fn as_ptr(&self) -> *mut bindings::page { 170 self.page.as_ptr() 171 } 172 173 /// Get the node id containing this page. 174 pub fn nid(&self) -> i32 { 175 // SAFETY: Always safe to call with a valid page. 176 unsafe { bindings::page_to_nid(self.as_ptr()) } 177 } 178 179 /// Runs a piece of code with this page mapped to an address. 180 /// 181 /// The page is unmapped when this call returns. 182 /// 183 /// # Using the raw pointer 184 /// 185 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 186 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might 187 /// only be mapped on the current thread, and when that is the case, dereferencing it on other 188 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't 189 /// cause data races, the memory may be uninitialized, and so on. 190 /// 191 /// If multiple threads map the same page at the same time, then they may reference with 192 /// different addresses. However, even if the addresses are different, the underlying memory is 193 /// still the same for these purposes (e.g., it's still a data race if they both write to the 194 /// same underlying byte at the same time). 195 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T { 196 // SAFETY: `page` is valid due to the type invariants on `Page`. 197 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; 198 199 let res = f(mapped_addr.cast()); 200 201 // This unmaps the page mapped above. 202 // 203 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner 204 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. 205 // 206 // In other words, if this call to `kunmap_local` happens when a different page should be 207 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the 208 // call just above in `with_page_mapped` that made that possible. In this case, it is the 209 // unsafe block that wraps that other call that is incorrect. 210 unsafe { bindings::kunmap_local(mapped_addr) }; 211 212 res 213 } 214 215 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. 216 /// 217 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the 218 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on 219 /// this task, as this method uses a local mapping. 220 /// 221 /// If `off` and `len` refers to a region outside of this page, then this method returns 222 /// [`EINVAL`] and does not call `f`. 223 /// 224 /// # Using the raw pointer 225 /// 226 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 227 /// `len` bytes and for the duration in which the closure is called. The pointer might only be 228 /// mapped on the current thread, and when that is the case, dereferencing it on other threads 229 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause 230 /// data races, the memory may be uninitialized, and so on. 231 /// 232 /// If multiple threads map the same page at the same time, then they may reference with 233 /// different addresses. However, even if the addresses are different, the underlying memory is 234 /// still the same for these purposes (e.g., it's still a data race if they both write to the 235 /// same underlying byte at the same time). 236 fn with_pointer_into_page<T>( 237 &self, 238 off: usize, 239 len: usize, 240 f: impl FnOnce(*mut u8) -> Result<T>, 241 ) -> Result<T> { 242 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE; 243 244 if bounds_ok { 245 self.with_page_mapped(move |page_addr| { 246 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will 247 // result in a pointer that is in bounds or one off the end of the page. 248 f(unsafe { page_addr.add(off) }) 249 }) 250 } else { 251 Err(EINVAL) 252 } 253 } 254 255 /// Maps the page and reads from it into the given buffer. 256 /// 257 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 258 /// outside of the page, then this call returns [`EINVAL`]. 259 /// 260 /// # Safety 261 /// 262 /// * Callers must ensure that `dst` is valid for writing `len` bytes. 263 /// * Callers must ensure that this call does not race with a write to the same page that 264 /// overlaps with this read. 265 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { 266 self.with_pointer_into_page(offset, len, move |src| { 267 // SAFETY: If `with_pointer_into_page` calls into this closure, then 268 // it has performed a bounds check and guarantees that `src` is 269 // valid for `len` bytes. 270 // 271 // There caller guarantees that there is no data race. 272 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 273 Ok(()) 274 }) 275 } 276 277 /// Maps the page and writes into it from the given buffer. 278 /// 279 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 280 /// outside of the page, then this call returns [`EINVAL`]. 281 /// 282 /// # Safety 283 /// 284 /// * Callers must ensure that `src` is valid for reading `len` bytes. 285 /// * Callers must ensure that this call does not race with a read or write to the same page 286 /// that overlaps with this write. 287 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { 288 self.with_pointer_into_page(offset, len, move |dst| { 289 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 290 // bounds check and guarantees that `dst` is valid for `len` bytes. 291 // 292 // There caller guarantees that there is no data race. 293 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 294 Ok(()) 295 }) 296 } 297 298 /// Maps the page and zeroes the given slice. 299 /// 300 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 301 /// outside of the page, then this call returns [`EINVAL`]. 302 /// 303 /// # Safety 304 /// 305 /// Callers must ensure that this call does not race with a read or write to the same page that 306 /// overlaps with this write. 307 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { 308 self.with_pointer_into_page(offset, len, move |dst| { 309 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 310 // bounds check and guarantees that `dst` is valid for `len` bytes. 311 // 312 // There caller guarantees that there is no data race. 313 unsafe { ptr::write_bytes(dst, 0u8, len) }; 314 Ok(()) 315 }) 316 } 317 318 /// Copies data from userspace into this page. 319 /// 320 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 321 /// outside of the page, then this call returns [`EINVAL`]. 322 /// 323 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. 324 /// However, they are not allowed on the page you are copying into. 325 /// 326 /// # Safety 327 /// 328 /// Callers must ensure that this call does not race with a read or write to the same page that 329 /// overlaps with this write. 330 pub unsafe fn copy_from_user_slice_raw( 331 &self, 332 reader: &mut UserSliceReader, 333 offset: usize, 334 len: usize, 335 ) -> Result { 336 self.with_pointer_into_page(offset, len, move |dst| { 337 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 338 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have 339 // exclusive access to the slice since the caller guarantees that there are no races. 340 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) 341 }) 342 } 343 } 344 345 impl Drop for Page { 346 #[inline] 347 fn drop(&mut self) { 348 // SAFETY: By the type invariants, we have ownership of the page and can free it. 349 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; 350 } 351 } 352