1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Kernel page allocation and management. 4 5 use crate::{ 6 alloc::{AllocError, Flags}, 7 bindings, 8 error::code::*, 9 error::Result, 10 uaccess::UserSliceReader, 11 }; 12 use core::{ 13 marker::PhantomData, 14 mem::ManuallyDrop, 15 ops::Deref, 16 ptr::{self, NonNull}, 17 }; 18 19 /// A bitwise shift for the page size. 20 pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize; 21 22 /// The number of bytes in a page. 23 pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; 24 25 /// A bitmask that gives the page containing a given address. 26 pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); 27 28 /// Round up the given number to the next multiple of [`PAGE_SIZE`]. 29 /// 30 /// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a 31 /// [`usize`]. 32 pub const fn page_align(addr: usize) -> usize { 33 // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong 34 // cases. 35 (addr + (PAGE_SIZE - 1)) & PAGE_MASK 36 } 37 38 /// Representation of a non-owning reference to a [`Page`]. 39 /// 40 /// This type provides a borrowed version of a [`Page`] that is owned by some other entity, e.g. a 41 /// [`Vmalloc`] allocation such as [`VBox`]. 42 /// 43 /// # Example 44 /// 45 /// ``` 46 /// # use kernel::{bindings, prelude::*}; 47 /// use kernel::page::{BorrowedPage, Page, PAGE_SIZE}; 48 /// # use core::{mem::MaybeUninit, ptr, ptr::NonNull }; 49 /// 50 /// fn borrow_page<'a>(vbox: &'a mut VBox<MaybeUninit<[u8; PAGE_SIZE]>>) -> BorrowedPage<'a> { 51 /// let ptr = ptr::from_ref(&**vbox); 52 /// 53 /// // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory. 54 /// let page = unsafe { bindings::vmalloc_to_page(ptr.cast()) }; 55 /// 56 /// // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid 57 /// // pointer to `Vmalloc` memory. 58 /// let page = unsafe { NonNull::new_unchecked(page) }; 59 /// 60 /// // SAFETY: 61 /// // - `self.0` is a valid pointer to a `struct page`. 62 /// // - `self.0` is valid for the entire lifetime of `self`. 63 /// unsafe { BorrowedPage::from_raw(page) } 64 /// } 65 /// 66 /// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; 67 /// let page = borrow_page(&mut vbox); 68 /// 69 /// // SAFETY: There is no concurrent read or write to this page. 70 /// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? }; 71 /// # Ok::<(), Error>(()) 72 /// ``` 73 /// 74 /// # Invariants 75 /// 76 /// The borrowed underlying pointer to a `struct page` is valid for the entire lifetime `'a`. 77 /// 78 /// [`VBox`]: kernel::alloc::VBox 79 /// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc 80 pub struct BorrowedPage<'a>(ManuallyDrop<Page>, PhantomData<&'a Page>); 81 82 impl<'a> BorrowedPage<'a> { 83 /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. 84 /// 85 /// # Safety 86 /// 87 /// - `ptr` must point to a valid `bindings::page`. 88 /// - `ptr` must remain valid for the entire lifetime `'a`. 89 pub unsafe fn from_raw(ptr: NonNull<bindings::page>) -> Self { 90 let page = Page { page: ptr }; 91 92 // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime 93 // `'a`. 94 Self(ManuallyDrop::new(page), PhantomData) 95 } 96 } 97 98 impl<'a> Deref for BorrowedPage<'a> { 99 type Target = Page; 100 101 fn deref(&self) -> &Self::Target { 102 &self.0 103 } 104 } 105 106 /// Trait to be implemented by types which provide an [`Iterator`] implementation of 107 /// [`BorrowedPage`] items, such as [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). 108 pub trait AsPageIter { 109 /// The [`Iterator`] type, e.g. [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). 110 type Iter<'a>: Iterator<Item = BorrowedPage<'a>> 111 where 112 Self: 'a; 113 114 /// Returns an [`Iterator`] of [`BorrowedPage`] items over all pages owned by `self`. 115 fn page_iter(&mut self) -> Self::Iter<'_>; 116 } 117 118 /// A pointer to a page that owns the page allocation. 119 /// 120 /// # Invariants 121 /// 122 /// The pointer is valid, and has ownership over the page. 123 pub struct Page { 124 page: NonNull<bindings::page>, 125 } 126 127 // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across 128 // threads is safe. 129 unsafe impl Send for Page {} 130 131 // SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing 132 // them concurrently is safe. 133 unsafe impl Sync for Page {} 134 135 impl Page { 136 /// Allocates a new page. 137 /// 138 /// # Examples 139 /// 140 /// Allocate memory for a page. 141 /// 142 /// ``` 143 /// use kernel::page::Page; 144 /// 145 /// let page = Page::alloc_page(GFP_KERNEL)?; 146 /// # Ok::<(), kernel::alloc::AllocError>(()) 147 /// ``` 148 /// 149 /// Allocate memory for a page and zero its contents. 150 /// 151 /// ``` 152 /// use kernel::page::Page; 153 /// 154 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; 155 /// # Ok::<(), kernel::alloc::AllocError>(()) 156 /// ``` 157 #[inline] 158 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> { 159 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it 160 // is always safe to call this method. 161 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; 162 let page = NonNull::new(page).ok_or(AllocError)?; 163 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly 164 // allocated page. We transfer that ownership to the new `Page` object. 165 Ok(Self { page }) 166 } 167 168 /// Returns a raw pointer to the page. 169 pub fn as_ptr(&self) -> *mut bindings::page { 170 self.page.as_ptr() 171 } 172 173 /// Runs a piece of code with this page mapped to an address. 174 /// 175 /// The page is unmapped when this call returns. 176 /// 177 /// # Using the raw pointer 178 /// 179 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 180 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might 181 /// only be mapped on the current thread, and when that is the case, dereferencing it on other 182 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't 183 /// cause data races, the memory may be uninitialized, and so on. 184 /// 185 /// If multiple threads map the same page at the same time, then they may reference with 186 /// different addresses. However, even if the addresses are different, the underlying memory is 187 /// still the same for these purposes (e.g., it's still a data race if they both write to the 188 /// same underlying byte at the same time). 189 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T { 190 // SAFETY: `page` is valid due to the type invariants on `Page`. 191 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; 192 193 let res = f(mapped_addr.cast()); 194 195 // This unmaps the page mapped above. 196 // 197 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner 198 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. 199 // 200 // In other words, if this call to `kunmap_local` happens when a different page should be 201 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the 202 // call just above in `with_page_mapped` that made that possible. In this case, it is the 203 // unsafe block that wraps that other call that is incorrect. 204 unsafe { bindings::kunmap_local(mapped_addr) }; 205 206 res 207 } 208 209 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. 210 /// 211 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the 212 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on 213 /// this task, as this method uses a local mapping. 214 /// 215 /// If `off` and `len` refers to a region outside of this page, then this method returns 216 /// [`EINVAL`] and does not call `f`. 217 /// 218 /// # Using the raw pointer 219 /// 220 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 221 /// `len` bytes and for the duration in which the closure is called. The pointer might only be 222 /// mapped on the current thread, and when that is the case, dereferencing it on other threads 223 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause 224 /// data races, the memory may be uninitialized, and so on. 225 /// 226 /// If multiple threads map the same page at the same time, then they may reference with 227 /// different addresses. However, even if the addresses are different, the underlying memory is 228 /// still the same for these purposes (e.g., it's still a data race if they both write to the 229 /// same underlying byte at the same time). 230 fn with_pointer_into_page<T>( 231 &self, 232 off: usize, 233 len: usize, 234 f: impl FnOnce(*mut u8) -> Result<T>, 235 ) -> Result<T> { 236 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE; 237 238 if bounds_ok { 239 self.with_page_mapped(move |page_addr| { 240 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will 241 // result in a pointer that is in bounds or one off the end of the page. 242 f(unsafe { page_addr.add(off) }) 243 }) 244 } else { 245 Err(EINVAL) 246 } 247 } 248 249 /// Maps the page and reads from it into the given buffer. 250 /// 251 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 252 /// outside of the page, then this call returns [`EINVAL`]. 253 /// 254 /// # Safety 255 /// 256 /// * Callers must ensure that `dst` is valid for writing `len` bytes. 257 /// * Callers must ensure that this call does not race with a write to the same page that 258 /// overlaps with this read. 259 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { 260 self.with_pointer_into_page(offset, len, move |src| { 261 // SAFETY: If `with_pointer_into_page` calls into this closure, then 262 // it has performed a bounds check and guarantees that `src` is 263 // valid for `len` bytes. 264 // 265 // There caller guarantees that there is no data race. 266 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 267 Ok(()) 268 }) 269 } 270 271 /// Maps the page and writes into it from the given buffer. 272 /// 273 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 274 /// outside of the page, then this call returns [`EINVAL`]. 275 /// 276 /// # Safety 277 /// 278 /// * Callers must ensure that `src` is valid for reading `len` bytes. 279 /// * Callers must ensure that this call does not race with a read or write to the same page 280 /// that overlaps with this write. 281 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { 282 self.with_pointer_into_page(offset, len, move |dst| { 283 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 284 // bounds check and guarantees that `dst` is valid for `len` bytes. 285 // 286 // There caller guarantees that there is no data race. 287 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 288 Ok(()) 289 }) 290 } 291 292 /// Maps the page and zeroes the given slice. 293 /// 294 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 295 /// outside of the page, then this call returns [`EINVAL`]. 296 /// 297 /// # Safety 298 /// 299 /// Callers must ensure that this call does not race with a read or write to the same page that 300 /// overlaps with this write. 301 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { 302 self.with_pointer_into_page(offset, len, move |dst| { 303 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 304 // bounds check and guarantees that `dst` is valid for `len` bytes. 305 // 306 // There caller guarantees that there is no data race. 307 unsafe { ptr::write_bytes(dst, 0u8, len) }; 308 Ok(()) 309 }) 310 } 311 312 /// Copies data from userspace into this page. 313 /// 314 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 315 /// outside of the page, then this call returns [`EINVAL`]. 316 /// 317 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. 318 /// However, they are not allowed on the page you are copying into. 319 /// 320 /// # Safety 321 /// 322 /// Callers must ensure that this call does not race with a read or write to the same page that 323 /// overlaps with this write. 324 pub unsafe fn copy_from_user_slice_raw( 325 &self, 326 reader: &mut UserSliceReader, 327 offset: usize, 328 len: usize, 329 ) -> Result { 330 self.with_pointer_into_page(offset, len, move |dst| { 331 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 332 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have 333 // exclusive access to the slice since the caller guarantees that there are no races. 334 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) 335 }) 336 } 337 } 338 339 impl Drop for Page { 340 #[inline] 341 fn drop(&mut self) { 342 // SAFETY: By the type invariants, we have ownership of the page and can free it. 343 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; 344 } 345 } 346