1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Kernel page allocation and management. 4 5 use crate::{ 6 alloc::{AllocError, Flags}, 7 bindings, 8 error::code::*, 9 error::Result, 10 uaccess::UserSliceReader, 11 }; 12 use core::ptr::{self, NonNull}; 13 14 /// A bitwise shift for the page size. 15 pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize; 16 17 /// The number of bytes in a page. 18 pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; 19 20 /// A bitmask that gives the page containing a given address. 21 pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); 22 23 /// Round up the given number to the next multiple of [`PAGE_SIZE`]. 24 /// 25 /// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a 26 /// [`usize`]. 27 pub const fn page_align(addr: usize) -> usize { 28 // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong 29 // cases. 30 (addr + (PAGE_SIZE - 1)) & PAGE_MASK 31 } 32 33 /// A pointer to a page that owns the page allocation. 34 /// 35 /// # Invariants 36 /// 37 /// The pointer is valid, and has ownership over the page. 38 pub struct Page { 39 page: NonNull<bindings::page>, 40 } 41 42 // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across 43 // threads is safe. 44 unsafe impl Send for Page {} 45 46 // SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing 47 // them concurrently is safe. 48 unsafe impl Sync for Page {} 49 50 impl Page { 51 /// Allocates a new page. 52 /// 53 /// # Examples 54 /// 55 /// Allocate memory for a page. 56 /// 57 /// ``` 58 /// use kernel::page::Page; 59 /// 60 /// let page = Page::alloc_page(GFP_KERNEL)?; 61 /// # Ok::<(), kernel::alloc::AllocError>(()) 62 /// ``` 63 /// 64 /// Allocate memory for a page and zero its contents. 65 /// 66 /// ``` 67 /// use kernel::page::Page; 68 /// 69 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; 70 /// # Ok::<(), kernel::alloc::AllocError>(()) 71 /// ``` 72 #[inline] 73 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> { 74 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it 75 // is always safe to call this method. 76 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; 77 let page = NonNull::new(page).ok_or(AllocError)?; 78 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly 79 // allocated page. We transfer that ownership to the new `Page` object. 80 Ok(Self { page }) 81 } 82 83 /// Returns a raw pointer to the page. 84 pub fn as_ptr(&self) -> *mut bindings::page { 85 self.page.as_ptr() 86 } 87 88 /// Get the node id containing this page. 89 pub fn nid(&self) -> i32 { 90 // SAFETY: Always safe to call with a valid page. 91 unsafe { bindings::page_to_nid(self.as_ptr()) } 92 } 93 94 /// Runs a piece of code with this page mapped to an address. 95 /// 96 /// The page is unmapped when this call returns. 97 /// 98 /// # Using the raw pointer 99 /// 100 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 101 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might 102 /// only be mapped on the current thread, and when that is the case, dereferencing it on other 103 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't 104 /// cause data races, the memory may be uninitialized, and so on. 105 /// 106 /// If multiple threads map the same page at the same time, then they may reference with 107 /// different addresses. However, even if the addresses are different, the underlying memory is 108 /// still the same for these purposes (e.g., it's still a data race if they both write to the 109 /// same underlying byte at the same time). 110 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T { 111 // SAFETY: `page` is valid due to the type invariants on `Page`. 112 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; 113 114 let res = f(mapped_addr.cast()); 115 116 // This unmaps the page mapped above. 117 // 118 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner 119 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. 120 // 121 // In other words, if this call to `kunmap_local` happens when a different page should be 122 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the 123 // call just above in `with_page_mapped` that made that possible. In this case, it is the 124 // unsafe block that wraps that other call that is incorrect. 125 unsafe { bindings::kunmap_local(mapped_addr) }; 126 127 res 128 } 129 130 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. 131 /// 132 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the 133 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on 134 /// this task, as this method uses a local mapping. 135 /// 136 /// If `off` and `len` refers to a region outside of this page, then this method returns 137 /// [`EINVAL`] and does not call `f`. 138 /// 139 /// # Using the raw pointer 140 /// 141 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 142 /// `len` bytes and for the duration in which the closure is called. The pointer might only be 143 /// mapped on the current thread, and when that is the case, dereferencing it on other threads 144 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause 145 /// data races, the memory may be uninitialized, and so on. 146 /// 147 /// If multiple threads map the same page at the same time, then they may reference with 148 /// different addresses. However, even if the addresses are different, the underlying memory is 149 /// still the same for these purposes (e.g., it's still a data race if they both write to the 150 /// same underlying byte at the same time). 151 fn with_pointer_into_page<T>( 152 &self, 153 off: usize, 154 len: usize, 155 f: impl FnOnce(*mut u8) -> Result<T>, 156 ) -> Result<T> { 157 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE; 158 159 if bounds_ok { 160 self.with_page_mapped(move |page_addr| { 161 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will 162 // result in a pointer that is in bounds or one off the end of the page. 163 f(unsafe { page_addr.add(off) }) 164 }) 165 } else { 166 Err(EINVAL) 167 } 168 } 169 170 /// Maps the page and reads from it into the given buffer. 171 /// 172 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 173 /// outside of the page, then this call returns [`EINVAL`]. 174 /// 175 /// # Safety 176 /// 177 /// * Callers must ensure that `dst` is valid for writing `len` bytes. 178 /// * Callers must ensure that this call does not race with a write to the same page that 179 /// overlaps with this read. 180 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { 181 self.with_pointer_into_page(offset, len, move |src| { 182 // SAFETY: If `with_pointer_into_page` calls into this closure, then 183 // it has performed a bounds check and guarantees that `src` is 184 // valid for `len` bytes. 185 // 186 // There caller guarantees that there is no data race. 187 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 188 Ok(()) 189 }) 190 } 191 192 /// Maps the page and writes into it from the given buffer. 193 /// 194 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 195 /// outside of the page, then this call returns [`EINVAL`]. 196 /// 197 /// # Safety 198 /// 199 /// * Callers must ensure that `src` is valid for reading `len` bytes. 200 /// * Callers must ensure that this call does not race with a read or write to the same page 201 /// that overlaps with this write. 202 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { 203 self.with_pointer_into_page(offset, len, move |dst| { 204 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 205 // bounds check and guarantees that `dst` is valid for `len` bytes. 206 // 207 // There caller guarantees that there is no data race. 208 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 209 Ok(()) 210 }) 211 } 212 213 /// Maps the page and zeroes the given slice. 214 /// 215 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 216 /// outside of the page, then this call returns [`EINVAL`]. 217 /// 218 /// # Safety 219 /// 220 /// Callers must ensure that this call does not race with a read or write to the same page that 221 /// overlaps with this write. 222 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { 223 self.with_pointer_into_page(offset, len, move |dst| { 224 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 225 // bounds check and guarantees that `dst` is valid for `len` bytes. 226 // 227 // There caller guarantees that there is no data race. 228 unsafe { ptr::write_bytes(dst, 0u8, len) }; 229 Ok(()) 230 }) 231 } 232 233 /// Copies data from userspace into this page. 234 /// 235 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 236 /// outside of the page, then this call returns [`EINVAL`]. 237 /// 238 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. 239 /// However, they are not allowed on the page you are copying into. 240 /// 241 /// # Safety 242 /// 243 /// Callers must ensure that this call does not race with a read or write to the same page that 244 /// overlaps with this write. 245 pub unsafe fn copy_from_user_slice_raw( 246 &self, 247 reader: &mut UserSliceReader, 248 offset: usize, 249 len: usize, 250 ) -> Result { 251 self.with_pointer_into_page(offset, len, move |dst| { 252 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 253 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have 254 // exclusive access to the slice since the caller guarantees that there are no races. 255 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) 256 }) 257 } 258 } 259 260 impl Drop for Page { 261 #[inline] 262 fn drop(&mut self) { 263 // SAFETY: By the type invariants, we have ownership of the page and can free it. 264 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; 265 } 266 } 267