1 // SPDX-License-Identifier: GPL-2.0 2 3 //! Kernel page allocation and management. 4 5 use crate::{ 6 alloc::{AllocError, Flags}, 7 bindings, 8 error::code::*, 9 error::Result, 10 uaccess::UserSliceReader, 11 }; 12 use core::ptr::{self, NonNull}; 13 14 /// A bitwise shift for the page size. 15 pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize; 16 17 /// The number of bytes in a page. 18 pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; 19 20 /// A bitmask that gives the page containing a given address. 21 pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); 22 23 /// A pointer to a page that owns the page allocation. 24 /// 25 /// # Invariants 26 /// 27 /// The pointer is valid, and has ownership over the page. 28 pub struct Page { 29 page: NonNull<bindings::page>, 30 } 31 32 // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across 33 // threads is safe. 34 unsafe impl Send for Page {} 35 36 // SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing 37 // them concurrently is safe. 38 unsafe impl Sync for Page {} 39 40 impl Page { 41 /// Allocates a new page. 42 /// 43 /// # Examples 44 /// 45 /// Allocate memory for a page. 46 /// 47 /// ``` 48 /// use kernel::page::Page; 49 /// 50 /// # fn dox() -> Result<(), kernel::alloc::AllocError> { 51 /// let page = Page::alloc_page(GFP_KERNEL)?; 52 /// # Ok(()) } 53 /// ``` 54 /// 55 /// Allocate memory for a page and zero its contents. 56 /// 57 /// ``` 58 /// use kernel::page::Page; 59 /// 60 /// # fn dox() -> Result<(), kernel::alloc::AllocError> { 61 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; 62 /// # Ok(()) } 63 /// ``` alloc_page(flags: Flags) -> Result<Self, AllocError>64 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> { 65 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it 66 // is always safe to call this method. 67 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; 68 let page = NonNull::new(page).ok_or(AllocError)?; 69 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly 70 // allocated page. We transfer that ownership to the new `Page` object. 71 Ok(Self { page }) 72 } 73 74 /// Returns a raw pointer to the page. as_ptr(&self) -> *mut bindings::page75 pub fn as_ptr(&self) -> *mut bindings::page { 76 self.page.as_ptr() 77 } 78 79 /// Runs a piece of code with this page mapped to an address. 80 /// 81 /// The page is unmapped when this call returns. 82 /// 83 /// # Using the raw pointer 84 /// 85 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 86 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might 87 /// only be mapped on the current thread, and when that is the case, dereferencing it on other 88 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't 89 /// cause data races, the memory may be uninitialized, and so on. 90 /// 91 /// If multiple threads map the same page at the same time, then they may reference with 92 /// different addresses. However, even if the addresses are different, the underlying memory is 93 /// still the same for these purposes (e.g., it's still a data race if they both write to the 94 /// same underlying byte at the same time). with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T95 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T { 96 // SAFETY: `page` is valid due to the type invariants on `Page`. 97 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; 98 99 let res = f(mapped_addr.cast()); 100 101 // This unmaps the page mapped above. 102 // 103 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner 104 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. 105 // 106 // In other words, if this call to `kunmap_local` happens when a different page should be 107 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the 108 // call just above in `with_page_mapped` that made that possible. In this case, it is the 109 // unsafe block that wraps that other call that is incorrect. 110 unsafe { bindings::kunmap_local(mapped_addr) }; 111 112 res 113 } 114 115 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. 116 /// 117 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the 118 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on 119 /// this task, as this method uses a local mapping. 120 /// 121 /// If `off` and `len` refers to a region outside of this page, then this method returns 122 /// [`EINVAL`] and does not call `f`. 123 /// 124 /// # Using the raw pointer 125 /// 126 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for 127 /// `len` bytes and for the duration in which the closure is called. The pointer might only be 128 /// mapped on the current thread, and when that is the case, dereferencing it on other threads 129 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause 130 /// data races, the memory may be uninitialized, and so on. 131 /// 132 /// If multiple threads map the same page at the same time, then they may reference with 133 /// different addresses. However, even if the addresses are different, the underlying memory is 134 /// still the same for these purposes (e.g., it's still a data race if they both write to the 135 /// same underlying byte at the same time). with_pointer_into_page<T>( &self, off: usize, len: usize, f: impl FnOnce(*mut u8) -> Result<T>, ) -> Result<T>136 fn with_pointer_into_page<T>( 137 &self, 138 off: usize, 139 len: usize, 140 f: impl FnOnce(*mut u8) -> Result<T>, 141 ) -> Result<T> { 142 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE; 143 144 if bounds_ok { 145 self.with_page_mapped(move |page_addr| { 146 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will 147 // result in a pointer that is in bounds or one off the end of the page. 148 f(unsafe { page_addr.add(off) }) 149 }) 150 } else { 151 Err(EINVAL) 152 } 153 } 154 155 /// Maps the page and reads from it into the given buffer. 156 /// 157 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 158 /// outside of the page, then this call returns [`EINVAL`]. 159 /// 160 /// # Safety 161 /// 162 /// * Callers must ensure that `dst` is valid for writing `len` bytes. 163 /// * Callers must ensure that this call does not race with a write to the same page that 164 /// overlaps with this read. read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result165 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { 166 self.with_pointer_into_page(offset, len, move |src| { 167 // SAFETY: If `with_pointer_into_page` calls into this closure, then 168 // it has performed a bounds check and guarantees that `src` is 169 // valid for `len` bytes. 170 // 171 // There caller guarantees that there is no data race. 172 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 173 Ok(()) 174 }) 175 } 176 177 /// Maps the page and writes into it from the given buffer. 178 /// 179 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 180 /// outside of the page, then this call returns [`EINVAL`]. 181 /// 182 /// # Safety 183 /// 184 /// * Callers must ensure that `src` is valid for reading `len` bytes. 185 /// * Callers must ensure that this call does not race with a read or write to the same page 186 /// that overlaps with this write. write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result187 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { 188 self.with_pointer_into_page(offset, len, move |dst| { 189 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 190 // bounds check and guarantees that `dst` is valid for `len` bytes. 191 // 192 // There caller guarantees that there is no data race. 193 unsafe { ptr::copy_nonoverlapping(src, dst, len) }; 194 Ok(()) 195 }) 196 } 197 198 /// Maps the page and zeroes the given slice. 199 /// 200 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 201 /// outside of the page, then this call returns [`EINVAL`]. 202 /// 203 /// # Safety 204 /// 205 /// Callers must ensure that this call does not race with a read or write to the same page that 206 /// overlaps with this write. fill_zero_raw(&self, offset: usize, len: usize) -> Result207 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { 208 self.with_pointer_into_page(offset, len, move |dst| { 209 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 210 // bounds check and guarantees that `dst` is valid for `len` bytes. 211 // 212 // There caller guarantees that there is no data race. 213 unsafe { ptr::write_bytes(dst, 0u8, len) }; 214 Ok(()) 215 }) 216 } 217 218 /// Copies data from userspace into this page. 219 /// 220 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes 221 /// outside of the page, then this call returns [`EINVAL`]. 222 /// 223 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. 224 /// However, they are not allowed on the page you are copying into. 225 /// 226 /// # Safety 227 /// 228 /// Callers must ensure that this call does not race with a read or write to the same page that 229 /// overlaps with this write. copy_from_user_slice_raw( &self, reader: &mut UserSliceReader, offset: usize, len: usize, ) -> Result230 pub unsafe fn copy_from_user_slice_raw( 231 &self, 232 reader: &mut UserSliceReader, 233 offset: usize, 234 len: usize, 235 ) -> Result { 236 self.with_pointer_into_page(offset, len, move |dst| { 237 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a 238 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have 239 // exclusive access to the slice since the caller guarantees that there are no races. 240 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) 241 }) 242 } 243 } 244 245 impl Drop for Page { drop(&mut self)246 fn drop(&mut self) { 247 // SAFETY: By the type invariants, we have ownership of the page and can free it. 248 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; 249 } 250 } 251