1 // SPDX-License-Identifier: GPL-2.0
2
3 //! Kernel page allocation and management.
4
5 use crate::{
6 alloc::{AllocError, Flags},
7 bindings,
8 error::code::*,
9 error::Result,
10 uaccess::UserSliceReader,
11 };
12 use core::ptr::{self, NonNull};
13
14 /// A bitwise shift for the page size.
15 pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;
16
17 /// The number of bytes in a page.
18 pub const PAGE_SIZE: usize = bindings::PAGE_SIZE;
19
20 /// A bitmask that gives the page containing a given address.
21 pub const PAGE_MASK: usize = !(PAGE_SIZE - 1);
22
23 /// Round up the given number to the next multiple of [`PAGE_SIZE`].
24 ///
25 /// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a
26 /// [`usize`].
page_align(addr: usize) -> usize27 pub const fn page_align(addr: usize) -> usize {
28 // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong
29 // cases.
30 (addr + (PAGE_SIZE - 1)) & PAGE_MASK
31 }
32
33 /// A pointer to a page that owns the page allocation.
34 ///
35 /// # Invariants
36 ///
37 /// The pointer is valid, and has ownership over the page.
38 pub struct Page {
39 page: NonNull<bindings::page>,
40 }
41
42 // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across
43 // threads is safe.
44 unsafe impl Send for Page {}
45
46 // SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing
47 // them concurrently is safe.
48 unsafe impl Sync for Page {}
49
50 impl Page {
51 /// Allocates a new page.
52 ///
53 /// # Examples
54 ///
55 /// Allocate memory for a page.
56 ///
57 /// ```
58 /// use kernel::page::Page;
59 ///
60 /// let page = Page::alloc_page(GFP_KERNEL)?;
61 /// # Ok::<(), kernel::alloc::AllocError>(())
62 /// ```
63 ///
64 /// Allocate memory for a page and zero its contents.
65 ///
66 /// ```
67 /// use kernel::page::Page;
68 ///
69 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?;
70 /// # Ok::<(), kernel::alloc::AllocError>(())
71 /// ```
72 #[inline]
alloc_page(flags: Flags) -> Result<Self, AllocError>73 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> {
74 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it
75 // is always safe to call this method.
76 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) };
77 let page = NonNull::new(page).ok_or(AllocError)?;
78 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly
79 // allocated page. We transfer that ownership to the new `Page` object.
80 Ok(Self { page })
81 }
82
83 /// Returns a raw pointer to the page.
as_ptr(&self) -> *mut bindings::page84 pub fn as_ptr(&self) -> *mut bindings::page {
85 self.page.as_ptr()
86 }
87
88 /// Runs a piece of code with this page mapped to an address.
89 ///
90 /// The page is unmapped when this call returns.
91 ///
92 /// # Using the raw pointer
93 ///
94 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
95 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might
96 /// only be mapped on the current thread, and when that is the case, dereferencing it on other
97 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't
98 /// cause data races, the memory may be uninitialized, and so on.
99 ///
100 /// If multiple threads map the same page at the same time, then they may reference with
101 /// different addresses. However, even if the addresses are different, the underlying memory is
102 /// still the same for these purposes (e.g., it's still a data race if they both write to the
103 /// same underlying byte at the same time).
with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T104 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T {
105 // SAFETY: `page` is valid due to the type invariants on `Page`.
106 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };
107
108 let res = f(mapped_addr.cast());
109
110 // This unmaps the page mapped above.
111 //
112 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner
113 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`.
114 //
115 // In other words, if this call to `kunmap_local` happens when a different page should be
116 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the
117 // call just above in `with_page_mapped` that made that possible. In this case, it is the
118 // unsafe block that wraps that other call that is incorrect.
119 unsafe { bindings::kunmap_local(mapped_addr) };
120
121 res
122 }
123
124 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking.
125 ///
126 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the
127 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on
128 /// this task, as this method uses a local mapping.
129 ///
130 /// If `off` and `len` refers to a region outside of this page, then this method returns
131 /// [`EINVAL`] and does not call `f`.
132 ///
133 /// # Using the raw pointer
134 ///
135 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
136 /// `len` bytes and for the duration in which the closure is called. The pointer might only be
137 /// mapped on the current thread, and when that is the case, dereferencing it on other threads
138 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause
139 /// data races, the memory may be uninitialized, and so on.
140 ///
141 /// If multiple threads map the same page at the same time, then they may reference with
142 /// different addresses. However, even if the addresses are different, the underlying memory is
143 /// still the same for these purposes (e.g., it's still a data race if they both write to the
144 /// same underlying byte at the same time).
with_pointer_into_page<T>( &self, off: usize, len: usize, f: impl FnOnce(*mut u8) -> Result<T>, ) -> Result<T>145 fn with_pointer_into_page<T>(
146 &self,
147 off: usize,
148 len: usize,
149 f: impl FnOnce(*mut u8) -> Result<T>,
150 ) -> Result<T> {
151 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;
152
153 if bounds_ok {
154 self.with_page_mapped(move |page_addr| {
155 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will
156 // result in a pointer that is in bounds or one off the end of the page.
157 f(unsafe { page_addr.add(off) })
158 })
159 } else {
160 Err(EINVAL)
161 }
162 }
163
164 /// Maps the page and reads from it into the given buffer.
165 ///
166 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
167 /// outside of the page, then this call returns [`EINVAL`].
168 ///
169 /// # Safety
170 ///
171 /// * Callers must ensure that `dst` is valid for writing `len` bytes.
172 /// * Callers must ensure that this call does not race with a write to the same page that
173 /// overlaps with this read.
read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result174 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result {
175 self.with_pointer_into_page(offset, len, move |src| {
176 // SAFETY: If `with_pointer_into_page` calls into this closure, then
177 // it has performed a bounds check and guarantees that `src` is
178 // valid for `len` bytes.
179 //
180 // There caller guarantees that there is no data race.
181 unsafe { ptr::copy_nonoverlapping(src, dst, len) };
182 Ok(())
183 })
184 }
185
186 /// Maps the page and writes into it from the given buffer.
187 ///
188 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
189 /// outside of the page, then this call returns [`EINVAL`].
190 ///
191 /// # Safety
192 ///
193 /// * Callers must ensure that `src` is valid for reading `len` bytes.
194 /// * Callers must ensure that this call does not race with a read or write to the same page
195 /// that overlaps with this write.
write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result196 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result {
197 self.with_pointer_into_page(offset, len, move |dst| {
198 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
199 // bounds check and guarantees that `dst` is valid for `len` bytes.
200 //
201 // There caller guarantees that there is no data race.
202 unsafe { ptr::copy_nonoverlapping(src, dst, len) };
203 Ok(())
204 })
205 }
206
207 /// Maps the page and zeroes the given slice.
208 ///
209 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
210 /// outside of the page, then this call returns [`EINVAL`].
211 ///
212 /// # Safety
213 ///
214 /// Callers must ensure that this call does not race with a read or write to the same page that
215 /// overlaps with this write.
fill_zero_raw(&self, offset: usize, len: usize) -> Result216 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result {
217 self.with_pointer_into_page(offset, len, move |dst| {
218 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
219 // bounds check and guarantees that `dst` is valid for `len` bytes.
220 //
221 // There caller guarantees that there is no data race.
222 unsafe { ptr::write_bytes(dst, 0u8, len) };
223 Ok(())
224 })
225 }
226
227 /// Copies data from userspace into this page.
228 ///
229 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
230 /// outside of the page, then this call returns [`EINVAL`].
231 ///
232 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address.
233 /// However, they are not allowed on the page you are copying into.
234 ///
235 /// # Safety
236 ///
237 /// Callers must ensure that this call does not race with a read or write to the same page that
238 /// overlaps with this write.
copy_from_user_slice_raw( &self, reader: &mut UserSliceReader, offset: usize, len: usize, ) -> Result239 pub unsafe fn copy_from_user_slice_raw(
240 &self,
241 reader: &mut UserSliceReader,
242 offset: usize,
243 len: usize,
244 ) -> Result {
245 self.with_pointer_into_page(offset, len, move |dst| {
246 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
247 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have
248 // exclusive access to the slice since the caller guarantees that there are no races.
249 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) })
250 })
251 }
252 }
253
254 impl Drop for Page {
255 #[inline]
drop(&mut self)256 fn drop(&mut self) {
257 // SAFETY: By the type invariants, we have ownership of the page and can free it.
258 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };
259 }
260 }
261