1 // SPDX-License-Identifier: GPL-2.0 2 3 //! IOMMU page table management. 4 //! 5 //! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h) 6 7 use core::{ 8 marker::PhantomData, 9 ptr::NonNull, // 10 }; 11 12 use crate::{ 13 alloc, 14 bindings, 15 device::{ 16 Bound, 17 Device, // 18 }, 19 devres::Devres, 20 error::to_result, 21 io::PhysAddr, 22 prelude::*, // 23 }; 24 25 use bindings::io_pgtable_fmt; 26 27 /// Protection flags used with IOMMU mappings. 28 pub mod prot { 29 /// Read access. 30 pub const READ: u32 = bindings::IOMMU_READ; 31 /// Write access. 32 pub const WRITE: u32 = bindings::IOMMU_WRITE; 33 /// Request cache coherency. 34 pub const CACHE: u32 = bindings::IOMMU_CACHE; 35 /// Request no-execute permission. 36 pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC; 37 /// MMIO peripheral mapping. 38 pub const MMIO: u32 = bindings::IOMMU_MMIO; 39 /// Privileged mapping. 40 pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV; 41 } 42 43 /// Represents a requested `io_pgtable` configuration. 44 pub struct Config { 45 /// Quirk bitmask (type-specific). 46 pub quirks: usize, 47 /// Valid page sizes, as a bitmask of powers of two. 48 pub pgsize_bitmap: usize, 49 /// Input address space size in bits. 50 pub ias: u32, 51 /// Output address space size in bits. 52 pub oas: u32, 53 /// IOMMU uses coherent accesses for page table walks. 54 pub coherent_walk: bool, 55 } 56 57 /// An io page table using a specific format. 58 /// 59 /// # Invariants 60 /// 61 /// The pointer references a valid io page table. 62 pub struct IoPageTable<F: IoPageTableFmt> { 63 ptr: NonNull<bindings::io_pgtable_ops>, 64 _marker: PhantomData<F>, 65 } 66 67 // SAFETY: `struct io_pgtable_ops` is not restricted to a single thread. 68 unsafe impl<F: IoPageTableFmt> Send for IoPageTable<F> {} 69 // SAFETY: `struct io_pgtable_ops` may be accessed concurrently. 70 unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<F> {} 71 72 /// The format used by this page table. 73 pub trait IoPageTableFmt: 'static { 74 /// The value representing this format. 75 const FORMAT: io_pgtable_fmt; 76 } 77 78 impl<F: IoPageTableFmt> IoPageTable<F> { 79 /// Create a new `IoPageTable` as a device resource. 80 #[inline] 81 pub fn new( 82 dev: &Device<Bound>, 83 config: Config, 84 ) -> impl PinInit<Devres<IoPageTable<F>>, Error> + '_ { 85 // SAFETY: Devres ensures that the value is dropped during device unbind. 86 Devres::new(dev, unsafe { Self::new_raw(dev, config) }) 87 } 88 89 /// Create a new `IoPageTable`. 90 /// 91 /// # Safety 92 /// 93 /// If successful, then the returned `IoPageTable` must be dropped before the device is 94 /// unbound. 95 #[inline] 96 pub unsafe fn new_raw(dev: &Device<Bound>, config: Config) -> Result<IoPageTable<F>> { 97 let mut raw_cfg = bindings::io_pgtable_cfg { 98 quirks: config.quirks, 99 pgsize_bitmap: config.pgsize_bitmap, 100 ias: config.ias, 101 oas: config.oas, 102 coherent_walk: config.coherent_walk, 103 tlb: &raw const NOOP_FLUSH_OPS, 104 iommu_dev: dev.as_raw(), 105 ..Zeroable::zeroed() 106 }; 107 108 // SAFETY: 109 // * The raw_cfg pointer is valid for the duration of this call. 110 // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer 111 // as cookie. 112 // * The caller ensures that the io pgtable does not outlive the device. 113 let ops = unsafe { 114 bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut()) 115 }; 116 117 // INVARIANT: We successfully created a valid page table. 118 Ok(IoPageTable { 119 ptr: NonNull::new(ops).ok_or(ENOMEM)?, 120 _marker: PhantomData, 121 }) 122 } 123 124 /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`. 125 #[inline] 126 pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops { 127 self.ptr.as_ptr() 128 } 129 130 /// Obtain a raw pointer to the underlying `struct io_pgtable`. 131 #[inline] 132 pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable { 133 // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable. 134 unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) } 135 } 136 137 /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`. 138 #[inline] 139 pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg { 140 // SAFETY: The `raw_pgtable()` method returns a valid pointer. 141 unsafe { &raw mut (*self.raw_pgtable()).cfg } 142 } 143 144 /// Map a physically contiguous range of pages of the same size. 145 /// 146 /// Even if successful, this operation may not map the entire range. In that case, only a 147 /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In 148 /// this case, the caller will usually call `map_pages` again for the remaining range. 149 /// 150 /// The returned [`Result`] indicates whether an error was encountered while mapping pages. 151 /// Note that this may return a non-zero length even if an error was encountered. The caller 152 /// will usually [unmap the relevant pages](Self::unmap_pages) on error. 153 /// 154 /// The caller must flush the TLB before using the pgtable to access the newly created mapping. 155 /// 156 /// # Safety 157 /// 158 /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while 159 /// this `map_pages` operation executes. 160 /// * This page table must not contain any mapping that overlaps with the mapping created by 161 /// this call. 162 /// * If this page table is live, then the caller must ensure that it's okay to access the 163 /// physical address being mapped for the duration in which it is mapped. 164 #[inline] 165 pub unsafe fn map_pages( 166 &self, 167 iova: usize, 168 paddr: PhysAddr, 169 pgsize: usize, 170 pgcount: usize, 171 prot: u32, 172 flags: alloc::Flags, 173 ) -> (usize, Result) { 174 let mut mapped: usize = 0; 175 176 // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null. 177 let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() }; 178 179 // SAFETY: The safety requirements of this method are sufficient to call `map_pages`. 180 let ret = to_result(unsafe { 181 (map_pages)( 182 self.raw_ops(), 183 iova, 184 paddr, 185 pgsize, 186 pgcount, 187 prot as i32, 188 flags.as_raw(), 189 &mut mapped, 190 ) 191 }); 192 193 (mapped, ret) 194 } 195 196 /// Unmap a range of virtually contiguous pages of the same size. 197 /// 198 /// This may not unmap the entire range, and returns the length of the unmapped prefix in 199 /// bytes. 200 /// 201 /// # Safety 202 /// 203 /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while 204 /// this `unmap_pages` operation executes. 205 /// * This page table must contain one or more consecutive mappings starting at `iova` whose 206 /// total size is `pgcount * pgsize`. 207 #[inline] 208 #[must_use] 209 pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize { 210 // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null. 211 let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() }; 212 213 // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`. 214 unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) } 215 } 216 } 217 218 // For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all 219 // required invalidations using a range. There is no need for it get ARM style invalidation 220 // instructions from the page table code. 221 // 222 // Support for flushing the TLB with ARM style invalidation instructions may be added in the 223 // future. 224 static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops { 225 tlb_flush_all: Some(rust_tlb_flush_all_noop), 226 tlb_flush_walk: Some(rust_tlb_flush_walk_noop), 227 tlb_add_page: None, 228 }; 229 230 #[no_mangle] 231 extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {} 232 233 #[no_mangle] 234 extern "C" fn rust_tlb_flush_walk_noop( 235 _iova: usize, 236 _size: usize, 237 _granule: usize, 238 _cookie: *mut core::ffi::c_void, 239 ) { 240 } 241 242 impl<F: IoPageTableFmt> Drop for IoPageTable<F> { 243 fn drop(&mut self) { 244 // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this 245 // destructor runs. 246 unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) }; 247 } 248 } 249 250 /// The `ARM_64_LPAE_S1` page table format. 251 pub enum ARM64LPAES1 {} 252 253 impl IoPageTableFmt for ARM64LPAES1 { 254 const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt; 255 } 256 257 impl IoPageTable<ARM64LPAES1> { 258 /// Access the `ttbr` field of the configuration. 259 /// 260 /// This is the physical address of the page table, which may be passed to the device that 261 /// needs to use it. 262 /// 263 /// # Safety 264 /// 265 /// The caller must ensure that the device stops using the page table before dropping it. 266 #[inline] 267 pub unsafe fn ttbr(&self) -> u64 { 268 // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. 269 unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr } 270 } 271 272 /// Access the `mair` field of the configuration. 273 #[inline] 274 pub fn mair(&self) -> u64 { 275 // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. 276 unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair } 277 } 278 } 279