1 // SPDX-License-Identifier: GPL-2.0 2 3 //! IOMMU page table management. 4 //! 5 //! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h) 6 7 use core::{ 8 marker::PhantomData, 9 ptr::NonNull, // 10 }; 11 12 use crate::{ 13 alloc, 14 bindings, 15 device::{ 16 Bound, 17 Device, // 18 }, 19 error::to_result, 20 io::PhysAddr, 21 prelude::*, // 22 }; 23 24 use bindings::io_pgtable_fmt; 25 26 /// Protection flags used with IOMMU mappings. 27 pub mod prot { 28 /// Read access. 29 pub const READ: u32 = bindings::IOMMU_READ; 30 /// Write access. 31 pub const WRITE: u32 = bindings::IOMMU_WRITE; 32 /// Request cache coherency. 33 pub const CACHE: u32 = bindings::IOMMU_CACHE; 34 /// Request no-execute permission. 35 pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC; 36 /// MMIO peripheral mapping. 37 pub const MMIO: u32 = bindings::IOMMU_MMIO; 38 /// Privileged mapping. 39 pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV; 40 } 41 42 /// Represents a requested `io_pgtable` configuration. 43 pub struct Config { 44 /// Quirk bitmask (type-specific). 45 pub quirks: usize, 46 /// Valid page sizes, as a bitmask of powers of two. 47 pub pgsize_bitmap: usize, 48 /// Input address space size in bits. 49 pub ias: u32, 50 /// Output address space size in bits. 51 pub oas: u32, 52 /// IOMMU uses coherent accesses for page table walks. 53 pub coherent_walk: bool, 54 } 55 56 /// An io page table using a specific format. 57 /// 58 /// # Invariants 59 /// 60 /// The pointer references a valid io page table. 61 pub struct IoPageTable<'a, F: IoPageTableFmt> { 62 ptr: NonNull<bindings::io_pgtable_ops>, 63 _dev: PhantomData<&'a Device<Bound>>, 64 _marker: PhantomData<F>, 65 } 66 67 // SAFETY: `struct io_pgtable_ops` is not restricted to a single thread. 68 unsafe impl<F: IoPageTableFmt> Send for IoPageTable<'_, F> {} 69 // SAFETY: `struct io_pgtable_ops` may be accessed concurrently. 70 unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<'_, F> {} 71 72 /// The format used by this page table. 73 pub trait IoPageTableFmt: 'static { 74 /// The value representing this format. 75 const FORMAT: io_pgtable_fmt; 76 } 77 78 impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> { 79 /// Create a new `IoPageTable`. 80 #[inline] 81 pub fn new(dev: &'a Device<Bound>, config: Config) -> Result<IoPageTable<'a, F>> { 82 let mut raw_cfg = bindings::io_pgtable_cfg { 83 quirks: config.quirks, 84 pgsize_bitmap: config.pgsize_bitmap, 85 ias: config.ias, 86 oas: config.oas, 87 coherent_walk: config.coherent_walk, 88 tlb: &raw const NOOP_FLUSH_OPS, 89 iommu_dev: dev.as_raw(), 90 // SAFETY: All zeroes is a valid value for `struct io_pgtable_cfg`. 91 ..unsafe { core::mem::zeroed() } 92 }; 93 94 // SAFETY: 95 // * The raw_cfg pointer is valid for the duration of this call. 96 // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer 97 // as cookie. 98 // * The caller ensures that the io pgtable does not outlive the device. 99 let ops = unsafe { 100 bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut()) 101 }; 102 103 // INVARIANT: We successfully created a valid page table. 104 Ok(IoPageTable { 105 ptr: NonNull::new(ops).ok_or(ENOMEM)?, 106 _dev: PhantomData, 107 _marker: PhantomData, 108 }) 109 } 110 111 /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`. 112 #[inline] 113 pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops { 114 self.ptr.as_ptr() 115 } 116 117 /// Obtain a raw pointer to the underlying `struct io_pgtable`. 118 #[inline] 119 pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable { 120 // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable. 121 unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) } 122 } 123 124 /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`. 125 #[inline] 126 pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg { 127 // SAFETY: The `raw_pgtable()` method returns a valid pointer. 128 unsafe { &raw mut (*self.raw_pgtable()).cfg } 129 } 130 131 /// Map a physically contiguous range of pages of the same size. 132 /// 133 /// Even if successful, this operation may not map the entire range. In that case, only a 134 /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In 135 /// this case, the caller will usually call `map_pages` again for the remaining range. 136 /// 137 /// The returned [`Result`] indicates whether an error was encountered while mapping pages. 138 /// Note that this may return a non-zero length even if an error was encountered. The caller 139 /// will usually [unmap the relevant pages](Self::unmap_pages) on error. 140 /// 141 /// The caller must flush the TLB before using the pgtable to access the newly created mapping. 142 /// 143 /// # Safety 144 /// 145 /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while 146 /// this `map_pages` operation executes. 147 /// * This page table must not contain any mapping that overlaps with the mapping created by 148 /// this call. 149 /// * If this page table is live, then the caller must ensure that it's okay to access the 150 /// physical address being mapped for the duration in which it is mapped. 151 #[inline] 152 pub unsafe fn map_pages( 153 &self, 154 iova: usize, 155 paddr: PhysAddr, 156 pgsize: usize, 157 pgcount: usize, 158 prot: u32, 159 flags: alloc::Flags, 160 ) -> (usize, Result) { 161 let mut mapped: usize = 0; 162 163 // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null. 164 let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() }; 165 166 // SAFETY: The safety requirements of this method are sufficient to call `map_pages`. 167 let ret = to_result(unsafe { 168 (map_pages)( 169 self.raw_ops(), 170 iova, 171 paddr, 172 pgsize, 173 pgcount, 174 prot as i32, 175 flags.as_raw(), 176 &mut mapped, 177 ) 178 }); 179 180 (mapped, ret) 181 } 182 183 /// Unmap a range of virtually contiguous pages of the same size. 184 /// 185 /// This may not unmap the entire range, and returns the length of the unmapped prefix in 186 /// bytes. 187 /// 188 /// # Safety 189 /// 190 /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while 191 /// this `unmap_pages` operation executes. 192 /// * This page table must contain one or more consecutive mappings starting at `iova` whose 193 /// total size is `pgcount * pgsize`. 194 #[inline] 195 #[must_use] 196 pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize { 197 // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null. 198 let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() }; 199 200 // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`. 201 unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) } 202 } 203 } 204 205 // For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all 206 // required invalidations using a range. There is no need for it get ARM style invalidation 207 // instructions from the page table code. 208 // 209 // Support for flushing the TLB with ARM style invalidation instructions may be added in the 210 // future. 211 static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops { 212 tlb_flush_all: Some(rust_tlb_flush_all_noop), 213 tlb_flush_walk: Some(rust_tlb_flush_walk_noop), 214 tlb_add_page: None, 215 }; 216 217 #[no_mangle] 218 extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {} 219 220 #[no_mangle] 221 extern "C" fn rust_tlb_flush_walk_noop( 222 _iova: usize, 223 _size: usize, 224 _granule: usize, 225 _cookie: *mut core::ffi::c_void, 226 ) { 227 } 228 229 impl<F: IoPageTableFmt> Drop for IoPageTable<'_, F> { 230 fn drop(&mut self) { 231 // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this 232 // destructor runs. 233 unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) }; 234 } 235 } 236 237 /// The `ARM_64_LPAE_S1` page table format. 238 pub enum ARM64LPAES1 {} 239 240 impl IoPageTableFmt for ARM64LPAES1 { 241 const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt; 242 } 243 244 impl IoPageTable<'_, ARM64LPAES1> { 245 /// Access the `ttbr` field of the configuration. 246 /// 247 /// This is the physical address of the page table, which may be passed to the device that 248 /// needs to use it. 249 /// 250 /// # Safety 251 /// 252 /// The caller must ensure that the device stops using the page table before dropping it. 253 #[inline] 254 pub unsafe fn ttbr(&self) -> u64 { 255 // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. 256 unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr } 257 } 258 259 /// Access the `mair` field of the configuration. 260 #[inline] 261 pub fn mair(&self) -> u64 { 262 // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. 263 unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair } 264 } 265 } 266