1 // SPDX-License-Identifier: GPL-2.0 2 3 //! IOMMU page table management. 4 //! 5 //! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h) 6 7 use core::{ 8 marker::PhantomData, 9 ptr::NonNull, // 10 }; 11 12 use crate::{ 13 alloc, 14 bindings, 15 device::{ 16 Bound, 17 Device, // 18 }, 19 error::to_result, 20 io::PhysAddr, 21 prelude::*, // 22 }; 23 24 use bindings::io_pgtable_fmt; 25 26 /// Protection flags used with IOMMU mappings. 27 pub mod prot { 28 /// Read access. 29 pub const READ: u32 = bindings::IOMMU_READ; 30 /// Write access. 31 pub const WRITE: u32 = bindings::IOMMU_WRITE; 32 /// Request cache coherency. 33 pub const CACHE: u32 = bindings::IOMMU_CACHE; 34 /// Request no-execute permission. 35 pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC; 36 /// MMIO peripheral mapping. 37 pub const MMIO: u32 = bindings::IOMMU_MMIO; 38 /// Privileged mapping. 39 pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV; 40 } 41 42 /// Represents a requested `io_pgtable` configuration. 43 pub struct Config { 44 /// Quirk bitmask (type-specific). 45 pub quirks: usize, 46 /// Valid page sizes, as a bitmask of powers of two. 47 pub pgsize_bitmap: usize, 48 /// Input address space size in bits. 49 pub ias: u32, 50 /// Output address space size in bits. 51 pub oas: u32, 52 /// IOMMU uses coherent accesses for page table walks. 53 pub coherent_walk: bool, 54 } 55 56 /// An io page table using a specific format. 57 /// 58 /// # Invariants 59 /// 60 /// The pointer references a valid io page table. 61 pub struct IoPageTable<'a, F: IoPageTableFmt> { 62 ptr: NonNull<bindings::io_pgtable_ops>, 63 _dev: PhantomData<&'a Device<Bound>>, 64 _marker: PhantomData<F>, 65 } 66 67 // SAFETY: `struct io_pgtable_ops` is not restricted to a single thread. 68 unsafe impl<F: IoPageTableFmt> Send for IoPageTable<'_, F> {} 69 // SAFETY: `struct io_pgtable_ops` may be accessed concurrently. 70 unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<'_, F> {} 71 72 /// The format used by this page table. 73 pub trait IoPageTableFmt: 'static { 74 /// The value representing this format. 75 const FORMAT: io_pgtable_fmt; 76 } 77 78 impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> { 79 /// Create a new `IoPageTable`. 80 #[inline] 81 pub fn new(dev: &'a Device<Bound>, config: Config) -> Result<IoPageTable<'a, F>> { 82 let mut raw_cfg = bindings::io_pgtable_cfg { 83 quirks: config.quirks, 84 pgsize_bitmap: config.pgsize_bitmap, 85 ias: config.ias, 86 oas: config.oas, 87 coherent_walk: config.coherent_walk, 88 tlb: &raw const NOOP_FLUSH_OPS, 89 iommu_dev: dev.as_raw(), 90 ..Zeroable::zeroed() 91 }; 92 93 // SAFETY: 94 // * The raw_cfg pointer is valid for the duration of this call. 95 // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer 96 // as cookie. 97 // * The caller ensures that the io pgtable does not outlive the device. 98 let ops = unsafe { 99 bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut()) 100 }; 101 102 // INVARIANT: We successfully created a valid page table. 103 Ok(IoPageTable { 104 ptr: NonNull::new(ops).ok_or(ENOMEM)?, 105 _dev: PhantomData, 106 _marker: PhantomData, 107 }) 108 } 109 110 /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`. 111 #[inline] 112 pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops { 113 self.ptr.as_ptr() 114 } 115 116 /// Obtain a raw pointer to the underlying `struct io_pgtable`. 117 #[inline] 118 pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable { 119 // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable. 120 unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) } 121 } 122 123 /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`. 124 #[inline] 125 pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg { 126 // SAFETY: The `raw_pgtable()` method returns a valid pointer. 127 unsafe { &raw mut (*self.raw_pgtable()).cfg } 128 } 129 130 /// Map a physically contiguous range of pages of the same size. 131 /// 132 /// Even if successful, this operation may not map the entire range. In that case, only a 133 /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In 134 /// this case, the caller will usually call `map_pages` again for the remaining range. 135 /// 136 /// The returned [`Result`] indicates whether an error was encountered while mapping pages. 137 /// Note that this may return a non-zero length even if an error was encountered. The caller 138 /// will usually [unmap the relevant pages](Self::unmap_pages) on error. 139 /// 140 /// The caller must flush the TLB before using the pgtable to access the newly created mapping. 141 /// 142 /// # Safety 143 /// 144 /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while 145 /// this `map_pages` operation executes. 146 /// * This page table must not contain any mapping that overlaps with the mapping created by 147 /// this call. 148 /// * If this page table is live, then the caller must ensure that it's okay to access the 149 /// physical address being mapped for the duration in which it is mapped. 150 #[inline] 151 pub unsafe fn map_pages( 152 &self, 153 iova: usize, 154 paddr: PhysAddr, 155 pgsize: usize, 156 pgcount: usize, 157 prot: u32, 158 flags: alloc::Flags, 159 ) -> (usize, Result) { 160 let mut mapped: usize = 0; 161 162 // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null. 163 let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() }; 164 165 // SAFETY: The safety requirements of this method are sufficient to call `map_pages`. 166 let ret = to_result(unsafe { 167 (map_pages)( 168 self.raw_ops(), 169 iova, 170 paddr, 171 pgsize, 172 pgcount, 173 prot as i32, 174 flags.as_raw(), 175 &mut mapped, 176 ) 177 }); 178 179 (mapped, ret) 180 } 181 182 /// Unmap a range of virtually contiguous pages of the same size. 183 /// 184 /// This may not unmap the entire range, and returns the length of the unmapped prefix in 185 /// bytes. 186 /// 187 /// # Safety 188 /// 189 /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while 190 /// this `unmap_pages` operation executes. 191 /// * This page table must contain one or more consecutive mappings starting at `iova` whose 192 /// total size is `pgcount * pgsize`. 193 #[inline] 194 #[must_use] 195 pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize { 196 // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null. 197 let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() }; 198 199 // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`. 200 unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) } 201 } 202 } 203 204 // For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all 205 // required invalidations using a range. There is no need for it get ARM style invalidation 206 // instructions from the page table code. 207 // 208 // Support for flushing the TLB with ARM style invalidation instructions may be added in the 209 // future. 210 static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops { 211 tlb_flush_all: Some(rust_tlb_flush_all_noop), 212 tlb_flush_walk: Some(rust_tlb_flush_walk_noop), 213 tlb_add_page: None, 214 }; 215 216 #[no_mangle] 217 extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {} 218 219 #[no_mangle] 220 extern "C" fn rust_tlb_flush_walk_noop( 221 _iova: usize, 222 _size: usize, 223 _granule: usize, 224 _cookie: *mut core::ffi::c_void, 225 ) { 226 } 227 228 impl<F: IoPageTableFmt> Drop for IoPageTable<'_, F> { 229 fn drop(&mut self) { 230 // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this 231 // destructor runs. 232 unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) }; 233 } 234 } 235 236 /// The `ARM_64_LPAE_S1` page table format. 237 pub enum ARM64LPAES1 {} 238 239 impl IoPageTableFmt for ARM64LPAES1 { 240 const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt; 241 } 242 243 impl IoPageTable<'_, ARM64LPAES1> { 244 /// Access the `ttbr` field of the configuration. 245 /// 246 /// This is the physical address of the page table, which may be passed to the device that 247 /// needs to use it. 248 /// 249 /// # Safety 250 /// 251 /// The caller must ensure that the device stops using the page table before dropping it. 252 #[inline] 253 pub unsafe fn ttbr(&self) -> u64 { 254 // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. 255 unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr } 256 } 257 258 /// Access the `mair` field of the configuration. 259 #[inline] 260 pub fn mair(&self) -> u64 { 261 // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`. 262 unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair } 263 } 264 } 265