1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 3 #![cfg(CONFIG_RUST_DRM_GPUVM)] 4 5 //! DRM GPUVM in immediate mode 6 //! 7 //! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated 8 //! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM 9 //! and the GPU's virtual address space has the same state at all times. 10 //! 11 //! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) 12 13 use kernel::{ 14 alloc::{ 15 AllocError, 16 Flags as AllocFlags, // 17 }, 18 bindings, 19 drm, 20 drm::gem::IntoGEMObject, 21 error::to_result, 22 prelude::*, 23 sync::aref::{ 24 ARef, 25 AlwaysRefCounted, // 26 }, 27 types::Opaque, // 28 }; 29 30 use core::{ 31 cell::UnsafeCell, 32 marker::PhantomData, 33 mem::{ 34 ManuallyDrop, 35 MaybeUninit, // 36 }, 37 ops::{ 38 Deref, 39 DerefMut, 40 Range, // 41 }, 42 ptr::{ 43 self, 44 NonNull, // 45 }, // 46 }; 47 48 mod sm_ops; 49 pub use self::sm_ops::*; 50 51 mod vm_bo; 52 pub use self::vm_bo::*; 53 54 mod va; 55 pub use self::va::*; 56 57 /// A DRM GPU VA manager. 58 /// 59 /// This object is refcounted, but the locations of mapped ranges may only be accessed or changed 60 /// via the special unique handle [`UniqueRefGpuVm`]. 61 /// 62 /// # Invariants 63 /// 64 /// * Stored in an allocation managed by the refcount in `self.vm`. 65 /// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. 66 /// * Does not contain any sparse [`GpuVa<T>`] instances. 67 #[pin_data] 68 pub struct GpuVm<T: DriverGpuVm> { 69 #[pin] 70 vm: Opaque<bindings::drm_gpuvm>, 71 /// Accessed only through the [`UniqueRefGpuVm`] reference. 72 data: UnsafeCell<T>, 73 } 74 75 // SAFETY: It is safe to send a `GpuVm<T>` to another thread: all data reachable through it 76 // (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds. 77 unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {} 78 // SAFETY: It is safe to share a `&GpuVm<T>` between threads: `&self` methods only alias data 79 // that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the 80 // reference and ultimately drop `T`, which the same bounds make `Send`. 81 unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {} 82 83 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 84 unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> { 85 fn inc_ref(&self) { 86 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 87 unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; 88 } 89 90 unsafe fn dec_ref(obj: NonNull<Self>) { 91 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 92 unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; 93 } 94 } 95 96 impl<T: DriverGpuVm> PartialEq for GpuVm<T> { 97 #[inline] 98 fn eq(&self, other: &Self) -> bool { 99 core::ptr::eq(self.as_raw(), other.as_raw()) 100 } 101 } 102 impl<T: DriverGpuVm> Eq for GpuVm<T> {} 103 104 impl<T: DriverGpuVm> GpuVm<T> { 105 const fn vtable() -> &'static bindings::drm_gpuvm_ops { 106 &bindings::drm_gpuvm_ops { 107 vm_free: Some(Self::vm_free), 108 op_alloc: None, 109 op_free: None, 110 vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN, 111 vm_bo_free: GpuVmBo::<T>::FREE_FN, 112 vm_bo_validate: None, 113 sm_step_map: Some(Self::sm_step_map), 114 sm_step_unmap: Some(Self::sm_step_unmap), 115 sm_step_remap: Some(Self::sm_step_remap), 116 } 117 } 118 119 /// Creates a GPUVM instance. 120 #[expect(clippy::new_ret_no_self)] 121 pub fn new<E, Ctx: drm::DeviceContext>( 122 name: &'static CStr, 123 dev: &drm::Device<T::Driver, Ctx>, 124 r_obj: &T::Object, 125 range: Range<u64>, 126 reserve_range: Range<u64>, 127 data: T, 128 ) -> Result<UniqueRefGpuVm<T>, E> 129 where 130 E: From<AllocError>, 131 E: From<core::convert::Infallible>, 132 { 133 let obj = KBox::try_pin_init::<E>( 134 try_pin_init!(Self { 135 data: UnsafeCell::new(data), 136 vm <- Opaque::ffi_init(|vm| { 137 // SAFETY: These arguments are valid. `vm` is valid until refcount drops to 138 // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag 139 // below. 140 unsafe { 141 bindings::drm_gpuvm_init( 142 vm, 143 name.as_char_ptr(), 144 bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE 145 | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, 146 dev.as_raw(), 147 r_obj.as_raw(), 148 range.start, 149 range.end - range.start, 150 reserve_range.start, 151 reserve_range.end - reserve_range.start, 152 const { Self::vtable() }, 153 ) 154 } 155 }), 156 }? E), 157 GFP_KERNEL | __GFP_ZERO, 158 )?; 159 // SAFETY: This transfers the initial refcount to the ARef. 160 let aref = unsafe { 161 ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( 162 Pin::into_inner_unchecked(obj), 163 ))) 164 }; 165 // INVARIANT: This reference is unique. 166 Ok(UniqueRefGpuVm(aref)) 167 } 168 169 /// Access this [`GpuVm`] from a raw pointer. 170 /// 171 /// # Safety 172 /// 173 /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains 174 /// valid for at least `'a`. 175 #[inline] 176 pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { 177 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the 178 // pointer is valid for 'a. 179 unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } 180 } 181 182 /// Returns a raw pointer to the embedded `struct drm_gpuvm`. 183 #[inline] 184 pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { 185 self.vm.get() 186 } 187 188 /// The start of the VA space. 189 #[inline] 190 pub fn va_start(&self) -> u64 { 191 // SAFETY: The `mm_start` field is immutable. 192 unsafe { (*self.as_raw()).mm_start } 193 } 194 195 /// The length of the GPU's virtual address space. 196 #[inline] 197 pub fn va_length(&self) -> u64 { 198 // SAFETY: The `mm_range` field is immutable. 199 unsafe { (*self.as_raw()).mm_range } 200 } 201 202 /// Returns the range of the GPU virtual address space. 203 #[inline] 204 pub fn va_range(&self) -> Range<u64> { 205 let start = self.va_start(); 206 // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail. 207 let end = start + self.va_length(); 208 Range { start, end } 209 } 210 211 /// Get or create the [`GpuVmBo`] for this gem object. 212 #[inline] 213 pub fn obtain( 214 &self, 215 obj: &T::Object, 216 data: impl PinInit<T::VmBoData>, 217 ) -> Result<ARef<GpuVmBo<T>>, AllocError> { 218 Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) 219 } 220 221 /// Clean up buffer objects that are no longer used. 222 #[inline] 223 pub fn deferred_cleanup(&self) { 224 // SAFETY: This GPUVM uses immediate mode. 225 unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } 226 } 227 228 /// Check if this GEM object is an external object for this GPUVM. 229 #[inline] 230 pub fn is_extobj(&self, obj: &T::Object) -> bool { 231 // SAFETY: We may call this with any GPUVM and GEM object. 232 unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } 233 } 234 235 /// Free this GPUVM. 236 /// 237 /// # Safety 238 /// 239 /// Called when refcount hits zero. 240 unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { 241 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. 242 let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; 243 // SAFETY: By type invariants we can free it when refcount hits zero. 244 drop(unsafe { KBox::from_raw(me) }) 245 } 246 247 #[inline] 248 fn raw_resv(&self) -> *mut bindings::dma_resv { 249 // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. 250 unsafe { (*(*self.as_raw()).r_obj).resv } 251 } 252 } 253 254 /// The manager for a GPUVM. 255 pub trait DriverGpuVm: Sized + Send + Sync { 256 /// Parent `Driver` for this object. 257 type Driver: drm::Driver; 258 259 /// The kind of GEM object stored in this GPUVM. 260 type Object: drm::driver::AllocImpl<Driver = Self::Driver> + Send + Sync; 261 262 /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). 263 /// 264 /// Only `Send` is required: the data has a single owner at all times, moving 265 /// between threads by value (handed back as a [`GpuVaRemoved`]) but never 266 /// accessed by two threads concurrently. 267 type VaData: Send; 268 269 /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). 270 type VmBoData: Send + Sync; 271 272 /// The private data passed to callbacks. 273 type SmContext<'ctx> 274 where 275 Self: 'ctx; 276 277 /// Indicates that a new mapping should be created. 278 fn sm_step_map<'op, 'ctx>( 279 &mut self, 280 op: OpMap<'op, Self>, 281 context: &mut Self::SmContext<'ctx>, 282 ) -> Result<OpMapped<'op, Self>, Error>; 283 284 /// Indicates that an existing mapping should be removed. 285 fn sm_step_unmap<'op, 'ctx>( 286 &mut self, 287 op: OpUnmap<'op, Self>, 288 context: &mut Self::SmContext<'ctx>, 289 ) -> Result<OpUnmapped<'op, Self>, Error>; 290 291 /// Indicates that an existing mapping should be split up. 292 fn sm_step_remap<'op, 'ctx>( 293 &mut self, 294 op: OpRemap<'op, Self>, 295 context: &mut Self::SmContext<'ctx>, 296 ) -> Result<OpRemapped<'op, Self>, Error>; 297 } 298 299 /// The core of the DRM GPU VA manager. 300 /// 301 /// This object is a unique reference to the VM that can access the interval tree and the Rust 302 /// `data` field. 303 /// 304 /// # Invariants 305 /// 306 /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. 307 // `Send`/`Sync` derive from `ARef<GpuVm<T>>`; the trait bounds make them correct for the unique 308 // handle's `&mut T` access. 309 pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>); 310 311 impl<T: DriverGpuVm> UniqueRefGpuVm<T> { 312 /// Access the data owned by this `UniqueRefGpuVm` immutably. 313 #[inline] 314 pub fn data_ref(&self) -> &T { 315 // SAFETY: By the type invariants we may access `data`. 316 unsafe { &*self.0.data.get() } 317 } 318 319 /// Access the data owned by this `UniqueRefGpuVm` mutably. 320 #[inline] 321 pub fn data(&mut self) -> &mut T { 322 // SAFETY: By the type invariants we may access `data`. 323 unsafe { &mut *self.0.data.get() } 324 } 325 } 326 327 impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> { 328 type Target = GpuVm<T>; 329 330 #[inline] 331 fn deref(&self) -> &GpuVm<T> { 332 &self.0 333 } 334 } 335