1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 3 #![cfg(CONFIG_RUST_DRM_GPUVM)] 4 5 //! DRM GPUVM in immediate mode 6 //! 7 //! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated 8 //! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM 9 //! and the GPU's virtual address space has the same state at all times. 10 //! 11 //! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) 12 13 use kernel::{ 14 alloc::{ 15 AllocError, 16 Flags as AllocFlags, // 17 }, 18 bindings, 19 drm, 20 drm::gem::IntoGEMObject, 21 error::to_result, 22 prelude::*, 23 sync::aref::{ 24 ARef, 25 AlwaysRefCounted, // 26 }, 27 types::Opaque, // 28 }; 29 30 use core::{ 31 cell::UnsafeCell, 32 marker::PhantomData, 33 mem::{ 34 ManuallyDrop, 35 MaybeUninit, // 36 }, 37 ops::{ 38 Deref, 39 DerefMut, 40 Range, // 41 }, 42 ptr::{ 43 self, 44 NonNull, // 45 }, // 46 }; 47 48 mod sm_ops; 49 pub use self::sm_ops::*; 50 51 mod vm_bo; 52 pub use self::vm_bo::*; 53 54 mod va; 55 pub use self::va::*; 56 57 /// A DRM GPU VA manager. 58 /// 59 /// This object is refcounted, but the locations of mapped ranges may only be accessed or changed 60 /// via the special unique handle [`UniqueRefGpuVm`]. 61 /// 62 /// # Invariants 63 /// 64 /// * Stored in an allocation managed by the refcount in `self.vm`. 65 /// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. 66 /// * Does not contain any sparse [`GpuVa<T>`] instances. 67 #[pin_data] 68 pub struct GpuVm<T: DriverGpuVm> { 69 #[pin] 70 vm: Opaque<bindings::drm_gpuvm>, 71 /// Accessed only through the [`UniqueRefGpuVm`] reference. 72 data: UnsafeCell<T>, 73 } 74 75 // SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will 76 // drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. 77 unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {} 78 // SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. 79 unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {} 80 81 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 82 unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> { 83 fn inc_ref(&self) { 84 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 85 unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; 86 } 87 88 unsafe fn dec_ref(obj: NonNull<Self>) { 89 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 90 unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; 91 } 92 } 93 94 impl<T: DriverGpuVm> PartialEq for GpuVm<T> { 95 #[inline] 96 fn eq(&self, other: &Self) -> bool { 97 core::ptr::eq(self.as_raw(), other.as_raw()) 98 } 99 } 100 impl<T: DriverGpuVm> Eq for GpuVm<T> {} 101 102 impl<T: DriverGpuVm> GpuVm<T> { 103 const fn vtable() -> &'static bindings::drm_gpuvm_ops { 104 &bindings::drm_gpuvm_ops { 105 vm_free: Some(Self::vm_free), 106 op_alloc: None, 107 op_free: None, 108 vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN, 109 vm_bo_free: GpuVmBo::<T>::FREE_FN, 110 vm_bo_validate: None, 111 sm_step_map: Some(Self::sm_step_map), 112 sm_step_unmap: Some(Self::sm_step_unmap), 113 sm_step_remap: Some(Self::sm_step_remap), 114 } 115 } 116 117 /// Creates a GPUVM instance. 118 #[expect(clippy::new_ret_no_self)] 119 pub fn new<E>( 120 name: &'static CStr, 121 dev: &drm::Device<T::Driver>, 122 r_obj: &T::Object, 123 range: Range<u64>, 124 reserve_range: Range<u64>, 125 data: T, 126 ) -> Result<UniqueRefGpuVm<T>, E> 127 where 128 E: From<AllocError>, 129 E: From<core::convert::Infallible>, 130 { 131 let obj = KBox::try_pin_init::<E>( 132 try_pin_init!(Self { 133 data: UnsafeCell::new(data), 134 vm <- Opaque::ffi_init(|vm| { 135 // SAFETY: These arguments are valid. `vm` is valid until refcount drops to 136 // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag 137 // below. 138 unsafe { 139 bindings::drm_gpuvm_init( 140 vm, 141 name.as_char_ptr(), 142 bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE 143 | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, 144 dev.as_raw(), 145 r_obj.as_raw(), 146 range.start, 147 range.end - range.start, 148 reserve_range.start, 149 reserve_range.end - reserve_range.start, 150 const { Self::vtable() }, 151 ) 152 } 153 }), 154 }? E), 155 GFP_KERNEL | __GFP_ZERO, 156 )?; 157 // SAFETY: This transfers the initial refcount to the ARef. 158 let aref = unsafe { 159 ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( 160 Pin::into_inner_unchecked(obj), 161 ))) 162 }; 163 // INVARIANT: This reference is unique. 164 Ok(UniqueRefGpuVm(aref)) 165 } 166 167 /// Access this [`GpuVm`] from a raw pointer. 168 /// 169 /// # Safety 170 /// 171 /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains 172 /// valid for at least `'a`. 173 #[inline] 174 pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { 175 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the 176 // pointer is valid for 'a. 177 unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } 178 } 179 180 /// Returns a raw pointer to the embedded `struct drm_gpuvm`. 181 #[inline] 182 pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { 183 self.vm.get() 184 } 185 186 /// The start of the VA space. 187 #[inline] 188 pub fn va_start(&self) -> u64 { 189 // SAFETY: The `mm_start` field is immutable. 190 unsafe { (*self.as_raw()).mm_start } 191 } 192 193 /// The length of the GPU's virtual address space. 194 #[inline] 195 pub fn va_length(&self) -> u64 { 196 // SAFETY: The `mm_range` field is immutable. 197 unsafe { (*self.as_raw()).mm_range } 198 } 199 200 /// Returns the range of the GPU virtual address space. 201 #[inline] 202 pub fn va_range(&self) -> Range<u64> { 203 let start = self.va_start(); 204 // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail. 205 let end = start + self.va_length(); 206 Range { start, end } 207 } 208 209 /// Get or create the [`GpuVmBo`] for this gem object. 210 #[inline] 211 pub fn obtain( 212 &self, 213 obj: &T::Object, 214 data: impl PinInit<T::VmBoData>, 215 ) -> Result<ARef<GpuVmBo<T>>, AllocError> { 216 Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) 217 } 218 219 /// Clean up buffer objects that are no longer used. 220 #[inline] 221 pub fn deferred_cleanup(&self) { 222 // SAFETY: This GPUVM uses immediate mode. 223 unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } 224 } 225 226 /// Check if this GEM object is an external object for this GPUVM. 227 #[inline] 228 pub fn is_extobj(&self, obj: &T::Object) -> bool { 229 // SAFETY: We may call this with any GPUVM and GEM object. 230 unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } 231 } 232 233 /// Free this GPUVM. 234 /// 235 /// # Safety 236 /// 237 /// Called when refcount hits zero. 238 unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { 239 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. 240 let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; 241 // SAFETY: By type invariants we can free it when refcount hits zero. 242 drop(unsafe { KBox::from_raw(me) }) 243 } 244 245 #[inline] 246 fn raw_resv(&self) -> *mut bindings::dma_resv { 247 // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. 248 unsafe { (*(*self.as_raw()).r_obj).resv } 249 } 250 } 251 252 /// The manager for a GPUVM. 253 pub trait DriverGpuVm: Sized + Send { 254 /// Parent `Driver` for this object. 255 type Driver: drm::Driver<Object = Self::Object>; 256 257 /// The kind of GEM object stored in this GPUVM. 258 type Object: IntoGEMObject; 259 260 /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). 261 type VaData; 262 263 /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). 264 type VmBoData; 265 266 /// The private data passed to callbacks. 267 type SmContext<'ctx>; 268 269 /// Indicates that a new mapping should be created. 270 fn sm_step_map<'op, 'ctx>( 271 &mut self, 272 op: OpMap<'op, Self>, 273 context: &mut Self::SmContext<'ctx>, 274 ) -> Result<OpMapped<'op, Self>, Error>; 275 276 /// Indicates that an existing mapping should be removed. 277 fn sm_step_unmap<'op, 'ctx>( 278 &mut self, 279 op: OpUnmap<'op, Self>, 280 context: &mut Self::SmContext<'ctx>, 281 ) -> Result<OpUnmapped<'op, Self>, Error>; 282 283 /// Indicates that an existing mapping should be split up. 284 fn sm_step_remap<'op, 'ctx>( 285 &mut self, 286 op: OpRemap<'op, Self>, 287 context: &mut Self::SmContext<'ctx>, 288 ) -> Result<OpRemapped<'op, Self>, Error>; 289 } 290 291 /// The core of the DRM GPU VA manager. 292 /// 293 /// This object is a unique reference to the VM that can access the interval tree and the Rust 294 /// `data` field. 295 /// 296 /// # Invariants 297 /// 298 /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. 299 pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>); 300 301 // SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and 302 // concurrent access to `data` is safe due to the `T: Sync` requirement. 303 unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {} 304 305 impl<T: DriverGpuVm> UniqueRefGpuVm<T> { 306 /// Access the data owned by this `UniqueRefGpuVm` immutably. 307 #[inline] 308 pub fn data_ref(&self) -> &T { 309 // SAFETY: By the type invariants we may access `data`. 310 unsafe { &*self.0.data.get() } 311 } 312 313 /// Access the data owned by this `UniqueRefGpuVm` mutably. 314 #[inline] 315 pub fn data(&mut self) -> &mut T { 316 // SAFETY: By the type invariants we may access `data`. 317 unsafe { &mut *self.0.data.get() } 318 } 319 } 320 321 impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> { 322 type Target = GpuVm<T>; 323 324 #[inline] 325 fn deref(&self) -> &GpuVm<T> { 326 &self.0 327 } 328 } 329