1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 3 #![cfg(CONFIG_DRM_GPUVM = "y")] 4 5 //! DRM GPUVM in immediate mode 6 //! 7 //! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated 8 //! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM 9 //! and the GPU's virtual address space has the same state at all times. 10 //! 11 //! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) 12 13 use kernel::{ 14 alloc::{ 15 AllocError, 16 Flags as AllocFlags, // 17 }, 18 bindings, 19 drm, 20 drm::gem::IntoGEMObject, 21 prelude::*, 22 sync::aref::{ 23 ARef, 24 AlwaysRefCounted, // 25 }, 26 types::Opaque, // 27 }; 28 29 use core::{ 30 cell::UnsafeCell, 31 mem::{ 32 ManuallyDrop, 33 MaybeUninit, // 34 }, 35 ops::{ 36 Deref, 37 DerefMut, 38 Range, // 39 }, 40 ptr::{ 41 self, 42 NonNull, // 43 }, // 44 }; 45 46 mod va; 47 pub use self::va::*; 48 49 mod vm_bo; 50 pub use self::vm_bo::*; 51 52 /// A DRM GPU VA manager. 53 /// 54 /// This object is refcounted, but the locations of mapped ranges may only be accessed or changed 55 /// via the special unique handle [`UniqueRefGpuVm`]. 56 /// 57 /// # Invariants 58 /// 59 /// * Stored in an allocation managed by the refcount in `self.vm`. 60 /// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. 61 /// * Does not contain any sparse [`GpuVa<T>`] instances. 62 #[pin_data] 63 pub struct GpuVm<T: DriverGpuVm> { 64 #[pin] 65 vm: Opaque<bindings::drm_gpuvm>, 66 /// Accessed only through the [`UniqueRefGpuVm`] reference. 67 data: UnsafeCell<T>, 68 } 69 70 // SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will 71 // drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. 72 unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {} 73 // SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. 74 unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {} 75 76 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 77 unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> { 78 fn inc_ref(&self) { 79 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 80 unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; 81 } 82 83 unsafe fn dec_ref(obj: NonNull<Self>) { 84 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 85 unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; 86 } 87 } 88 89 impl<T: DriverGpuVm> PartialEq for GpuVm<T> { 90 #[inline] 91 fn eq(&self, other: &Self) -> bool { 92 core::ptr::eq(self.as_raw(), other.as_raw()) 93 } 94 } 95 impl<T: DriverGpuVm> Eq for GpuVm<T> {} 96 97 impl<T: DriverGpuVm> GpuVm<T> { 98 const fn vtable() -> &'static bindings::drm_gpuvm_ops { 99 &bindings::drm_gpuvm_ops { 100 vm_free: Some(Self::vm_free), 101 op_alloc: None, 102 op_free: None, 103 vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN, 104 vm_bo_free: GpuVmBo::<T>::FREE_FN, 105 vm_bo_validate: None, 106 sm_step_map: None, 107 sm_step_unmap: None, 108 sm_step_remap: None, 109 } 110 } 111 112 /// Creates a GPUVM instance. 113 #[expect(clippy::new_ret_no_self)] 114 pub fn new<E>( 115 name: &'static CStr, 116 dev: &drm::Device<T::Driver>, 117 r_obj: &T::Object, 118 range: Range<u64>, 119 reserve_range: Range<u64>, 120 data: T, 121 ) -> Result<UniqueRefGpuVm<T>, E> 122 where 123 E: From<AllocError>, 124 E: From<core::convert::Infallible>, 125 { 126 let obj = KBox::try_pin_init::<E>( 127 try_pin_init!(Self { 128 data: UnsafeCell::new(data), 129 vm <- Opaque::ffi_init(|vm| { 130 // SAFETY: These arguments are valid. `vm` is valid until refcount drops to 131 // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag 132 // below. 133 unsafe { 134 bindings::drm_gpuvm_init( 135 vm, 136 name.as_char_ptr(), 137 bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE 138 | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, 139 dev.as_raw(), 140 r_obj.as_raw(), 141 range.start, 142 range.end - range.start, 143 reserve_range.start, 144 reserve_range.end - reserve_range.start, 145 const { Self::vtable() }, 146 ) 147 } 148 }), 149 }? E), 150 GFP_KERNEL | __GFP_ZERO, 151 )?; 152 // SAFETY: This transfers the initial refcount to the ARef. 153 let aref = unsafe { 154 ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( 155 Pin::into_inner_unchecked(obj), 156 ))) 157 }; 158 // INVARIANT: This reference is unique. 159 Ok(UniqueRefGpuVm(aref)) 160 } 161 162 /// Access this [`GpuVm`] from a raw pointer. 163 /// 164 /// # Safety 165 /// 166 /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains 167 /// valid for at least `'a`. 168 #[inline] 169 pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { 170 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the 171 // pointer is valid for 'a. 172 unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } 173 } 174 175 /// Returns a raw pointer to the embedded `struct drm_gpuvm`. 176 #[inline] 177 pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { 178 self.vm.get() 179 } 180 181 /// The start of the VA space. 182 #[inline] 183 pub fn va_start(&self) -> u64 { 184 // SAFETY: The `mm_start` field is immutable. 185 unsafe { (*self.as_raw()).mm_start } 186 } 187 188 /// The length of the GPU's virtual address space. 189 #[inline] 190 pub fn va_length(&self) -> u64 { 191 // SAFETY: The `mm_range` field is immutable. 192 unsafe { (*self.as_raw()).mm_range } 193 } 194 195 /// Returns the range of the GPU virtual address space. 196 #[inline] 197 pub fn va_range(&self) -> Range<u64> { 198 let start = self.va_start(); 199 // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail. 200 let end = start + self.va_length(); 201 Range { start, end } 202 } 203 204 /// Get or create the [`GpuVmBo`] for this gem object. 205 #[inline] 206 pub fn obtain( 207 &self, 208 obj: &T::Object, 209 data: impl PinInit<T::VmBoData>, 210 ) -> Result<ARef<GpuVmBo<T>>, AllocError> { 211 Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) 212 } 213 214 /// Clean up buffer objects that are no longer used. 215 #[inline] 216 pub fn deferred_cleanup(&self) { 217 // SAFETY: This GPUVM uses immediate mode. 218 unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } 219 } 220 221 /// Check if this GEM object is an external object for this GPUVM. 222 #[inline] 223 pub fn is_extobj(&self, obj: &T::Object) -> bool { 224 // SAFETY: We may call this with any GPUVM and GEM object. 225 unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } 226 } 227 228 /// Free this GPUVM. 229 /// 230 /// # Safety 231 /// 232 /// Called when refcount hits zero. 233 unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { 234 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. 235 let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; 236 // SAFETY: By type invariants we can free it when refcount hits zero. 237 drop(unsafe { KBox::from_raw(me) }) 238 } 239 240 #[inline] 241 fn raw_resv(&self) -> *mut bindings::dma_resv { 242 // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. 243 unsafe { (*(*self.as_raw()).r_obj).resv } 244 } 245 } 246 247 /// The manager for a GPUVM. 248 pub trait DriverGpuVm: Sized + Send { 249 /// Parent `Driver` for this object. 250 type Driver: drm::Driver<Object = Self::Object>; 251 252 /// The kind of GEM object stored in this GPUVM. 253 type Object: IntoGEMObject; 254 255 /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). 256 type VaData; 257 258 /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). 259 type VmBoData; 260 } 261 262 /// The core of the DRM GPU VA manager. 263 /// 264 /// This object is a unique reference to the VM that can access the interval tree and the Rust 265 /// `data` field. 266 /// 267 /// # Invariants 268 /// 269 /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. 270 pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>); 271 272 // SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and 273 // concurrent access to `data` is safe due to the `T: Sync` requirement. 274 unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {} 275 276 impl<T: DriverGpuVm> UniqueRefGpuVm<T> { 277 /// Access the data owned by this `UniqueRefGpuVm` immutably. 278 #[inline] 279 pub fn data_ref(&self) -> &T { 280 // SAFETY: By the type invariants we may access `data`. 281 unsafe { &*self.0.data.get() } 282 } 283 284 /// Access the data owned by this `UniqueRefGpuVm` mutably. 285 #[inline] 286 pub fn data(&mut self) -> &mut T { 287 // SAFETY: By the type invariants we may access `data`. 288 unsafe { &mut *self.0.data.get() } 289 } 290 } 291 292 impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> { 293 type Target = GpuVm<T>; 294 295 #[inline] 296 fn deref(&self) -> &GpuVm<T> { 297 &self.0 298 } 299 } 300