1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 3 #![cfg(CONFIG_DRM_GPUVM = "y")] 4 5 //! DRM GPUVM in immediate mode 6 //! 7 //! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated 8 //! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM 9 //! and the GPU's virtual address space has the same state at all times. 10 //! 11 //! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) 12 13 use kernel::{ 14 alloc::AllocError, 15 bindings, 16 drm, 17 drm::gem::IntoGEMObject, 18 prelude::*, 19 sync::aref::{ 20 ARef, 21 AlwaysRefCounted, // 22 }, 23 types::Opaque, // 24 }; 25 26 use core::{ 27 cell::UnsafeCell, 28 mem::ManuallyDrop, 29 ops::{ 30 Deref, 31 Range, // 32 }, 33 ptr::{ 34 self, 35 NonNull, // 36 }, // 37 }; 38 39 mod vm_bo; 40 pub use self::vm_bo::*; 41 42 /// A DRM GPU VA manager. 43 /// 44 /// This object is refcounted, but the locations of mapped ranges may only be accessed or changed 45 /// via the special unique handle [`UniqueRefGpuVm`]. 46 /// 47 /// # Invariants 48 /// 49 /// * Stored in an allocation managed by the refcount in `self.vm`. 50 /// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. 51 /// * Does not contain any sparse `GpuVa` instances. 52 #[pin_data] 53 pub struct GpuVm<T: DriverGpuVm> { 54 #[pin] 55 vm: Opaque<bindings::drm_gpuvm>, 56 /// Accessed only through the [`UniqueRefGpuVm`] reference. 57 data: UnsafeCell<T>, 58 } 59 60 // SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will 61 // drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. 62 unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {} 63 // SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. 64 unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {} 65 66 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 67 unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> { 68 fn inc_ref(&self) { 69 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 70 unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; 71 } 72 73 unsafe fn dec_ref(obj: NonNull<Self>) { 74 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 75 unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; 76 } 77 } 78 79 impl<T: DriverGpuVm> PartialEq for GpuVm<T> { 80 #[inline] 81 fn eq(&self, other: &Self) -> bool { 82 core::ptr::eq(self.as_raw(), other.as_raw()) 83 } 84 } 85 impl<T: DriverGpuVm> Eq for GpuVm<T> {} 86 87 impl<T: DriverGpuVm> GpuVm<T> { 88 const fn vtable() -> &'static bindings::drm_gpuvm_ops { 89 &bindings::drm_gpuvm_ops { 90 vm_free: Some(Self::vm_free), 91 op_alloc: None, 92 op_free: None, 93 vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN, 94 vm_bo_free: GpuVmBo::<T>::FREE_FN, 95 vm_bo_validate: None, 96 sm_step_map: None, 97 sm_step_unmap: None, 98 sm_step_remap: None, 99 } 100 } 101 102 /// Creates a GPUVM instance. 103 #[expect(clippy::new_ret_no_self)] 104 pub fn new<E>( 105 name: &'static CStr, 106 dev: &drm::Device<T::Driver>, 107 r_obj: &T::Object, 108 range: Range<u64>, 109 reserve_range: Range<u64>, 110 data: T, 111 ) -> Result<UniqueRefGpuVm<T>, E> 112 where 113 E: From<AllocError>, 114 E: From<core::convert::Infallible>, 115 { 116 let obj = KBox::try_pin_init::<E>( 117 try_pin_init!(Self { 118 data: UnsafeCell::new(data), 119 vm <- Opaque::ffi_init(|vm| { 120 // SAFETY: These arguments are valid. `vm` is valid until refcount drops to 121 // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag 122 // below. 123 unsafe { 124 bindings::drm_gpuvm_init( 125 vm, 126 name.as_char_ptr(), 127 bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE 128 | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, 129 dev.as_raw(), 130 r_obj.as_raw(), 131 range.start, 132 range.end - range.start, 133 reserve_range.start, 134 reserve_range.end - reserve_range.start, 135 const { Self::vtable() }, 136 ) 137 } 138 }), 139 }? E), 140 GFP_KERNEL | __GFP_ZERO, 141 )?; 142 // SAFETY: This transfers the initial refcount to the ARef. 143 let aref = unsafe { 144 ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( 145 Pin::into_inner_unchecked(obj), 146 ))) 147 }; 148 // INVARIANT: This reference is unique. 149 Ok(UniqueRefGpuVm(aref)) 150 } 151 152 /// Access this [`GpuVm`] from a raw pointer. 153 /// 154 /// # Safety 155 /// 156 /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains 157 /// valid for at least `'a`. 158 #[inline] 159 pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { 160 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the 161 // pointer is valid for 'a. 162 unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } 163 } 164 165 /// Returns a raw pointer to the embedded `struct drm_gpuvm`. 166 #[inline] 167 pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { 168 self.vm.get() 169 } 170 171 /// The start of the VA space. 172 #[inline] 173 pub fn va_start(&self) -> u64 { 174 // SAFETY: The `mm_start` field is immutable. 175 unsafe { (*self.as_raw()).mm_start } 176 } 177 178 /// The length of the GPU's virtual address space. 179 #[inline] 180 pub fn va_length(&self) -> u64 { 181 // SAFETY: The `mm_range` field is immutable. 182 unsafe { (*self.as_raw()).mm_range } 183 } 184 185 /// Returns the range of the GPU virtual address space. 186 #[inline] 187 pub fn va_range(&self) -> Range<u64> { 188 let start = self.va_start(); 189 // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail. 190 let end = start + self.va_length(); 191 Range { start, end } 192 } 193 194 /// Get or create the [`GpuVmBo`] for this gem object. 195 #[inline] 196 pub fn obtain( 197 &self, 198 obj: &T::Object, 199 data: impl PinInit<T::VmBoData>, 200 ) -> Result<ARef<GpuVmBo<T>>, AllocError> { 201 Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) 202 } 203 204 /// Clean up buffer objects that are no longer used. 205 #[inline] 206 pub fn deferred_cleanup(&self) { 207 // SAFETY: This GPUVM uses immediate mode. 208 unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } 209 } 210 211 /// Check if this GEM object is an external object for this GPUVM. 212 #[inline] 213 pub fn is_extobj(&self, obj: &T::Object) -> bool { 214 // SAFETY: We may call this with any GPUVM and GEM object. 215 unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } 216 } 217 218 /// Free this GPUVM. 219 /// 220 /// # Safety 221 /// 222 /// Called when refcount hits zero. 223 unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { 224 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. 225 let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; 226 // SAFETY: By type invariants we can free it when refcount hits zero. 227 drop(unsafe { KBox::from_raw(me) }) 228 } 229 230 #[inline] 231 fn raw_resv(&self) -> *mut bindings::dma_resv { 232 // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. 233 unsafe { (*(*self.as_raw()).r_obj).resv } 234 } 235 } 236 237 /// The manager for a GPUVM. 238 pub trait DriverGpuVm: Sized + Send { 239 /// Parent `Driver` for this object. 240 type Driver: drm::Driver<Object = Self::Object>; 241 242 /// The kind of GEM object stored in this GPUVM. 243 type Object: IntoGEMObject; 244 245 /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). 246 type VmBoData; 247 } 248 249 /// The core of the DRM GPU VA manager. 250 /// 251 /// This object is a unique reference to the VM that can access the interval tree and the Rust 252 /// `data` field. 253 /// 254 /// # Invariants 255 /// 256 /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. 257 pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>); 258 259 // SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and 260 // concurrent access to `data` is safe due to the `T: Sync` requirement. 261 unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {} 262 263 impl<T: DriverGpuVm> UniqueRefGpuVm<T> { 264 /// Access the data owned by this `UniqueRefGpuVm` immutably. 265 #[inline] 266 pub fn data_ref(&self) -> &T { 267 // SAFETY: By the type invariants we may access `data`. 268 unsafe { &*self.0.data.get() } 269 } 270 271 /// Access the data owned by this `UniqueRefGpuVm` mutably. 272 #[inline] 273 pub fn data(&mut self) -> &mut T { 274 // SAFETY: By the type invariants we may access `data`. 275 unsafe { &mut *self.0.data.get() } 276 } 277 } 278 279 impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> { 280 type Target = GpuVm<T>; 281 282 #[inline] 283 fn deref(&self) -> &GpuVm<T> { 284 &self.0 285 } 286 } 287