1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 3 #![cfg(CONFIG_DRM_GPUVM = "y")] 4 5 //! DRM GPUVM in immediate mode 6 //! 7 //! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated 8 //! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM 9 //! and the GPU's virtual address space has the same state at all times. 10 //! 11 //! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) 12 13 use kernel::{ 14 alloc::AllocError, 15 bindings, 16 drm, 17 drm::gem::IntoGEMObject, 18 prelude::*, 19 sync::aref::{ 20 ARef, 21 AlwaysRefCounted, // 22 }, 23 types::Opaque, // 24 }; 25 26 use core::{ 27 cell::UnsafeCell, 28 ops::{ 29 Deref, 30 Range, // 31 }, 32 ptr::NonNull, // 33 }; 34 35 /// A DRM GPU VA manager. 36 /// 37 /// This object is refcounted, but the locations of mapped ranges may only be accessed or changed 38 /// via the special unique handle [`UniqueRefGpuVm`]. 39 /// 40 /// # Invariants 41 /// 42 /// * Stored in an allocation managed by the refcount in `self.vm`. 43 /// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. 44 /// * Does not contain any sparse `GpuVa` instances. 45 #[pin_data] 46 pub struct GpuVm<T: DriverGpuVm> { 47 #[pin] 48 vm: Opaque<bindings::drm_gpuvm>, 49 /// Accessed only through the [`UniqueRefGpuVm`] reference. 50 data: UnsafeCell<T>, 51 } 52 53 // SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will 54 // drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. 55 unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {} 56 // SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. 57 unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {} 58 59 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 60 unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> { 61 fn inc_ref(&self) { 62 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 63 unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; 64 } 65 66 unsafe fn dec_ref(obj: NonNull<Self>) { 67 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. 68 unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; 69 } 70 } 71 72 impl<T: DriverGpuVm> PartialEq for GpuVm<T> { 73 #[inline] 74 fn eq(&self, other: &Self) -> bool { 75 core::ptr::eq(self.as_raw(), other.as_raw()) 76 } 77 } 78 impl<T: DriverGpuVm> Eq for GpuVm<T> {} 79 80 impl<T: DriverGpuVm> GpuVm<T> { 81 const fn vtable() -> &'static bindings::drm_gpuvm_ops { 82 &bindings::drm_gpuvm_ops { 83 vm_free: Some(Self::vm_free), 84 op_alloc: None, 85 op_free: None, 86 vm_bo_alloc: None, 87 vm_bo_free: None, 88 vm_bo_validate: None, 89 sm_step_map: None, 90 sm_step_unmap: None, 91 sm_step_remap: None, 92 } 93 } 94 95 /// Creates a GPUVM instance. 96 #[expect(clippy::new_ret_no_self)] 97 pub fn new<E>( 98 name: &'static CStr, 99 dev: &drm::Device<T::Driver>, 100 r_obj: &T::Object, 101 range: Range<u64>, 102 reserve_range: Range<u64>, 103 data: T, 104 ) -> Result<UniqueRefGpuVm<T>, E> 105 where 106 E: From<AllocError>, 107 E: From<core::convert::Infallible>, 108 { 109 let obj = KBox::try_pin_init::<E>( 110 try_pin_init!(Self { 111 data: UnsafeCell::new(data), 112 vm <- Opaque::ffi_init(|vm| { 113 // SAFETY: These arguments are valid. `vm` is valid until refcount drops to 114 // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag 115 // below. 116 unsafe { 117 bindings::drm_gpuvm_init( 118 vm, 119 name.as_char_ptr(), 120 bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE 121 | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, 122 dev.as_raw(), 123 r_obj.as_raw(), 124 range.start, 125 range.end - range.start, 126 reserve_range.start, 127 reserve_range.end - reserve_range.start, 128 const { Self::vtable() }, 129 ) 130 } 131 }), 132 }? E), 133 GFP_KERNEL | __GFP_ZERO, 134 )?; 135 // SAFETY: This transfers the initial refcount to the ARef. 136 let aref = unsafe { 137 ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( 138 Pin::into_inner_unchecked(obj), 139 ))) 140 }; 141 // INVARIANT: This reference is unique. 142 Ok(UniqueRefGpuVm(aref)) 143 } 144 145 /// Access this [`GpuVm`] from a raw pointer. 146 /// 147 /// # Safety 148 /// 149 /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains 150 /// valid for at least `'a`. 151 #[inline] 152 pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { 153 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the 154 // pointer is valid for 'a. 155 unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } 156 } 157 158 /// Returns a raw pointer to the embedded `struct drm_gpuvm`. 159 #[inline] 160 pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { 161 self.vm.get() 162 } 163 164 /// The start of the VA space. 165 #[inline] 166 pub fn va_start(&self) -> u64 { 167 // SAFETY: The `mm_start` field is immutable. 168 unsafe { (*self.as_raw()).mm_start } 169 } 170 171 /// The length of the GPU's virtual address space. 172 #[inline] 173 pub fn va_length(&self) -> u64 { 174 // SAFETY: The `mm_range` field is immutable. 175 unsafe { (*self.as_raw()).mm_range } 176 } 177 178 /// Returns the range of the GPU virtual address space. 179 #[inline] 180 pub fn va_range(&self) -> Range<u64> { 181 let start = self.va_start(); 182 // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail. 183 let end = start + self.va_length(); 184 Range { start, end } 185 } 186 187 /// Clean up buffer objects that are no longer used. 188 #[inline] 189 pub fn deferred_cleanup(&self) { 190 // SAFETY: This GPUVM uses immediate mode. 191 unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } 192 } 193 194 /// Check if this GEM object is an external object for this GPUVM. 195 #[inline] 196 pub fn is_extobj(&self, obj: &T::Object) -> bool { 197 // SAFETY: We may call this with any GPUVM and GEM object. 198 unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } 199 } 200 201 /// Free this GPUVM. 202 /// 203 /// # Safety 204 /// 205 /// Called when refcount hits zero. 206 unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { 207 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. 208 let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; 209 // SAFETY: By type invariants we can free it when refcount hits zero. 210 drop(unsafe { KBox::from_raw(me) }) 211 } 212 } 213 214 /// The manager for a GPUVM. 215 pub trait DriverGpuVm: Sized + Send { 216 /// Parent `Driver` for this object. 217 type Driver: drm::Driver<Object = Self::Object>; 218 219 /// The kind of GEM object stored in this GPUVM. 220 type Object: IntoGEMObject; 221 } 222 223 /// The core of the DRM GPU VA manager. 224 /// 225 /// This object is a unique reference to the VM that can access the interval tree and the Rust 226 /// `data` field. 227 /// 228 /// # Invariants 229 /// 230 /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. 231 pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>); 232 233 // SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and 234 // concurrent access to `data` is safe due to the `T: Sync` requirement. 235 unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {} 236 237 impl<T: DriverGpuVm> UniqueRefGpuVm<T> { 238 /// Access the data owned by this `UniqueRefGpuVm` immutably. 239 #[inline] 240 pub fn data_ref(&self) -> &T { 241 // SAFETY: By the type invariants we may access `data`. 242 unsafe { &*self.0.data.get() } 243 } 244 245 /// Access the data owned by this `UniqueRefGpuVm` mutably. 246 #[inline] 247 pub fn data(&mut self) -> &mut T { 248 // SAFETY: By the type invariants we may access `data`. 249 unsafe { &mut *self.0.data.get() } 250 } 251 } 252 253 impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> { 254 type Target = GpuVm<T>; 255 256 #[inline] 257 fn deref(&self) -> &GpuVm<T> { 258 &self.0 259 } 260 } 261