1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_exec.h" 7 8 #include <drm/drm_device.h> 9 #include <drm/drm_exec.h> 10 #include <drm/drm_file.h> 11 #include <uapi/drm/xe_drm.h> 12 #include <linux/delay.h> 13 14 #include "xe_bo.h" 15 #include "xe_device.h" 16 #include "xe_exec_queue.h" 17 #include "xe_hw_engine_group.h" 18 #include "xe_macros.h" 19 #include "xe_ring_ops_types.h" 20 #include "xe_sched_job.h" 21 #include "xe_sync.h" 22 #include "xe_vm.h" 23 24 /** 25 * DOC: Execbuf (User GPU command submission) 26 * 27 * Execs have historically been rather complicated in DRM drivers (at least in 28 * the i915) because a few things: 29 * 30 * - Passing in a list BO which are read / written to creating implicit syncs 31 * - Binding at exec time 32 * - Flow controlling the ring at exec time 33 * 34 * In XE we avoid all of this complication by not allowing a BO list to be 35 * passed into an exec, using the dma-buf implicit sync uAPI, have binds as 36 * seperate operations, and using the DRM scheduler to flow control the ring. 37 * Let's deep dive on each of these. 38 * 39 * We can get away from a BO list by forcing the user to use in / out fences on 40 * every exec rather than the kernel tracking dependencies of BO (e.g. if the 41 * user knows an exec writes to a BO and reads from the BO in the next exec, it 42 * is the user's responsibility to pass in / out fence between the two execs). 43 * 44 * We do not allow a user to trigger a bind at exec time rather we have a VM 45 * bind IOCTL which uses the same in / out fence interface as exec. In that 46 * sense, a VM bind is basically the same operation as an exec from the user 47 * perspective. e.g. If an exec depends on a VM bind use the in / out fence 48 * interface (struct drm_xe_sync) to synchronize like syncing between two 49 * dependent execs. 50 * 51 * Although a user cannot trigger a bind, we still have to rebind userptrs in 52 * the VM that have been invalidated since the last exec, likewise we also have 53 * to rebind BOs that have been evicted by the kernel. We schedule these rebinds 54 * behind any pending kernel operations on any external BOs in VM or any BOs 55 * private to the VM. This is accomplished by the rebinds waiting on BOs 56 * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs 57 * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and 58 * for external BOs). 59 * 60 * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute 61 * mode VMs we use preempt fences and a rebind worker (TODO: add link). 62 * 63 * There is no need to flow control the ring in the exec as we write the ring at 64 * submission time and set the DRM scheduler max job limit SIZE_OF_RING / 65 * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the 66 * ring is available. 67 * 68 * All of this results in a rather simple exec implementation. 69 * 70 * Flow 71 * ~~~~ 72 * 73 * .. code-block:: 74 * 75 * Parse input arguments 76 * Wait for any async VM bind passed as in-fences to start 77 * <----------------------------------------------------------------------| 78 * Lock global VM lock in read mode | 79 * Pin userptrs (also finds userptr invalidated since last exec) | 80 * Lock exec (VM dma-resv lock, external BOs dma-resv locks) | 81 * Validate BOs that have been evicted | 82 * Create job | 83 * Rebind invalidated userptrs + evicted BOs (non-compute-mode) | 84 * Add rebind fence dependency to job | 85 * Add job VM dma-resv bookkeeping slot (non-compute mode) | 86 * Add job to external BOs dma-resv write slots (non-compute mode) | 87 * Check if any userptrs invalidated since pin ------ Drop locks ---------| 88 * Install in / out fences for job 89 * Submit job 90 * Unlock all 91 */ 92 93 /* 94 * Add validation and rebinding to the drm_exec locking loop, since both can 95 * trigger eviction which may require sleeping dma_resv locks. 96 */ 97 static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec) 98 { 99 struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm); 100 101 /* The fence slot added here is intended for the exec sched job. */ 102 return xe_vm_validate_rebind(vm, &vm_exec->exec, 1); 103 } 104 105 int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 106 { 107 struct xe_device *xe = to_xe_device(dev); 108 struct xe_file *xef = to_xe_file(file); 109 struct drm_xe_exec *args = data; 110 struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); 111 u64 __user *addresses_user = u64_to_user_ptr(args->address); 112 struct xe_exec_queue *q; 113 struct xe_sync_entry *syncs = NULL; 114 u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; 115 struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn}; 116 struct drm_exec *exec = &vm_exec.exec; 117 u32 i, num_syncs, num_ufence = 0; 118 struct xe_sched_job *job; 119 struct xe_vm *vm; 120 bool write_locked, skip_retry = false; 121 ktime_t end = 0; 122 int err = 0; 123 struct xe_hw_engine_group *group; 124 enum xe_hw_engine_group_execution_mode mode, previous_mode; 125 126 if (XE_IOCTL_DBG(xe, args->extensions) || 127 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || 128 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 129 return -EINVAL; 130 131 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 132 if (XE_IOCTL_DBG(xe, !q)) 133 return -ENOENT; 134 135 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) 136 return -EINVAL; 137 138 if (XE_IOCTL_DBG(xe, args->num_batch_buffer && 139 q->width != args->num_batch_buffer)) 140 return -EINVAL; 141 142 if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) { 143 err = -ECANCELED; 144 goto err_exec_queue; 145 } 146 147 if (args->num_syncs) { 148 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); 149 if (!syncs) { 150 err = -ENOMEM; 151 goto err_exec_queue; 152 } 153 } 154 155 vm = q->vm; 156 157 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { 158 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], 159 &syncs_user[num_syncs], SYNC_PARSE_FLAG_EXEC | 160 (xe_vm_in_lr_mode(vm) ? 161 SYNC_PARSE_FLAG_LR_MODE : 0)); 162 if (err) 163 goto err_syncs; 164 165 if (xe_sync_is_ufence(&syncs[num_syncs])) 166 num_ufence++; 167 } 168 169 if (XE_IOCTL_DBG(xe, num_ufence > 1)) { 170 err = -EINVAL; 171 goto err_syncs; 172 } 173 174 if (xe_exec_queue_is_parallel(q)) { 175 err = __copy_from_user(addresses, addresses_user, sizeof(u64) * 176 q->width); 177 if (err) { 178 err = -EFAULT; 179 goto err_syncs; 180 } 181 } 182 183 group = q->hwe->hw_engine_group; 184 mode = xe_hw_engine_group_find_exec_mode(q); 185 186 if (mode == EXEC_MODE_DMA_FENCE) { 187 err = xe_hw_engine_group_get_mode(group, mode, &previous_mode); 188 if (err) 189 goto err_syncs; 190 } 191 192 retry: 193 if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) { 194 err = down_write_killable(&vm->lock); 195 write_locked = true; 196 } else { 197 /* We don't allow execs while the VM is in error state */ 198 err = down_read_interruptible(&vm->lock); 199 write_locked = false; 200 } 201 if (err) 202 goto err_syncs; 203 204 if (write_locked) { 205 err = xe_vm_userptr_pin(vm); 206 downgrade_write(&vm->lock); 207 write_locked = false; 208 if (err) 209 goto err_hw_exec_mode; 210 } 211 212 if (!args->num_batch_buffer) { 213 err = xe_vm_lock(vm, true); 214 if (err) 215 goto err_unlock_list; 216 217 if (!xe_vm_in_lr_mode(vm)) { 218 struct dma_fence *fence; 219 220 fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm); 221 if (IS_ERR(fence)) { 222 err = PTR_ERR(fence); 223 goto err_unlock_list; 224 } 225 for (i = 0; i < num_syncs; i++) 226 xe_sync_entry_signal(&syncs[i], fence); 227 xe_exec_queue_last_fence_set(q, vm, fence); 228 dma_fence_put(fence); 229 } 230 231 xe_vm_unlock(vm); 232 goto err_unlock_list; 233 } 234 235 vm_exec.vm = &vm->gpuvm; 236 vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT; 237 if (xe_vm_in_lr_mode(vm)) { 238 drm_exec_init(exec, vm_exec.flags, 0); 239 } else { 240 err = drm_gpuvm_exec_lock(&vm_exec); 241 if (err) { 242 if (xe_vm_validate_should_retry(exec, err, &end)) 243 err = -EAGAIN; 244 goto err_unlock_list; 245 } 246 } 247 248 if (xe_vm_is_closed_or_banned(q->vm)) { 249 drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n"); 250 err = -ECANCELED; 251 goto err_exec; 252 } 253 254 if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) { 255 err = -EWOULDBLOCK; /* Aliased to -EAGAIN */ 256 skip_retry = true; 257 goto err_exec; 258 } 259 260 job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? 261 addresses : &args->address); 262 if (IS_ERR(job)) { 263 err = PTR_ERR(job); 264 goto err_exec; 265 } 266 267 /* Wait behind rebinds */ 268 if (!xe_vm_in_lr_mode(vm)) { 269 err = xe_sched_job_add_deps(job, 270 xe_vm_resv(vm), 271 DMA_RESV_USAGE_KERNEL); 272 if (err) 273 goto err_put_job; 274 } 275 276 for (i = 0; i < num_syncs && !err; i++) 277 err = xe_sync_entry_add_deps(&syncs[i], job); 278 if (err) 279 goto err_put_job; 280 281 if (!xe_vm_in_lr_mode(vm)) { 282 err = xe_sched_job_last_fence_add_dep(job, vm); 283 if (err) 284 goto err_put_job; 285 286 err = down_read_interruptible(&vm->userptr.notifier_lock); 287 if (err) 288 goto err_put_job; 289 290 err = __xe_vm_userptr_needs_repin(vm); 291 if (err) 292 goto err_repin; 293 } 294 295 /* 296 * Point of no return, if we error after this point just set an error on 297 * the job and let the DRM scheduler / backend clean up the job. 298 */ 299 xe_sched_job_arm(job); 300 if (!xe_vm_in_lr_mode(vm)) 301 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished, 302 DMA_RESV_USAGE_BOOKKEEP, 303 DMA_RESV_USAGE_BOOKKEEP); 304 305 for (i = 0; i < num_syncs; i++) { 306 xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished); 307 xe_sched_job_init_user_fence(job, &syncs[i]); 308 } 309 310 if (xe_exec_queue_is_lr(q)) 311 q->ring_ops->emit_job(job); 312 if (!xe_vm_in_lr_mode(vm)) 313 xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished); 314 xe_sched_job_push(job); 315 xe_vm_reactivate_rebind(vm); 316 317 if (!err && !xe_vm_in_lr_mode(vm)) { 318 spin_lock(&xe->ttm.lru_lock); 319 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 320 spin_unlock(&xe->ttm.lru_lock); 321 } 322 323 if (mode == EXEC_MODE_LR) 324 xe_hw_engine_group_resume_faulting_lr_jobs(group); 325 326 err_repin: 327 if (!xe_vm_in_lr_mode(vm)) 328 up_read(&vm->userptr.notifier_lock); 329 err_put_job: 330 if (err) 331 xe_sched_job_put(job); 332 err_exec: 333 drm_exec_fini(exec); 334 err_unlock_list: 335 up_read(&vm->lock); 336 if (err == -EAGAIN && !skip_retry) 337 goto retry; 338 err_hw_exec_mode: 339 if (mode == EXEC_MODE_DMA_FENCE) 340 xe_hw_engine_group_put(group); 341 err_syncs: 342 while (num_syncs--) 343 xe_sync_entry_cleanup(&syncs[num_syncs]); 344 kfree(syncs); 345 err_exec_queue: 346 xe_exec_queue_put(q); 347 348 return err; 349 } 350