1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_exec.h" 7 8 #include <drm/drm_device.h> 9 #include <drm/drm_exec.h> 10 #include <drm/drm_file.h> 11 #include <uapi/drm/xe_drm.h> 12 #include <linux/delay.h> 13 14 #include "xe_bo.h" 15 #include "xe_device.h" 16 #include "xe_exec_queue.h" 17 #include "xe_hw_engine_group.h" 18 #include "xe_macros.h" 19 #include "xe_ring_ops_types.h" 20 #include "xe_sched_job.h" 21 #include "xe_sync.h" 22 #include "xe_svm.h" 23 #include "xe_vm.h" 24 25 /** 26 * DOC: Execbuf (User GPU command submission) 27 * 28 * Execs have historically been rather complicated in DRM drivers (at least in 29 * the i915) because a few things: 30 * 31 * - Passing in a list BO which are read / written to creating implicit syncs 32 * - Binding at exec time 33 * - Flow controlling the ring at exec time 34 * 35 * In XE we avoid all of this complication by not allowing a BO list to be 36 * passed into an exec, using the dma-buf implicit sync uAPI, have binds as 37 * separate operations, and using the DRM scheduler to flow control the ring. 38 * Let's deep dive on each of these. 39 * 40 * We can get away from a BO list by forcing the user to use in / out fences on 41 * every exec rather than the kernel tracking dependencies of BO (e.g. if the 42 * user knows an exec writes to a BO and reads from the BO in the next exec, it 43 * is the user's responsibility to pass in / out fence between the two execs). 44 * 45 * We do not allow a user to trigger a bind at exec time rather we have a VM 46 * bind IOCTL which uses the same in / out fence interface as exec. In that 47 * sense, a VM bind is basically the same operation as an exec from the user 48 * perspective. e.g. If an exec depends on a VM bind use the in / out fence 49 * interface (struct drm_xe_sync) to synchronize like syncing between two 50 * dependent execs. 51 * 52 * Although a user cannot trigger a bind, we still have to rebind userptrs in 53 * the VM that have been invalidated since the last exec, likewise we also have 54 * to rebind BOs that have been evicted by the kernel. We schedule these rebinds 55 * behind any pending kernel operations on any external BOs in VM or any BOs 56 * private to the VM. This is accomplished by the rebinds waiting on BOs 57 * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs 58 * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and 59 * for external BOs). 60 * 61 * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute 62 * mode VMs we use preempt fences and a rebind worker (TODO: add link). 63 * 64 * There is no need to flow control the ring in the exec as we write the ring at 65 * submission time and set the DRM scheduler max job limit SIZE_OF_RING / 66 * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the 67 * ring is available. 68 * 69 * All of this results in a rather simple exec implementation. 70 * 71 * Flow 72 * ~~~~ 73 * 74 * .. code-block:: 75 * 76 * Parse input arguments 77 * Wait for any async VM bind passed as in-fences to start 78 * <----------------------------------------------------------------------| 79 * Lock global VM lock in read mode | 80 * Pin userptrs (also finds userptr invalidated since last exec) | 81 * Lock exec (VM dma-resv lock, external BOs dma-resv locks) | 82 * Validate BOs that have been evicted | 83 * Create job | 84 * Rebind invalidated userptrs + evicted BOs (non-compute-mode) | 85 * Add rebind fence dependency to job | 86 * Add job VM dma-resv bookkeeping slot (non-compute mode) | 87 * Add job to external BOs dma-resv write slots (non-compute mode) | 88 * Check if any userptrs invalidated since pin ------ Drop locks ---------| 89 * Install in / out fences for job 90 * Submit job 91 * Unlock all 92 */ 93 94 /* 95 * Add validation and rebinding to the drm_exec locking loop, since both can 96 * trigger eviction which may require sleeping dma_resv locks. 97 */ 98 static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec) 99 { 100 struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm); 101 int ret; 102 103 /* The fence slot added here is intended for the exec sched job. */ 104 xe_vm_set_validation_exec(vm, &vm_exec->exec); 105 ret = xe_vm_validate_rebind(vm, &vm_exec->exec, 1); 106 xe_vm_set_validation_exec(vm, NULL); 107 return ret; 108 } 109 110 int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 111 { 112 struct xe_device *xe = to_xe_device(dev); 113 struct xe_file *xef = to_xe_file(file); 114 struct drm_xe_exec *args = data; 115 struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); 116 u64 __user *addresses_user = u64_to_user_ptr(args->address); 117 struct xe_exec_queue *q; 118 struct xe_sync_entry *syncs = NULL; 119 u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; 120 struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn}; 121 struct drm_exec *exec = &vm_exec.exec; 122 u32 i, num_syncs, num_ufence = 0; 123 struct xe_validation_ctx ctx; 124 struct xe_sched_job *job; 125 struct xe_vm *vm; 126 bool write_locked, skip_retry = false; 127 int err = 0; 128 struct xe_hw_engine_group *group; 129 enum xe_hw_engine_group_execution_mode mode, previous_mode; 130 131 if (XE_IOCTL_DBG(xe, args->extensions) || 132 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || 133 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 134 return -EINVAL; 135 136 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 137 if (XE_IOCTL_DBG(xe, !q)) 138 return -ENOENT; 139 140 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) { 141 err = -EINVAL; 142 goto err_exec_queue; 143 } 144 145 if (XE_IOCTL_DBG(xe, args->num_batch_buffer && 146 q->width != args->num_batch_buffer)) { 147 err = -EINVAL; 148 goto err_exec_queue; 149 } 150 151 if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) { 152 err = -ECANCELED; 153 goto err_exec_queue; 154 } 155 156 if (args->num_syncs) { 157 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); 158 if (!syncs) { 159 err = -ENOMEM; 160 goto err_exec_queue; 161 } 162 } 163 164 vm = q->vm; 165 166 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { 167 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], 168 &syncs_user[num_syncs], NULL, 0, 169 SYNC_PARSE_FLAG_EXEC | 170 (xe_vm_in_lr_mode(vm) ? 171 SYNC_PARSE_FLAG_LR_MODE : 0)); 172 if (err) 173 goto err_syncs; 174 175 if (xe_sync_is_ufence(&syncs[num_syncs])) 176 num_ufence++; 177 } 178 179 if (XE_IOCTL_DBG(xe, num_ufence > 1)) { 180 err = -EINVAL; 181 goto err_syncs; 182 } 183 184 if (xe_exec_queue_is_parallel(q)) { 185 err = copy_from_user(addresses, addresses_user, sizeof(u64) * 186 q->width); 187 if (err) { 188 err = -EFAULT; 189 goto err_syncs; 190 } 191 } 192 193 group = q->hwe->hw_engine_group; 194 mode = xe_hw_engine_group_find_exec_mode(q); 195 196 if (mode == EXEC_MODE_DMA_FENCE) { 197 err = xe_hw_engine_group_get_mode(group, mode, &previous_mode); 198 if (err) 199 goto err_syncs; 200 } 201 202 retry: 203 if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) { 204 err = down_write_killable(&vm->lock); 205 write_locked = true; 206 } else { 207 /* We don't allow execs while the VM is in error state */ 208 err = down_read_interruptible(&vm->lock); 209 write_locked = false; 210 } 211 if (err) 212 goto err_hw_exec_mode; 213 214 if (write_locked) { 215 err = xe_vm_userptr_pin(vm); 216 downgrade_write(&vm->lock); 217 write_locked = false; 218 if (err) 219 goto err_unlock_list; 220 } 221 222 if (!args->num_batch_buffer) { 223 err = xe_vm_lock(vm, true); 224 if (err) 225 goto err_unlock_list; 226 227 if (!xe_vm_in_lr_mode(vm)) { 228 struct dma_fence *fence; 229 230 fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm); 231 if (IS_ERR(fence)) { 232 err = PTR_ERR(fence); 233 xe_vm_unlock(vm); 234 goto err_unlock_list; 235 } 236 for (i = 0; i < num_syncs; i++) 237 xe_sync_entry_signal(&syncs[i], fence); 238 xe_exec_queue_last_fence_set(q, vm, fence); 239 dma_fence_put(fence); 240 } 241 242 xe_vm_unlock(vm); 243 goto err_unlock_list; 244 } 245 246 /* 247 * It's OK to block interruptible here with the vm lock held, since 248 * on task freezing during suspend / hibernate, the call will 249 * return -ERESTARTSYS and the IOCTL will be rerun. 250 */ 251 err = wait_for_completion_interruptible(&xe->pm_block); 252 if (err) 253 goto err_unlock_list; 254 255 if (!xe_vm_in_lr_mode(vm)) { 256 vm_exec.vm = &vm->gpuvm; 257 vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT; 258 err = xe_validation_exec_lock(&ctx, &vm_exec, &xe->val); 259 if (err) 260 goto err_unlock_list; 261 } 262 263 if (xe_vm_is_closed_or_banned(q->vm)) { 264 drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n"); 265 err = -ECANCELED; 266 goto err_exec; 267 } 268 269 if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) { 270 err = -EWOULDBLOCK; /* Aliased to -EAGAIN */ 271 skip_retry = true; 272 goto err_exec; 273 } 274 275 if (xe_exec_queue_uses_pxp(q)) { 276 err = xe_vm_validate_protected(q->vm); 277 if (err) 278 goto err_exec; 279 } 280 281 job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? 282 addresses : &args->address); 283 if (IS_ERR(job)) { 284 err = PTR_ERR(job); 285 goto err_exec; 286 } 287 288 /* Wait behind rebinds */ 289 if (!xe_vm_in_lr_mode(vm)) { 290 err = xe_sched_job_add_deps(job, 291 xe_vm_resv(vm), 292 DMA_RESV_USAGE_KERNEL); 293 if (err) 294 goto err_put_job; 295 } 296 297 for (i = 0; i < num_syncs && !err; i++) 298 err = xe_sync_entry_add_deps(&syncs[i], job); 299 if (err) 300 goto err_put_job; 301 302 if (!xe_vm_in_lr_mode(vm)) { 303 err = xe_sched_job_last_fence_add_dep(job, vm); 304 if (err) 305 goto err_put_job; 306 307 err = xe_svm_notifier_lock_interruptible(vm); 308 if (err) 309 goto err_put_job; 310 311 err = __xe_vm_userptr_needs_repin(vm); 312 if (err) 313 goto err_repin; 314 } 315 316 /* 317 * Point of no return, if we error after this point just set an error on 318 * the job and let the DRM scheduler / backend clean up the job. 319 */ 320 xe_sched_job_arm(job); 321 if (!xe_vm_in_lr_mode(vm)) 322 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished, 323 DMA_RESV_USAGE_BOOKKEEP, 324 DMA_RESV_USAGE_BOOKKEEP); 325 326 for (i = 0; i < num_syncs; i++) { 327 xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished); 328 xe_sched_job_init_user_fence(job, &syncs[i]); 329 } 330 331 if (xe_exec_queue_is_lr(q)) 332 q->ring_ops->emit_job(job); 333 if (!xe_vm_in_lr_mode(vm)) 334 xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished); 335 xe_sched_job_push(job); 336 xe_vm_reactivate_rebind(vm); 337 338 if (!err && !xe_vm_in_lr_mode(vm)) { 339 spin_lock(&xe->ttm.lru_lock); 340 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 341 spin_unlock(&xe->ttm.lru_lock); 342 } 343 344 if (mode == EXEC_MODE_LR) 345 xe_hw_engine_group_resume_faulting_lr_jobs(group); 346 347 err_repin: 348 if (!xe_vm_in_lr_mode(vm)) 349 xe_svm_notifier_unlock(vm); 350 err_put_job: 351 if (err) 352 xe_sched_job_put(job); 353 err_exec: 354 if (!xe_vm_in_lr_mode(vm)) 355 xe_validation_ctx_fini(&ctx); 356 err_unlock_list: 357 up_read(&vm->lock); 358 if (err == -EAGAIN && !skip_retry) 359 goto retry; 360 err_hw_exec_mode: 361 if (mode == EXEC_MODE_DMA_FENCE) 362 xe_hw_engine_group_put(group); 363 err_syncs: 364 while (num_syncs--) 365 xe_sync_entry_cleanup(&syncs[num_syncs]); 366 kfree(syncs); 367 err_exec_queue: 368 xe_exec_queue_put(q); 369 370 return err; 371 } 372