1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_exec.h" 7 8 #include <drm/drm_device.h> 9 #include <drm/drm_exec.h> 10 #include <drm/drm_file.h> 11 #include <uapi/drm/xe_drm.h> 12 #include <linux/delay.h> 13 14 #include "xe_device.h" 15 #include "xe_exec_queue.h" 16 #include "xe_hw_engine_group.h" 17 #include "xe_macros.h" 18 #include "xe_pm.h" 19 #include "xe_ring_ops_types.h" 20 #include "xe_sched_job.h" 21 #include "xe_sync.h" 22 #include "xe_svm.h" 23 #include "xe_trace.h" 24 #include "xe_vm.h" 25 26 /** 27 * DOC: Execbuf (User GPU command submission) 28 * 29 * Execs have historically been rather complicated in DRM drivers (at least in 30 * the i915) because a few things: 31 * 32 * - Passing in a list BO which are read / written to creating implicit syncs 33 * - Binding at exec time 34 * - Flow controlling the ring at exec time 35 * 36 * In Xe we avoid all of this complication by not allowing a BO list to be 37 * passed into an exec, using the dma-buf implicit sync uAPI, have binds as 38 * separate operations, and using the DRM scheduler to flow control the ring. 39 * Let's deep dive on each of these. 40 * 41 * We can get away from a BO list by forcing the user to use in / out fences on 42 * every exec rather than the kernel tracking dependencies of BO (e.g. if the 43 * user knows an exec writes to a BO and reads from the BO in the next exec, it 44 * is the user's responsibility to pass in / out fence between the two execs). 45 * 46 * We do not allow a user to trigger a bind at exec time rather we have a VM 47 * bind IOCTL which uses the same in / out fence interface as exec. In that 48 * sense, a VM bind is basically the same operation as an exec from the user 49 * perspective. e.g. If an exec depends on a VM bind use the in / out fence 50 * interface (struct drm_xe_sync) to synchronize like syncing between two 51 * dependent execs. 52 * 53 * Although a user cannot trigger a bind, we still have to rebind userptrs in 54 * the VM that have been invalidated since the last exec, likewise we also have 55 * to rebind BOs that have been evicted by the kernel. We schedule these rebinds 56 * behind any pending kernel operations on any external BOs in VM or any BOs 57 * private to the VM. This is accomplished by the rebinds waiting on BOs 58 * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs 59 * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and 60 * for external BOs). 61 * 62 * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute 63 * mode VMs we use preempt fences and a rebind worker (TODO: add link). 64 * 65 * There is no need to flow control the ring in the exec as we write the ring at 66 * submission time and set the DRM scheduler max job limit SIZE_OF_RING / 67 * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the 68 * ring is available. 69 * 70 * All of this results in a rather simple exec implementation. 71 * 72 * Flow 73 * ~~~~ 74 * 75 * .. code-block:: 76 * 77 * Parse input arguments 78 * Wait for any async VM bind passed as in-fences to start 79 * <----------------------------------------------------------------------| 80 * Lock global VM lock in read mode | 81 * Pin userptrs (also finds userptr invalidated since last exec) | 82 * Lock exec (VM dma-resv lock, external BOs dma-resv locks) | 83 * Validate BOs that have been evicted | 84 * Create job | 85 * Rebind invalidated userptrs + evicted BOs (non-compute-mode) | 86 * Add rebind fence dependency to job | 87 * Add job VM dma-resv bookkeeping slot (non-compute mode) | 88 * Add job to external BOs dma-resv write slots (non-compute mode) | 89 * Check if any userptrs invalidated since pin ------ Drop locks ---------| 90 * Install in / out fences for job 91 * Submit job 92 * Unlock all 93 */ 94 95 /* 96 * Add validation and rebinding to the drm_exec locking loop, since both can 97 * trigger eviction which may require sleeping dma_resv locks. 98 */ 99 static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec) 100 { 101 struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm); 102 int ret; 103 104 /* The fence slot added here is intended for the exec sched job. */ 105 xe_vm_set_validation_exec(vm, &vm_exec->exec); 106 ret = xe_vm_validate_rebind(vm, &vm_exec->exec, 1); 107 xe_vm_set_validation_exec(vm, NULL); 108 return ret; 109 } 110 111 int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 112 { 113 struct xe_device *xe = to_xe_device(dev); 114 struct xe_file *xef = to_xe_file(file); 115 struct drm_xe_exec *args = data; 116 struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); 117 u64 __user *addresses_user = u64_to_user_ptr(args->address); 118 struct xe_exec_queue *q; 119 struct xe_sync_entry *syncs = NULL; 120 u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; 121 struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn}; 122 struct drm_exec *exec = &vm_exec.exec; 123 u32 i, num_syncs, num_in_sync = 0, num_ufence = 0; 124 struct xe_validation_ctx ctx; 125 struct xe_sched_job *job; 126 struct xe_vm *vm; 127 bool write_locked; 128 int err = 0; 129 struct xe_hw_engine_group *group; 130 enum xe_hw_engine_group_execution_mode mode, previous_mode; 131 132 if (XE_IOCTL_DBG(xe, args->extensions) || 133 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || 134 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) || 135 XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) 136 return -EINVAL; 137 138 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 139 if (XE_IOCTL_DBG(xe, !q)) 140 return -ENOENT; 141 142 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) { 143 err = -EINVAL; 144 goto err_exec_queue; 145 } 146 147 if (XE_IOCTL_DBG(xe, args->num_batch_buffer && 148 q->width != args->num_batch_buffer)) { 149 err = -EINVAL; 150 goto err_exec_queue; 151 } 152 153 if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) { 154 err = -ECANCELED; 155 goto err_exec_queue; 156 } 157 158 if (atomic_read(&q->job_cnt) >= XE_MAX_JOB_COUNT_PER_EXEC_QUEUE) { 159 trace_xe_exec_queue_reach_max_job_count(q, XE_MAX_JOB_COUNT_PER_EXEC_QUEUE); 160 err = -EAGAIN; 161 goto err_exec_queue; 162 } 163 164 if (args->num_syncs) { 165 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); 166 if (!syncs) { 167 err = -ENOMEM; 168 goto err_exec_queue; 169 } 170 } 171 172 vm = q->vm; 173 174 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { 175 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], 176 &syncs_user[num_syncs], NULL, 0, 177 SYNC_PARSE_FLAG_EXEC | 178 (xe_vm_in_lr_mode(vm) ? 179 SYNC_PARSE_FLAG_LR_MODE : 0)); 180 if (err) 181 goto err_syncs; 182 183 if (xe_sync_is_ufence(&syncs[num_syncs])) 184 num_ufence++; 185 186 if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs])) 187 num_in_sync++; 188 } 189 190 if (XE_IOCTL_DBG(xe, num_ufence > 1)) { 191 err = -EINVAL; 192 goto err_syncs; 193 } 194 195 if (xe_exec_queue_is_parallel(q)) { 196 err = copy_from_user(addresses, addresses_user, sizeof(u64) * 197 q->width); 198 if (err) { 199 err = -EFAULT; 200 goto err_syncs; 201 } 202 } 203 204 group = q->hwe->hw_engine_group; 205 mode = xe_hw_engine_group_find_exec_mode(q); 206 207 if (mode == EXEC_MODE_DMA_FENCE) { 208 err = xe_hw_engine_group_get_mode(group, mode, &previous_mode, 209 syncs, num_in_sync ? 210 num_syncs : 0); 211 if (err) 212 goto err_syncs; 213 } 214 215 retry: 216 if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) { 217 err = down_write_killable(&vm->lock); 218 write_locked = true; 219 } else { 220 /* We don't allow execs while the VM is in error state */ 221 err = down_read_interruptible(&vm->lock); 222 write_locked = false; 223 } 224 if (err) 225 goto err_hw_exec_mode; 226 227 if (write_locked) { 228 err = xe_vm_userptr_pin(vm); 229 downgrade_write(&vm->lock); 230 write_locked = false; 231 if (err) 232 goto err_unlock_list; 233 } 234 235 if (!args->num_batch_buffer) { 236 err = xe_vm_lock(vm, true); 237 if (err) 238 goto err_unlock_list; 239 240 if (!xe_vm_in_lr_mode(vm)) { 241 struct dma_fence *fence; 242 243 fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm); 244 if (IS_ERR(fence)) { 245 err = PTR_ERR(fence); 246 xe_vm_unlock(vm); 247 goto err_unlock_list; 248 } 249 for (i = 0; i < num_syncs; i++) 250 xe_sync_entry_signal(&syncs[i], fence); 251 xe_exec_queue_last_fence_set(q, vm, fence); 252 dma_fence_put(fence); 253 } 254 255 xe_vm_unlock(vm); 256 goto err_unlock_list; 257 } 258 259 /* 260 * It's OK to block interruptible here with the vm lock held, since 261 * on task freezing during suspend / hibernate, the call will 262 * return -ERESTARTSYS and the IOCTL will be rerun. 263 */ 264 err = xe_pm_block_on_suspend(xe); 265 if (err) 266 goto err_unlock_list; 267 268 if (!xe_vm_in_lr_mode(vm)) { 269 vm_exec.vm = &vm->gpuvm; 270 vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT; 271 err = xe_validation_exec_lock(&ctx, &vm_exec, &xe->val); 272 if (err) 273 goto err_unlock_list; 274 } 275 276 if (xe_vm_is_closed_or_banned(q->vm)) { 277 drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n"); 278 err = -ECANCELED; 279 goto err_exec; 280 } 281 282 if (xe_exec_queue_uses_pxp(q)) { 283 err = xe_vm_validate_protected(q->vm); 284 if (err) 285 goto err_exec; 286 } 287 288 job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? 289 addresses : &args->address); 290 if (IS_ERR(job)) { 291 err = PTR_ERR(job); 292 goto err_exec; 293 } 294 295 /* Wait behind rebinds */ 296 if (!xe_vm_in_lr_mode(vm)) { 297 err = xe_sched_job_add_deps(job, 298 xe_vm_resv(vm), 299 DMA_RESV_USAGE_KERNEL); 300 if (err) 301 goto err_put_job; 302 } 303 304 for (i = 0; i < num_syncs && !err; i++) 305 err = xe_sync_entry_add_deps(&syncs[i], job); 306 if (err) 307 goto err_put_job; 308 309 if (!xe_vm_in_lr_mode(vm)) { 310 err = xe_svm_notifier_lock_interruptible(vm); 311 if (err) 312 goto err_put_job; 313 314 err = __xe_vm_userptr_needs_repin(vm); 315 if (err) 316 goto err_repin; 317 } 318 319 /* 320 * Point of no return, if we error after this point just set an error on 321 * the job and let the DRM scheduler / backend clean up the job. 322 */ 323 xe_sched_job_arm(job); 324 if (!xe_vm_in_lr_mode(vm)) 325 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished, 326 DMA_RESV_USAGE_BOOKKEEP, 327 DMA_RESV_USAGE_BOOKKEEP); 328 329 for (i = 0; i < num_syncs; i++) { 330 xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished); 331 xe_sched_job_init_user_fence(job, &syncs[i]); 332 } 333 334 if (!xe_vm_in_lr_mode(vm)) 335 xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished); 336 xe_sched_job_push(job); 337 xe_vm_reactivate_rebind(vm); 338 339 if (!err && !xe_vm_in_lr_mode(vm)) { 340 spin_lock(&xe->ttm.lru_lock); 341 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 342 spin_unlock(&xe->ttm.lru_lock); 343 } 344 345 if (mode == EXEC_MODE_LR) 346 xe_hw_engine_group_resume_faulting_lr_jobs(group); 347 348 err_repin: 349 if (!xe_vm_in_lr_mode(vm)) 350 xe_svm_notifier_unlock(vm); 351 err_put_job: 352 if (err) 353 xe_sched_job_put(job); 354 err_exec: 355 if (!xe_vm_in_lr_mode(vm)) 356 xe_validation_ctx_fini(&ctx); 357 err_unlock_list: 358 up_read(&vm->lock); 359 if (err == -EAGAIN) 360 goto retry; 361 err_hw_exec_mode: 362 if (mode == EXEC_MODE_DMA_FENCE) 363 xe_hw_engine_group_put(group); 364 err_syncs: 365 while (num_syncs--) 366 xe_sync_entry_cleanup(&syncs[num_syncs]); 367 kfree(syncs); 368 err_exec_queue: 369 xe_exec_queue_put(q); 370 371 return err; 372 } 373