1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_exec.h" 7 8 #include <drm/drm_device.h> 9 #include <drm/drm_exec.h> 10 #include <drm/drm_file.h> 11 #include <drm/xe_drm.h> 12 #include <linux/delay.h> 13 14 #include "xe_bo.h" 15 #include "xe_device.h" 16 #include "xe_exec_queue.h" 17 #include "xe_macros.h" 18 #include "xe_ring_ops_types.h" 19 #include "xe_sched_job.h" 20 #include "xe_sync.h" 21 #include "xe_vm.h" 22 23 /** 24 * DOC: Execbuf (User GPU command submission) 25 * 26 * Execs have historically been rather complicated in DRM drivers (at least in 27 * the i915) because a few things: 28 * 29 * - Passing in a list BO which are read / written to creating implicit syncs 30 * - Binding at exec time 31 * - Flow controlling the ring at exec time 32 * 33 * In XE we avoid all of this complication by not allowing a BO list to be 34 * passed into an exec, using the dma-buf implicit sync uAPI, have binds as 35 * seperate operations, and using the DRM scheduler to flow control the ring. 36 * Let's deep dive on each of these. 37 * 38 * We can get away from a BO list by forcing the user to use in / out fences on 39 * every exec rather than the kernel tracking dependencies of BO (e.g. if the 40 * user knows an exec writes to a BO and reads from the BO in the next exec, it 41 * is the user's responsibility to pass in / out fence between the two execs). 42 * 43 * Implicit dependencies for external BOs are handled by using the dma-buf 44 * implicit dependency uAPI (TODO: add link). To make this works each exec must 45 * install the job's fence into the DMA_RESV_USAGE_WRITE slot of every external 46 * BO mapped in the VM. 47 * 48 * We do not allow a user to trigger a bind at exec time rather we have a VM 49 * bind IOCTL which uses the same in / out fence interface as exec. In that 50 * sense, a VM bind is basically the same operation as an exec from the user 51 * perspective. e.g. If an exec depends on a VM bind use the in / out fence 52 * interface (struct drm_xe_sync) to synchronize like syncing between two 53 * dependent execs. 54 * 55 * Although a user cannot trigger a bind, we still have to rebind userptrs in 56 * the VM that have been invalidated since the last exec, likewise we also have 57 * to rebind BOs that have been evicted by the kernel. We schedule these rebinds 58 * behind any pending kernel operations on any external BOs in VM or any BOs 59 * private to the VM. This is accomplished by the rebinds waiting on BOs 60 * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs 61 * slots (inflight execs are in the DMA_RESV_USAGE_BOOKING for private BOs and 62 * in DMA_RESV_USAGE_WRITE for external BOs). 63 * 64 * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute 65 * mode VMs we use preempt fences and a rebind worker (TODO: add link). 66 * 67 * There is no need to flow control the ring in the exec as we write the ring at 68 * submission time and set the DRM scheduler max job limit SIZE_OF_RING / 69 * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the 70 * ring is available. 71 * 72 * All of this results in a rather simple exec implementation. 73 * 74 * Flow 75 * ~~~~ 76 * 77 * .. code-block:: 78 * 79 * Parse input arguments 80 * Wait for any async VM bind passed as in-fences to start 81 * <----------------------------------------------------------------------| 82 * Lock global VM lock in read mode | 83 * Pin userptrs (also finds userptr invalidated since last exec) | 84 * Lock exec (VM dma-resv lock, external BOs dma-resv locks) | 85 * Validate BOs that have been evicted | 86 * Create job | 87 * Rebind invalidated userptrs + evicted BOs (non-compute-mode) | 88 * Add rebind fence dependency to job | 89 * Add job VM dma-resv bookkeeping slot (non-compute mode) | 90 * Add job to external BOs dma-resv write slots (non-compute mode) | 91 * Check if any userptrs invalidated since pin ------ Drop locks ---------| 92 * Install in / out fences for job 93 * Submit job 94 * Unlock all 95 */ 96 97 static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec) 98 { 99 return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec); 100 } 101 102 int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 103 { 104 struct xe_device *xe = to_xe_device(dev); 105 struct xe_file *xef = to_xe_file(file); 106 struct drm_xe_exec *args = data; 107 struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); 108 u64 __user *addresses_user = u64_to_user_ptr(args->address); 109 struct xe_exec_queue *q; 110 struct xe_sync_entry *syncs = NULL; 111 u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; 112 struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn}; 113 struct drm_exec *exec = &vm_exec.exec; 114 u32 i, num_syncs = 0, num_ufence = 0; 115 struct xe_sched_job *job; 116 struct dma_fence *rebind_fence; 117 struct xe_vm *vm; 118 bool write_locked, skip_retry = false; 119 ktime_t end = 0; 120 int err = 0; 121 122 if (XE_IOCTL_DBG(xe, args->extensions) || 123 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || 124 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 125 return -EINVAL; 126 127 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 128 if (XE_IOCTL_DBG(xe, !q)) 129 return -ENOENT; 130 131 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) 132 return -EINVAL; 133 134 if (XE_IOCTL_DBG(xe, args->num_batch_buffer && 135 q->width != args->num_batch_buffer)) 136 return -EINVAL; 137 138 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_BANNED)) { 139 err = -ECANCELED; 140 goto err_exec_queue; 141 } 142 143 if (args->num_syncs) { 144 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); 145 if (!syncs) { 146 err = -ENOMEM; 147 goto err_exec_queue; 148 } 149 } 150 151 vm = q->vm; 152 153 for (i = 0; i < args->num_syncs; i++) { 154 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs++], 155 &syncs_user[i], SYNC_PARSE_FLAG_EXEC | 156 (xe_vm_in_lr_mode(vm) ? 157 SYNC_PARSE_FLAG_LR_MODE : 0)); 158 if (err) 159 goto err_syncs; 160 161 if (xe_sync_is_ufence(&syncs[i])) 162 num_ufence++; 163 } 164 165 if (XE_IOCTL_DBG(xe, num_ufence > 1)) { 166 err = -EINVAL; 167 goto err_syncs; 168 } 169 170 if (xe_exec_queue_is_parallel(q)) { 171 err = __copy_from_user(addresses, addresses_user, sizeof(u64) * 172 q->width); 173 if (err) { 174 err = -EFAULT; 175 goto err_syncs; 176 } 177 } 178 179 retry: 180 if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) { 181 err = down_write_killable(&vm->lock); 182 write_locked = true; 183 } else { 184 /* We don't allow execs while the VM is in error state */ 185 err = down_read_interruptible(&vm->lock); 186 write_locked = false; 187 } 188 if (err) 189 goto err_syncs; 190 191 if (write_locked) { 192 err = xe_vm_userptr_pin(vm); 193 downgrade_write(&vm->lock); 194 write_locked = false; 195 if (err) 196 goto err_unlock_list; 197 } 198 199 vm_exec.vm = &vm->gpuvm; 200 vm_exec.num_fences = 1 + vm->xe->info.tile_count; 201 vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT; 202 if (xe_vm_in_lr_mode(vm)) { 203 drm_exec_init(exec, vm_exec.flags, 0); 204 } else { 205 err = drm_gpuvm_exec_lock(&vm_exec); 206 if (err) { 207 if (xe_vm_validate_should_retry(exec, err, &end)) 208 err = -EAGAIN; 209 goto err_unlock_list; 210 } 211 } 212 213 if (xe_vm_is_closed_or_banned(q->vm)) { 214 drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n"); 215 err = -ECANCELED; 216 goto err_exec; 217 } 218 219 if (!args->num_batch_buffer) { 220 if (!xe_vm_in_lr_mode(vm)) { 221 struct dma_fence *fence; 222 223 fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm); 224 if (IS_ERR(fence)) { 225 err = PTR_ERR(fence); 226 goto err_exec; 227 } 228 for (i = 0; i < num_syncs; i++) 229 xe_sync_entry_signal(&syncs[i], NULL, fence); 230 xe_exec_queue_last_fence_set(q, vm, fence); 231 dma_fence_put(fence); 232 } 233 234 goto err_exec; 235 } 236 237 if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) { 238 err = -EWOULDBLOCK; /* Aliased to -EAGAIN */ 239 skip_retry = true; 240 goto err_exec; 241 } 242 243 job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? 244 addresses : &args->address); 245 if (IS_ERR(job)) { 246 err = PTR_ERR(job); 247 goto err_exec; 248 } 249 250 /* 251 * Rebind any invalidated userptr or evicted BOs in the VM, non-compute 252 * VM mode only. 253 */ 254 rebind_fence = xe_vm_rebind(vm, false); 255 if (IS_ERR(rebind_fence)) { 256 err = PTR_ERR(rebind_fence); 257 goto err_put_job; 258 } 259 260 /* 261 * We store the rebind_fence in the VM so subsequent execs don't get 262 * scheduled before the rebinds of userptrs / evicted BOs is complete. 263 */ 264 if (rebind_fence) { 265 dma_fence_put(vm->rebind_fence); 266 vm->rebind_fence = rebind_fence; 267 } 268 if (vm->rebind_fence) { 269 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 270 &vm->rebind_fence->flags)) { 271 dma_fence_put(vm->rebind_fence); 272 vm->rebind_fence = NULL; 273 } else { 274 dma_fence_get(vm->rebind_fence); 275 err = drm_sched_job_add_dependency(&job->drm, 276 vm->rebind_fence); 277 if (err) 278 goto err_put_job; 279 } 280 } 281 282 /* Wait behind munmap style rebinds */ 283 if (!xe_vm_in_lr_mode(vm)) { 284 err = drm_sched_job_add_resv_dependencies(&job->drm, 285 xe_vm_resv(vm), 286 DMA_RESV_USAGE_KERNEL); 287 if (err) 288 goto err_put_job; 289 } 290 291 for (i = 0; i < num_syncs && !err; i++) 292 err = xe_sync_entry_add_deps(&syncs[i], job); 293 if (err) 294 goto err_put_job; 295 296 if (!xe_vm_in_lr_mode(vm)) { 297 err = xe_sched_job_last_fence_add_dep(job, vm); 298 if (err) 299 goto err_put_job; 300 301 err = down_read_interruptible(&vm->userptr.notifier_lock); 302 if (err) 303 goto err_put_job; 304 305 err = __xe_vm_userptr_needs_repin(vm); 306 if (err) 307 goto err_repin; 308 } 309 310 /* 311 * Point of no return, if we error after this point just set an error on 312 * the job and let the DRM scheduler / backend clean up the job. 313 */ 314 xe_sched_job_arm(job); 315 if (!xe_vm_in_lr_mode(vm)) 316 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished, 317 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE); 318 319 for (i = 0; i < num_syncs; i++) 320 xe_sync_entry_signal(&syncs[i], job, 321 &job->drm.s_fence->finished); 322 323 if (xe_exec_queue_is_lr(q)) 324 q->ring_ops->emit_job(job); 325 if (!xe_vm_in_lr_mode(vm)) 326 xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished); 327 xe_sched_job_push(job); 328 xe_vm_reactivate_rebind(vm); 329 330 if (!err && !xe_vm_in_lr_mode(vm)) { 331 spin_lock(&xe->ttm.lru_lock); 332 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 333 spin_unlock(&xe->ttm.lru_lock); 334 } 335 336 err_repin: 337 if (!xe_vm_in_lr_mode(vm)) 338 up_read(&vm->userptr.notifier_lock); 339 err_put_job: 340 if (err) 341 xe_sched_job_put(job); 342 err_exec: 343 drm_exec_fini(exec); 344 err_unlock_list: 345 if (write_locked) 346 up_write(&vm->lock); 347 else 348 up_read(&vm->lock); 349 if (err == -EAGAIN && !skip_retry) 350 goto retry; 351 err_syncs: 352 for (i = 0; i < num_syncs; i++) 353 xe_sync_entry_cleanup(&syncs[i]); 354 kfree(syncs); 355 err_exec_queue: 356 xe_exec_queue_put(q); 357 358 return err; 359 } 360