1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_exec.h"
7
8 #include <drm/drm_device.h>
9 #include <drm/drm_exec.h>
10 #include <drm/drm_file.h>
11 #include <uapi/drm/xe_drm.h>
12 #include <linux/delay.h>
13
14 #include "xe_bo.h"
15 #include "xe_device.h"
16 #include "xe_exec_queue.h"
17 #include "xe_hw_engine_group.h"
18 #include "xe_macros.h"
19 #include "xe_ring_ops_types.h"
20 #include "xe_sched_job.h"
21 #include "xe_sync.h"
22 #include "xe_svm.h"
23 #include "xe_vm.h"
24
25 /**
26 * DOC: Execbuf (User GPU command submission)
27 *
28 * Execs have historically been rather complicated in DRM drivers (at least in
29 * the i915) because a few things:
30 *
31 * - Passing in a list BO which are read / written to creating implicit syncs
32 * - Binding at exec time
33 * - Flow controlling the ring at exec time
34 *
35 * In XE we avoid all of this complication by not allowing a BO list to be
36 * passed into an exec, using the dma-buf implicit sync uAPI, have binds as
37 * separate operations, and using the DRM scheduler to flow control the ring.
38 * Let's deep dive on each of these.
39 *
40 * We can get away from a BO list by forcing the user to use in / out fences on
41 * every exec rather than the kernel tracking dependencies of BO (e.g. if the
42 * user knows an exec writes to a BO and reads from the BO in the next exec, it
43 * is the user's responsibility to pass in / out fence between the two execs).
44 *
45 * We do not allow a user to trigger a bind at exec time rather we have a VM
46 * bind IOCTL which uses the same in / out fence interface as exec. In that
47 * sense, a VM bind is basically the same operation as an exec from the user
48 * perspective. e.g. If an exec depends on a VM bind use the in / out fence
49 * interface (struct drm_xe_sync) to synchronize like syncing between two
50 * dependent execs.
51 *
52 * Although a user cannot trigger a bind, we still have to rebind userptrs in
53 * the VM that have been invalidated since the last exec, likewise we also have
54 * to rebind BOs that have been evicted by the kernel. We schedule these rebinds
55 * behind any pending kernel operations on any external BOs in VM or any BOs
56 * private to the VM. This is accomplished by the rebinds waiting on BOs
57 * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs
58 * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and
59 * for external BOs).
60 *
61 * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute
62 * mode VMs we use preempt fences and a rebind worker (TODO: add link).
63 *
64 * There is no need to flow control the ring in the exec as we write the ring at
65 * submission time and set the DRM scheduler max job limit SIZE_OF_RING /
66 * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the
67 * ring is available.
68 *
69 * All of this results in a rather simple exec implementation.
70 *
71 * Flow
72 * ~~~~
73 *
74 * .. code-block::
75 *
76 * Parse input arguments
77 * Wait for any async VM bind passed as in-fences to start
78 * <----------------------------------------------------------------------|
79 * Lock global VM lock in read mode |
80 * Pin userptrs (also finds userptr invalidated since last exec) |
81 * Lock exec (VM dma-resv lock, external BOs dma-resv locks) |
82 * Validate BOs that have been evicted |
83 * Create job |
84 * Rebind invalidated userptrs + evicted BOs (non-compute-mode) |
85 * Add rebind fence dependency to job |
86 * Add job VM dma-resv bookkeeping slot (non-compute mode) |
87 * Add job to external BOs dma-resv write slots (non-compute mode) |
88 * Check if any userptrs invalidated since pin ------ Drop locks ---------|
89 * Install in / out fences for job
90 * Submit job
91 * Unlock all
92 */
93
94 /*
95 * Add validation and rebinding to the drm_exec locking loop, since both can
96 * trigger eviction which may require sleeping dma_resv locks.
97 */
xe_exec_fn(struct drm_gpuvm_exec * vm_exec)98 static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec)
99 {
100 struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm);
101 int ret;
102
103 /* The fence slot added here is intended for the exec sched job. */
104 xe_vm_set_validation_exec(vm, &vm_exec->exec);
105 ret = xe_vm_validate_rebind(vm, &vm_exec->exec, 1);
106 xe_vm_set_validation_exec(vm, NULL);
107 return ret;
108 }
109
xe_exec_ioctl(struct drm_device * dev,void * data,struct drm_file * file)110 int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
111 {
112 struct xe_device *xe = to_xe_device(dev);
113 struct xe_file *xef = to_xe_file(file);
114 struct drm_xe_exec *args = data;
115 struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs);
116 u64 __user *addresses_user = u64_to_user_ptr(args->address);
117 struct xe_exec_queue *q;
118 struct xe_sync_entry *syncs = NULL;
119 u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
120 struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
121 struct drm_exec *exec = &vm_exec.exec;
122 u32 i, num_syncs, num_ufence = 0;
123 struct xe_validation_ctx ctx;
124 struct xe_sched_job *job;
125 struct xe_vm *vm;
126 bool write_locked, skip_retry = false;
127 int err = 0;
128 struct xe_hw_engine_group *group;
129 enum xe_hw_engine_group_execution_mode mode, previous_mode;
130
131 if (XE_IOCTL_DBG(xe, args->extensions) ||
132 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||
133 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
134 return -EINVAL;
135
136 q = xe_exec_queue_lookup(xef, args->exec_queue_id);
137 if (XE_IOCTL_DBG(xe, !q))
138 return -ENOENT;
139
140 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) {
141 err = -EINVAL;
142 goto err_exec_queue;
143 }
144
145 if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&
146 q->width != args->num_batch_buffer)) {
147 err = -EINVAL;
148 goto err_exec_queue;
149 }
150
151 if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) {
152 err = -ECANCELED;
153 goto err_exec_queue;
154 }
155
156 if (args->num_syncs) {
157 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL);
158 if (!syncs) {
159 err = -ENOMEM;
160 goto err_exec_queue;
161 }
162 }
163
164 vm = q->vm;
165
166 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
167 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
168 &syncs_user[num_syncs], NULL, 0,
169 SYNC_PARSE_FLAG_EXEC |
170 (xe_vm_in_lr_mode(vm) ?
171 SYNC_PARSE_FLAG_LR_MODE : 0));
172 if (err)
173 goto err_syncs;
174
175 if (xe_sync_is_ufence(&syncs[num_syncs]))
176 num_ufence++;
177 }
178
179 if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
180 err = -EINVAL;
181 goto err_syncs;
182 }
183
184 if (xe_exec_queue_is_parallel(q)) {
185 err = copy_from_user(addresses, addresses_user, sizeof(u64) *
186 q->width);
187 if (err) {
188 err = -EFAULT;
189 goto err_syncs;
190 }
191 }
192
193 group = q->hwe->hw_engine_group;
194 mode = xe_hw_engine_group_find_exec_mode(q);
195
196 if (mode == EXEC_MODE_DMA_FENCE) {
197 err = xe_hw_engine_group_get_mode(group, mode, &previous_mode);
198 if (err)
199 goto err_syncs;
200 }
201
202 retry:
203 if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) {
204 err = down_write_killable(&vm->lock);
205 write_locked = true;
206 } else {
207 /* We don't allow execs while the VM is in error state */
208 err = down_read_interruptible(&vm->lock);
209 write_locked = false;
210 }
211 if (err)
212 goto err_hw_exec_mode;
213
214 if (write_locked) {
215 err = xe_vm_userptr_pin(vm);
216 downgrade_write(&vm->lock);
217 write_locked = false;
218 if (err)
219 goto err_unlock_list;
220 }
221
222 if (!args->num_batch_buffer) {
223 err = xe_vm_lock(vm, true);
224 if (err)
225 goto err_unlock_list;
226
227 if (!xe_vm_in_lr_mode(vm)) {
228 struct dma_fence *fence;
229
230 fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm);
231 if (IS_ERR(fence)) {
232 err = PTR_ERR(fence);
233 xe_vm_unlock(vm);
234 goto err_unlock_list;
235 }
236 for (i = 0; i < num_syncs; i++)
237 xe_sync_entry_signal(&syncs[i], fence);
238 xe_exec_queue_last_fence_set(q, vm, fence);
239 dma_fence_put(fence);
240 }
241
242 xe_vm_unlock(vm);
243 goto err_unlock_list;
244 }
245
246 /*
247 * It's OK to block interruptible here with the vm lock held, since
248 * on task freezing during suspend / hibernate, the call will
249 * return -ERESTARTSYS and the IOCTL will be rerun.
250 */
251 err = wait_for_completion_interruptible(&xe->pm_block);
252 if (err)
253 goto err_unlock_list;
254
255 if (!xe_vm_in_lr_mode(vm)) {
256 vm_exec.vm = &vm->gpuvm;
257 vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT;
258 err = xe_validation_exec_lock(&ctx, &vm_exec, &xe->val);
259 if (err)
260 goto err_unlock_list;
261 }
262
263 if (xe_vm_is_closed_or_banned(q->vm)) {
264 drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n");
265 err = -ECANCELED;
266 goto err_exec;
267 }
268
269 if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) {
270 err = -EWOULDBLOCK; /* Aliased to -EAGAIN */
271 skip_retry = true;
272 goto err_exec;
273 }
274
275 if (xe_exec_queue_uses_pxp(q)) {
276 err = xe_vm_validate_protected(q->vm);
277 if (err)
278 goto err_exec;
279 }
280
281 job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ?
282 addresses : &args->address);
283 if (IS_ERR(job)) {
284 err = PTR_ERR(job);
285 goto err_exec;
286 }
287
288 /* Wait behind rebinds */
289 if (!xe_vm_in_lr_mode(vm)) {
290 err = xe_sched_job_add_deps(job,
291 xe_vm_resv(vm),
292 DMA_RESV_USAGE_KERNEL);
293 if (err)
294 goto err_put_job;
295 }
296
297 for (i = 0; i < num_syncs && !err; i++)
298 err = xe_sync_entry_add_deps(&syncs[i], job);
299 if (err)
300 goto err_put_job;
301
302 if (!xe_vm_in_lr_mode(vm)) {
303 err = xe_sched_job_last_fence_add_dep(job, vm);
304 if (err)
305 goto err_put_job;
306
307 err = xe_svm_notifier_lock_interruptible(vm);
308 if (err)
309 goto err_put_job;
310
311 err = __xe_vm_userptr_needs_repin(vm);
312 if (err)
313 goto err_repin;
314 }
315
316 /*
317 * Point of no return, if we error after this point just set an error on
318 * the job and let the DRM scheduler / backend clean up the job.
319 */
320 xe_sched_job_arm(job);
321 if (!xe_vm_in_lr_mode(vm))
322 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,
323 DMA_RESV_USAGE_BOOKKEEP,
324 DMA_RESV_USAGE_BOOKKEEP);
325
326 for (i = 0; i < num_syncs; i++) {
327 xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished);
328 xe_sched_job_init_user_fence(job, &syncs[i]);
329 }
330
331 if (xe_exec_queue_is_lr(q))
332 q->ring_ops->emit_job(job);
333 if (!xe_vm_in_lr_mode(vm))
334 xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished);
335 xe_sched_job_push(job);
336 xe_vm_reactivate_rebind(vm);
337
338 if (!err && !xe_vm_in_lr_mode(vm)) {
339 spin_lock(&xe->ttm.lru_lock);
340 ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
341 spin_unlock(&xe->ttm.lru_lock);
342 }
343
344 if (mode == EXEC_MODE_LR)
345 xe_hw_engine_group_resume_faulting_lr_jobs(group);
346
347 err_repin:
348 if (!xe_vm_in_lr_mode(vm))
349 xe_svm_notifier_unlock(vm);
350 err_put_job:
351 if (err)
352 xe_sched_job_put(job);
353 err_exec:
354 if (!xe_vm_in_lr_mode(vm))
355 xe_validation_ctx_fini(&ctx);
356 err_unlock_list:
357 up_read(&vm->lock);
358 if (err == -EAGAIN && !skip_retry)
359 goto retry;
360 err_hw_exec_mode:
361 if (mode == EXEC_MODE_DMA_FENCE)
362 xe_hw_engine_group_put(group);
363 err_syncs:
364 while (num_syncs--)
365 xe_sync_entry_cleanup(&syncs[num_syncs]);
366 kfree(syncs);
367 err_exec_queue:
368 xe_exec_queue_put(q);
369
370 return err;
371 }
372