xref: /linux/drivers/gpu/drm/xe/xe_hw_engine_group.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2024 Intel Corporation
4  */
5 
6 #include <linux/error-injection.h>
7 
8 #include <drm/drm_managed.h>
9 
10 #include "xe_assert.h"
11 #include "xe_device_types.h"
12 #include "xe_exec_queue.h"
13 #include "xe_gt.h"
14 #include "xe_gt_stats.h"
15 #include "xe_hw_engine_group.h"
16 #include "xe_sync.h"
17 #include "xe_vm.h"
18 
19 static void
20 hw_engine_group_resume_lr_jobs_func(struct work_struct *w)
21 {
22 	struct xe_exec_queue *q;
23 	struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work);
24 	int err;
25 	enum xe_hw_engine_group_execution_mode previous_mode;
26 
27 	err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode,
28 					  NULL, 0);
29 	if (err)
30 		return;
31 
32 	if (previous_mode == EXEC_MODE_LR)
33 		goto put;
34 
35 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
36 		if (!xe_vm_in_fault_mode(q->vm))
37 			continue;
38 
39 		/*
40 		 * Only resume queues that were actually suspended. A queue whose
41 		 * suspend() failed (e.g. killed/banned/wedged) was never
42 		 * suspended, so it must not be resumed.
43 		 */
44 		if (!READ_ONCE(q->lr.suspended))
45 			continue;
46 
47 		WRITE_ONCE(q->lr.suspended, false);
48 		q->ops->resume(q);
49 	}
50 
51 put:
52 	xe_hw_engine_group_put(group);
53 }
54 
55 static struct xe_hw_engine_group *
56 hw_engine_group_alloc(struct xe_device *xe)
57 {
58 	struct xe_hw_engine_group *group;
59 	int err;
60 
61 	group = drmm_kzalloc(&xe->drm, sizeof(*group), GFP_KERNEL);
62 	if (!group)
63 		return ERR_PTR(-ENOMEM);
64 
65 	group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", WQ_PERCPU,
66 					   0);
67 	if (!group->resume_wq)
68 		return ERR_PTR(-ENOMEM);
69 
70 	err = drmm_add_action_or_reset(&xe->drm, __drmm_workqueue_release, group->resume_wq);
71 	if (err)
72 		return ERR_PTR(err);
73 
74 	init_rwsem(&group->mode_sem);
75 	INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func);
76 	INIT_LIST_HEAD(&group->exec_queue_list);
77 
78 	return group;
79 }
80 
81 /**
82  * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt
83  * @gt: The gt for which groups are setup
84  *
85  * Return: 0 on success, negative error code on error.
86  */
87 int xe_hw_engine_setup_groups(struct xe_gt *gt)
88 {
89 	struct xe_hw_engine *hwe;
90 	enum xe_hw_engine_id id;
91 	struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs;
92 	struct xe_device *xe = gt_to_xe(gt);
93 
94 	group_rcs_ccs = hw_engine_group_alloc(xe);
95 	if (IS_ERR(group_rcs_ccs))
96 		return PTR_ERR(group_rcs_ccs);
97 
98 	group_bcs = hw_engine_group_alloc(xe);
99 	if (IS_ERR(group_bcs))
100 		return PTR_ERR(group_bcs);
101 
102 	group_vcs_vecs = hw_engine_group_alloc(xe);
103 	if (IS_ERR(group_vcs_vecs))
104 		return PTR_ERR(group_vcs_vecs);
105 
106 	for_each_hw_engine(hwe, gt, id) {
107 		switch (hwe->class) {
108 		case XE_ENGINE_CLASS_COPY:
109 			hwe->hw_engine_group = group_bcs;
110 			break;
111 		case XE_ENGINE_CLASS_RENDER:
112 		case XE_ENGINE_CLASS_COMPUTE:
113 			hwe->hw_engine_group = group_rcs_ccs;
114 			break;
115 		case XE_ENGINE_CLASS_VIDEO_DECODE:
116 		case XE_ENGINE_CLASS_VIDEO_ENHANCE:
117 			hwe->hw_engine_group = group_vcs_vecs;
118 			break;
119 		case XE_ENGINE_CLASS_OTHER:
120 			break;
121 		case XE_ENGINE_CLASS_MAX:
122 			xe_gt_assert(gt, false);
123 		}
124 	}
125 
126 	return 0;
127 }
128 
129 /**
130  * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group
131  * @group: The hw engine group
132  * @q: The exec_queue
133  *
134  * Return: 0 on success,
135  *	    -EINTR if the lock could not be acquired
136  */
137 int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q)
138 {
139 	int err;
140 	struct xe_device *xe = gt_to_xe(q->gt);
141 
142 	xe_assert(xe, group);
143 	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM));
144 	xe_assert(xe, q->vm);
145 
146 	if (xe_vm_in_preempt_fence_mode(q->vm))
147 		return 0;
148 
149 	err = down_write_killable(&group->mode_sem);
150 	if (err)
151 		return err;
152 
153 	if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) {
154 		/*
155 		 * suspend() can fail (e.g. killed/banned/wedged), leaving the
156 		 * queue un-suspended. Propagate the failure so the queue is not
157 		 * added; on failure nothing was suspended, so there is nothing to
158 		 * undo. Only record the queue as suspended (and later resume it)
159 		 * once suspend() has succeeded.
160 		 */
161 		err = q->ops->suspend(q);
162 		if (err)
163 			goto err_suspend;
164 
165 		WRITE_ONCE(q->lr.suspended, true);
166 		err = q->ops->suspend_wait(q);
167 		if (err)
168 			goto err_suspend;
169 
170 		xe_hw_engine_group_resume_faulting_lr_jobs(group);
171 	}
172 
173 	list_add(&q->hw_engine_group_link, &group->exec_queue_list);
174 	up_write(&group->mode_sem);
175 
176 	return 0;
177 
178 err_suspend:
179 	up_write(&group->mode_sem);
180 	return err;
181 }
182 ALLOW_ERROR_INJECTION(xe_hw_engine_group_add_exec_queue, ERRNO);
183 
184 /**
185  * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group
186  * @group: The hw engine group
187  * @q: The exec_queue
188  */
189 void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q)
190 {
191 	struct xe_device *xe = gt_to_xe(q->gt);
192 
193 	xe_assert(xe, group);
194 	xe_assert(xe, q->vm);
195 
196 	down_write(&group->mode_sem);
197 
198 	if (!list_empty(&q->hw_engine_group_link))
199 		list_del(&q->hw_engine_group_link);
200 
201 	up_write(&group->mode_sem);
202 }
203 
204 /**
205  * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's
206  * faulting LR jobs
207  * @group: The hw engine group
208  */
209 void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group)
210 {
211 	queue_work(group->resume_wq, &group->resume_work);
212 }
213 
214 /**
215  * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group
216  * @group: The hw engine group
217  * @has_deps: dma-fence job triggering suspend has dependencies
218  *
219  * Return: 0 on success, negative error code on error.
220  */
221 static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group,
222 						       bool has_deps)
223 {
224 	int err;
225 	struct xe_exec_queue *q;
226 	struct xe_gt *gt = NULL;
227 	bool need_resume = false;
228 	ktime_t start = xe_gt_stats_ktime_get();
229 
230 	lockdep_assert_held_write(&group->mode_sem);
231 
232 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
233 
234 		if (!xe_vm_in_fault_mode(q->vm))
235 			continue;
236 
237 		if (has_deps)
238 			return -EAGAIN;
239 
240 		xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1);
241 		/*
242 		 * suspend() only fails when the queue is killed/banned/wedged.
243 		 * Such a queue is being torn down (its removal from HW is handled
244 		 * by the kill/ban teardown), so it is not a live fault-mode
245 		 * context the mode switch must preempt. Skip it rather than
246 		 * failing the switch, otherwise one dying sibling would block a
247 		 * dma-fence submission on the healthy queues in the group. Only
248 		 * queues recorded as suspended below are later waited on and
249 		 * resumed.
250 		 */
251 		err = q->ops->suspend(q);
252 		if (err)
253 			continue;
254 
255 		WRITE_ONCE(q->lr.suspended, true);
256 		need_resume = true;
257 		gt = q->gt;
258 	}
259 
260 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
261 		if (!xe_vm_in_fault_mode(q->vm))
262 			continue;
263 
264 		/* Only wait on queues that were actually suspended above. */
265 		if (!READ_ONCE(q->lr.suspended))
266 			continue;
267 
268 		err = q->ops->suspend_wait(q);
269 		if (err)
270 			goto err_resume;
271 	}
272 
273 	if (gt) {
274 		xe_gt_stats_incr(gt,
275 				 XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
276 				 xe_gt_stats_ktime_us_delta(start));
277 	}
278 
279 	if (need_resume)
280 		xe_hw_engine_group_resume_faulting_lr_jobs(group);
281 
282 	return 0;
283 
284 err_resume:
285 	/*
286 	 * A suspend_wait() failed partway through the mode switch. Resume the
287 	 * sibling queues that were already suspended in this call so they are
288 	 * not left suspended forever.
289 	 *
290 	 * resume() requires the suspend to have completed (suspend_pending
291 	 * cleared) or it trips the !suspend_pending assert. So skip the resume
292 	 * when either:
293 	 *  - suspend_wait_blocking() fails: on a GuC timeout it bans the queue
294 	 *    and triggers cleanup, so the queue is being torn down; or
295 	 *  - reset_status() is true: the queue was reset/killed/banned/wedged.
296 	 *    suspend_wait() can return success in this case via its killed/
297 	 *    stopped wait condition while suspend_pending is still set, and the
298 	 *    queue is being torn down anyway, so its state is resolved by
299 	 *    teardown rather than by a resume here.
300 	 * In either case leave the queue marked suspended.
301 	 *
302 	 * Use the *blocking* (uninterruptible) wait here: the queues resumed on
303 	 * this path may belong to a different process than the one that
304 	 * triggered the mode switch. An interruptible suspend_wait() would
305 	 * return -ERESTARTSYS if the triggering task is signalled, skip the
306 	 * resume, and leave the other process's queue suspended forever
307 	 * (cross-process DoS).
308 	 */
309 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
310 		if (!xe_vm_in_fault_mode(q->vm))
311 			continue;
312 
313 		if (!READ_ONCE(q->lr.suspended))
314 			continue;
315 
316 		if (q->ops->suspend_wait_blocking(q) || q->ops->reset_status(q))
317 			continue;
318 
319 		WRITE_ONCE(q->lr.suspended, false);
320 		q->ops->resume(q);
321 	}
322 
323 	return err;
324 }
325 
326 /**
327  * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete
328  * @group: The hw engine group
329  *
330  * This function is not meant to be called directly from a user IOCTL as dma_fence_wait()
331  * is not interruptible.
332  *
333  * Return: 0 on success,
334  *	   -ETIME if waiting for one job failed
335  */
336 static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group)
337 {
338 	long timeout;
339 	struct xe_exec_queue *q;
340 	struct xe_gt *gt = NULL;
341 	struct dma_fence *fence;
342 	ktime_t start = xe_gt_stats_ktime_get();
343 
344 	lockdep_assert_held_write(&group->mode_sem);
345 
346 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
347 		if (xe_vm_in_lr_mode(q->vm))
348 			continue;
349 
350 		xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1);
351 		fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);
352 		timeout = dma_fence_wait(fence, false);
353 		dma_fence_put(fence);
354 		gt = q->gt;
355 
356 		if (timeout < 0)
357 			return -ETIME;
358 	}
359 
360 	if (gt) {
361 		xe_gt_stats_incr(gt,
362 				 XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
363 				 xe_gt_stats_ktime_us_delta(start));
364 	}
365 
366 	return 0;
367 }
368 
369 static int switch_mode(struct xe_hw_engine_group *group, bool has_deps)
370 {
371 	int err = 0;
372 	enum xe_hw_engine_group_execution_mode new_mode;
373 
374 	lockdep_assert_held_write(&group->mode_sem);
375 
376 	switch (group->cur_mode) {
377 	case EXEC_MODE_LR:
378 		new_mode = EXEC_MODE_DMA_FENCE;
379 		err = xe_hw_engine_group_suspend_faulting_lr_jobs(group,
380 								  has_deps);
381 		break;
382 	case EXEC_MODE_DMA_FENCE:
383 		new_mode = EXEC_MODE_LR;
384 		err = xe_hw_engine_group_wait_for_dma_fence_jobs(group);
385 		break;
386 	}
387 
388 	if (err)
389 		return err;
390 
391 	group->cur_mode = new_mode;
392 
393 	return 0;
394 }
395 
396 static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs)
397 {
398 	int err, i;
399 
400 	for (i = 0; i < num_syncs; ++i) {
401 		err = xe_sync_entry_wait(syncs + i);
402 		if (err)
403 			return err;
404 	}
405 
406 	return 0;
407 }
408 
409 /**
410  * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode
411  * @group: The hw engine group
412  * @new_mode: The new execution mode
413  * @previous_mode: Pointer to the previous mode provided for use by caller
414  * @syncs: Syncs from exec IOCTL
415  * @num_syncs: Number of syncs from exec IOCTL
416  *
417  * Return: 0 if successful, -EINTR if locking failed.
418  */
419 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
420 				enum xe_hw_engine_group_execution_mode new_mode,
421 				enum xe_hw_engine_group_execution_mode *previous_mode,
422 				struct xe_sync_entry *syncs, int num_syncs)
423 __acquires(&group->mode_sem)
424 {
425 	bool has_deps = !!num_syncs;
426 	int err = down_read_interruptible(&group->mode_sem);
427 
428 	if (err)
429 		return err;
430 
431 	*previous_mode = group->cur_mode;
432 
433 	if (new_mode != group->cur_mode) {
434 		up_read(&group->mode_sem);
435 retry:
436 		err = down_write_killable(&group->mode_sem);
437 		if (err)
438 			return err;
439 
440 		if (new_mode != group->cur_mode) {
441 			err = switch_mode(group, has_deps);
442 			if (err) {
443 				up_write(&group->mode_sem);
444 
445 				if (err != -EAGAIN)
446 					return err;
447 
448 				err = wait_syncs(syncs, num_syncs);
449 				if (err)
450 					return err;
451 
452 				has_deps = false;
453 				goto retry;
454 			}
455 		}
456 		downgrade_write(&group->mode_sem);
457 	}
458 
459 	return err;
460 }
461 
462 /**
463  * xe_hw_engine_group_put() - Put the group
464  * @group: The hw engine group
465  */
466 void xe_hw_engine_group_put(struct xe_hw_engine_group *group)
467 __releases(&group->mode_sem)
468 {
469 	up_read(&group->mode_sem);
470 }
471 
472 /**
473  * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue
474  * @q: The exec_queue
475  */
476 enum xe_hw_engine_group_execution_mode
477 xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q)
478 {
479 	if (xe_vm_in_fault_mode(q->vm))
480 		return EXEC_MODE_LR;
481 	else
482 		return EXEC_MODE_DMA_FENCE;
483 }
484