xref: /linux/drivers/gpu/drm/xe/xe_guc_submit.c (revision f831b84713b57cbd67f228a7f10147791e5ab13d)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_guc_submit.h"
7 
8 #include <linux/bitfield.h>
9 #include <uapi/drm/xe_drm.h>
10 #include <linux/bitmap.h>
11 #include <linux/circ_buf.h>
12 #include <linux/dma-fence-array.h>
13 
14 #include <drm/drm_drv.h>
15 #include <drm/drm_managed.h>
16 
17 #include "abi/guc_actions_abi.h"
18 #include "abi/guc_actions_slpc_abi.h"
19 #include "abi/guc_klvs_abi.h"
20 #include "xe_assert.h"
21 #include "xe_bo.h"
22 #include "xe_cpu_bind.h"
23 #include "xe_devcoredump.h"
24 #include "xe_device.h"
25 #include "xe_exec_queue.h"
26 #include "xe_force_wake.h"
27 #include "xe_gpu_scheduler.h"
28 #include "xe_gt.h"
29 #include "xe_gt_clock.h"
30 #include "xe_gt_printk.h"
31 #include "xe_guc.h"
32 #include "xe_guc_capture.h"
33 #include "xe_guc_ct.h"
34 #include "xe_guc_exec_queue_types.h"
35 #include "xe_guc_id_mgr.h"
36 #include "xe_guc_klv_helpers.h"
37 #include "xe_guc_submit_types.h"
38 #include "xe_hw_engine.h"
39 #include "xe_log.h"
40 #include "xe_lrc.h"
41 #include "xe_macros.h"
42 #include "xe_map.h"
43 #include "xe_mocs.h"
44 #include "xe_module.h"
45 #include "xe_pm.h"
46 #include "xe_pt.h"
47 #include "xe_ring_ops_types.h"
48 #include "xe_sched_job.h"
49 #include "xe_sleep.h"
50 #include "xe_trace.h"
51 #include "xe_uc_fw.h"
52 #include "xe_vm.h"
53 
54 #define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN		6
55 
56 static int guc_submit_reset_prepare(struct xe_guc *guc);
57 
58 static struct xe_guc *
59 exec_queue_to_guc(struct xe_exec_queue *q)
60 {
61 	return &q->gt->uc.guc;
62 }
63 
64 /*
65  * Helpers for engine state, using an atomic as some of the bits can transition
66  * as the same time (e.g. a suspend can be happning at the same time as schedule
67  * engine done being processed).
68  */
69 #define EXEC_QUEUE_STATE_REGISTERED		(1 << 0)
70 #define EXEC_QUEUE_STATE_ENABLED		(1 << 1)
71 #define EXEC_QUEUE_STATE_PENDING_ENABLE		(1 << 2)
72 #define EXEC_QUEUE_STATE_PENDING_DISABLE	(1 << 3)
73 #define EXEC_QUEUE_STATE_DESTROYED		(1 << 4)
74 #define EXEC_QUEUE_STATE_SUSPENDED		(1 << 5)
75 #define EXEC_QUEUE_STATE_RESET			(1 << 6)
76 #define EXEC_QUEUE_STATE_KILLED			(1 << 7)
77 #define EXEC_QUEUE_STATE_WEDGED			(1 << 8)
78 #define EXEC_QUEUE_STATE_BANNED			(1 << 9)
79 #define EXEC_QUEUE_STATE_PENDING_RESUME		(1 << 10)
80 
81 static bool exec_queue_registered(struct xe_exec_queue *q)
82 {
83 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED;
84 }
85 
86 static void set_exec_queue_registered(struct xe_exec_queue *q)
87 {
88 	atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);
89 }
90 
91 static void clear_exec_queue_registered(struct xe_exec_queue *q)
92 {
93 	atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);
94 }
95 
96 static bool exec_queue_enabled(struct xe_exec_queue *q)
97 {
98 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED;
99 }
100 
101 static void set_exec_queue_enabled(struct xe_exec_queue *q)
102 {
103 	atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state);
104 }
105 
106 static void clear_exec_queue_enabled(struct xe_exec_queue *q)
107 {
108 	atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state);
109 }
110 
111 static bool exec_queue_pending_enable(struct xe_exec_queue *q)
112 {
113 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE;
114 }
115 
116 static void set_exec_queue_pending_enable(struct xe_exec_queue *q)
117 {
118 	atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);
119 }
120 
121 static void clear_exec_queue_pending_enable(struct xe_exec_queue *q)
122 {
123 	atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);
124 }
125 
126 static bool exec_queue_pending_disable(struct xe_exec_queue *q)
127 {
128 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE;
129 }
130 
131 static void set_exec_queue_pending_disable(struct xe_exec_queue *q)
132 {
133 	atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);
134 }
135 
136 static void clear_exec_queue_pending_disable(struct xe_exec_queue *q)
137 {
138 	atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);
139 }
140 
141 static bool exec_queue_destroyed(struct xe_exec_queue *q)
142 {
143 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED;
144 }
145 
146 static void set_exec_queue_destroyed(struct xe_exec_queue *q)
147 {
148 	atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state);
149 }
150 
151 static void clear_exec_queue_destroyed(struct xe_exec_queue *q)
152 {
153 	atomic_and(~EXEC_QUEUE_STATE_DESTROYED, &q->guc->state);
154 }
155 
156 static bool exec_queue_banned(struct xe_exec_queue *q)
157 {
158 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED;
159 }
160 
161 static void set_exec_queue_banned(struct xe_exec_queue *q)
162 {
163 	atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state);
164 }
165 
166 static void clear_exec_queue_banned(struct xe_exec_queue *q)
167 {
168 	atomic_andnot(EXEC_QUEUE_STATE_BANNED, &q->guc->state);
169 }
170 
171 static bool exec_queue_suspended(struct xe_exec_queue *q)
172 {
173 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED;
174 }
175 
176 static void set_exec_queue_suspended(struct xe_exec_queue *q)
177 {
178 	atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state);
179 }
180 
181 static void clear_exec_queue_suspended(struct xe_exec_queue *q)
182 {
183 	atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state);
184 }
185 
186 static bool exec_queue_reset(struct xe_exec_queue *q)
187 {
188 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET;
189 }
190 
191 static void set_exec_queue_reset(struct xe_exec_queue *q)
192 {
193 	atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state);
194 }
195 
196 static bool exec_queue_killed(struct xe_exec_queue *q)
197 {
198 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED;
199 }
200 
201 static void set_exec_queue_killed(struct xe_exec_queue *q)
202 {
203 	atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state);
204 }
205 
206 static bool exec_queue_wedged(struct xe_exec_queue *q)
207 {
208 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED;
209 }
210 
211 static void set_exec_queue_wedged(struct xe_exec_queue *q)
212 {
213 	atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state);
214 }
215 
216 static bool exec_queue_pending_resume(struct xe_exec_queue *q)
217 {
218 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_RESUME;
219 }
220 
221 static void set_exec_queue_pending_resume(struct xe_exec_queue *q)
222 {
223 	atomic_or(EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state);
224 }
225 
226 static void clear_exec_queue_pending_resume(struct xe_exec_queue *q)
227 {
228 	atomic_and(~EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state);
229 }
230 
231 static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)
232 {
233 	return (atomic_read(&q->guc->state) &
234 		(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED |
235 		 EXEC_QUEUE_STATE_BANNED));
236 }
237 
238 static void guc_submit_sw_fini(struct drm_device *drm, void *arg)
239 {
240 	struct xe_guc *guc = arg;
241 	struct xe_gt *gt = guc_to_gt(guc);
242 
243 	xe_gt_assert(gt, xa_empty(&guc->submission_state.exec_queue_lookup));
244 
245 	xa_destroy(&guc->submission_state.exec_queue_lookup);
246 }
247 
248 static void guc_submit_fini(void *arg)
249 {
250 	struct xe_guc *guc = arg;
251 	struct xe_exec_queue *q;
252 	unsigned long index;
253 
254 	/* Drop any wedged queue refs */
255 	mutex_lock(&guc->submission_state.lock);
256 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
257 		if (exec_queue_wedged(q)) {
258 			mutex_unlock(&guc->submission_state.lock);
259 			xe_exec_queue_put(q);
260 			mutex_lock(&guc->submission_state.lock);
261 		}
262 	}
263 	mutex_unlock(&guc->submission_state.lock);
264 
265 	/* Forcefully kill any remaining exec queues */
266 	xe_guc_ct_stop(&guc->ct);
267 	guc_submit_reset_prepare(guc);
268 	xe_guc_softreset(guc);
269 	xe_guc_submit_stop(guc);
270 	xe_uc_fw_sanitize(&guc->fw);
271 	xe_guc_submit_pause_abort(guc);
272 }
273 
274 static const struct xe_exec_queue_ops guc_exec_queue_ops;
275 
276 static void primelockdep(struct xe_guc *guc)
277 {
278 	if (!IS_ENABLED(CONFIG_LOCKDEP))
279 		return;
280 
281 	fs_reclaim_acquire(GFP_KERNEL);
282 
283 	mutex_lock(&guc->submission_state.lock);
284 	mutex_unlock(&guc->submission_state.lock);
285 
286 	fs_reclaim_release(GFP_KERNEL);
287 }
288 
289 /**
290  * xe_guc_submit_init() - Initialize GuC submission.
291  * @guc: the &xe_guc to initialize
292  * @num_ids: number of GuC context IDs to use
293  *
294  * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all
295  * GuC context IDs supported by the GuC firmware should be used for submission.
296  *
297  * Only VF drivers will have to provide explicit number of GuC context IDs
298  * that they can use for submission.
299  *
300  * Return: 0 on success or a negative error code on failure.
301  */
302 int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids)
303 {
304 	struct xe_device *xe = guc_to_xe(guc);
305 	struct xe_gt *gt = guc_to_gt(guc);
306 	int err;
307 
308 	err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock);
309 	if (err)
310 		return err;
311 
312 	err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids);
313 	if (err)
314 		return err;
315 
316 	gt->exec_queue_ops = &guc_exec_queue_ops;
317 
318 	xa_init(&guc->submission_state.exec_queue_lookup);
319 
320 	primelockdep(guc);
321 
322 	guc->submission_state.initialized = true;
323 
324 	err = drmm_add_action_or_reset(&xe->drm, guc_submit_sw_fini, guc);
325 	if (err)
326 		return err;
327 
328 	return devm_add_action_or_reset(xe->drm.dev, guc_submit_fini, guc);
329 }
330 
331 /*
332  * Given that we want to guarantee enough RCS throughput to avoid missing
333  * frames, we set the yield policy to 20% of each 80ms interval.
334  */
335 #define RC_YIELD_DURATION	80	/* in ms */
336 #define RC_YIELD_RATIO		20	/* in percent */
337 static u32 *emit_render_compute_yield_klv(u32 *emit)
338 {
339 	*emit++ = PREP_GUC_KLV_TAG(SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD);
340 	*emit++ = RC_YIELD_DURATION;
341 	*emit++ = RC_YIELD_RATIO;
342 
343 	return emit;
344 }
345 
346 #define SCHEDULING_POLICY_MAX_DWORDS 16
347 static int guc_init_global_schedule_policy(struct xe_guc *guc)
348 {
349 	u32 data[SCHEDULING_POLICY_MAX_DWORDS];
350 	u32 *emit = data;
351 	u32 count = 0;
352 	int ret;
353 
354 	if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0))
355 		return 0;
356 
357 	*emit++ = XE_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV;
358 
359 	if (CCS_INSTANCES(guc_to_gt(guc)))
360 		emit = emit_render_compute_yield_klv(emit);
361 
362 	count = emit - data;
363 	if (count > 1) {
364 		xe_assert(guc_to_xe(guc), count <= SCHEDULING_POLICY_MAX_DWORDS);
365 
366 		ret = xe_guc_ct_send_block(&guc->ct, data, count);
367 		if (ret < 0) {
368 			xe_gt_err(guc_to_gt(guc),
369 				  "failed to enable GuC scheduling policies: %pe\n",
370 				  ERR_PTR(ret));
371 			return ret;
372 		}
373 	}
374 
375 	return 0;
376 }
377 
378 int xe_guc_submit_enable(struct xe_guc *guc)
379 {
380 	int ret;
381 
382 	ret = guc_init_global_schedule_policy(guc);
383 	if (ret)
384 		return ret;
385 
386 	guc->submission_state.enabled = true;
387 
388 	return 0;
389 }
390 
391 void xe_guc_submit_disable(struct xe_guc *guc)
392 {
393 	guc->submission_state.enabled = false;
394 }
395 
396 static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q,
397 			     int count)
398 {
399 	int i;
400 
401 	mutex_lock(&guc->submission_state.lock);
402 
403 	for (i = 0; i < count; ++i)
404 		xa_erase(&guc->submission_state.exec_queue_lookup,
405 			 q->guc->id + i);
406 
407 	xe_guc_id_mgr_release_locked(&guc->submission_state.idm,
408 				     q->guc->id, q->width);
409 
410 	mutex_unlock(&guc->submission_state.lock);
411 }
412 
413 static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
414 {
415 	int ret, i;
416 
417 	mutex_lock(&guc->submission_state.lock);
418 	ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm,
419 					   q->width);
420 	mutex_unlock(&guc->submission_state.lock);
421 	if (ret < 0)
422 		return ret;
423 
424 	q->guc->id = ret;
425 
426 	/* Reserve empty slots. */
427 	for (i = 0; i < q->width; ++i) {
428 		ret = xa_insert(&guc->submission_state.exec_queue_lookup,
429 				 q->guc->id + i, NULL, GFP_KERNEL);
430 		if (ret)
431 			goto err_release;
432 	}
433 
434 	return 0;
435 
436 err_release:
437 	__release_guc_id(guc, q, i);
438 
439 	return ret;
440 }
441 
442 static void publish_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
443 {
444 	int i;
445 
446 	lockdep_assert_held(&guc->submission_state.lock);
447 
448 	for (i = 0; i < q->width; ++i) {
449 		void *old;
450 
451 		old = xa_store(&guc->submission_state.exec_queue_lookup,
452 			       q->guc->id + i, q, GFP_NOWAIT);
453 		XE_WARN_ON(old || xa_is_err(old));
454 	}
455 }
456 
457 static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
458 {
459 	__release_guc_id(guc, q, q->width);
460 }
461 
462 struct exec_queue_policy {
463 	u32 count;
464 	struct guc_update_exec_queue_policy h2g;
465 };
466 
467 static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy)
468 {
469 	size_t bytes = sizeof(policy->h2g.header) +
470 		       (sizeof(policy->h2g.klv[0]) * policy->count);
471 
472 	return bytes / sizeof(u32);
473 }
474 
475 static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy,
476 					      u16 guc_id)
477 {
478 	policy->h2g.header.action =
479 		XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;
480 	policy->h2g.header.guc_id = guc_id;
481 	policy->count = 0;
482 }
483 
484 #define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \
485 static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \
486 					   u32 data) \
487 { \
488 	XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \
489 \
490 	policy->h2g.klv[policy->count].kl = \
491 		FIELD_PREP(GUC_KLV_0_KEY, \
492 			   GUC_CONTEXT_POLICIES_KLV_ID_##id) | \
493 		FIELD_PREP(GUC_KLV_0_LEN, 1); \
494 	policy->h2g.klv[policy->count].value = data; \
495 	policy->count++; \
496 }
497 
498 MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)
499 MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)
500 MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY)
501 MAKE_EXEC_QUEUE_POLICY_ADD(slpc_exec_queue_freq_req, SLPM_GT_FREQUENCY)
502 #undef MAKE_EXEC_QUEUE_POLICY_ADD
503 
504 static const int xe_exec_queue_prio_to_guc[] = {
505 	[XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL,
506 	[XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL,
507 	[XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH,
508 	[XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH,
509 };
510 
511 static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q)
512 {
513 	struct exec_queue_policy policy;
514 	enum xe_exec_queue_priority prio = q->sched_props.priority;
515 	u32 timeslice_us = q->sched_props.timeslice_us;
516 	u32 slpc_exec_queue_freq_req = 0;
517 	u32 preempt_timeout_us = q->sched_props.preempt_timeout_us;
518 
519 	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) &&
520 		     !xe_exec_queue_is_multi_queue_secondary(q));
521 
522 	if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY)
523 		slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE;
524 
525 	__guc_exec_queue_policy_start_klv(&policy, q->guc->id);
526 	__guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]);
527 	__guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us);
528 	__guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us);
529 	__guc_exec_queue_policy_add_slpc_exec_queue_freq_req(&policy,
530 							     slpc_exec_queue_freq_req);
531 
532 	xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,
533 		       __guc_exec_queue_policy_action_size(&policy), 0, 0);
534 }
535 
536 static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q)
537 {
538 	struct exec_queue_policy policy;
539 
540 	xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q));
541 
542 	__guc_exec_queue_policy_start_klv(&policy, q->guc->id);
543 	__guc_exec_queue_policy_add_preemption_timeout(&policy, 1);
544 
545 	xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,
546 		       __guc_exec_queue_policy_action_size(&policy), 0, 0);
547 }
548 
549 static bool vf_recovery(struct xe_guc *guc)
550 {
551 	return xe_gt_recovery_pending(guc_to_gt(guc));
552 }
553 
554 static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
555 {
556 	struct xe_guc *guc = exec_queue_to_guc(q);
557 	struct xe_device *xe = guc_to_xe(guc);
558 
559 	/** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
560 	wake_up_all(&xe->ufence_wq);
561 
562 	xe_sched_tdr_queue_imm(&q->guc->sched);
563 }
564 
565 static void xe_guc_exec_queue_group_stop(struct xe_exec_queue *q)
566 {
567 	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
568 	struct xe_exec_queue_group *group = q->multi_queue.group;
569 	struct xe_exec_queue *eq, *next;
570 	LIST_HEAD(tmp);
571 
572 	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
573 		     xe_exec_queue_is_multi_queue(q));
574 
575 	mutex_lock(&group->list_lock);
576 
577 	/*
578 	 * Stop all future queues being from executing while group is stopped.
579 	 */
580 	group->stopped = true;
581 
582 	list_for_each_entry_safe(eq, next, &group->list, multi_queue.link)
583 		/*
584 		 * Refcount prevents an attempted removal from &group->list,
585 		 * temporary list allows safe iteration after dropping
586 		 * &group->list_lock.
587 		 */
588 		if (xe_exec_queue_get_unless_zero(eq))
589 			list_move_tail(&eq->multi_queue.link, &tmp);
590 
591 	mutex_unlock(&group->list_lock);
592 
593 	/* We cannot stop under list lock without getting inversions */
594 	xe_sched_submission_stop(&primary->guc->sched);
595 	list_for_each_entry(eq, &tmp, multi_queue.link)
596 		xe_sched_submission_stop(&eq->guc->sched);
597 
598 	mutex_lock(&group->list_lock);
599 	list_for_each_entry_safe(eq, next, &tmp, multi_queue.link) {
600 		/*
601 		 * Corner where we got banned while stopping and not on
602 		 * &group->list
603 		 */
604 		if (READ_ONCE(group->banned))
605 			xe_guc_exec_queue_trigger_cleanup(eq);
606 
607 		list_move_tail(&eq->multi_queue.link, &group->list);
608 		xe_exec_queue_put(eq);
609 	}
610 	mutex_unlock(&group->list_lock);
611 }
612 
613 static void xe_guc_exec_queue_group_start(struct xe_exec_queue *q)
614 {
615 	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
616 	struct xe_exec_queue_group *group = q->multi_queue.group;
617 	struct xe_exec_queue *eq;
618 
619 	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
620 		     xe_exec_queue_is_multi_queue(q));
621 
622 	xe_sched_submission_start(&primary->guc->sched);
623 
624 	mutex_lock(&group->list_lock);
625 	group->stopped = false;
626 	list_for_each_entry(eq, &group->list, multi_queue.link)
627 		xe_sched_submission_start(&eq->guc->sched);
628 	mutex_unlock(&group->list_lock);
629 }
630 
631 static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q)
632 {
633 	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
634 	struct xe_exec_queue_group *group = q->multi_queue.group;
635 	struct xe_exec_queue *eq;
636 
637 	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
638 		     xe_exec_queue_is_multi_queue(q));
639 
640 	/* Group banned, skip timeout check in TDR */
641 	WRITE_ONCE(group->banned, true);
642 	xe_guc_exec_queue_trigger_cleanup(primary);
643 
644 	mutex_lock(&group->list_lock);
645 	list_for_each_entry(eq, &group->list, multi_queue.link)
646 		xe_guc_exec_queue_trigger_cleanup(eq);
647 	mutex_unlock(&group->list_lock);
648 }
649 
650 static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q)
651 {
652 	if (xe_exec_queue_is_multi_queue(q)) {
653 		struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
654 		struct xe_exec_queue_group *group = q->multi_queue.group;
655 		struct xe_exec_queue *eq;
656 
657 		/* Group banned, skip timeout check in TDR */
658 		WRITE_ONCE(group->banned, true);
659 
660 		set_exec_queue_reset(primary);
661 		if (!exec_queue_banned(primary))
662 			xe_guc_exec_queue_trigger_cleanup(primary);
663 
664 		mutex_lock(&group->list_lock);
665 		list_for_each_entry(eq, &group->list, multi_queue.link) {
666 			set_exec_queue_reset(eq);
667 			if (!exec_queue_banned(eq))
668 				xe_guc_exec_queue_trigger_cleanup(eq);
669 		}
670 		mutex_unlock(&group->list_lock);
671 	} else {
672 		set_exec_queue_reset(q);
673 		if (!exec_queue_banned(q))
674 			xe_guc_exec_queue_trigger_cleanup(q);
675 	}
676 }
677 
678 static void set_exec_queue_group_banned(struct xe_exec_queue *q)
679 {
680 	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
681 	struct xe_exec_queue_group *group = q->multi_queue.group;
682 	struct xe_exec_queue *eq;
683 
684 	/* Ban all queues of the multi-queue group */
685 	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
686 		     xe_exec_queue_is_multi_queue(q));
687 	set_exec_queue_banned(primary);
688 
689 	mutex_lock(&group->list_lock);
690 	list_for_each_entry(eq, &group->list, multi_queue.link)
691 		set_exec_queue_banned(eq);
692 	mutex_unlock(&group->list_lock);
693 }
694 
695 /* Helper for context registration H2G */
696 struct guc_ctxt_registration_info {
697 	u32 flags;
698 	u32 context_idx;
699 	u32 engine_class;
700 	u32 engine_submit_mask;
701 	u32 wq_desc_lo;
702 	u32 wq_desc_hi;
703 	u32 wq_base_lo;
704 	u32 wq_base_hi;
705 	u32 wq_size;
706 	u32 cgp_lo;
707 	u32 cgp_hi;
708 	u32 hwlrca_lo;
709 	u32 hwlrca_hi;
710 };
711 
712 #define parallel_read(xe_, map_, field_) \
713 	xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
714 			field_)
715 #define parallel_write(xe_, map_, field_, val_) \
716 	xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
717 			field_, val_)
718 
719 /**
720  * DOC: Multi Queue Group GuC interface
721  *
722  * The multi queue group coordination between KMD and GuC is through a software
723  * construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page
724  * allocated in the global GTT.
725  *
726  * CGP format:
727  *
728  * +-----------+---------------------------+---------------------------------------------+
729  * | DWORD     | Name                      | Description                                 |
730  * +-----------+---------------------------+---------------------------------------------+
731  * | 0         | Version                   | Bits [15:8]=Major ver, [7:0]=Minor ver      |
732  * +-----------+---------------------------+---------------------------------------------+
733  * | 1..15     | RESERVED                  | MBZ                                         |
734  * +-----------+---------------------------+---------------------------------------------+
735  * | 16        | KMD_QUEUE_UPDATE_MASK_DW0 | KMD queue mask for queues 31..0             |
736  * +-----------+---------------------------+---------------------------------------------+
737  * | 17        | KMD_QUEUE_UPDATE_MASK_DW1 | KMD queue mask for queues 63..32            |
738  * +-----------+---------------------------+---------------------------------------------+
739  * | 18..31    | RESERVED                  | MBZ                                         |
740  * +-----------+---------------------------+---------------------------------------------+
741  * | 32        | Q0CD_DW0                  | Queue 0 context LRC descriptor lower DWORD  |
742  * +-----------+---------------------------+---------------------------------------------+
743  * | 33        | Q0ContextIndex            | Context ID for Queue 0                      |
744  * +-----------+---------------------------+---------------------------------------------+
745  * | 34        | Q1CD_DW0                  | Queue 1 context LRC descriptor lower DWORD  |
746  * +-----------+---------------------------+---------------------------------------------+
747  * | 35        | Q1ContextIndex            | Context ID for Queue 1                      |
748  * +-----------+---------------------------+---------------------------------------------+
749  * | ...       |...                        | ...                                         |
750  * +-----------+---------------------------+---------------------------------------------+
751  * | 158       | Q63CD_DW0                 | Queue 63 context LRC descriptor lower DWORD |
752  * +-----------+---------------------------+---------------------------------------------+
753  * | 159       | Q63ContextIndex           | Context ID for Queue 63                     |
754  * +-----------+---------------------------+---------------------------------------------+
755  * | 160..1024 | RESERVED                  | MBZ                                         |
756  * +-----------+---------------------------+---------------------------------------------+
757  *
758  * While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified
759  * through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies
760  * the CGP address. When the secondary queues are added to the group, the CGP is
761  * updated with entry for that queue and GuC is notified through the H2G interface
762  * XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages
763  * with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also
764  * sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any
765  * error in the CGP. Only one of these CGP update messages can be outstanding
766  * (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW*
767  * fields indicate which queue entry is being updated in the CGP.
768  *
769  * The primary queue (Q0) represents the multi queue group context in GuC and
770  * submission on any queue of the group must be through Q0 GuC interface only.
771  *
772  * As it is not required to register secondary queues with GuC, the secondary queue
773  * context ids in the CGP are populated with Q0 context id.
774  */
775 
776 #define CGP_VERSION_MAJOR_SHIFT	8
777 
778 static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe,
779 					       struct xe_exec_queue *q)
780 {
781 	struct xe_exec_queue_group *group = q->multi_queue.group;
782 	u32 guc_id = group->primary->guc->id;
783 
784 	/* Currently implementing CGP version 1.0 */
785 	xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32,
786 		  1 << CGP_VERSION_MAJOR_SHIFT);
787 
788 	xe_map_wr(xe, &group->cgp_bo->vmap,
789 		  (32 + q->multi_queue.pos * 2) * sizeof(u32),
790 		  u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0])));
791 
792 	xe_map_wr(xe, &group->cgp_bo->vmap,
793 		  (33 + q->multi_queue.pos * 2) * sizeof(u32),
794 		  u32, guc_id);
795 
796 	if (q->multi_queue.pos / 32) {
797 		xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32),
798 			  u32, BIT(q->multi_queue.pos % 32));
799 		xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0);
800 	} else {
801 		xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32),
802 			  u32, BIT(q->multi_queue.pos));
803 		xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0);
804 	}
805 }
806 
807 #define CGP_SYNC_REGISTRATION			BIT(0)
808 
809 static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
810 					     struct xe_exec_queue *q,
811 					     const u32 *action, u32 len,
812 					     unsigned int flags)
813 {
814 	struct xe_exec_queue_group *group = q->multi_queue.group;
815 	struct xe_device *xe = guc_to_xe(guc);
816 	enum xe_multi_queue_priority priority;
817 	long ret;
818 
819 	/*
820 	 * As all queues of a multi queue group use single drm scheduler
821 	 * submit workqueue, CGP synchronization with GuC are serialized.
822 	 * Hence, no locking is required here.
823 	 * Wait for any pending CGP_SYNC_DONE response before updating the
824 	 * CGP page and sending CGP_SYNC message.
825 	 */
826 	ret = wait_event_timeout(guc->ct.wq,
827 				 !READ_ONCE(group->sync_pending) ||
828 				 xe_guc_read_stopped(guc) || vf_recovery(guc),
829 				 HZ);
830 	if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) {
831 		/* CGP_SYNC failed. Reset gt, cleanup the group */
832 		xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n");
833 		set_exec_queue_group_banned(q);
834 		xe_gt_reset_async(q->gt);
835 		xe_guc_exec_queue_group_trigger_cleanup(q);
836 		return;
837 	}
838 
839 	/*
840 	 * If woken by VF migration recovery, do not touch the CGP or send: the
841 	 * message would be lost and, for a registration, GuC must (re-)register
842 	 * the context before its CGP entry may be read. Flag the queue so revert
843 	 * replays it - a registration by re-registration, a dynamic update by a
844 	 * replayed CGP_SYNC - and bail.
845 	 */
846 	if (vf_recovery(guc)) {
847 		if (flags & CGP_SYNC_REGISTRATION)
848 			q->guc->multi_queue.re_register = true;
849 		else
850 			q->guc->multi_queue.re_update = true;
851 		return;
852 	}
853 
854 	scoped_guard(spinlock, &q->multi_queue.lock)
855 		priority = q->multi_queue.priority;
856 
857 	xe_lrc_set_multi_queue_priority(q->lrc[0], priority);
858 	xe_guc_exec_queue_group_cgp_update(xe, q);
859 
860 	/*
861 	 * Record the nature of this outstanding sync so revert can replay it if
862 	 * its CGP_SYNC_DONE is lost across a migration: a registration is
863 	 * recovered by re-registration, a dynamic update by a replayed CGP_SYNC.
864 	 */
865 	if (flags & CGP_SYNC_REGISTRATION) {
866 		q->guc->multi_queue.registering_cgp = true;
867 		q->guc->multi_queue.updating_cgp = false;
868 	} else {
869 		q->guc->multi_queue.updating_cgp = true;
870 		q->guc->multi_queue.registering_cgp = false;
871 	}
872 	WRITE_ONCE(group->cgp_update_q, q);
873 	WRITE_ONCE(group->sync_pending, true);
874 	xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1);
875 }
876 
877 static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q, unsigned int flags)
878 {
879 #define MAX_MULTI_QUEUE_CGP_SYNC_SIZE	(2)
880 	struct xe_guc *guc = exec_queue_to_guc(q);
881 	struct xe_exec_queue_group *group = q->multi_queue.group;
882 	u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
883 	int len = 0;
884 
885 	action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
886 	action[len++] = group->primary->guc->id;
887 
888 	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
889 #undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
890 
891 	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len, flags);
892 }
893 
894 static void __register_exec_queue_group(struct xe_exec_queue *q,
895 					struct guc_ctxt_registration_info *info)
896 {
897 	struct xe_guc *guc = exec_queue_to_guc(q);
898 #define MAX_MULTI_QUEUE_REG_SIZE	(8)
899 	u32 action[MAX_MULTI_QUEUE_REG_SIZE];
900 	int len = 0;
901 
902 	action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE;
903 	action[len++] = info->flags;
904 	action[len++] = info->context_idx;
905 	action[len++] = info->engine_class;
906 	action[len++] = info->engine_submit_mask;
907 	action[len++] = 0; /* Reserved */
908 	action[len++] = info->cgp_lo;
909 	action[len++] = info->cgp_hi;
910 
911 	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE);
912 #undef MAX_MULTI_QUEUE_REG_SIZE
913 
914 	/*
915 	 * The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a
916 	 * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
917 	 * from guc.
918 	 */
919 	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len,
920 					 CGP_SYNC_REGISTRATION);
921 }
922 
923 static void __register_mlrc_exec_queue(struct xe_guc *guc,
924 				       struct xe_exec_queue *q,
925 				       struct guc_ctxt_registration_info *info)
926 {
927 #define MAX_MLRC_REG_SIZE      (13 + XE_HW_ENGINE_MAX_INSTANCE * 2)
928 	u32 action[MAX_MLRC_REG_SIZE];
929 	int len = 0;
930 	int i;
931 
932 	xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_parallel(q));
933 
934 	action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
935 	action[len++] = info->flags;
936 	action[len++] = info->context_idx;
937 	action[len++] = info->engine_class;
938 	action[len++] = info->engine_submit_mask;
939 	action[len++] = info->wq_desc_lo;
940 	action[len++] = info->wq_desc_hi;
941 	action[len++] = info->wq_base_lo;
942 	action[len++] = info->wq_base_hi;
943 	action[len++] = info->wq_size;
944 	action[len++] = q->width;
945 	action[len++] = info->hwlrca_lo;
946 	action[len++] = info->hwlrca_hi;
947 
948 	for (i = 1; i < q->width; ++i) {
949 		struct xe_lrc *lrc = q->lrc[i];
950 
951 		action[len++] = lower_32_bits(xe_lrc_descriptor(lrc));
952 		action[len++] = upper_32_bits(xe_lrc_descriptor(lrc));
953 	}
954 
955 	/* explicitly checks some fields that we might fixup later */
956 	xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo ==
957 		     action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER]);
958 	xe_gt_assert(guc_to_gt(guc), info->wq_base_lo ==
959 		     action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER]);
960 	xe_gt_assert(guc_to_gt(guc), q->width ==
961 		     action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS]);
962 	xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo ==
963 		     action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR]);
964 	xe_gt_assert(guc_to_gt(guc), len <= MAX_MLRC_REG_SIZE);
965 #undef MAX_MLRC_REG_SIZE
966 
967 	xe_guc_ct_send(&guc->ct, action, len, 0, 0);
968 }
969 
970 static void __register_exec_queue(struct xe_guc *guc,
971 				  struct guc_ctxt_registration_info *info)
972 {
973 	u32 action[] = {
974 		XE_GUC_ACTION_REGISTER_CONTEXT,
975 		info->flags,
976 		info->context_idx,
977 		info->engine_class,
978 		info->engine_submit_mask,
979 		info->wq_desc_lo,
980 		info->wq_desc_hi,
981 		info->wq_base_lo,
982 		info->wq_base_hi,
983 		info->wq_size,
984 		info->hwlrca_lo,
985 		info->hwlrca_hi,
986 	};
987 
988 	/* explicitly checks some fields that we might fixup later */
989 	xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo ==
990 		     action[XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER]);
991 	xe_gt_assert(guc_to_gt(guc), info->wq_base_lo ==
992 		     action[XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER]);
993 	xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo ==
994 		     action[XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR]);
995 
996 	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
997 }
998 
999 static u32 xe_hwe_guc_logical_to_submit_mask(struct xe_hw_engine *hwe, u32 logical_mask)
1000 {
1001 	struct xe_gt *gt = hwe->gt;
1002 
1003 	if (xe_gt_is_usm_hwe(gt, hwe)) {
1004 		int shift = gt->usm.paging_hwe0->logical_instance;
1005 		u32 paging_logical_mask = gt->usm.paging_logical_mask;
1006 
1007 		xe_gt_assert(gt, (logical_mask & paging_logical_mask) == logical_mask);
1008 
1009 		/*
1010 		 * Remap to GUC_PAGING_CLASS logical instance mask, if
1011 		 * applicable.
1012 		 */
1013 		if (xe_guc_has_paging_engine(&hwe->gt->uc.guc))
1014 			return logical_mask >> shift;
1015 	}
1016 
1017 	return logical_mask;
1018 }
1019 
1020 static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
1021 {
1022 	struct xe_guc *guc = exec_queue_to_guc(q);
1023 	struct xe_device *xe = guc_to_xe(guc);
1024 	struct xe_lrc *lrc = q->lrc[0];
1025 	struct guc_ctxt_registration_info info;
1026 
1027 	xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q));
1028 	xe_gt_assert(guc_to_gt(guc), ctx_type < GUC_CONTEXT_COUNT);
1029 
1030 	memset(&info, 0, sizeof(info));
1031 	info.context_idx = q->guc->id;
1032 	info.engine_class = xe_hwe_to_guc_class(q->hwe);
1033 	info.engine_submit_mask =
1034 		xe_hwe_guc_logical_to_submit_mask(q->hwe, q->logical_mask);
1035 	info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc));
1036 	info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc));
1037 	info.flags = CONTEXT_REGISTRATION_FLAG_KMD |
1038 		FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type);
1039 
1040 	if (xe_exec_queue_is_multi_queue(q)) {
1041 		struct xe_exec_queue_group *group = q->multi_queue.group;
1042 
1043 		info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo);
1044 		info.cgp_hi = 0;
1045 	}
1046 
1047 	if (xe_exec_queue_is_parallel(q)) {
1048 		u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);
1049 		struct iosys_map map = xe_lrc_parallel_map(lrc);
1050 
1051 		info.wq_desc_lo = lower_32_bits(ggtt_addr +
1052 			offsetof(struct guc_submit_parallel_scratch, wq_desc));
1053 		info.wq_desc_hi = upper_32_bits(ggtt_addr +
1054 			offsetof(struct guc_submit_parallel_scratch, wq_desc));
1055 		info.wq_base_lo = lower_32_bits(ggtt_addr +
1056 			offsetof(struct guc_submit_parallel_scratch, wq[0]));
1057 		info.wq_base_hi = upper_32_bits(ggtt_addr +
1058 			offsetof(struct guc_submit_parallel_scratch, wq[0]));
1059 		info.wq_size = WQ_SIZE;
1060 
1061 		q->guc->wqi_head = 0;
1062 		q->guc->wqi_tail = 0;
1063 		xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE);
1064 		parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE);
1065 	}
1066 
1067 	set_exec_queue_registered(q);
1068 	trace_xe_exec_queue_register(q);
1069 	if (xe_exec_queue_is_multi_queue_primary(q))
1070 		__register_exec_queue_group(q, &info);
1071 	else if (xe_exec_queue_is_parallel(q))
1072 		__register_mlrc_exec_queue(guc, q, &info);
1073 	else if (!xe_exec_queue_is_multi_queue_secondary(q))
1074 		__register_exec_queue(guc, &info);
1075 
1076 	if (!xe_exec_queue_is_multi_queue_secondary(q))
1077 		init_policies(guc, q);
1078 
1079 	if (xe_exec_queue_is_multi_queue_secondary(q))
1080 		guc_exec_queue_send_cgp_sync(q, CGP_SYNC_REGISTRATION);
1081 }
1082 
1083 static u32 wq_space_until_wrap(struct xe_exec_queue *q)
1084 {
1085 	return (WQ_SIZE - q->guc->wqi_tail);
1086 }
1087 
1088 static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size)
1089 {
1090 	struct xe_guc *guc = exec_queue_to_guc(q);
1091 	struct xe_device *xe = guc_to_xe(guc);
1092 	struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
1093 	unsigned int sleep_period_ms = 1, sleep_total_ms = 0;
1094 
1095 #define AVAILABLE_SPACE \
1096 	CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE)
1097 	if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) {
1098 try_again:
1099 		q->guc->wqi_head = parallel_read(xe, map, wq_desc.head);
1100 		if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) {
1101 			if (sleep_total_ms > 2000) {
1102 				xe_gt_reset_async(q->gt);
1103 				return -ENODEV;
1104 			}
1105 
1106 			sleep_total_ms += xe_sleep_exponential_ms(&sleep_period_ms, 64);
1107 			goto try_again;
1108 		}
1109 	}
1110 #undef AVAILABLE_SPACE
1111 
1112 	return 0;
1113 }
1114 
1115 static int wq_noop_append(struct xe_exec_queue *q)
1116 {
1117 	struct xe_guc *guc = exec_queue_to_guc(q);
1118 	struct xe_device *xe = guc_to_xe(guc);
1119 	struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
1120 	u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1;
1121 
1122 	if (wq_wait_for_space(q, wq_space_until_wrap(q)))
1123 		return -ENODEV;
1124 
1125 	xe_gt_assert(guc_to_gt(guc), FIELD_FIT(WQ_LEN_MASK, len_dw));
1126 
1127 	parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)],
1128 		       FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |
1129 		       FIELD_PREP(WQ_LEN_MASK, len_dw));
1130 	q->guc->wqi_tail = 0;
1131 
1132 	return 0;
1133 }
1134 
1135 static void wq_item_append(struct xe_exec_queue *q)
1136 {
1137 	struct xe_guc *guc = exec_queue_to_guc(q);
1138 	struct xe_device *xe = guc_to_xe(guc);
1139 	struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
1140 #define WQ_HEADER_SIZE	4	/* Includes 1 LRC address too */
1141 	u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)];
1142 	u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32);
1143 	u32 len_dw = (wqi_size / sizeof(u32)) - 1;
1144 	int i = 0, j;
1145 
1146 	if (wqi_size > wq_space_until_wrap(q)) {
1147 		if (wq_noop_append(q))
1148 			return;
1149 	}
1150 	if (wq_wait_for_space(q, wqi_size))
1151 		return;
1152 
1153 	wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) |
1154 		FIELD_PREP(WQ_LEN_MASK, len_dw);
1155 	wqi[i++] = xe_lrc_descriptor(q->lrc[0]);
1156 	wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) |
1157 		FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64));
1158 	wqi[i++] = 0;
1159 	for (j = 1; j < q->width; ++j) {
1160 		struct xe_lrc *lrc = q->lrc[j];
1161 
1162 		wqi[i++] = lrc->ring.tail / sizeof(u64);
1163 	}
1164 
1165 	xe_gt_assert(guc_to_gt(guc), i == wqi_size / sizeof(u32));
1166 
1167 	iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch,
1168 				      wq[q->guc->wqi_tail / sizeof(u32)]));
1169 	xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size);
1170 	q->guc->wqi_tail += wqi_size;
1171 	xe_gt_assert(guc_to_gt(guc), q->guc->wqi_tail <= WQ_SIZE);
1172 
1173 	xe_device_wmb(xe);
1174 
1175 	map = xe_lrc_parallel_map(q->lrc[0]);
1176 	parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail);
1177 }
1178 
1179 #define RESUME_PENDING	~0x0ull
1180 static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
1181 {
1182 	struct xe_guc *guc = exec_queue_to_guc(q);
1183 	struct xe_lrc *lrc = q->lrc[0];
1184 	u32 action[3];
1185 	u32 g2h_len = 0;
1186 	u32 num_g2h = 0;
1187 	int len = 0;
1188 	bool extra_submit = false;
1189 
1190 	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
1191 
1192 	if (!job->restore_replay || job->last_replay) {
1193 		/*
1194 		 * A chained ULLS job relies on its predecessor's postamble to
1195 		 * publish its ring tail. After a reset the context comes back
1196 		 * disabled with that postamble gone, so publish it here.
1197 		 */
1198 		if (xe_exec_queue_is_parallel(q))
1199 			wq_item_append(q);
1200 		else if (!xe_sched_job_ulls_is_chained(job) ||
1201 			 !exec_queue_enabled(q))
1202 			xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
1203 		job->last_replay = false;
1204 	}
1205 
1206 	if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q))
1207 		return;
1208 
1209 	/*
1210 	 * All queues in a multi-queue group will use the primary queue
1211 	 * of the group to interface with GuC. If primay is suspended,
1212 	 * just return. Jobs will get scheduled once primary is resumed.
1213 	 */
1214 	q = xe_exec_queue_multi_queue_primary(q);
1215 	if (exec_queue_suspended(q))
1216 		return;
1217 
1218 	if (xe_sched_job_ulls_is_chained(job))
1219 		xe_lrc_set_ulls_semaphore(lrc, xe_sched_job_lrc_seqno(job));
1220 
1221 	if (!exec_queue_enabled(q)) {
1222 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
1223 		action[len++] = q->guc->id;
1224 		action[len++] = GUC_CONTEXT_ENABLE;
1225 		g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET;
1226 		num_g2h = 1;
1227 		if (xe_exec_queue_is_parallel(q))
1228 			extra_submit = true;
1229 
1230 		q->guc->resume_time = RESUME_PENDING;
1231 		set_exec_queue_pending_enable(q);
1232 		set_exec_queue_enabled(q);
1233 		trace_xe_exec_queue_scheduling_enable(q);
1234 	} else if (!xe_sched_job_ulls_is_chained(job)) {
1235 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;
1236 		action[len++] = q->guc->id;
1237 		trace_xe_exec_queue_submit(q);
1238 	}
1239 
1240 	if (!xe_sched_job_ulls_is_chained(job) || num_g2h)
1241 		xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);
1242 
1243 	if (extra_submit) {
1244 		len = 0;
1245 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;
1246 		action[len++] = q->guc->id;
1247 		trace_xe_exec_queue_submit(q);
1248 
1249 		xe_guc_ct_send(&guc->ct, action, len, 0, 0);
1250 	}
1251 }
1252 
1253 static bool is_pt_job(struct xe_sched_job *job)
1254 {
1255 	return job->is_pt_job;
1256 }
1257 
1258 static void run_pt_job(struct xe_device *xe, struct xe_sched_job *job,
1259 		       bool force_clear)
1260 {
1261 	struct xe_tile *tile;
1262 	int id;
1263 
1264 	for_each_tile(tile, xe, id) {
1265 		struct xe_pt_job_ops *pt_job_ops =
1266 			job->pt_update[0].pt_job_ops[id];
1267 
1268 		if (!pt_job_ops || !pt_job_ops->current_op)
1269 			continue;
1270 
1271 		xe_cpu_bind_update_pgtables_execute(job->pt_update[0].vm, tile,
1272 						    job->pt_update[0].ops,
1273 						    pt_job_ops->ops,
1274 						    pt_job_ops->current_op,
1275 						    force_clear);
1276 	}
1277 }
1278 
1279 static void put_pt_job(struct xe_device *xe, struct xe_sched_job *job)
1280 {
1281 	struct xe_tile *tile;
1282 	int id;
1283 
1284 	for_each_tile(tile, xe, id) {
1285 		struct xe_pt_job_ops *pt_job_ops =
1286 			job->pt_update[0].pt_job_ops[id];
1287 
1288 		xe_pt_job_ops_put(pt_job_ops);
1289 	}
1290 }
1291 
1292 static struct dma_fence *
1293 guc_exec_queue_run_job(struct drm_sched_job *drm_job)
1294 {
1295 	struct xe_sched_job *job = to_xe_sched_job(drm_job);
1296 	struct xe_exec_queue *q = job->q;
1297 	struct xe_guc *guc = exec_queue_to_guc(q);
1298 	bool killed_or_banned_or_wedged_or_error  =
1299 		exec_queue_killed_or_banned_or_wedged(q) ||
1300 		xe_sched_job_is_error(job);
1301 
1302 	xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) ||
1303 		     exec_queue_banned(q) || exec_queue_suspended(q));
1304 
1305 	trace_xe_sched_job_run(job);
1306 
1307 	if (is_pt_job(job)) {
1308 		xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q));
1309 
1310 		run_pt_job(guc_to_xe(guc), job,
1311 			   killed_or_banned_or_wedged_or_error);
1312 		put_pt_job(guc_to_xe(guc), job);
1313 		dma_fence_put(job->fence);	/* Drop ref from xe_sched_job_arm */
1314 
1315 		return NULL;
1316 	} else if (!killed_or_banned_or_wedged_or_error) {
1317 		if (xe_exec_queue_is_multi_queue_secondary(q)) {
1318 			struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
1319 
1320 			if (exec_queue_killed_or_banned_or_wedged(primary))
1321 				goto run_job_out;
1322 
1323 			if (!exec_queue_registered(primary))
1324 				register_exec_queue(primary, GUC_CONTEXT_NORMAL);
1325 		}
1326 
1327 		if (!exec_queue_registered(q))
1328 			register_exec_queue(q, GUC_CONTEXT_NORMAL);
1329 		if (!job->restore_replay)
1330 			q->ring_ops->emit_job(job);
1331 		submit_exec_queue(q, job);
1332 		job->restore_replay = false;
1333 	}
1334 
1335 run_job_out:
1336 
1337 	return job->fence;
1338 }
1339 
1340 static void guc_exec_queue_free_job(struct drm_sched_job *drm_job)
1341 {
1342 	struct xe_sched_job *job = to_xe_sched_job(drm_job);
1343 
1344 	trace_xe_sched_job_free(job);
1345 	xe_sched_job_put(job);
1346 }
1347 
1348 int xe_guc_read_stopped(struct xe_guc *guc)
1349 {
1350 	return atomic_read(&guc->submission_state.stopped);
1351 }
1352 
1353 static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
1354 						    struct xe_exec_queue *q,
1355 						    u32 runnable_state);
1356 static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q);
1357 
1358 #define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable)			\
1359 	u32 action[] = {						\
1360 		XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET,			\
1361 		q->guc->id,						\
1362 		GUC_CONTEXT_##enable_disable,				\
1363 	}
1364 
1365 static void disable_scheduling_deregister(struct xe_guc *guc,
1366 					  struct xe_exec_queue *q)
1367 {
1368 	MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
1369 	int ret;
1370 
1371 	if (!xe_exec_queue_is_multi_queue_secondary(q))
1372 		set_min_preemption_timeout(guc, q);
1373 
1374 	smp_rmb();
1375 	ret = wait_event_timeout(guc->ct.wq,
1376 				 (!exec_queue_pending_enable(q) &&
1377 				  !exec_queue_pending_disable(q)) ||
1378 					 xe_guc_read_stopped(guc) ||
1379 					 vf_recovery(guc),
1380 				 HZ * 5);
1381 	if (!ret && !vf_recovery(guc)) {
1382 		struct xe_gpu_scheduler *sched = &q->guc->sched;
1383 
1384 		xe_gt_warn(q->gt, "Pending enable/disable failed to respond\n");
1385 		xe_sched_submission_start(sched);
1386 		xe_gt_reset_async(q->gt);
1387 		xe_sched_tdr_queue_imm(sched);
1388 		return;
1389 	}
1390 
1391 	clear_exec_queue_enabled(q);
1392 	set_exec_queue_pending_disable(q);
1393 	set_exec_queue_destroyed(q);
1394 	trace_xe_exec_queue_scheduling_disable(q);
1395 
1396 	/*
1397 	 * Reserve space for both G2H here as the 2nd G2H is sent from a G2H
1398 	 * handler and we are not allowed to reserved G2H space in handlers.
1399 	 */
1400 	if (xe_exec_queue_is_multi_queue_secondary(q))
1401 		handle_multi_queue_secondary_sched_done(guc, q, 0);
1402 	else
1403 		xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1404 			       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +
1405 			       G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
1406 }
1407 
1408 /**
1409  * xe_guc_submit_wedge() - Wedge GuC submission
1410  * @guc: the GuC object
1411  *
1412  * Save exec queue's registered with GuC state by taking a ref to each queue.
1413  * Register a DRMM handler to drop refs upon driver unload.
1414  */
1415 void xe_guc_submit_wedge(struct xe_guc *guc)
1416 {
1417 	struct xe_device *xe = guc_to_xe(guc);
1418 	struct xe_exec_queue *q;
1419 	unsigned long index;
1420 
1421 	xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);
1422 
1423 	/*
1424 	 * If device is being wedged even before submission_state is
1425 	 * initialized, there's nothing to do here.
1426 	 */
1427 	if (!guc->submission_state.initialized)
1428 		return;
1429 
1430 	if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) {
1431 		mutex_lock(&guc->submission_state.lock);
1432 		xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
1433 			if (xe_exec_queue_get_unless_zero(q))
1434 				set_exec_queue_wedged(q);
1435 		mutex_unlock(&guc->submission_state.lock);
1436 	} else {
1437 		/* Forcefully kill any remaining exec queues, signal fences */
1438 		guc_submit_reset_prepare(guc);
1439 		xe_guc_submit_stop(guc);
1440 		xe_guc_softreset(guc);
1441 		xe_uc_fw_sanitize(&guc->fw);
1442 		xe_guc_submit_pause_abort(guc);
1443 	}
1444 }
1445 
1446 static bool guc_submit_hint_wedged(struct xe_guc *guc)
1447 {
1448 	struct xe_device *xe = guc_to_xe(guc);
1449 
1450 	if (xe->wedged.mode != XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET)
1451 		return false;
1452 
1453 	if (xe_device_wedged(xe))
1454 		return true;
1455 
1456 	xe_device_declare_wedged(xe, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET);
1457 
1458 	return true;
1459 }
1460 
1461 #define ADJUST_FIVE_PERCENT(__t)	mul_u64_u32_div(__t, 105, 100)
1462 
1463 static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job)
1464 {
1465 	struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q));
1466 	u32 ctx_timestamp, ctx_job_timestamp;
1467 	u32 timeout_ms = q->sched_props.job_timeout_ms;
1468 	u32 diff;
1469 	u64 running_time_ms;
1470 
1471 	if (!xe_sched_job_started(job)) {
1472 		xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started",
1473 			   xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
1474 			   q->guc->id);
1475 
1476 		/* GuC never scheduled this job - let the caller trigger a GT reset. */
1477 		return true;
1478 	}
1479 
1480 	ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0]));
1481 	if (ctx_timestamp == job->sample_timestamp) {
1482 		if (IS_SRIOV_VF(gt_to_xe(gt)))
1483 			xe_gt_notice(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck",
1484 				     xe_sched_job_seqno(job),
1485 				     xe_sched_job_lrc_seqno(job), q->guc->id);
1486 		else
1487 			xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck",
1488 				   xe_sched_job_seqno(job),
1489 				   xe_sched_job_lrc_seqno(job), q->guc->id);
1490 
1491 		return xe_sched_invalidate_job(job, 0);
1492 	}
1493 
1494 	job->sample_timestamp = ctx_timestamp;
1495 	ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]);
1496 
1497 	/*
1498 	 * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch
1499 	 * possible overflows with a high timeout.
1500 	 */
1501 	xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC);
1502 
1503 	diff = ctx_timestamp - ctx_job_timestamp;
1504 
1505 	/*
1506 	 * Ensure timeout is within 5% to account for an GuC scheduling latency
1507 	 */
1508 	running_time_ms =
1509 		ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff));
1510 
1511 	xe_gt_dbg(gt,
1512 		  "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x",
1513 		  xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
1514 		  q->guc->id, running_time_ms, timeout_ms, diff);
1515 
1516 	return running_time_ms >= timeout_ms;
1517 }
1518 
1519 static void enable_scheduling(struct xe_exec_queue *q)
1520 {
1521 	MAKE_SCHED_CONTEXT_ACTION(q, ENABLE);
1522 	struct xe_guc *guc = exec_queue_to_guc(q);
1523 	int ret;
1524 
1525 	xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
1526 	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
1527 	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
1528 	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q));
1529 
1530 	set_exec_queue_pending_enable(q);
1531 	set_exec_queue_enabled(q);
1532 	trace_xe_exec_queue_scheduling_enable(q);
1533 
1534 	if (xe_exec_queue_is_multi_queue_secondary(q))
1535 		handle_multi_queue_secondary_sched_done(guc, q, 1);
1536 	else
1537 		xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1538 			       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1539 
1540 	ret = wait_event_timeout(guc->ct.wq,
1541 				 !exec_queue_pending_enable(q) ||
1542 				 xe_guc_read_stopped(guc) ||
1543 				 vf_recovery(guc), HZ * 5);
1544 	if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) {
1545 		xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond");
1546 		set_exec_queue_banned(q);
1547 		xe_gt_reset_async(q->gt);
1548 		xe_sched_tdr_queue_imm(&q->guc->sched);
1549 	}
1550 }
1551 
1552 static void disable_scheduling(struct xe_exec_queue *q, bool immediate)
1553 {
1554 	MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
1555 	struct xe_guc *guc = exec_queue_to_guc(q);
1556 
1557 	xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
1558 	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
1559 	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
1560 
1561 	if (immediate && !xe_exec_queue_is_multi_queue_secondary(q))
1562 		set_min_preemption_timeout(guc, q);
1563 	clear_exec_queue_enabled(q);
1564 	set_exec_queue_pending_disable(q);
1565 	trace_xe_exec_queue_scheduling_disable(q);
1566 
1567 	if (xe_exec_queue_is_multi_queue_secondary(q))
1568 		handle_multi_queue_secondary_sched_done(guc, q, 0);
1569 	else
1570 		xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1571 			       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1572 }
1573 
1574 /*
1575  * Recover via GT reset for a kernel queue, or for a GuC scheduling failure (job
1576  * never started) on a queue that was not already killed or banned. An already
1577  * banned queue must stay banned, so its unstarted jobs do not clear the ban or
1578  * trigger a reset.
1579  */
1580 static bool timeout_needs_gt_reset(struct xe_exec_queue *q, struct xe_sched_job *job,
1581 				   bool skip_timeout_check)
1582 {
1583 	if (q->flags & EXEC_QUEUE_FLAG_KERNEL)
1584 		return true;
1585 
1586 	return !skip_timeout_check && !xe_sched_job_started(job);
1587 }
1588 
1589 static enum drm_gpu_sched_stat
1590 guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
1591 {
1592 	struct xe_sched_job *job = to_xe_sched_job(drm_job);
1593 	struct drm_sched_job *tmp_job;
1594 	struct xe_exec_queue *q = job->q, *primary;
1595 	struct xe_gpu_scheduler *sched = &q->guc->sched;
1596 	struct xe_guc *guc = exec_queue_to_guc(q);
1597 	const char *process_name = "no process";
1598 	struct xe_device *xe = guc_to_xe(guc);
1599 	int err = -ETIME;
1600 	pid_t pid = -1;
1601 	bool wedged = false, wedge_device = false, skip_timeout_check;
1602 	int idx;
1603 
1604 	xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
1605 
1606 	primary = xe_exec_queue_multi_queue_primary(q);
1607 
1608 	/*
1609 	 * TDR has fired before free job worker. Common if exec queue
1610 	 * immediately closed after last fence signaled. Add back to pending
1611 	 * list so job can be freed and kick scheduler ensuring free job is not
1612 	 * lost.
1613 	 */
1614 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags) ||
1615 	    vf_recovery(guc))
1616 		return DRM_GPU_SCHED_STAT_NO_HANG;
1617 
1618 	if (!drm_dev_enter(&xe->drm, &idx)) {
1619 		xe_sched_job_set_error(job, -ENODEV);
1620 		return DRM_GPU_SCHED_STAT_NO_HANG;
1621 	}
1622 
1623 	/* Kill the run_job entry point */
1624 	if (xe_exec_queue_is_multi_queue(q))
1625 		xe_guc_exec_queue_group_stop(q);
1626 	else
1627 		xe_sched_submission_stop(sched);
1628 
1629 	/* Must check all state after stopping scheduler */
1630 	skip_timeout_check = exec_queue_reset(q) ||
1631 		exec_queue_killed_or_banned_or_wedged(q);
1632 
1633 	/* Skip timeout check if multi-queue group is banned */
1634 	if (xe_exec_queue_is_multi_queue(q) &&
1635 	    READ_ONCE(q->multi_queue.group->banned))
1636 		skip_timeout_check = true;
1637 
1638 	/* LR jobs can only get here if queue has been killed or hit an error */
1639 	if (xe_exec_queue_is_lr(q))
1640 		xe_gt_assert(guc_to_gt(guc), skip_timeout_check);
1641 
1642 	/*
1643 	 * If devcoredump not captured and GuC capture for the job is not ready
1644 	 * do manual capture first and decide later if we need to use it
1645 	 */
1646 	if (!xe_device_is_in_reset(xe) && !exec_queue_killed(q) && !xe->devcoredump.captured &&
1647 	    !xe_guc_capture_get_matching_and_lock(q)) {
1648 		/* take force wake before engine register manual capture */
1649 		CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
1650 		if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
1651 			xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n");
1652 
1653 		xe_engine_snapshot_capture_for_queue(q);
1654 	}
1655 
1656 	/*
1657 	 * Check if job is actually timed out, if so restart job execution and TDR
1658 	 */
1659 	if (!skip_timeout_check && !check_timeout(q, job))
1660 		goto rearm;
1661 
1662 	/*
1663 	 * Killed queues must not newly wedge the device, but preserve an
1664 	 * already-wedged state to avoid warning on teardown timeouts.
1665 	 */
1666 	if (!exec_queue_killed(q))
1667 		wedged = guc_submit_hint_wedged(exec_queue_to_guc(q));
1668 	else
1669 		wedged = xe_device_wedged(xe);
1670 
1671 	/*
1672 	 * Only tag as GPU hang if this is the original timeout, not a
1673 	 * consequence of a prior kill (e.g., page-offline).
1674 	 */
1675 	if (!exec_queue_killed(q))
1676 		atomic_or(DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG, &q->ban_reason);
1677 	set_exec_queue_banned(q);
1678 
1679 	/* Kick job / queue off hardware */
1680 	if (!xe_device_is_in_reset(xe) && !wedged &&
1681 	    (exec_queue_enabled(primary) || exec_queue_pending_disable(primary))) {
1682 		int ret;
1683 
1684 		if (exec_queue_reset(primary))
1685 			err = -EIO;
1686 
1687 		if (xe_uc_fw_is_running(&guc->fw)) {
1688 			/*
1689 			 * Wait for any pending G2H to flush out before
1690 			 * modifying state
1691 			 */
1692 			ret = wait_event_timeout(guc->ct.wq,
1693 						 (!exec_queue_pending_enable(primary) &&
1694 						  !exec_queue_pending_disable(primary)) ||
1695 						 xe_guc_read_stopped(guc) ||
1696 						 vf_recovery(guc), HZ * 5);
1697 			if (vf_recovery(guc))
1698 				goto handle_vf_resume;
1699 			if (!ret || xe_guc_read_stopped(guc))
1700 				goto trigger_reset;
1701 
1702 			disable_scheduling(primary, skip_timeout_check);
1703 		}
1704 
1705 		/*
1706 		 * Must wait for scheduling to be disabled before signalling
1707 		 * any fences, if GT broken the GT reset code should signal us.
1708 		 *
1709 		 * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault
1710 		 * error) messages which can cause the schedule disable to get
1711 		 * lost. If this occurs, trigger a GT reset to recover.
1712 		 */
1713 		smp_rmb();
1714 		ret = wait_event_timeout(guc->ct.wq,
1715 					 !xe_uc_fw_is_running(&guc->fw) ||
1716 					 !exec_queue_pending_disable(primary) ||
1717 					 xe_guc_read_stopped(guc) ||
1718 					 vf_recovery(guc), HZ * 5);
1719 		if (vf_recovery(guc))
1720 			goto handle_vf_resume;
1721 		if (!ret || xe_guc_read_stopped(guc)) {
1722 trigger_reset:
1723 			if (!ret)
1724 				xe_gt_warn(guc_to_gt(guc),
1725 					   "Schedule disable failed to respond, guc_id=%d",
1726 					   primary->guc->id);
1727 			xe_devcoredump(primary, job,
1728 				       "Schedule disable failed to respond, guc_id=%d, ret=%d, guc_read=%d",
1729 				       primary->guc->id, ret, xe_guc_read_stopped(guc));
1730 			xe_gt_reset_async(primary->gt);
1731 			xe_sched_tdr_queue_imm(sched);
1732 			goto rearm;
1733 		}
1734 	}
1735 
1736 	if (q->vm && q->vm->xef) {
1737 		process_name = q->vm->xef->process_name;
1738 		pid = q->vm->xef->pid;
1739 	}
1740 
1741 	if (!exec_queue_killed(q))
1742 		xe_gt_notice(guc_to_gt(guc),
1743 			     "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]",
1744 			     xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
1745 			     q->guc->id, q->flags, process_name, pid);
1746 
1747 	trace_xe_sched_job_timedout(job);
1748 
1749 	/* Do not access device if in reset */
1750 	if (!xe_device_is_in_reset(xe) && !exec_queue_killed(q))
1751 		xe_devcoredump(q, job,
1752 			       "Timedout job - seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx",
1753 			       xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
1754 			       q->guc->id, q->flags);
1755 
1756 	if (!wedged) {
1757 		if (timeout_needs_gt_reset(q, job, skip_timeout_check)) {
1758 			if (!xe_sched_invalidate_job(job, 2)) {
1759 				clear_exec_queue_banned(q);
1760 				/* protect concurrent page offline reasons */
1761 				atomic_andnot(DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG,
1762 					      &q->ban_reason);
1763 				xe_gt_reset_async(q->gt);
1764 				goto rearm;
1765 			}
1766 			if (q->flags & EXEC_QUEUE_FLAG_KERNEL) {
1767 				xe_gt_WARN(q->gt, true, "Kernel-submitted job timed out\n");
1768 				wedge_device = true;
1769 			}
1770 		} else if (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)) {
1771 			xe_gt_WARN(q->gt, true, "VM job timed out on non-killed execqueue\n");
1772 		}
1773 	}
1774 
1775 	/* Mark all outstanding jobs as bad, thus completing them */
1776 	xe_sched_job_set_error(job, err);
1777 	drm_sched_for_each_pending_job(tmp_job, &sched->base, NULL)
1778 		xe_sched_job_set_error(to_xe_sched_job(tmp_job), -ECANCELED);
1779 
1780 	if (xe_exec_queue_is_multi_queue(q)) {
1781 		xe_guc_exec_queue_group_start(q);
1782 		xe_guc_exec_queue_group_trigger_cleanup(q);
1783 	} else {
1784 		xe_sched_submission_start(sched);
1785 		xe_guc_exec_queue_trigger_cleanup(q);
1786 	}
1787 
1788 	if (wedge_device)
1789 		xe_device_declare_wedged(gt_to_xe(q->gt),
1790 					 DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET);
1791 
1792 	/*
1793 	 * We want the job added back to the pending list so it gets freed; this
1794 	 * is what DRM_GPU_SCHED_STAT_NO_HANG does.
1795 	 */
1796 	drm_dev_exit(idx);
1797 	return DRM_GPU_SCHED_STAT_NO_HANG;
1798 
1799 rearm:
1800 	/*
1801 	 * XXX: Ideally want to adjust timeout based on current execution time
1802 	 * but there is not currently an easy way to do in DRM scheduler. With
1803 	 * some thought, do this in a follow up.
1804 	 */
1805 	if (xe_exec_queue_is_multi_queue(q))
1806 		xe_guc_exec_queue_group_start(q);
1807 	else
1808 		xe_sched_submission_start(sched);
1809 handle_vf_resume:
1810 	drm_dev_exit(idx);
1811 	return DRM_GPU_SCHED_STAT_NO_HANG;
1812 }
1813 
1814 static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q);
1815 static int guc_exec_queue_suspend_wait_blocking(struct xe_exec_queue *q);
1816 
1817 static void guc_exec_queue_fini(struct xe_exec_queue *q)
1818 {
1819 	struct xe_guc_exec_queue *ge = q->guc;
1820 	struct xe_guc *guc = exec_queue_to_guc(q);
1821 	struct drm_device *drm = &guc_to_xe(guc)->drm;
1822 
1823 	/*
1824 	 * A secondary can leave the group while still preempt suspended (e.g.
1825 	 * xe_vm_remove_compute_exec_queue() forces its preempt fence to signal,
1826 	 * which suspends it). It holds one forwarded suspend reference on the
1827 	 * primary, so drop it and resume the primary if it was the last member
1828 	 * that had it suspended. Primaries forward to nobody, so they don't need
1829 	 * this.
1830 	 *
1831 	 * First make sure the primary's forwarded suspend has completed. If the
1832 	 * secondary was killed/reset before its preempt fence worker ran, that
1833 	 * worker skips suspend_wait() (see preempt_fence_work_func()), leaving
1834 	 * the primary's suspend possibly in flight. drop_suspend() runs under a
1835 	 * spinlock and cannot wait, so drain it here with the uninterruptible
1836 	 * blocking wait; otherwise resuming the primary in drop_suspend() could
1837 	 * trip the !suspend_pending assert.
1838 	 */
1839 	if (xe_exec_queue_is_multi_queue_secondary(q)) {
1840 		if (READ_ONCE(q->guc->suspend_count))
1841 			guc_exec_queue_suspend_wait_blocking(q);
1842 		guc_exec_queue_multi_queue_drop_suspend(q);
1843 	}
1844 
1845 	if (xe_exec_queue_is_multi_queue_secondary(q)) {
1846 		struct xe_exec_queue_group *group = q->multi_queue.group;
1847 
1848 		mutex_lock(&group->list_lock);
1849 		list_del(&q->multi_queue.link);
1850 		mutex_unlock(&group->list_lock);
1851 	}
1852 
1853 	release_guc_id(guc, q);
1854 	xe_sched_entity_fini(&ge->entity);
1855 	xe_sched_fini(&ge->sched);
1856 
1857 	/*
1858 	 * RCU free due sched being exported via DRM scheduler fences
1859 	 * (timeline name).
1860 	 */
1861 	kfree_rcu(ge, rcu);
1862 
1863 	drm_dev_put(drm);
1864 }
1865 
1866 static void guc_exec_queue_do_destroy(struct xe_exec_queue *q)
1867 {
1868 	struct xe_guc_exec_queue *ge = q->guc;
1869 	struct xe_guc *guc = exec_queue_to_guc(q);
1870 	struct xe_device *xe = guc_to_xe(guc);
1871 	struct drm_device *drm = &xe->drm;
1872 
1873 	/*
1874 	 * guc_exec_queue_fini() drops the queue's drm_device ref.
1875 	 * Keep the device alive until the PM-runtime guard unwinds.
1876 	 */
1877 	drm_dev_get(drm);
1878 
1879 	scoped_guard(xe_pm_runtime, xe) {
1880 		trace_xe_exec_queue_destroy(q);
1881 
1882 		/* Confirm no work left behind accessing device structures */
1883 		cancel_delayed_work_sync(&ge->sched.base.work_tdr);
1884 
1885 		xe_exec_queue_fini(q);
1886 	}
1887 
1888 	drm_dev_put(drm);
1889 }
1890 
1891 static void __guc_exec_queue_destroy_async(struct work_struct *w)
1892 {
1893 	struct xe_guc_exec_queue *ge =
1894 		container_of(w, struct xe_guc_exec_queue, destroy_async);
1895 
1896 	guc_exec_queue_do_destroy(ge->q);
1897 }
1898 
1899 static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)
1900 {
1901 	INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async);
1902 	xe_destroy_wq_queue(&q->guc->destroy_async);
1903 }
1904 
1905 static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg,
1906 						 bool bound)
1907 {
1908 	struct xe_exec_queue *q = msg->private_data;
1909 	struct xe_guc *guc = exec_queue_to_guc(q);
1910 
1911 	trace_xe_exec_queue_cleanup_entity(q);
1912 
1913 	/*
1914 	 * Expected state transitions for cleanup:
1915 	 * - If the exec queue is registered and GuC firmware is running, we must first
1916 	 *   disable scheduling and deregister the queue to ensure proper teardown and
1917 	 *   resource release in the GuC, then destroy the exec queue on driver side.
1918 	 * - If the GuC is already stopped (e.g., during driver unload or GPU reset),
1919 	 *   we cannot expect a response for the deregister request. In this case,
1920 	 *   it is safe to directly destroy the exec queue on driver side, as the GuC
1921 	 *   will not process further requests and all resources must be cleaned up locally.
1922 	 */
1923 	/* A wedged GuC won't answer the H2G, so tear down on the driver side. */
1924 	if (bound && !exec_queue_wedged(q) && exec_queue_registered(q) &&
1925 	    xe_uc_fw_is_running(&guc->fw))
1926 		disable_scheduling_deregister(guc, q);
1927 	else
1928 		guc_exec_queue_destroy_async(q);
1929 }
1930 
1931 static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q)
1932 {
1933 	return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q);
1934 }
1935 
1936 static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg,
1937 							 bool bound)
1938 {
1939 	struct xe_exec_queue *q = msg->private_data;
1940 	struct xe_guc *guc = exec_queue_to_guc(q);
1941 
1942 	if (guc_exec_queue_allowed_to_change_state(q) && bound)
1943 		init_policies(guc, q);
1944 	kfree(msg);
1945 }
1946 
1947 static void suspend_wait_wake(struct xe_exec_queue *q)
1948 {
1949 	struct xe_guc *guc = exec_queue_to_guc(q);
1950 	struct xe_device *xe = guc_to_xe(guc);
1951 
1952 	/*
1953 	 * We use a GuC shared wait queue for VFs because the VF resfix start
1954 	 * interrupt must be able to wake all instances of suspend_wait. This
1955 	 * prevents the VF migration worker from being starved during
1956 	 * scheduling.
1957 	 */
1958 	if (IS_SRIOV_VF(xe))
1959 		wake_up_all(&guc->ct.wq);
1960 	else
1961 		wake_up(&q->guc->suspend_wait);
1962 }
1963 
1964 static void __suspend_fence_signal(struct xe_exec_queue *q)
1965 {
1966 	if (!q->guc->suspend_pending)
1967 		return;
1968 
1969 	WRITE_ONCE(q->guc->suspend_pending, false);
1970 	suspend_wait_wake(q);
1971 }
1972 
1973 static void suspend_fence_signal(struct xe_exec_queue *q)
1974 {
1975 	struct xe_guc *guc = exec_queue_to_guc(q);
1976 
1977 	xe_gt_assert(guc_to_gt(guc), exec_queue_suspended(q) || exec_queue_killed(q) ||
1978 		     xe_guc_read_stopped(guc));
1979 	xe_gt_assert(guc_to_gt(guc), q->guc->suspend_pending);
1980 
1981 	__suspend_fence_signal(q);
1982 }
1983 
1984 static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg,
1985 						 bool bound)
1986 {
1987 	struct xe_exec_queue *q = msg->private_data;
1988 	struct xe_guc *guc = exec_queue_to_guc(q);
1989 
1990 	if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) &&
1991 	    exec_queue_enabled(q) && bound) {
1992 		wait_event(guc->ct.wq, vf_recovery(guc) ||
1993 			   ((q->guc->resume_time != RESUME_PENDING ||
1994 			   xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q)));
1995 
1996 		if (!xe_guc_read_stopped(guc)) {
1997 			s64 since_resume_ms =
1998 				ktime_ms_delta(ktime_get(),
1999 					       q->guc->resume_time);
2000 			s64 wait_ms = q->vm->preempt.min_run_period_ms -
2001 				since_resume_ms;
2002 
2003 			if (wait_ms > 0 && q->guc->resume_time)
2004 				xe_sleep_relaxed_ms(wait_ms);
2005 
2006 			set_exec_queue_suspended(q);
2007 			disable_scheduling(q, false);
2008 		}
2009 	} else if (q->guc->suspend_pending) {
2010 		set_exec_queue_suspended(q);
2011 		suspend_fence_signal(q);
2012 	}
2013 }
2014 
2015 static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg,
2016 						bool bound)
2017 {
2018 	struct xe_exec_queue *q = msg->private_data;
2019 
2020 	if (guc_exec_queue_allowed_to_change_state(q) && bound) {
2021 		clear_exec_queue_suspended(q);
2022 		if (!exec_queue_enabled(q)) {
2023 			q->guc->resume_time = RESUME_PENDING;
2024 			set_exec_queue_pending_resume(q);
2025 			enable_scheduling(q);
2026 		}
2027 	} else {
2028 		clear_exec_queue_suspended(q);
2029 	}
2030 }
2031 
2032 static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg,
2033 								  bool bound)
2034 {
2035 	struct xe_exec_queue *q = msg->private_data;
2036 
2037 	if (guc_exec_queue_allowed_to_change_state(q) && bound)
2038 		guc_exec_queue_send_cgp_sync(q, 0);
2039 
2040 	kfree(msg);
2041 }
2042 
2043 static void __guc_exec_queue_process_msg_cgp_sync(struct xe_sched_msg *msg,
2044 						  bool bound)
2045 {
2046 	struct xe_exec_queue *q = msg->private_data;
2047 
2048 	/*
2049 	 * Replay a dynamic CGP update lost across VF migration by re-issuing the
2050 	 * CGP update + CGP_SYNC (re-applies the current priority from
2051 	 * q->multi_queue.priority).
2052 	 */
2053 	if (guc_exec_queue_allowed_to_change_state(q) && bound)
2054 		guc_exec_queue_send_cgp_sync(q, 0);
2055 }
2056 
2057 #define CLEANUP				1	/* Non-zero values to catch uninitialized msg */
2058 #define SET_SCHED_PROPS			2
2059 #define SUSPEND				3
2060 #define RESUME				4
2061 #define SET_MULTI_QUEUE_PRIORITY	5
2062 #define CGP_SYNC_MSG			6
2063 #define OPCODE_MASK	0xf
2064 #define MSG_LOCKED	BIT(8)
2065 #define MSG_HEAD	BIT(9)
2066 #define MSG_PM_REF	BIT(10)
2067 
2068 static void guc_exec_queue_process_msg(struct xe_sched_msg *msg)
2069 {
2070 	struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data));
2071 	int idx;
2072 	bool pm_ref = !!(msg->opcode & MSG_PM_REF);
2073 	bool bound = drm_dev_enter(&xe->drm, &idx);
2074 
2075 	trace_xe_sched_msg_recv(msg);
2076 
2077 	switch (msg->opcode & OPCODE_MASK) {
2078 	case CLEANUP:
2079 		__guc_exec_queue_process_msg_cleanup(msg, bound);
2080 		break;
2081 	case SET_SCHED_PROPS:
2082 		__guc_exec_queue_process_msg_set_sched_props(msg, bound);
2083 		break;
2084 	case SUSPEND:
2085 		__guc_exec_queue_process_msg_suspend(msg, bound);
2086 		break;
2087 	case RESUME:
2088 		__guc_exec_queue_process_msg_resume(msg, bound);
2089 		break;
2090 	case SET_MULTI_QUEUE_PRIORITY:
2091 		__guc_exec_queue_process_msg_set_multi_queue_priority(msg, bound);
2092 		break;
2093 	case CGP_SYNC_MSG:
2094 		__guc_exec_queue_process_msg_cgp_sync(msg, bound);
2095 		break;
2096 	default:
2097 		XE_WARN_ON("Unknown message type");
2098 	}
2099 
2100 	if (pm_ref)
2101 		xe_pm_runtime_put(xe);
2102 
2103 	if (bound)
2104 		drm_dev_exit(idx);
2105 }
2106 
2107 static const struct drm_sched_backend_ops drm_sched_ops = {
2108 	.run_job = guc_exec_queue_run_job,
2109 	.free_job = guc_exec_queue_free_job,
2110 	.timedout_job = guc_exec_queue_timedout_job,
2111 };
2112 
2113 static const struct xe_sched_backend_ops xe_sched_ops = {
2114 	.process_msg = guc_exec_queue_process_msg,
2115 };
2116 
2117 static int guc_exec_queue_init(struct xe_exec_queue *q)
2118 {
2119 	struct xe_gpu_scheduler *sched;
2120 	struct xe_guc *guc = exec_queue_to_guc(q);
2121 	struct drm_device *drm = &guc_to_xe(guc)->drm;
2122 	struct workqueue_struct *submit_wq = NULL;
2123 	struct xe_guc_exec_queue *ge;
2124 	long timeout;
2125 	int err, i;
2126 	int max_jobs = (xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES);
2127 
2128 	xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc)));
2129 
2130 	ge = kzalloc_obj(*ge);
2131 	if (!ge)
2132 		return -ENOMEM;
2133 
2134 	drm_dev_get(drm);
2135 
2136 	q->guc = ge;
2137 	ge->q = q;
2138 	init_rcu_head(&ge->rcu);
2139 	init_waitqueue_head(&ge->suspend_wait);
2140 
2141 	for (i = 0; i < MAX_STATIC_MSG_TYPE; ++i)
2142 		INIT_LIST_HEAD(&ge->static_msgs[i].link);
2143 
2144 	timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT :
2145 		  msecs_to_jiffies(q->sched_props.job_timeout_ms);
2146 
2147 	err = alloc_guc_id(guc, q);
2148 	if (err)
2149 		goto err_free;
2150 
2151 	xe_exec_queue_assign_name(q, q->guc->id);
2152 
2153 	strscpy(ge->name, q->name, sizeof(ge->name));
2154 
2155 	/*
2156 	 * Use primary queue's submit_wq for all secondary queues of a
2157 	 * multi queue group. This serialization avoids any locking around
2158 	 * CGP synchronization with GuC.
2159 	 */
2160 	if (xe_exec_queue_is_multi_queue_secondary(q)) {
2161 		struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
2162 
2163 		submit_wq = primary->guc->sched.base.submit_wq;
2164 	}
2165 
2166 	if (q->vm && q->vm->flags & XE_VM_FLAG_MIGRATION)
2167 		max_jobs = min(max_jobs, LRC_MIGRATION_ULLS_SEMAPHORE_COUNT - 1);
2168 
2169 	err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
2170 			    submit_wq, max_jobs, 64,
2171 			    timeout, guc_to_gt(guc)->ordered_wq, NULL,
2172 			    ge->name, gt_to_xe(q->gt)->drm.dev);
2173 	if (err)
2174 		goto err_release_id;
2175 
2176 	sched = &ge->sched;
2177 	err = xe_sched_entity_init(&ge->entity, sched);
2178 	if (err)
2179 		goto err_sched;
2180 
2181 	q->entity = &ge->entity;
2182 
2183 	mutex_lock(&guc->submission_state.lock);
2184 	if (xe_guc_read_stopped(guc) || vf_recovery(guc))
2185 		xe_sched_stop(sched);
2186 	publish_guc_id(guc, q);
2187 	mutex_unlock(&guc->submission_state.lock);
2188 
2189 	/*
2190 	 * Maintain secondary queues of the multi queue group in a list
2191 	 * for handling dependencies across the queues in the group.
2192 	 */
2193 	if (xe_exec_queue_is_multi_queue_secondary(q)) {
2194 		struct xe_exec_queue_group *group = q->multi_queue.group;
2195 
2196 		INIT_LIST_HEAD(&q->multi_queue.link);
2197 		mutex_lock(&group->list_lock);
2198 		if (group->stopped)
2199 			WRITE_ONCE(q->guc->sched.base.pause_submit, true);
2200 		list_add_tail(&q->multi_queue.link, &group->list);
2201 		mutex_unlock(&group->list_lock);
2202 	}
2203 
2204 	if (xe_exec_queue_is_multi_queue(q))
2205 		trace_xe_exec_queue_create_multi_queue(q);
2206 	else
2207 		trace_xe_exec_queue_create(q);
2208 
2209 	return 0;
2210 
2211 err_sched:
2212 	xe_sched_fini(&ge->sched);
2213 err_release_id:
2214 	release_guc_id(guc, q);
2215 err_free:
2216 	kfree(ge);
2217 	drm_dev_put(drm);
2218 
2219 	return err;
2220 }
2221 
2222 static void guc_exec_queue_kill(struct xe_exec_queue *q)
2223 {
2224 	trace_xe_exec_queue_kill(q);
2225 	set_exec_queue_killed(q);
2226 	/*
2227 	 * Do not signal the suspend fence here, the queue may still be running
2228 	 * on the hardware. Only wake any suspend waiter so it observes the
2229 	 * killed state instead of waiting out the full timeout, suspend_pending
2230 	 * stays set until the queue is torn down.
2231 	 */
2232 	suspend_wait_wake(q);
2233 	xe_guc_exec_queue_trigger_cleanup(q);
2234 }
2235 
2236 static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg,
2237 				   u32 opcode)
2238 {
2239 	struct xe_device *xe = guc_to_xe(exec_queue_to_guc(q));
2240 	int idx;
2241 	bool bound = drm_dev_enter(&xe->drm, &idx);
2242 
2243 	INIT_LIST_HEAD(&msg->link);
2244 	msg->opcode = opcode & OPCODE_MASK;
2245 	if (bound) {
2246 		xe_pm_runtime_get_noresume(xe);
2247 		msg->opcode |= MSG_PM_REF;
2248 	}
2249 	msg->private_data = q;
2250 
2251 	trace_xe_sched_msg_add(msg);
2252 	if (opcode & MSG_HEAD)
2253 		xe_sched_add_msg_head(&q->guc->sched, msg);
2254 	else if (opcode & MSG_LOCKED)
2255 		xe_sched_add_msg_locked(&q->guc->sched, msg);
2256 	else
2257 		xe_sched_add_msg(&q->guc->sched, msg);
2258 
2259 	if (bound)
2260 		drm_dev_exit(idx);
2261 }
2262 
2263 static void guc_exec_queue_try_add_msg_head(struct xe_exec_queue *q,
2264 					    struct xe_sched_msg *msg,
2265 					    u32 opcode)
2266 {
2267 	if (!list_empty(&msg->link))
2268 		return;
2269 
2270 	guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED | MSG_HEAD);
2271 }
2272 
2273 static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q,
2274 				       struct xe_sched_msg *msg,
2275 				       u32 opcode)
2276 {
2277 	if (!list_empty(&msg->link))
2278 		return false;
2279 
2280 	guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED);
2281 
2282 	return true;
2283 }
2284 
2285 #define STATIC_MSG_CLEANUP	0
2286 #define STATIC_MSG_SUSPEND	1
2287 #define STATIC_MSG_RESUME	2
2288 #define STATIC_MSG_CGP_SYNC	3
2289 static void guc_exec_queue_destroy(struct xe_exec_queue *q)
2290 {
2291 	struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;
2292 
2293 	guc_exec_queue_add_msg(q, msg, CLEANUP);
2294 }
2295 
2296 static int guc_exec_queue_set_priority(struct xe_exec_queue *q,
2297 				       enum xe_exec_queue_priority priority)
2298 {
2299 	struct xe_sched_msg *msg;
2300 
2301 	if (q->sched_props.priority == priority ||
2302 	    exec_queue_killed_or_banned_or_wedged(q))
2303 		return 0;
2304 
2305 	msg = kmalloc_obj(*msg);
2306 	if (!msg)
2307 		return -ENOMEM;
2308 
2309 	q->sched_props.priority = priority;
2310 	guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);
2311 
2312 	return 0;
2313 }
2314 
2315 static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us)
2316 {
2317 	struct xe_sched_msg *msg;
2318 
2319 	if (q->sched_props.timeslice_us == timeslice_us ||
2320 	    exec_queue_killed_or_banned_or_wedged(q))
2321 		return 0;
2322 
2323 	msg = kmalloc_obj(*msg);
2324 	if (!msg)
2325 		return -ENOMEM;
2326 
2327 	q->sched_props.timeslice_us = timeslice_us;
2328 	guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);
2329 
2330 	return 0;
2331 }
2332 
2333 static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q,
2334 					      u32 preempt_timeout_us)
2335 {
2336 	struct xe_sched_msg *msg;
2337 
2338 	if (q->sched_props.preempt_timeout_us == preempt_timeout_us ||
2339 	    exec_queue_killed_or_banned_or_wedged(q))
2340 		return 0;
2341 
2342 	msg = kmalloc_obj(*msg);
2343 	if (!msg)
2344 		return -ENOMEM;
2345 
2346 	q->sched_props.preempt_timeout_us = preempt_timeout_us;
2347 	guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);
2348 
2349 	return 0;
2350 }
2351 
2352 static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q,
2353 						   enum xe_multi_queue_priority priority)
2354 {
2355 	struct xe_sched_msg *msg;
2356 
2357 	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q));
2358 
2359 	if (exec_queue_killed_or_banned_or_wedged(q))
2360 		return 0;
2361 
2362 	msg = kmalloc_obj(*msg);
2363 	if (!msg)
2364 		return -ENOMEM;
2365 
2366 	scoped_guard(spinlock, &q->multi_queue.lock) {
2367 		if (q->multi_queue.priority == priority) {
2368 			kfree(msg);
2369 			return 0;
2370 		}
2371 
2372 		q->multi_queue.priority = priority;
2373 	}
2374 
2375 	guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY);
2376 
2377 	return 0;
2378 }
2379 
2380 /*
2381  * Core suspend: take a suspend reference on @q and, on the first reference,
2382  * disable its GuC context so the GPU is actually preempted. Caller must have
2383  * ensured @q is not killed/banned/wedged. Returns true if this was the first
2384  * suspend reference (the 0->1 transition).
2385  */
2386 static bool __guc_exec_queue_suspend(struct xe_exec_queue *q)
2387 {
2388 	struct xe_guc_exec_queue *ge = q->guc;
2389 	struct xe_gpu_scheduler *sched = &ge->sched;
2390 	struct xe_sched_msg *msg = ge->static_msgs + STATIC_MSG_SUSPEND;
2391 	bool first;
2392 
2393 	xe_sched_msg_lock(sched);
2394 	first = (++ge->suspend_count == 1);
2395 	if (first) {
2396 		bool added = guc_exec_queue_try_add_msg(q, msg, SUSPEND);
2397 
2398 		/* slot must be free at 0->1 */
2399 		xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), added);
2400 		ge->suspend_pending = true;
2401 	}
2402 	xe_sched_msg_unlock(sched);
2403 
2404 	return first;
2405 }
2406 
2407 /*
2408  * Core resume: drop a suspend reference on @q and, on the last reference,
2409  * re-enable its GuC context. Returns true if this dropped the last suspend
2410  * reference (the 1->0 transition).
2411  */
2412 static bool __guc_exec_queue_resume(struct xe_exec_queue *q)
2413 {
2414 	struct xe_guc_exec_queue *ge = q->guc;
2415 	struct xe_gpu_scheduler *sched = &ge->sched;
2416 	struct xe_sched_msg *msg = ge->static_msgs + STATIC_MSG_RESUME;
2417 	struct xe_guc *guc = exec_queue_to_guc(q);
2418 	bool last;
2419 
2420 	xe_sched_msg_lock(sched);
2421 	/*
2422 	 * A killed, banned or wedged queue keeps suspend_pending set until it
2423 	 * is torn down, so only assert the flag is clear otherwise.
2424 	 */
2425 	xe_gt_assert(guc_to_gt(guc), !ge->suspend_pending ||
2426 		     exec_queue_killed_or_banned_or_wedged(q));
2427 	xe_gt_assert(guc_to_gt(guc), ge->suspend_count > 0);
2428 	last = (--ge->suspend_count == 0);
2429 	if (last) {
2430 		bool added = guc_exec_queue_try_add_msg(q, msg, RESUME);
2431 
2432 		/* slot must be free at 1->0 */
2433 		xe_gt_assert(guc_to_gt(guc), added);
2434 	}
2435 	xe_sched_msg_unlock(sched);
2436 
2437 	return last;
2438 }
2439 
2440 static int guc_exec_queue_suspend(struct xe_exec_queue *q)
2441 {
2442 	if (exec_queue_killed_or_banned_or_wedged(q))
2443 		return -EINVAL;
2444 
2445 	/*
2446 	 * Non-multi-queue queues and multi-queue primaries suspend themselves
2447 	 * directly: their own msg_lock makes the suspend_count 0->1 transition
2448 	 * and the suspend_pending update atomic, so no group level serialization
2449 	 * is needed.
2450 	 */
2451 	if (!xe_exec_queue_is_multi_queue_secondary(q)) {
2452 		__guc_exec_queue_suspend(q);
2453 		return 0;
2454 	}
2455 
2456 	/*
2457 	 * A secondary's suspend is meaningless once the primary - which owns the
2458 	 * group's GuC context - is gone, so fail it too. This keeps the
2459 	 * secondary's effective state consistent with guc_exec_queue_reset_status(),
2460 	 * which already reports the primary's killed/banned/wedged state for
2461 	 * secondaries. A primary killed *after* this check is still handled at
2462 	 * message-processing time, where the SUSPEND is a no-op for a killed
2463 	 * context; this only covers an already-dead primary.
2464 	 */
2465 	if (exec_queue_killed_or_banned_or_wedged(xe_exec_queue_multi_queue_primary(q)))
2466 		return -EINVAL;
2467 
2468 	/*
2469 	 * A secondary doesn't interface with GuC: suspend it like any other
2470 	 * queue (its own suspend_count drives its internally handled scheduler
2471 	 * state) and, only on its own 0->1 transition, forward the suspend to the
2472 	 * primary so the GPU is actually preempted. Hold @suspend_lock so that
2473 	 * observing the secondary's transition and forwarding it to the primary
2474 	 * happen atomically; this keeps the primary's refcount paired with member
2475 	 * transitions even if the same secondary is suspended and resumed
2476 	 * concurrently across rebind cycles.
2477 	 */
2478 	scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) {
2479 		if (__guc_exec_queue_suspend(q))
2480 			__guc_exec_queue_suspend(xe_exec_queue_multi_queue_primary(q));
2481 	}
2482 
2483 	return 0;
2484 }
2485 
2486 static void guc_exec_queue_suspend_timeout_ban(struct xe_exec_queue *q)
2487 {
2488 	struct xe_guc *guc = exec_queue_to_guc(q);
2489 
2490 	xe_gt_warn(guc_to_gt(guc),
2491 		   "Suspend fence, guc_id=%d, failed to respond, banning queue",
2492 		   q->guc->id);
2493 	/*
2494 	 * The GuC failed to respond to the suspend within the timeout. This is
2495 	 * not recoverable for this context, so ban it and tear it down via
2496 	 * cleanup rather than leave it suspended forever. The suspend fence is
2497 	 * not signalled here, the queue may still be running on the hardware,
2498 	 * so suspend_pending stays set until teardown. The waiter that timed
2499 	 * out is the caller, so no wake up is needed either.
2500 	 *
2501 	 * @q is the primary here; it owns the group's GuC context, so a failure
2502 	 * to suspend it wedges the whole group. Ban and tear down the entire
2503 	 * group in the multi-queue case.
2504 	 */
2505 	if (xe_exec_queue_is_multi_queue(q)) {
2506 		set_exec_queue_group_banned(q);
2507 		xe_guc_exec_queue_group_trigger_cleanup(q);
2508 	} else {
2509 		set_exec_queue_banned(q);
2510 		xe_guc_exec_queue_trigger_cleanup(q);
2511 	}
2512 }
2513 
2514 /*
2515  * Wait for @q's own suspend to complete: suspend_pending cleared, or the queue
2516  * killed / GuC stopped. With @blocking, wait uninterruptibly and do not handle
2517  * VF recovery (for callers that must complete on behalf of a possibly
2518  * cross-process queue); otherwise wait interruptibly.
2519  *
2520  * Returns 0 on completion or -ETIME on timeout. Interruptible waits may also
2521  * return -EAGAIN (VF recovery in progress, retry) or -ERESTARTSYS (aborted by a
2522  * signal; suspend_pending may still be set, so callers must not resume()
2523  * without re-confirming the suspend).
2524  */
2525 static int guc_exec_queue_wait_suspend_done(struct xe_exec_queue *q, bool blocking)
2526 {
2527 	struct xe_guc *guc = exec_queue_to_guc(q);
2528 	struct xe_device *xe = guc_to_xe(guc);
2529 	int ret;
2530 
2531 	/*
2532 	 * A kill does not clear suspend_pending, the queue may still be running
2533 	 * on the hardware, so exec_queue_killed() is what ends the wait in that
2534 	 * case and the queue is torn down asynchronously.
2535 	 */
2536 #define WAIT_COND \
2537 	(!READ_ONCE(q->guc->suspend_pending) ||	exec_queue_killed(q) || \
2538 	 xe_guc_read_stopped(guc))
2539 
2540 retry:
2541 	if (blocking) {
2542 		if (IS_SRIOV_VF(xe))
2543 			ret = wait_event_timeout(guc->ct.wq, WAIT_COND, HZ * 5);
2544 		else
2545 			ret = wait_event_timeout(q->guc->suspend_wait, WAIT_COND,
2546 						 HZ * 5);
2547 	} else if (IS_SRIOV_VF(xe)) {
2548 		ret = wait_event_interruptible_timeout(guc->ct.wq, WAIT_COND ||
2549 						       vf_recovery(guc), HZ * 5);
2550 	} else {
2551 		ret = wait_event_interruptible_timeout(q->guc->suspend_wait,
2552 						       WAIT_COND, HZ * 5);
2553 	}
2554 
2555 	if (!blocking && vf_recovery(guc) && !xe_device_wedged(xe))
2556 		return -EAGAIN;
2557 
2558 	if (!ret)
2559 		return -ETIME;
2560 	else if (!blocking && IS_SRIOV_VF(xe) && !WAIT_COND)
2561 		/* Corner case on RESFIX DONE where vf_recovery() changes */
2562 		goto retry;
2563 
2564 #undef WAIT_COND
2565 
2566 	return ret < 0 ? ret : 0;
2567 }
2568 
2569 static int guc_exec_queue_suspend_wait_common(struct xe_exec_queue *q, bool blocking)
2570 {
2571 	int ret;
2572 
2573 	/*
2574 	 * A secondary's suspend rides the sched-message worker (short-circuited,
2575 	 * no GuC round-trip) and so is not synchronous with
2576 	 * guc_exec_queue_suspend(): its own suspend_pending may still be set
2577 	 * here. Waiting on the primary alone is not sufficient - if the primary
2578 	 * was already suspended, the forward is a refcount-only transition that
2579 	 * queues no new primary SUSPEND and leaves the primary's suspend_pending
2580 	 * clear, so the primary wait would return immediately while the
2581 	 * secondary's suspend is still in flight, and a later resume() would trip
2582 	 * the secondary's !suspend_pending assert. So first wait for the
2583 	 * secondary's own suspend to complete, then wait on the primary.
2584 	 *
2585 	 * A timeout on either bans the queue (being multi-queue, that tears down
2586 	 * the whole group). A secondary suspend has no real GuC round-trip, so
2587 	 * its timeout is a software scheduler stall rather than a GuC fault, but
2588 	 * banning is still the safe recovery: otherwise the queue is left with
2589 	 * suspend_pending set and a subsequent resume() trips the !suspend_pending
2590 	 * assert.
2591 	 */
2592 	if (xe_exec_queue_is_multi_queue_secondary(q)) {
2593 		ret = guc_exec_queue_wait_suspend_done(q, blocking);
2594 		if (ret == -ETIME)
2595 			guc_exec_queue_suspend_timeout_ban(q);
2596 		if (ret)
2597 			return ret;
2598 	}
2599 
2600 	q = xe_exec_queue_multi_queue_primary(q);
2601 	ret = guc_exec_queue_wait_suspend_done(q, blocking);
2602 	if (ret == -ETIME)
2603 		guc_exec_queue_suspend_timeout_ban(q);
2604 
2605 	return ret;
2606 }
2607 
2608 static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q)
2609 {
2610 	return guc_exec_queue_suspend_wait_common(q, false);
2611 }
2612 
2613 /*
2614  * Uninterruptible variant of guc_exec_queue_suspend_wait() for callers that
2615  * must complete the wait on behalf of a queue possibly owned by a different
2616  * process (e.g. cleanup/undo paths). An interruptible wait could return
2617  * -ERESTARTSYS if the calling task is signalled, leaving that queue suspended
2618  * forever (cross-process DoS). VF recovery is deliberately not handled (no
2619  * -EAGAIN) since a blocking caller cannot retry.
2620  */
2621 static int guc_exec_queue_suspend_wait_blocking(struct xe_exec_queue *q)
2622 {
2623 	return guc_exec_queue_suspend_wait_common(q, true);
2624 }
2625 
2626 static void guc_exec_queue_resume(struct xe_exec_queue *q)
2627 {
2628 	/*
2629 	 * Non-multi-queue queues and multi-queue primaries resume themselves
2630 	 * directly; their own msg_lock is sufficient.
2631 	 */
2632 	if (!xe_exec_queue_is_multi_queue_secondary(q)) {
2633 		__guc_exec_queue_resume(q);
2634 		return;
2635 	}
2636 
2637 	/*
2638 	 * Mirror of guc_exec_queue_suspend(): resume the secondary like any
2639 	 * other queue and, only on its own 1->0 transition, forward the resume
2640 	 * to the primary so the primary's GuC context is re-enabled once the
2641 	 * last member that suspended it resumes. @suspend_lock keeps the
2642 	 * secondary transition and the primary forward atomic.
2643 	 */
2644 	scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) {
2645 		if (__guc_exec_queue_resume(q))
2646 			__guc_exec_queue_resume(xe_exec_queue_multi_queue_primary(q));
2647 	}
2648 }
2649 
2650 /*
2651  * Drop a leaving secondary's forwarded suspend reference on the primary and
2652  * resume the primary if this was the last member that had it suspended.
2653  * See guc_exec_queue_fini().
2654  */
2655 static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q)
2656 {
2657 	scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) {
2658 		struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
2659 
2660 		/*
2661 		 * A suspended secondary holds exactly one suspend reference on the
2662 		 * primary (forwarded on its 0->1 transition). If it leaves while
2663 		 * still suspended, release that reference so the primary is not
2664 		 * kept disabled forever.
2665 		 */
2666 		if (!READ_ONCE(q->guc->suspend_count))
2667 			break;
2668 
2669 		if (exec_queue_killed_or_banned_or_wedged(primary))
2670 			break;
2671 
2672 		/*
2673 		 * No suspend_wait() here (and we can't - suspend_lock is a
2674 		 * spinlock). guc_exec_queue_fini() has already drained the
2675 		 * primary's forwarded suspend with the blocking wait, so its
2676 		 * suspend has completed (suspend_pending cleared) by the time we
2677 		 * resume it here. __guc_exec_queue_resume() asserts this.
2678 		 */
2679 		__guc_exec_queue_resume(primary);
2680 	}
2681 }
2682 
2683 static u64 guc_exec_queue_reset_status(struct xe_exec_queue *q)
2684 {
2685 	/* TODO: In case of multiqueue, if a secondary queue is banned due to
2686 	 * page offlining, checking only the primary queue's GuC reset status
2687 	 * may mask the true reason or race with it.
2688 	 */
2689 	if (xe_exec_queue_is_multi_queue_secondary(q)) {
2690 		u64 status = guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q));
2691 
2692 		if (status)
2693 			return status;
2694 	}
2695 
2696 	if (exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q)) {
2697 		u64 reason = atomic_read_acquire(&q->ban_reason);
2698 
2699 		/* If no specific reason was recorded, default to GPU hang */
2700 		if (!reason)
2701 			reason = DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG;
2702 		return reason;
2703 	}
2704 
2705 	return 0;
2706 }
2707 
2708 /*
2709  * All of these functions are an abstraction layer which other parts of Xe can
2710  * use to trap into the GuC backend. All of these functions, aside from init,
2711  * really shouldn't do much other than trap into the DRM scheduler which
2712  * synchronizes these operations.
2713  */
2714 static const struct xe_exec_queue_ops guc_exec_queue_ops = {
2715 	.init = guc_exec_queue_init,
2716 	.kill = guc_exec_queue_kill,
2717 	.fini = guc_exec_queue_fini,
2718 	.destroy = guc_exec_queue_destroy,
2719 	.set_priority = guc_exec_queue_set_priority,
2720 	.set_timeslice = guc_exec_queue_set_timeslice,
2721 	.set_preempt_timeout = guc_exec_queue_set_preempt_timeout,
2722 	.set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority,
2723 	.suspend = guc_exec_queue_suspend,
2724 	.suspend_wait = guc_exec_queue_suspend_wait,
2725 	.suspend_wait_blocking = guc_exec_queue_suspend_wait_blocking,
2726 	.resume = guc_exec_queue_resume,
2727 	.reset_status = guc_exec_queue_reset_status,
2728 };
2729 
2730 static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
2731 {
2732 	struct xe_gpu_scheduler *sched = &q->guc->sched;
2733 	bool do_destroy = false;
2734 
2735 	/* Stop scheduling + flush any DRM scheduler operations */
2736 	xe_sched_submission_stop(sched);
2737 
2738 	/* Clean up lost G2H + reset engine state */
2739 	if (exec_queue_registered(q)) {
2740 		if (exec_queue_destroyed(q))
2741 			do_destroy = true;
2742 	}
2743 	xe_sched_msg_lock(sched);
2744 	if (q->guc->suspend_pending) {
2745 		set_exec_queue_suspended(q);
2746 		suspend_fence_signal(q);
2747 	}
2748 	xe_sched_msg_unlock(sched);
2749 	atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED |
2750 		   EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED |
2751 		   EXEC_QUEUE_STATE_SUSPENDED,
2752 		   &q->guc->state);
2753 	q->guc->resume_time = 0;
2754 	trace_xe_exec_queue_stop(q);
2755 
2756 	/*
2757 	 * Ban any engine (aside from kernel and engines used for VM ops) with a
2758 	 * started but not complete job or if a job has gone through a GT reset
2759 	 * more than twice.
2760 	 */
2761 	if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) {
2762 		struct xe_sched_job *job = xe_sched_first_pending_job(sched);
2763 		bool ban = false;
2764 
2765 		if (job) {
2766 			if ((xe_sched_job_started(job) &&
2767 			    !xe_sched_job_completed(job)) ||
2768 			    xe_sched_invalidate_job(job, 2)) {
2769 				trace_xe_sched_job_ban(job);
2770 				ban = true;
2771 			}
2772 		}
2773 
2774 		if (ban) {
2775 			set_exec_queue_banned(q);
2776 			xe_guc_exec_queue_trigger_cleanup(q);
2777 		}
2778 	}
2779 
2780 	if (do_destroy)
2781 		guc_exec_queue_destroy_async(q);
2782 }
2783 
2784 static int guc_submit_reset_prepare(struct xe_guc *guc)
2785 {
2786 	int ret;
2787 
2788 	/*
2789 	 * Using an atomic here rather than submission_state.lock as this
2790 	 * function can be called while holding the CT lock (engine reset
2791 	 * failure). submission_state.lock needs the CT lock to resubmit jobs.
2792 	 * Atomic is not ideal, but it works to prevent against concurrent reset
2793 	 * and releasing any TDRs waiting on guc->submission_state.stopped.
2794 	 */
2795 	ret = atomic_fetch_or(1, &guc->submission_state.stopped);
2796 	smp_wmb();
2797 	wake_up_all(&guc->ct.wq);
2798 
2799 	return ret;
2800 }
2801 
2802 int xe_guc_submit_reset_prepare(struct xe_guc *guc)
2803 {
2804 	if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc)))
2805 		return 0;
2806 
2807 	if (!guc->submission_state.initialized)
2808 		return 0;
2809 
2810 	return guc_submit_reset_prepare(guc);
2811 }
2812 
2813 void xe_guc_submit_reset_wait(struct xe_guc *guc)
2814 {
2815 	wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) ||
2816 		   !xe_guc_read_stopped(guc));
2817 }
2818 
2819 void xe_guc_submit_stop(struct xe_guc *guc)
2820 {
2821 	struct xe_exec_queue *q;
2822 	unsigned long index;
2823 
2824 	xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1);
2825 
2826 	mutex_lock(&guc->submission_state.lock);
2827 
2828 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
2829 		/* Prevent redundant attempts to stop parallel queues */
2830 		if (q->guc->id != index)
2831 			continue;
2832 
2833 		guc_exec_queue_stop(guc, q);
2834 	}
2835 
2836 	mutex_unlock(&guc->submission_state.lock);
2837 
2838 	/*
2839 	 * No one can enter the backend at this point, aside from new engine
2840 	 * creation which is protected by guc->submission_state.lock.
2841 	 */
2842 
2843 }
2844 
2845 static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc,
2846 						       struct xe_exec_queue *q)
2847 {
2848 	bool pending_enable, pending_disable, pending_resume;
2849 
2850 	pending_enable = exec_queue_pending_enable(q);
2851 	pending_resume = exec_queue_pending_resume(q);
2852 
2853 	if (pending_enable && pending_resume) {
2854 		q->guc->needs_resume = true;
2855 		xe_gt_dbg(guc_to_gt(guc), "Replay RESUME - guc_id=%d",
2856 			  q->guc->id);
2857 	}
2858 
2859 	if (pending_enable && !pending_resume) {
2860 		clear_exec_queue_registered(q);
2861 		xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER - guc_id=%d",
2862 			  q->guc->id);
2863 	}
2864 
2865 	if (pending_enable) {
2866 		clear_exec_queue_enabled(q);
2867 		clear_exec_queue_pending_resume(q);
2868 		clear_exec_queue_pending_enable(q);
2869 		xe_gt_dbg(guc_to_gt(guc), "Replay ENABLE - guc_id=%d",
2870 			  q->guc->id);
2871 	}
2872 
2873 	if (exec_queue_destroyed(q) && exec_queue_registered(q)) {
2874 		clear_exec_queue_destroyed(q);
2875 		q->guc->needs_cleanup = true;
2876 		xe_gt_dbg(guc_to_gt(guc), "Replay CLEANUP - guc_id=%d",
2877 			  q->guc->id);
2878 	}
2879 
2880 	pending_disable = exec_queue_pending_disable(q);
2881 
2882 	if (pending_disable && exec_queue_suspended(q)) {
2883 		clear_exec_queue_suspended(q);
2884 		q->guc->needs_suspend = true;
2885 		xe_gt_dbg(guc_to_gt(guc), "Replay SUSPEND - guc_id=%d",
2886 			  q->guc->id);
2887 	}
2888 
2889 	if (pending_disable) {
2890 		if (!pending_enable)
2891 			set_exec_queue_enabled(q);
2892 		clear_exec_queue_pending_disable(q);
2893 		xe_gt_dbg(guc_to_gt(guc), "Replay DISABLE - guc_id=%d",
2894 			  q->guc->id);
2895 	}
2896 
2897 	/*
2898 	 * A registration time CGP update that bailed when woken by VF recovery.
2899 	 * Re-register the queue.
2900 	 */
2901 	if (q->guc->multi_queue.re_register) {
2902 		clear_exec_queue_registered(q);
2903 		q->guc->multi_queue.re_register = false;
2904 		xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER (cgp) - guc_id=%d",
2905 			  q->guc->id);
2906 	}
2907 
2908 	/*
2909 	 * If a CGP update gets dropped during migration, CGP_SYNC_DONE will not
2910 	 * be received (sync_pending still set and this queue owns it). Recover
2911 	 * it the same way and clear the stuck sync_pending.
2912 	 */
2913 	if (xe_exec_queue_is_multi_queue(q)) {
2914 		struct xe_exec_queue_group *group = q->multi_queue.group;
2915 
2916 		if (q == READ_ONCE(group->cgp_update_q) &&
2917 		    READ_ONCE(group->sync_pending)) {
2918 			if (q->guc->multi_queue.registering_cgp) {
2919 				clear_exec_queue_registered(q);
2920 				xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER (cgp sync) - guc_id=%d",
2921 					  q->guc->id);
2922 			} else if (q->guc->multi_queue.updating_cgp) {
2923 				q->guc->multi_queue.needs_cgp_sync = true;
2924 				xe_gt_dbg(guc_to_gt(guc), "Replay CGP_SYNC - guc_id=%d",
2925 					  q->guc->id);
2926 			}
2927 			q->guc->multi_queue.registering_cgp = false;
2928 			q->guc->multi_queue.updating_cgp = false;
2929 			WRITE_ONCE(group->cgp_update_q, NULL);
2930 			WRITE_ONCE(group->sync_pending, false);
2931 		}
2932 	}
2933 
2934 	/*
2935 	 * A dynamic-time CGP update that bailed when woken by VF recovery.
2936 	 * Replay the dynamic CGP update unless the queue is registered or being
2937 	 * re-registered, which re-does the CGP anyway.
2938 	 */
2939 	if (q->guc->multi_queue.re_update) {
2940 		q->guc->multi_queue.re_update = false;
2941 		if (exec_queue_registered(q)) {
2942 			q->guc->multi_queue.needs_cgp_sync = true;
2943 			xe_gt_dbg(guc_to_gt(guc), "Replay CGP_SYNC (re-update) - guc_id=%d",
2944 				  q->guc->id);
2945 		}
2946 	}
2947 
2948 	q->guc->resume_time = 0;
2949 }
2950 
2951 static void lrc_parallel_clear(struct xe_lrc *lrc)
2952 {
2953 	struct xe_device *xe = gt_to_xe(lrc->gt);
2954 	struct iosys_map map = xe_lrc_parallel_map(lrc);
2955 	int i;
2956 
2957 	for (i = 0; i < WQ_SIZE / sizeof(u32); ++i)
2958 		parallel_write(xe, map, wq[i],
2959 			       FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |
2960 			       FIELD_PREP(WQ_LEN_MASK, 0));
2961 }
2962 
2963 /*
2964  * This function is quite complex but only real way to ensure no state is lost
2965  * during VF resume flows. The function scans the queue state, make adjustments
2966  * as needed, and queues jobs / messages which replayed upon unpause.
2967  */
2968 static void guc_exec_queue_pause_prepare(struct xe_guc *guc, struct xe_exec_queue *q)
2969 {
2970 	struct xe_gpu_scheduler *sched = &q->guc->sched;
2971 
2972 	lockdep_assert_held(&guc->submission_state.lock);
2973 
2974 	/* Stop scheduling + flush any DRM scheduler operations */
2975 	xe_sched_submission_stop(sched);
2976 	cancel_delayed_work_sync(&sched->base.work_tdr);
2977 }
2978 
2979 static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q)
2980 {
2981 	struct xe_gpu_scheduler *sched = &q->guc->sched;
2982 	struct xe_sched_job *job;
2983 	int i;
2984 
2985 	lockdep_assert_held(&guc->submission_state.lock);
2986 
2987 	guc_exec_queue_revert_pending_state_change(guc, q);
2988 
2989 	if (xe_exec_queue_is_parallel(q)) {
2990 		/* Pairs with WRITE_ONCE in __xe_exec_queue_init  */
2991 		struct xe_lrc *lrc = READ_ONCE(q->lrc[0]);
2992 
2993 		/*
2994 		 * NOP existing WQ commands that may contain stale GGTT
2995 		 * addresses. These will be replayed upon unpause. The hardware
2996 		 * seems to get confused if the WQ head/tail pointers are
2997 		 * adjusted.
2998 		 */
2999 		if (lrc)
3000 			lrc_parallel_clear(lrc);
3001 	}
3002 
3003 	job = xe_sched_first_pending_job(sched);
3004 	if (job) {
3005 		job->restore_replay = true;
3006 
3007 		/*
3008 		 * Adjust software tail so jobs submitted overwrite previous
3009 		 * position in ring buffer with new GGTT addresses.
3010 		 */
3011 		for (i = 0; i < q->width; ++i)
3012 			q->lrc[i]->ring.tail = job->ptrs[i].head;
3013 	}
3014 }
3015 
3016 /**
3017  * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC.
3018  * @guc: the &xe_guc struct instance whose scheduler is to be disabled
3019  */
3020 void xe_guc_submit_pause(struct xe_guc *guc)
3021 {
3022 	struct xe_exec_queue *q;
3023 	unsigned long index;
3024 
3025 	mutex_lock(&guc->submission_state.lock);
3026 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
3027 		xe_sched_submission_stop(&q->guc->sched);
3028 	mutex_unlock(&guc->submission_state.lock);
3029 }
3030 
3031 /**
3032  * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF.
3033  * @guc: the &xe_guc struct instance whose scheduler is to be disabled
3034  */
3035 void xe_guc_submit_pause_vf(struct xe_guc *guc)
3036 {
3037 	struct xe_exec_queue *q;
3038 	unsigned long index;
3039 
3040 	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
3041 	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
3042 
3043 	mutex_lock(&guc->submission_state.lock);
3044 	/*
3045 	 * Stop all schedulers before reverting any queue: in a multi-queue
3046 	 * group a secondary's run_job() can register the primary, which must
3047 	 * not race an in-progress revert.
3048 	 */
3049 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
3050 		/* Prevent redundant attempts to stop parallel queues */
3051 		if (q->guc->id != index)
3052 			continue;
3053 
3054 		guc_exec_queue_pause_prepare(guc, q);
3055 	}
3056 
3057 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
3058 		/* Prevent redundant attempts to stop parallel queues */
3059 		if (q->guc->id != index)
3060 			continue;
3061 
3062 		guc_exec_queue_pause(guc, q);
3063 	}
3064 	mutex_unlock(&guc->submission_state.lock);
3065 }
3066 
3067 static void guc_exec_queue_start(struct xe_exec_queue *q)
3068 {
3069 	struct xe_gpu_scheduler *sched = &q->guc->sched;
3070 
3071 	if (!exec_queue_killed_or_banned_or_wedged(q)) {
3072 		struct xe_sched_job *job = xe_sched_first_pending_job(sched);
3073 		int i;
3074 
3075 		trace_xe_exec_queue_resubmit(q);
3076 		if (job) {
3077 			for (i = 0; i < q->width; ++i) {
3078 				/*
3079 				 * The GuC context is unregistered at this point
3080 				 * time, adjusting software ring tail ensures
3081 				 * jobs are rewritten in original placement,
3082 				 * adjusting LRC tail ensures the newly loaded
3083 				 * GuC / contexts only view the LRC tail
3084 				 * increasing as jobs are written out.
3085 				 */
3086 				q->lrc[i]->ring.tail = job->ptrs[i].head;
3087 				/*
3088 				 * A ULLS job's slot can have been consumed by
3089 				 * the CS without the job running, leaving the
3090 				 * saved head past it. Rewind so the re-emitted
3091 				 * job is ahead of the head, not behind it.
3092 				 */
3093 				if (xe_sched_job_is_ulls(job))
3094 					xe_lrc_set_ring_head(q->lrc[i],
3095 							     job->ptrs[i].head);
3096 				xe_lrc_set_ring_tail(q->lrc[i],
3097 						     xe_lrc_ring_head(q->lrc[i]));
3098 			}
3099 		}
3100 		xe_sched_resubmit_jobs(sched);
3101 	}
3102 
3103 	xe_sched_submission_start(sched);
3104 	xe_sched_submission_resume_tdr(sched);
3105 }
3106 
3107 int xe_guc_submit_start(struct xe_guc *guc)
3108 {
3109 	struct xe_exec_queue *q;
3110 	unsigned long index;
3111 
3112 	xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1);
3113 
3114 	mutex_lock(&guc->submission_state.lock);
3115 	atomic_dec(&guc->submission_state.stopped);
3116 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
3117 		/* Prevent redundant attempts to start parallel queues */
3118 		if (q->guc->id != index)
3119 			continue;
3120 
3121 		guc_exec_queue_start(q);
3122 	}
3123 	mutex_unlock(&guc->submission_state.lock);
3124 
3125 	wake_up_all(&guc->ct.wq);
3126 
3127 	return 0;
3128 }
3129 
3130 static void guc_exec_queue_unpause_prepare(struct xe_guc *guc,
3131 					   struct xe_exec_queue *q)
3132 {
3133 	struct xe_gpu_scheduler *sched = &q->guc->sched;
3134 	struct xe_sched_job *job = NULL;
3135 	struct drm_sched_job *s_job;
3136 	bool restore_replay = false;
3137 
3138 	drm_sched_for_each_pending_job(s_job, &sched->base, NULL) {
3139 		job = to_xe_sched_job(s_job);
3140 		restore_replay |= job->restore_replay;
3141 		if (restore_replay) {
3142 			xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d",
3143 				  q->guc->id, xe_sched_job_seqno(job));
3144 
3145 			q->ring_ops->emit_job(job);
3146 			job->restore_replay = true;
3147 		}
3148 	}
3149 
3150 	if (job)
3151 		job->last_replay = true;
3152 }
3153 
3154 /**
3155  * xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF.
3156  * @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause
3157  */
3158 void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc)
3159 {
3160 	struct xe_exec_queue *q;
3161 	unsigned long index;
3162 
3163 	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
3164 	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
3165 
3166 	mutex_lock(&guc->submission_state.lock);
3167 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
3168 		/* Prevent redundant attempts to stop parallel queues */
3169 		if (q->guc->id != index)
3170 			continue;
3171 
3172 		guc_exec_queue_unpause_prepare(guc, q);
3173 	}
3174 	mutex_unlock(&guc->submission_state.lock);
3175 }
3176 
3177 static void guc_exec_queue_replay_pending_state_change(struct xe_exec_queue *q)
3178 {
3179 	struct xe_gpu_scheduler *sched = &q->guc->sched;
3180 	struct xe_sched_msg *msg;
3181 
3182 	if (q->guc->multi_queue.needs_cgp_sync) {
3183 		msg = q->guc->static_msgs + STATIC_MSG_CGP_SYNC;
3184 
3185 		xe_sched_msg_lock(sched);
3186 		guc_exec_queue_try_add_msg_head(q, msg, CGP_SYNC_MSG);
3187 		xe_sched_msg_unlock(sched);
3188 
3189 		q->guc->multi_queue.needs_cgp_sync = false;
3190 	}
3191 
3192 	if (q->guc->needs_cleanup) {
3193 		msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;
3194 
3195 		guc_exec_queue_add_msg(q, msg, CLEANUP);
3196 		q->guc->needs_cleanup = false;
3197 	}
3198 
3199 	if (q->guc->needs_suspend) {
3200 		msg = q->guc->static_msgs + STATIC_MSG_SUSPEND;
3201 
3202 		xe_sched_msg_lock(sched);
3203 		guc_exec_queue_try_add_msg_head(q, msg, SUSPEND);
3204 		xe_sched_msg_unlock(sched);
3205 
3206 		q->guc->needs_suspend = false;
3207 	}
3208 
3209 	/*
3210 	 * The resume must be in the message queue before the suspend as it is
3211 	 * not possible for a resume to be issued if a suspend pending is, but
3212 	 * the inverse is possible.
3213 	 */
3214 	if (q->guc->needs_resume) {
3215 		msg = q->guc->static_msgs + STATIC_MSG_RESUME;
3216 
3217 		xe_sched_msg_lock(sched);
3218 		guc_exec_queue_try_add_msg_head(q, msg, RESUME);
3219 		xe_sched_msg_unlock(sched);
3220 
3221 		q->guc->needs_resume = false;
3222 	}
3223 }
3224 
3225 static void guc_exec_queue_unpause(struct xe_guc *guc, struct xe_exec_queue *q)
3226 {
3227 	struct xe_gpu_scheduler *sched = &q->guc->sched;
3228 	bool needs_tdr = exec_queue_killed_or_banned_or_wedged(q);
3229 
3230 	lockdep_assert_held(&guc->submission_state.lock);
3231 
3232 	xe_sched_resubmit_jobs(sched);
3233 	guc_exec_queue_replay_pending_state_change(q);
3234 	xe_sched_submission_start(sched);
3235 	if (needs_tdr)
3236 		xe_guc_exec_queue_trigger_cleanup(q);
3237 	xe_sched_submission_resume_tdr(sched);
3238 }
3239 
3240 /**
3241  * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC.
3242  * @guc: the &xe_guc struct instance whose scheduler is to be enabled
3243  */
3244 void xe_guc_submit_unpause(struct xe_guc *guc)
3245 {
3246 	struct xe_exec_queue *q;
3247 	unsigned long index;
3248 
3249 	mutex_lock(&guc->submission_state.lock);
3250 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
3251 		xe_sched_submission_start(&q->guc->sched);
3252 	mutex_unlock(&guc->submission_state.lock);
3253 }
3254 
3255 /**
3256  * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF.
3257  * @guc: the &xe_guc struct instance whose scheduler is to be enabled
3258  */
3259 void xe_guc_submit_unpause_vf(struct xe_guc *guc)
3260 {
3261 	struct xe_exec_queue *q;
3262 	unsigned long index;
3263 
3264 	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
3265 
3266 	mutex_lock(&guc->submission_state.lock);
3267 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
3268 		/*
3269 		 * Prevent redundant attempts to stop parallel queues, or queues
3270 		 * created after resfix done.
3271 		 */
3272 		if (q->guc->id != index ||
3273 		    !drm_sched_is_stopped(&q->guc->sched.base))
3274 			continue;
3275 
3276 		guc_exec_queue_unpause(guc, q);
3277 	}
3278 	mutex_unlock(&guc->submission_state.lock);
3279 }
3280 
3281 /**
3282  * xe_guc_submit_pause_abort - Abort all paused submission task on given GuC.
3283  * @guc: the &xe_guc struct instance whose scheduler is to be aborted
3284  */
3285 void xe_guc_submit_pause_abort(struct xe_guc *guc)
3286 {
3287 	struct xe_exec_queue *q;
3288 	unsigned long index;
3289 
3290 	mutex_lock(&guc->submission_state.lock);
3291 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
3292 		struct xe_gpu_scheduler *sched = &q->guc->sched;
3293 
3294 		/* Prevent redundant attempts to stop parallel queues */
3295 		if (q->guc->id != index)
3296 			continue;
3297 
3298 		xe_sched_submission_start(sched);
3299 		guc_exec_queue_kill(q);
3300 	}
3301 	mutex_unlock(&guc->submission_state.lock);
3302 }
3303 
3304 static struct xe_exec_queue *
3305 g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id)
3306 {
3307 	struct xe_gt *gt = guc_to_gt(guc);
3308 	struct xe_exec_queue *q;
3309 
3310 	if (unlikely(guc_id >= GUC_ID_MAX)) {
3311 		xe_gt_err(gt, "Invalid guc_id %u\n", guc_id);
3312 		return NULL;
3313 	}
3314 
3315 	q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id);
3316 	if (unlikely(!q)) {
3317 		xe_gt_err(gt, "No exec queue found for guc_id %u\n", guc_id);
3318 		return NULL;
3319 	}
3320 
3321 	xe_gt_assert(guc_to_gt(guc), guc_id >= q->guc->id);
3322 	xe_gt_assert(guc_to_gt(guc), guc_id < (q->guc->id + q->width));
3323 
3324 	return q;
3325 }
3326 
3327 static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)
3328 {
3329 	u32 action[] = {
3330 		XE_GUC_ACTION_DEREGISTER_CONTEXT,
3331 		q->guc->id,
3332 	};
3333 
3334 	xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q));
3335 	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
3336 	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
3337 	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q));
3338 
3339 	trace_xe_exec_queue_deregister(q);
3340 
3341 	if (xe_exec_queue_is_multi_queue_secondary(q))
3342 		handle_deregister_done(guc, q);
3343 	else
3344 		xe_guc_ct_send_g2h_handler(&guc->ct, action,
3345 					   ARRAY_SIZE(action));
3346 }
3347 
3348 static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q,
3349 			      u32 runnable_state)
3350 {
3351 	trace_xe_exec_queue_scheduling_done(q);
3352 
3353 	if (runnable_state == 1) {
3354 		xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q));
3355 
3356 		q->guc->resume_time = ktime_get();
3357 		clear_exec_queue_pending_resume(q);
3358 		clear_exec_queue_pending_enable(q);
3359 		smp_wmb();
3360 		wake_up_all(&guc->ct.wq);
3361 	} else {
3362 		bool was_pending;
3363 
3364 		xe_gt_assert(guc_to_gt(guc), runnable_state == 0);
3365 		xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q));
3366 
3367 		/*
3368 		 * A TDR stops the scheduler, which can trap a queued SUSPEND
3369 		 * message, and then issues a disable of its own. suspend_pending
3370 		 * is still set in that case, so only attribute this SCHED_DONE
3371 		 * to a suspend if the SUSPEND message actually ran, i.e. it
3372 		 * marked the queue suspended before disabling scheduling. The
3373 		 * trapped message completes the suspend once submission
3374 		 * restarts.
3375 		 */
3376 		xe_sched_msg_lock(&q->guc->sched);
3377 		was_pending = q->guc->suspend_pending && exec_queue_suspended(q);
3378 		if (was_pending) {
3379 			clear_exec_queue_pending_disable(q);
3380 			suspend_fence_signal(q);
3381 		}
3382 		xe_sched_msg_unlock(&q->guc->sched);
3383 
3384 		if (!was_pending) {
3385 			/*
3386 			 * Make sure to clear the pending_disable only after
3387 			 * sampling the destroyed state. We want to ensure we
3388 			 * don't trigger the unregister too early with something
3389 			 * intending to only disable scheduling. The caller
3390 			 * doing the destroy must wait for an ongoing
3391 			 * pending_disable before marking as destroyed.
3392 			 */
3393 			bool destroyed = exec_queue_destroyed(q);
3394 
3395 			clear_exec_queue_pending_disable(q);
3396 			if (destroyed)
3397 				deregister_exec_queue(guc, q);
3398 		}
3399 
3400 		/*
3401 		 * Wake after pending_disable is cleared, not before, so waiters
3402 		 * such as the TDR see the updated state on the first re-check.
3403 		 */
3404 		smp_wmb();
3405 		wake_up_all(&guc->ct.wq);
3406 	}
3407 }
3408 
3409 static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
3410 						    struct xe_exec_queue *q,
3411 						    u32 runnable_state)
3412 {
3413 	/* Take CT lock here as handle_sched_done() do send a h2g message */
3414 	mutex_lock(&guc->ct.lock);
3415 	handle_sched_done(guc, q, runnable_state);
3416 	mutex_unlock(&guc->ct.lock);
3417 }
3418 
3419 int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
3420 {
3421 	struct xe_exec_queue *q;
3422 	u32 guc_id, runnable_state;
3423 
3424 	if (unlikely(len < 2))
3425 		return -EPROTO;
3426 
3427 	guc_id = msg[0];
3428 	runnable_state = msg[1];
3429 
3430 	q = g2h_exec_queue_lookup(guc, guc_id);
3431 	if (unlikely(!q))
3432 		return -EPROTO;
3433 
3434 	if (unlikely(!exec_queue_pending_enable(q) &&
3435 		     !exec_queue_pending_disable(q))) {
3436 		xe_gt_err(guc_to_gt(guc),
3437 			  "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u",
3438 			  atomic_read(&q->guc->state), q->guc->id,
3439 			  runnable_state);
3440 		return -EPROTO;
3441 	}
3442 
3443 	handle_sched_done(guc, q, runnable_state);
3444 
3445 	return 0;
3446 }
3447 
3448 static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q)
3449 {
3450 	trace_xe_exec_queue_deregister_done(q);
3451 
3452 	clear_exec_queue_registered(q);
3453 	guc_exec_queue_destroy_async(q);
3454 }
3455 
3456 int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
3457 {
3458 	struct xe_exec_queue *q;
3459 	u32 guc_id;
3460 
3461 	if (unlikely(len < 1))
3462 		return -EPROTO;
3463 
3464 	guc_id = msg[0];
3465 
3466 	q = g2h_exec_queue_lookup(guc, guc_id);
3467 	if (unlikely(!q))
3468 		return -EPROTO;
3469 
3470 	if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) ||
3471 	    exec_queue_pending_enable(q) || exec_queue_enabled(q)) {
3472 		xe_gt_err(guc_to_gt(guc),
3473 			  "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d",
3474 			  atomic_read(&q->guc->state), q->guc->id);
3475 		return -EPROTO;
3476 	}
3477 
3478 	handle_deregister_done(guc, q);
3479 
3480 	return 0;
3481 }
3482 
3483 int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len)
3484 {
3485 	struct xe_gt *gt = guc_to_gt(guc);
3486 	struct xe_exec_queue *q;
3487 	u32 guc_id;
3488 
3489 	if (unlikely(len < 1))
3490 		return -EPROTO;
3491 
3492 	guc_id = msg[0];
3493 
3494 	q = g2h_exec_queue_lookup(guc, guc_id);
3495 	if (unlikely(!q))
3496 		return -EPROTO;
3497 
3498 	if (!exec_queue_killed(q))
3499 		xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x",
3500 			   xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id,
3501 			   atomic_read(&q->guc->state));
3502 
3503 	trace_xe_exec_queue_reset(q);
3504 
3505 	/*
3506 	 * A banned engine is a NOP at this point (came from
3507 	 * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel
3508 	 * jobs by setting timeout of the job to the minimum value kicking
3509 	 * guc_exec_queue_timedout_job.
3510 	 */
3511 	xe_guc_exec_queue_reset_trigger_cleanup(q);
3512 
3513 	return 0;
3514 }
3515 
3516 /*
3517  * xe_guc_error_capture_handler - Handler of GuC captured message
3518  * @guc: The GuC object
3519  * @msg: Point to the message
3520  * @len: The message length
3521  *
3522  * When GuC captured data is ready, GuC will send message
3523  * XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION to host, this function will be
3524  * called 1st to check status before process the data comes with the message.
3525  *
3526  * Returns: error code. 0 if success
3527  */
3528 int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len)
3529 {
3530 	u32 status;
3531 
3532 	if (unlikely(len != XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION_DATA_LEN))
3533 		return -EPROTO;
3534 
3535 	status = msg[0] & XE_GUC_STATE_CAPTURE_EVENT_STATUS_MASK;
3536 	if (status == XE_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE)
3537 		xe_gt_warn(guc_to_gt(guc), "G2H-Error capture no space");
3538 
3539 	xe_guc_capture_process(guc);
3540 
3541 	return 0;
3542 }
3543 
3544 int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
3545 					       u32 len)
3546 {
3547 	struct xe_gt *gt = guc_to_gt(guc);
3548 	struct xe_exec_queue *q;
3549 	u32 guc_id;
3550 	u32 type = XE_GUC_CAT_ERR_TYPE_INVALID;
3551 
3552 	if (unlikely(!len || len > 2))
3553 		return -EPROTO;
3554 
3555 	guc_id = msg[0];
3556 
3557 	if (len == 2)
3558 		type = msg[1];
3559 
3560 	if (guc_id == GUC_ID_UNKNOWN) {
3561 		/*
3562 		 * GuC uses GUC_ID_UNKNOWN if it can not map the CAT fault to any PF/VF
3563 		 * context. In such case only PF will be notified about that fault.
3564 		 */
3565 		xe_gt_err_ratelimited(gt, "Memory CAT error reported by GuC!\n");
3566 		return 0;
3567 	}
3568 
3569 	q = g2h_exec_queue_lookup(guc, guc_id);
3570 	if (unlikely(!q))
3571 		return -EPROTO;
3572 
3573 	/*
3574 	 * The type is HW-defined and changes based on platform, so we don't
3575 	 * decode it in the kernel and only check if it is valid.
3576 	 * See bspec 54047 and 72187 for details.
3577 	 */
3578 	if (type != XE_GUC_CAT_ERR_TYPE_INVALID)
3579 		xe_gt_info(gt,
3580 			   "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d",
3581 			   type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
3582 	else
3583 		xe_gt_info(gt,
3584 			   "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d",
3585 			   xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
3586 
3587 	trace_xe_exec_queue_memory_cat_error(q);
3588 
3589 	/* Treat the same as engine reset */
3590 	xe_guc_exec_queue_reset_trigger_cleanup(q);
3591 
3592 	return 0;
3593 }
3594 
3595 int xe_guc_uncorrectable_error_handler(struct xe_guc *guc, u32 *msg, u32 len)
3596 {
3597 	struct xe_gt *gt = guc_to_gt(guc);
3598 	struct xe_exec_queue *q;
3599 	u32 guc_id;
3600 
3601 	if (unlikely(!len || len > 1))
3602 		return -EPROTO;
3603 
3604 	guc_id = msg[0];
3605 
3606 	if (guc_id == GUC_ID_UNKNOWN) {
3607 		xe_gt_err_ratelimited(gt, "GuC: Uncorrectable local error with unknown GuC id\n");
3608 		return 0;
3609 	}
3610 
3611 	q = g2h_exec_queue_lookup(guc, guc_id);
3612 	if (unlikely(!q))
3613 		return -EPROTO;
3614 
3615 	/*
3616 	 * On CRI, the hardware cannot distinguish GT translation faults
3617 	 * from true RAS conditions. Demote to debug to avoid false
3618 	 * positive hardware error logs.
3619 	 */
3620 	if (gt_to_xe(gt)->info.platform == XE_CRESCENTISLAND)
3621 		xe_gt_dbg(gt,
3622 			  "GuC: Uncorrectable local error! guc_id=%d class=%s, logical_mask=0x%x",
3623 			  guc_id, xe_hw_engine_class_to_str(q->class), q->logical_mask);
3624 	else
3625 		xe_gt_err(gt,
3626 			  "GuC: Uncorrectable local error! guc_id=%d class=%s, logical_mask=0x%x",
3627 			  guc_id, xe_hw_engine_class_to_str(q->class), q->logical_mask);
3628 
3629 	trace_xe_guc_uncorrectable_error(q);
3630 
3631 	/* Treat the same as engine reset */
3632 	xe_guc_exec_queue_reset_trigger_cleanup(q);
3633 
3634 	return 0;
3635 }
3636 
3637 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len)
3638 {
3639 	struct xe_gt *gt = guc_to_gt(guc);
3640 	u8 guc_class, instance;
3641 	u32 reason;
3642 
3643 	if (unlikely(len != 3))
3644 		return -EPROTO;
3645 
3646 	guc_class = msg[0];
3647 	instance = msg[1];
3648 	reason = msg[2];
3649 
3650 	/* Unexpected failure of a hardware feature, log an actual error */
3651 	xe_log_err(gt, GUCSUBMIT, -EIO,
3652 		   "engine reset failed on %u:%u, reason=%#x\n",
3653 		   guc_class, instance, reason);
3654 
3655 	xe_gt_reset_async(gt);
3656 
3657 	return 0;
3658 }
3659 
3660 int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg,
3661 						u32 len)
3662 {
3663 	struct xe_gt *gt = guc_to_gt(guc);
3664 	struct xe_device *xe = guc_to_xe(guc);
3665 	struct xe_exec_queue *q;
3666 	u32 guc_id = msg[2];
3667 
3668 	if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) {
3669 		drm_err(&xe->drm, "Invalid length %u", len);
3670 		return -EPROTO;
3671 	}
3672 
3673 	q = g2h_exec_queue_lookup(guc, guc_id);
3674 	if (unlikely(!q))
3675 		return -EPROTO;
3676 
3677 	xe_gt_dbg(gt,
3678 		  "CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x",
3679 		  msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]);
3680 
3681 	trace_xe_exec_queue_cgp_context_error(q);
3682 
3683 	/* Treat the same as engine reset */
3684 	xe_guc_exec_queue_reset_trigger_cleanup(q);
3685 
3686 	return 0;
3687 }
3688 
3689 /**
3690  * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler
3691  * @guc: guc
3692  * @msg: message indicating CGP sync done
3693  * @len: length of message
3694  *
3695  * Set multi queue group's sync_pending flag to false and wakeup anyone waiting
3696  * for CGP synchronization to complete.
3697  *
3698  * Return: 0 on success, -EPROTO for malformed messages.
3699  */
3700 int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
3701 {
3702 	struct xe_device *xe = guc_to_xe(guc);
3703 	struct xe_exec_queue_group *group;
3704 	struct xe_exec_queue *q, *upd_q;
3705 	u32 guc_id = msg[0];
3706 
3707 	if (unlikely(len < 1)) {
3708 		drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len);
3709 		return -EPROTO;
3710 	}
3711 
3712 	q = g2h_exec_queue_lookup(guc, guc_id);
3713 	if (unlikely(!q))
3714 		return -EPROTO;
3715 
3716 	if (!xe_exec_queue_is_multi_queue_primary(q)) {
3717 		drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response");
3718 		return -EPROTO;
3719 	}
3720 
3721 	/*
3722 	 * The outstanding CGP update is now confirmed; clear the owning queue's
3723 	 * tracking so a later migration does not needlessly replay it.
3724 	 */
3725 	group = q->multi_queue.group;
3726 	upd_q = READ_ONCE(group->cgp_update_q);
3727 	if (upd_q) {
3728 		upd_q->guc->multi_queue.registering_cgp = false;
3729 		upd_q->guc->multi_queue.updating_cgp = false;
3730 		WRITE_ONCE(group->cgp_update_q, NULL);
3731 	}
3732 
3733 	/* Wakeup the serialized cgp update wait */
3734 	WRITE_ONCE(group->sync_pending, false);
3735 	xe_guc_ct_wake_waiters(&guc->ct);
3736 
3737 	return 0;
3738 }
3739 
3740 static void
3741 guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q,
3742 				   struct xe_guc_submit_exec_queue_snapshot *snapshot)
3743 {
3744 	struct xe_guc *guc = exec_queue_to_guc(q);
3745 	struct xe_device *xe = guc_to_xe(guc);
3746 	struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
3747 	int i;
3748 
3749 	snapshot->guc.wqi_head = q->guc->wqi_head;
3750 	snapshot->guc.wqi_tail = q->guc->wqi_tail;
3751 	snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head);
3752 	snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail);
3753 	snapshot->parallel.wq_desc.status = parallel_read(xe, map,
3754 							  wq_desc.wq_status);
3755 
3756 	if (snapshot->parallel.wq_desc.head !=
3757 	    snapshot->parallel.wq_desc.tail) {
3758 		for (i = snapshot->parallel.wq_desc.head;
3759 		     i != snapshot->parallel.wq_desc.tail;
3760 		     i = (i + sizeof(u32)) % WQ_SIZE)
3761 			snapshot->parallel.wq[i / sizeof(u32)] =
3762 				parallel_read(xe, map, wq[i / sizeof(u32)]);
3763 	}
3764 }
3765 
3766 static void
3767 guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,
3768 				 struct drm_printer *p)
3769 {
3770 	int i;
3771 
3772 	drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n",
3773 		   snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head);
3774 	drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n",
3775 		   snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail);
3776 	drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status);
3777 
3778 	if (snapshot->parallel.wq_desc.head !=
3779 	    snapshot->parallel.wq_desc.tail) {
3780 		for (i = snapshot->parallel.wq_desc.head;
3781 		     i != snapshot->parallel.wq_desc.tail;
3782 		     i = (i + sizeof(u32)) % WQ_SIZE)
3783 			drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32),
3784 				   snapshot->parallel.wq[i / sizeof(u32)]);
3785 	}
3786 }
3787 
3788 /**
3789  * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine.
3790  * @q: faulty exec queue
3791  *
3792  * This can be printed out in a later stage like during dev_coredump
3793  * analysis.
3794  *
3795  * Returns: a GuC Submit Engine snapshot object that must be freed by the
3796  * caller, using `xe_guc_exec_queue_snapshot_free`.
3797  */
3798 struct xe_guc_submit_exec_queue_snapshot *
3799 xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q)
3800 {
3801 	struct xe_gpu_scheduler *sched = &q->guc->sched;
3802 	struct xe_guc_submit_exec_queue_snapshot *snapshot;
3803 	int i;
3804 
3805 	snapshot = kzalloc_obj(*snapshot, GFP_ATOMIC);
3806 
3807 	if (!snapshot)
3808 		return NULL;
3809 
3810 	snapshot->guc.id = q->guc->id;
3811 	memcpy(&snapshot->name, &q->name, sizeof(snapshot->name));
3812 	snapshot->class = q->class;
3813 	snapshot->logical_mask = q->logical_mask;
3814 	snapshot->width = q->width;
3815 	snapshot->refcount = kref_read(&q->refcount);
3816 	snapshot->jobcount = atomic_read(&q->job_cnt);
3817 	snapshot->sched_timeout = sched->base.timeout;
3818 	snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us;
3819 	snapshot->sched_props.preempt_timeout_us =
3820 		q->sched_props.preempt_timeout_us;
3821 
3822 	snapshot->lrc = kmalloc_objs(struct xe_lrc_snapshot *, q->width,
3823 				     GFP_ATOMIC);
3824 
3825 	if (snapshot->lrc) {
3826 		for (i = 0; i < q->width; ++i) {
3827 			struct xe_lrc *lrc = q->lrc[i];
3828 
3829 			snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc);
3830 		}
3831 	}
3832 
3833 	snapshot->schedule_state = atomic_read(&q->guc->state);
3834 	snapshot->exec_queue_flags = q->flags;
3835 
3836 	snapshot->parallel_execution = xe_exec_queue_is_parallel(q);
3837 	if (snapshot->parallel_execution)
3838 		guc_exec_queue_wq_snapshot_capture(q, snapshot);
3839 
3840 	if (xe_exec_queue_is_multi_queue(q)) {
3841 		snapshot->multi_queue.valid = true;
3842 		snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id;
3843 		snapshot->multi_queue.pos = q->multi_queue.pos;
3844 	}
3845 
3846 	return snapshot;
3847 }
3848 
3849 /**
3850  * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine.
3851  * @snapshot: Previously captured snapshot of job.
3852  *
3853  * This captures some data that requires taking some locks, so it cannot be done in signaling path.
3854  */
3855 void
3856 xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot)
3857 {
3858 	int i;
3859 
3860 	if (!snapshot || !snapshot->lrc)
3861 		return;
3862 
3863 	for (i = 0; i < snapshot->width; ++i)
3864 		xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]);
3865 }
3866 
3867 /**
3868  * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot.
3869  * @snapshot: GuC Submit Engine snapshot object.
3870  * @p: drm_printer where it will be printed out.
3871  *
3872  * This function prints out a given GuC Submit Engine snapshot object.
3873  */
3874 void
3875 xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,
3876 				 struct drm_printer *p)
3877 {
3878 	int i;
3879 
3880 	if (!snapshot)
3881 		return;
3882 
3883 	drm_printf(p, "GuC ID: %d\n", snapshot->guc.id);
3884 	drm_printf(p, "\tName: %s\n", snapshot->name);
3885 	drm_printf(p, "\tClass: %d\n", snapshot->class);
3886 	drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask);
3887 	drm_printf(p, "\tWidth: %d\n", snapshot->width);
3888 	drm_printf(p, "\tRef: %d\n", snapshot->refcount);
3889 	drm_printf(p, "\tJob count: %u\n", snapshot->jobcount);
3890 	drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout);
3891 	drm_printf(p, "\tTimeslice: %u (us)\n",
3892 		   snapshot->sched_props.timeslice_us);
3893 	drm_printf(p, "\tPreempt timeout: %u (us)\n",
3894 		   snapshot->sched_props.preempt_timeout_us);
3895 
3896 	for (i = 0; snapshot->lrc && i < snapshot->width; ++i)
3897 		xe_lrc_snapshot_print(snapshot->lrc[i], p);
3898 
3899 	drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state);
3900 	drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags);
3901 
3902 	if (snapshot->parallel_execution)
3903 		guc_exec_queue_wq_snapshot_print(snapshot, p);
3904 
3905 	if (snapshot->multi_queue.valid) {
3906 		drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary);
3907 		drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos);
3908 	}
3909 }
3910 
3911 /**
3912  * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given
3913  * snapshot.
3914  * @snapshot: GuC Submit Engine snapshot object.
3915  *
3916  * This function free all the memory that needed to be allocated at capture
3917  * time.
3918  */
3919 void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot)
3920 {
3921 	int i;
3922 
3923 	if (!snapshot)
3924 		return;
3925 
3926 	if (snapshot->lrc) {
3927 		for (i = 0; i < snapshot->width; i++)
3928 			xe_lrc_snapshot_free(snapshot->lrc[i]);
3929 		kfree(snapshot->lrc);
3930 	}
3931 	kfree(snapshot);
3932 }
3933 
3934 static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p)
3935 {
3936 	struct xe_guc_submit_exec_queue_snapshot *snapshot;
3937 
3938 	snapshot = xe_guc_exec_queue_snapshot_capture(q);
3939 	xe_guc_exec_queue_snapshot_print(snapshot, p);
3940 	xe_guc_exec_queue_snapshot_free(snapshot);
3941 }
3942 
3943 /**
3944  * xe_guc_register_vf_exec_queue - Register exec queue for a given context type.
3945  * @q: Execution queue
3946  * @ctx_type: Type of the context
3947  *
3948  * This function registers the execution queue with the guc. Special context
3949  * types like GUC_CONTEXT_COMPRESSION_SAVE and GUC_CONTEXT_COMPRESSION_RESTORE
3950  * are only applicable for IGPU and in the VF.
3951  * Submits the execution queue to GUC after registering it.
3952  *
3953  * Returns - None.
3954  */
3955 void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type)
3956 {
3957 	struct xe_guc *guc = exec_queue_to_guc(q);
3958 	struct xe_device *xe = guc_to_xe(guc);
3959 	struct xe_gt *gt = guc_to_gt(guc);
3960 
3961 	xe_gt_assert(gt, IS_SRIOV_VF(xe));
3962 	xe_gt_assert(gt, !IS_DGFX(xe));
3963 	xe_gt_assert(gt, ctx_type == GUC_CONTEXT_COMPRESSION_SAVE ||
3964 		     ctx_type == GUC_CONTEXT_COMPRESSION_RESTORE);
3965 	xe_gt_assert(gt, GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 23, 0));
3966 
3967 	register_exec_queue(q, ctx_type);
3968 	enable_scheduling(q);
3969 }
3970 
3971 /**
3972  * xe_guc_submit_print - GuC Submit Print.
3973  * @guc: GuC.
3974  * @p: drm_printer where it will be printed out.
3975  *
3976  * This function capture and prints snapshots of **all** GuC Engines.
3977  */
3978 void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p)
3979 {
3980 	struct xe_exec_queue *q;
3981 	unsigned long index;
3982 
3983 	if (!xe_device_uc_enabled(guc_to_xe(guc)))
3984 		return;
3985 
3986 	mutex_lock(&guc->submission_state.lock);
3987 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
3988 		guc_exec_queue_print(q, p);
3989 	mutex_unlock(&guc->submission_state.lock);
3990 }
3991 
3992 /**
3993  * xe_guc_has_registered_mlrc_queues - check whether there are any MLRC queues
3994  * registered with the GuC
3995  * @guc: GuC.
3996  *
3997  * Return: true if any MLRC queue is registered with the GuC, false otherwise.
3998  */
3999 bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc)
4000 {
4001 	struct xe_exec_queue *q;
4002 	unsigned long index;
4003 
4004 	guard(mutex)(&guc->submission_state.lock);
4005 
4006 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
4007 		if (q->width > 1)
4008 			return true;
4009 
4010 	return false;
4011 }
4012 
4013 /**
4014  * xe_guc_submit_active_multi_queue_lrca() - Resolve the LRCA of the active
4015  * queue in the multi-queue group currently running on an engine.
4016  * @guc: the &xe_guc managing the exec queues
4017  * @hwe: the &xe_hw_engine whose active queue is being resolved
4018  * @cur_lrca: value read from RING_CURRENT_LRCA, identifies the running group
4019  * @active_id: current Active Queue ID read from CSMQDEBUG (position in group)
4020  *
4021  * The running group is identified by matching @cur_lrca against the group's
4022  * primary LRCA; @active_id then selects the active queue within that group.
4023  *
4024  * Return: the LRCA of the active queue, or 0 if no matching queue is found.
4025  */
4026 u32 xe_guc_submit_active_multi_queue_lrca(struct xe_guc *guc,
4027 					  struct xe_hw_engine *hwe,
4028 					  u32 cur_lrca, u32 active_id)
4029 {
4030 	struct xe_exec_queue *q;
4031 	unsigned long index;
4032 	u32 lrca = 0;
4033 
4034 	/*
4035 	 * submission_state.lock also protects exec_queue teardown: an exec
4036 	 * queue is removed from exec_queue_lookup before its group/primary
4037 	 * are freed, so any q found in the xarray below has a live group
4038 	 * and primary for as long as we hold the lock.
4039 	 */
4040 	guard(mutex)(&guc->submission_state.lock);
4041 
4042 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
4043 		struct xe_exec_queue_group *group = q->multi_queue.group;
4044 		struct xe_lrc *active_lrc;
4045 		struct xe_lrc *primary_lrc;
4046 
4047 		if (!q->multi_queue.valid || !group || !group->primary)
4048 			continue;
4049 		/*
4050 		 * Multi-queue exec queues are bound to a hw engine class;
4051 		 * GuC dynamically schedules them onto one of the class's
4052 		 * physical instances, so there is no fixed queue-to-instance
4053 		 * mapping to filter on here.
4054 		 */
4055 		if (q->class != hwe->class)
4056 			continue;
4057 		if (q->multi_queue.pos != active_id)
4058 			continue;
4059 		/*
4060 		 * LRCAs are page-aligned (4K) addresses in GGTT; the low
4061 		 * bits reported by RING_CURRENT_LRCA are not meaningful, so
4062 		 * only compare bits [31:12].
4063 		 */
4064 		primary_lrc = xe_exec_queue_get_lrc(group->primary, 0);
4065 		if (!primary_lrc)
4066 			continue;
4067 
4068 		if ((xe_lrc_ggtt_addr(primary_lrc) ^ cur_lrca) & GENMASK(31, 12)) {
4069 			xe_lrc_put(primary_lrc);
4070 			continue;
4071 		}
4072 
4073 		active_lrc = xe_exec_queue_get_lrc(q, 0);
4074 		xe_lrc_put(primary_lrc);
4075 		if (!active_lrc)
4076 			continue;
4077 
4078 		lrca = xe_lrc_ggtt_addr(active_lrc);
4079 		xe_lrc_put(active_lrc);
4080 		break;
4081 	}
4082 
4083 	return lrca;
4084 }
4085 
4086 /**
4087  * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all
4088  * exec queues registered to given GuC.
4089  * @guc: the &xe_guc struct instance
4090  * @scratch: scratch buffer to be used as temporary storage
4091  *
4092  * Returns: zero on success, negative error code on failure.
4093  */
4094 int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch)
4095 {
4096 	struct xe_exec_queue *q;
4097 	unsigned long index;
4098 	int err = 0;
4099 
4100 	mutex_lock(&guc->submission_state.lock);
4101 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
4102 		/* Prevent redundant attempts to stop parallel queues */
4103 		if (q->guc->id != index)
4104 			continue;
4105 
4106 		err = xe_exec_queue_contexts_hwsp_rebase(q, scratch);
4107 		if (err)
4108 			break;
4109 	}
4110 	mutex_unlock(&guc->submission_state.lock);
4111 
4112 	return err;
4113 }
4114