xref: /linux/drivers/gpu/drm/xe/xe_guc_submit.c (revision 705c09bb3cdffb141986598ad4ff9c9b0a66c3bd)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_guc_submit.h"
7 
8 #include <linux/bitfield.h>
9 #include <linux/bitmap.h>
10 #include <linux/circ_buf.h>
11 #include <linux/delay.h>
12 #include <linux/dma-fence-array.h>
13 
14 #include <drm/drm_managed.h>
15 
16 #include "abi/guc_actions_abi.h"
17 #include "abi/guc_klvs_abi.h"
18 #include "regs/xe_lrc_layout.h"
19 #include "xe_assert.h"
20 #include "xe_devcoredump.h"
21 #include "xe_device.h"
22 #include "xe_exec_queue.h"
23 #include "xe_force_wake.h"
24 #include "xe_gpu_scheduler.h"
25 #include "xe_gt.h"
26 #include "xe_gt_printk.h"
27 #include "xe_guc.h"
28 #include "xe_guc_ct.h"
29 #include "xe_guc_exec_queue_types.h"
30 #include "xe_guc_submit_types.h"
31 #include "xe_hw_engine.h"
32 #include "xe_hw_fence.h"
33 #include "xe_lrc.h"
34 #include "xe_macros.h"
35 #include "xe_map.h"
36 #include "xe_mocs.h"
37 #include "xe_ring_ops_types.h"
38 #include "xe_sched_job.h"
39 #include "xe_trace.h"
40 #include "xe_vm.h"
41 
42 static struct xe_guc *
43 exec_queue_to_guc(struct xe_exec_queue *q)
44 {
45 	return &q->gt->uc.guc;
46 }
47 
48 /*
49  * Helpers for engine state, using an atomic as some of the bits can transition
50  * as the same time (e.g. a suspend can be happning at the same time as schedule
51  * engine done being processed).
52  */
53 #define EXEC_QUEUE_STATE_REGISTERED		(1 << 0)
54 #define ENGINE_STATE_ENABLED		(1 << 1)
55 #define EXEC_QUEUE_STATE_PENDING_ENABLE	(1 << 2)
56 #define EXEC_QUEUE_STATE_PENDING_DISABLE	(1 << 3)
57 #define EXEC_QUEUE_STATE_DESTROYED		(1 << 4)
58 #define ENGINE_STATE_SUSPENDED		(1 << 5)
59 #define EXEC_QUEUE_STATE_RESET		(1 << 6)
60 #define ENGINE_STATE_KILLED		(1 << 7)
61 
62 static bool exec_queue_registered(struct xe_exec_queue *q)
63 {
64 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED;
65 }
66 
67 static void set_exec_queue_registered(struct xe_exec_queue *q)
68 {
69 	atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);
70 }
71 
72 static void clear_exec_queue_registered(struct xe_exec_queue *q)
73 {
74 	atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);
75 }
76 
77 static bool exec_queue_enabled(struct xe_exec_queue *q)
78 {
79 	return atomic_read(&q->guc->state) & ENGINE_STATE_ENABLED;
80 }
81 
82 static void set_exec_queue_enabled(struct xe_exec_queue *q)
83 {
84 	atomic_or(ENGINE_STATE_ENABLED, &q->guc->state);
85 }
86 
87 static void clear_exec_queue_enabled(struct xe_exec_queue *q)
88 {
89 	atomic_and(~ENGINE_STATE_ENABLED, &q->guc->state);
90 }
91 
92 static bool exec_queue_pending_enable(struct xe_exec_queue *q)
93 {
94 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE;
95 }
96 
97 static void set_exec_queue_pending_enable(struct xe_exec_queue *q)
98 {
99 	atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);
100 }
101 
102 static void clear_exec_queue_pending_enable(struct xe_exec_queue *q)
103 {
104 	atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);
105 }
106 
107 static bool exec_queue_pending_disable(struct xe_exec_queue *q)
108 {
109 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE;
110 }
111 
112 static void set_exec_queue_pending_disable(struct xe_exec_queue *q)
113 {
114 	atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);
115 }
116 
117 static void clear_exec_queue_pending_disable(struct xe_exec_queue *q)
118 {
119 	atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);
120 }
121 
122 static bool exec_queue_destroyed(struct xe_exec_queue *q)
123 {
124 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED;
125 }
126 
127 static void set_exec_queue_destroyed(struct xe_exec_queue *q)
128 {
129 	atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state);
130 }
131 
132 static bool exec_queue_banned(struct xe_exec_queue *q)
133 {
134 	return (q->flags & EXEC_QUEUE_FLAG_BANNED);
135 }
136 
137 static void set_exec_queue_banned(struct xe_exec_queue *q)
138 {
139 	q->flags |= EXEC_QUEUE_FLAG_BANNED;
140 }
141 
142 static bool exec_queue_suspended(struct xe_exec_queue *q)
143 {
144 	return atomic_read(&q->guc->state) & ENGINE_STATE_SUSPENDED;
145 }
146 
147 static void set_exec_queue_suspended(struct xe_exec_queue *q)
148 {
149 	atomic_or(ENGINE_STATE_SUSPENDED, &q->guc->state);
150 }
151 
152 static void clear_exec_queue_suspended(struct xe_exec_queue *q)
153 {
154 	atomic_and(~ENGINE_STATE_SUSPENDED, &q->guc->state);
155 }
156 
157 static bool exec_queue_reset(struct xe_exec_queue *q)
158 {
159 	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET;
160 }
161 
162 static void set_exec_queue_reset(struct xe_exec_queue *q)
163 {
164 	atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state);
165 }
166 
167 static bool exec_queue_killed(struct xe_exec_queue *q)
168 {
169 	return atomic_read(&q->guc->state) & ENGINE_STATE_KILLED;
170 }
171 
172 static void set_exec_queue_killed(struct xe_exec_queue *q)
173 {
174 	atomic_or(ENGINE_STATE_KILLED, &q->guc->state);
175 }
176 
177 static bool exec_queue_killed_or_banned(struct xe_exec_queue *q)
178 {
179 	return exec_queue_killed(q) || exec_queue_banned(q);
180 }
181 
182 #ifdef CONFIG_PROVE_LOCKING
183 static int alloc_submit_wq(struct xe_guc *guc)
184 {
185 	int i;
186 
187 	for (i = 0; i < NUM_SUBMIT_WQ; ++i) {
188 		guc->submission_state.submit_wq_pool[i] =
189 			alloc_ordered_workqueue("submit_wq", 0);
190 		if (!guc->submission_state.submit_wq_pool[i])
191 			goto err_free;
192 	}
193 
194 	return 0;
195 
196 err_free:
197 	while (i)
198 		destroy_workqueue(guc->submission_state.submit_wq_pool[--i]);
199 
200 	return -ENOMEM;
201 }
202 
203 static void free_submit_wq(struct xe_guc *guc)
204 {
205 	int i;
206 
207 	for (i = 0; i < NUM_SUBMIT_WQ; ++i)
208 		destroy_workqueue(guc->submission_state.submit_wq_pool[i]);
209 }
210 
211 static struct workqueue_struct *get_submit_wq(struct xe_guc *guc)
212 {
213 	int idx = guc->submission_state.submit_wq_idx++ % NUM_SUBMIT_WQ;
214 
215 	return guc->submission_state.submit_wq_pool[idx];
216 }
217 #else
218 static int alloc_submit_wq(struct xe_guc *guc)
219 {
220 	return 0;
221 }
222 
223 static void free_submit_wq(struct xe_guc *guc)
224 {
225 
226 }
227 
228 static struct workqueue_struct *get_submit_wq(struct xe_guc *guc)
229 {
230 	return NULL;
231 }
232 #endif
233 
234 static void guc_submit_fini(struct drm_device *drm, void *arg)
235 {
236 	struct xe_guc *guc = arg;
237 
238 	xa_destroy(&guc->submission_state.exec_queue_lookup);
239 	ida_destroy(&guc->submission_state.guc_ids);
240 	bitmap_free(guc->submission_state.guc_ids_bitmap);
241 	free_submit_wq(guc);
242 	mutex_destroy(&guc->submission_state.lock);
243 }
244 
245 #define GUC_ID_MAX		65535
246 #define GUC_ID_NUMBER_MLRC	4096
247 #define GUC_ID_NUMBER_SLRC	(GUC_ID_MAX - GUC_ID_NUMBER_MLRC)
248 #define GUC_ID_START_MLRC	GUC_ID_NUMBER_SLRC
249 
250 static const struct xe_exec_queue_ops guc_exec_queue_ops;
251 
252 static void primelockdep(struct xe_guc *guc)
253 {
254 	if (!IS_ENABLED(CONFIG_LOCKDEP))
255 		return;
256 
257 	fs_reclaim_acquire(GFP_KERNEL);
258 
259 	mutex_lock(&guc->submission_state.lock);
260 	might_lock(&guc->submission_state.suspend.lock);
261 	mutex_unlock(&guc->submission_state.lock);
262 
263 	fs_reclaim_release(GFP_KERNEL);
264 }
265 
266 int xe_guc_submit_init(struct xe_guc *guc)
267 {
268 	struct xe_device *xe = guc_to_xe(guc);
269 	struct xe_gt *gt = guc_to_gt(guc);
270 	int err;
271 
272 	guc->submission_state.guc_ids_bitmap =
273 		bitmap_zalloc(GUC_ID_NUMBER_MLRC, GFP_KERNEL);
274 	if (!guc->submission_state.guc_ids_bitmap)
275 		return -ENOMEM;
276 
277 	err = alloc_submit_wq(guc);
278 	if (err) {
279 		bitmap_free(guc->submission_state.guc_ids_bitmap);
280 		return err;
281 	}
282 
283 	gt->exec_queue_ops = &guc_exec_queue_ops;
284 
285 	mutex_init(&guc->submission_state.lock);
286 	xa_init(&guc->submission_state.exec_queue_lookup);
287 	ida_init(&guc->submission_state.guc_ids);
288 
289 	spin_lock_init(&guc->submission_state.suspend.lock);
290 	guc->submission_state.suspend.context = dma_fence_context_alloc(1);
291 
292 	primelockdep(guc);
293 
294 	err = drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc);
295 	if (err)
296 		return err;
297 
298 	return 0;
299 }
300 
301 static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count)
302 {
303 	int i;
304 
305 	lockdep_assert_held(&guc->submission_state.lock);
306 
307 	for (i = 0; i < xa_count; ++i)
308 		xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i);
309 
310 	if (xe_exec_queue_is_parallel(q))
311 		bitmap_release_region(guc->submission_state.guc_ids_bitmap,
312 				      q->guc->id - GUC_ID_START_MLRC,
313 				      order_base_2(q->width));
314 	else
315 		ida_free(&guc->submission_state.guc_ids, q->guc->id);
316 }
317 
318 static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
319 {
320 	int ret;
321 	void *ptr;
322 	int i;
323 
324 	/*
325 	 * Must use GFP_NOWAIT as this lock is in the dma fence signalling path,
326 	 * worse case user gets -ENOMEM on engine create and has to try again.
327 	 *
328 	 * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent
329 	 * failure.
330 	 */
331 	lockdep_assert_held(&guc->submission_state.lock);
332 
333 	if (xe_exec_queue_is_parallel(q)) {
334 		void *bitmap = guc->submission_state.guc_ids_bitmap;
335 
336 		ret = bitmap_find_free_region(bitmap, GUC_ID_NUMBER_MLRC,
337 					      order_base_2(q->width));
338 	} else {
339 		ret = ida_alloc_max(&guc->submission_state.guc_ids,
340 				    GUC_ID_NUMBER_SLRC - 1, GFP_NOWAIT);
341 	}
342 	if (ret < 0)
343 		return ret;
344 
345 	q->guc->id = ret;
346 	if (xe_exec_queue_is_parallel(q))
347 		q->guc->id += GUC_ID_START_MLRC;
348 
349 	for (i = 0; i < q->width; ++i) {
350 		ptr = xa_store(&guc->submission_state.exec_queue_lookup,
351 			       q->guc->id + i, q, GFP_NOWAIT);
352 		if (IS_ERR(ptr)) {
353 			ret = PTR_ERR(ptr);
354 			goto err_release;
355 		}
356 	}
357 
358 	return 0;
359 
360 err_release:
361 	__release_guc_id(guc, q, i);
362 
363 	return ret;
364 }
365 
366 static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
367 {
368 	mutex_lock(&guc->submission_state.lock);
369 	__release_guc_id(guc, q, q->width);
370 	mutex_unlock(&guc->submission_state.lock);
371 }
372 
373 struct exec_queue_policy {
374 	u32 count;
375 	struct guc_update_exec_queue_policy h2g;
376 };
377 
378 static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy)
379 {
380 	size_t bytes = sizeof(policy->h2g.header) +
381 		       (sizeof(policy->h2g.klv[0]) * policy->count);
382 
383 	return bytes / sizeof(u32);
384 }
385 
386 static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy,
387 					      u16 guc_id)
388 {
389 	policy->h2g.header.action =
390 		XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;
391 	policy->h2g.header.guc_id = guc_id;
392 	policy->count = 0;
393 }
394 
395 #define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \
396 static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \
397 					   u32 data) \
398 { \
399 	XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \
400 \
401 	policy->h2g.klv[policy->count].kl = \
402 		FIELD_PREP(GUC_KLV_0_KEY, \
403 			   GUC_CONTEXT_POLICIES_KLV_ID_##id) | \
404 		FIELD_PREP(GUC_KLV_0_LEN, 1); \
405 	policy->h2g.klv[policy->count].value = data; \
406 	policy->count++; \
407 }
408 
409 MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)
410 MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)
411 MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY)
412 #undef MAKE_EXEC_QUEUE_POLICY_ADD
413 
414 static const int xe_exec_queue_prio_to_guc[] = {
415 	[XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL,
416 	[XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL,
417 	[XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH,
418 	[XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH,
419 };
420 
421 static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q)
422 {
423 	struct exec_queue_policy policy;
424 	struct xe_device *xe = guc_to_xe(guc);
425 	enum xe_exec_queue_priority prio = q->sched_props.priority;
426 	u32 timeslice_us = q->sched_props.timeslice_us;
427 	u32 preempt_timeout_us = q->sched_props.preempt_timeout_us;
428 
429 	xe_assert(xe, exec_queue_registered(q));
430 
431 	__guc_exec_queue_policy_start_klv(&policy, q->guc->id);
432 	__guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]);
433 	__guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us);
434 	__guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us);
435 
436 	xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,
437 		       __guc_exec_queue_policy_action_size(&policy), 0, 0);
438 }
439 
440 static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q)
441 {
442 	struct exec_queue_policy policy;
443 
444 	__guc_exec_queue_policy_start_klv(&policy, q->guc->id);
445 	__guc_exec_queue_policy_add_preemption_timeout(&policy, 1);
446 
447 	xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,
448 		       __guc_exec_queue_policy_action_size(&policy), 0, 0);
449 }
450 
451 #define parallel_read(xe_, map_, field_) \
452 	xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
453 			field_)
454 #define parallel_write(xe_, map_, field_, val_) \
455 	xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
456 			field_, val_)
457 
458 static void __register_mlrc_engine(struct xe_guc *guc,
459 				   struct xe_exec_queue *q,
460 				   struct guc_ctxt_registration_info *info)
461 {
462 #define MAX_MLRC_REG_SIZE      (13 + XE_HW_ENGINE_MAX_INSTANCE * 2)
463 	struct xe_device *xe = guc_to_xe(guc);
464 	u32 action[MAX_MLRC_REG_SIZE];
465 	int len = 0;
466 	int i;
467 
468 	xe_assert(xe, xe_exec_queue_is_parallel(q));
469 
470 	action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
471 	action[len++] = info->flags;
472 	action[len++] = info->context_idx;
473 	action[len++] = info->engine_class;
474 	action[len++] = info->engine_submit_mask;
475 	action[len++] = info->wq_desc_lo;
476 	action[len++] = info->wq_desc_hi;
477 	action[len++] = info->wq_base_lo;
478 	action[len++] = info->wq_base_hi;
479 	action[len++] = info->wq_size;
480 	action[len++] = q->width;
481 	action[len++] = info->hwlrca_lo;
482 	action[len++] = info->hwlrca_hi;
483 
484 	for (i = 1; i < q->width; ++i) {
485 		struct xe_lrc *lrc = q->lrc + i;
486 
487 		action[len++] = lower_32_bits(xe_lrc_descriptor(lrc));
488 		action[len++] = upper_32_bits(xe_lrc_descriptor(lrc));
489 	}
490 
491 	xe_assert(xe, len <= MAX_MLRC_REG_SIZE);
492 #undef MAX_MLRC_REG_SIZE
493 
494 	xe_guc_ct_send(&guc->ct, action, len, 0, 0);
495 }
496 
497 static void __register_engine(struct xe_guc *guc,
498 			      struct guc_ctxt_registration_info *info)
499 {
500 	u32 action[] = {
501 		XE_GUC_ACTION_REGISTER_CONTEXT,
502 		info->flags,
503 		info->context_idx,
504 		info->engine_class,
505 		info->engine_submit_mask,
506 		info->wq_desc_lo,
507 		info->wq_desc_hi,
508 		info->wq_base_lo,
509 		info->wq_base_hi,
510 		info->wq_size,
511 		info->hwlrca_lo,
512 		info->hwlrca_hi,
513 	};
514 
515 	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
516 }
517 
518 static void register_engine(struct xe_exec_queue *q)
519 {
520 	struct xe_guc *guc = exec_queue_to_guc(q);
521 	struct xe_device *xe = guc_to_xe(guc);
522 	struct xe_lrc *lrc = q->lrc;
523 	struct guc_ctxt_registration_info info;
524 
525 	xe_assert(xe, !exec_queue_registered(q));
526 
527 	memset(&info, 0, sizeof(info));
528 	info.context_idx = q->guc->id;
529 	info.engine_class = xe_engine_class_to_guc_class(q->class);
530 	info.engine_submit_mask = q->logical_mask;
531 	info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc));
532 	info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc));
533 	info.flags = CONTEXT_REGISTRATION_FLAG_KMD;
534 
535 	if (xe_exec_queue_is_parallel(q)) {
536 		u32 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);
537 		struct iosys_map map = xe_lrc_parallel_map(lrc);
538 
539 		info.wq_desc_lo = lower_32_bits(ggtt_addr +
540 			offsetof(struct guc_submit_parallel_scratch, wq_desc));
541 		info.wq_desc_hi = upper_32_bits(ggtt_addr +
542 			offsetof(struct guc_submit_parallel_scratch, wq_desc));
543 		info.wq_base_lo = lower_32_bits(ggtt_addr +
544 			offsetof(struct guc_submit_parallel_scratch, wq[0]));
545 		info.wq_base_hi = upper_32_bits(ggtt_addr +
546 			offsetof(struct guc_submit_parallel_scratch, wq[0]));
547 		info.wq_size = WQ_SIZE;
548 
549 		q->guc->wqi_head = 0;
550 		q->guc->wqi_tail = 0;
551 		xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE);
552 		parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE);
553 	}
554 
555 	/*
556 	 * We must keep a reference for LR engines if engine is registered with
557 	 * the GuC as jobs signal immediately and can't destroy an engine if the
558 	 * GuC has a reference to it.
559 	 */
560 	if (xe_exec_queue_is_lr(q))
561 		xe_exec_queue_get(q);
562 
563 	set_exec_queue_registered(q);
564 	trace_xe_exec_queue_register(q);
565 	if (xe_exec_queue_is_parallel(q))
566 		__register_mlrc_engine(guc, q, &info);
567 	else
568 		__register_engine(guc, &info);
569 	init_policies(guc, q);
570 }
571 
572 static u32 wq_space_until_wrap(struct xe_exec_queue *q)
573 {
574 	return (WQ_SIZE - q->guc->wqi_tail);
575 }
576 
577 static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size)
578 {
579 	struct xe_guc *guc = exec_queue_to_guc(q);
580 	struct xe_device *xe = guc_to_xe(guc);
581 	struct iosys_map map = xe_lrc_parallel_map(q->lrc);
582 	unsigned int sleep_period_ms = 1;
583 
584 #define AVAILABLE_SPACE \
585 	CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE)
586 	if (wqi_size > AVAILABLE_SPACE) {
587 try_again:
588 		q->guc->wqi_head = parallel_read(xe, map, wq_desc.head);
589 		if (wqi_size > AVAILABLE_SPACE) {
590 			if (sleep_period_ms == 1024) {
591 				xe_gt_reset_async(q->gt);
592 				return -ENODEV;
593 			}
594 
595 			msleep(sleep_period_ms);
596 			sleep_period_ms <<= 1;
597 			goto try_again;
598 		}
599 	}
600 #undef AVAILABLE_SPACE
601 
602 	return 0;
603 }
604 
605 static int wq_noop_append(struct xe_exec_queue *q)
606 {
607 	struct xe_guc *guc = exec_queue_to_guc(q);
608 	struct xe_device *xe = guc_to_xe(guc);
609 	struct iosys_map map = xe_lrc_parallel_map(q->lrc);
610 	u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1;
611 
612 	if (wq_wait_for_space(q, wq_space_until_wrap(q)))
613 		return -ENODEV;
614 
615 	xe_assert(xe, FIELD_FIT(WQ_LEN_MASK, len_dw));
616 
617 	parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)],
618 		       FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |
619 		       FIELD_PREP(WQ_LEN_MASK, len_dw));
620 	q->guc->wqi_tail = 0;
621 
622 	return 0;
623 }
624 
625 static void wq_item_append(struct xe_exec_queue *q)
626 {
627 	struct xe_guc *guc = exec_queue_to_guc(q);
628 	struct xe_device *xe = guc_to_xe(guc);
629 	struct iosys_map map = xe_lrc_parallel_map(q->lrc);
630 #define WQ_HEADER_SIZE	4	/* Includes 1 LRC address too */
631 	u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)];
632 	u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32);
633 	u32 len_dw = (wqi_size / sizeof(u32)) - 1;
634 	int i = 0, j;
635 
636 	if (wqi_size > wq_space_until_wrap(q)) {
637 		if (wq_noop_append(q))
638 			return;
639 	}
640 	if (wq_wait_for_space(q, wqi_size))
641 		return;
642 
643 	wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) |
644 		FIELD_PREP(WQ_LEN_MASK, len_dw);
645 	wqi[i++] = xe_lrc_descriptor(q->lrc);
646 	wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) |
647 		FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc->ring.tail / sizeof(u64));
648 	wqi[i++] = 0;
649 	for (j = 1; j < q->width; ++j) {
650 		struct xe_lrc *lrc = q->lrc + j;
651 
652 		wqi[i++] = lrc->ring.tail / sizeof(u64);
653 	}
654 
655 	xe_assert(xe, i == wqi_size / sizeof(u32));
656 
657 	iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch,
658 				      wq[q->guc->wqi_tail / sizeof(u32)]));
659 	xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size);
660 	q->guc->wqi_tail += wqi_size;
661 	xe_assert(xe, q->guc->wqi_tail <= WQ_SIZE);
662 
663 	xe_device_wmb(xe);
664 
665 	map = xe_lrc_parallel_map(q->lrc);
666 	parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail);
667 }
668 
669 #define RESUME_PENDING	~0x0ull
670 static void submit_exec_queue(struct xe_exec_queue *q)
671 {
672 	struct xe_guc *guc = exec_queue_to_guc(q);
673 	struct xe_device *xe = guc_to_xe(guc);
674 	struct xe_lrc *lrc = q->lrc;
675 	u32 action[3];
676 	u32 g2h_len = 0;
677 	u32 num_g2h = 0;
678 	int len = 0;
679 	bool extra_submit = false;
680 
681 	xe_assert(xe, exec_queue_registered(q));
682 
683 	if (xe_exec_queue_is_parallel(q))
684 		wq_item_append(q);
685 	else
686 		xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail);
687 
688 	if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q))
689 		return;
690 
691 	if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) {
692 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
693 		action[len++] = q->guc->id;
694 		action[len++] = GUC_CONTEXT_ENABLE;
695 		g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET;
696 		num_g2h = 1;
697 		if (xe_exec_queue_is_parallel(q))
698 			extra_submit = true;
699 
700 		q->guc->resume_time = RESUME_PENDING;
701 		set_exec_queue_pending_enable(q);
702 		set_exec_queue_enabled(q);
703 		trace_xe_exec_queue_scheduling_enable(q);
704 	} else {
705 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;
706 		action[len++] = q->guc->id;
707 		trace_xe_exec_queue_submit(q);
708 	}
709 
710 	xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);
711 
712 	if (extra_submit) {
713 		len = 0;
714 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;
715 		action[len++] = q->guc->id;
716 		trace_xe_exec_queue_submit(q);
717 
718 		xe_guc_ct_send(&guc->ct, action, len, 0, 0);
719 	}
720 }
721 
722 static struct dma_fence *
723 guc_exec_queue_run_job(struct drm_sched_job *drm_job)
724 {
725 	struct xe_sched_job *job = to_xe_sched_job(drm_job);
726 	struct xe_exec_queue *q = job->q;
727 	struct xe_guc *guc = exec_queue_to_guc(q);
728 	struct xe_device *xe = guc_to_xe(guc);
729 	bool lr = xe_exec_queue_is_lr(q);
730 
731 	xe_assert(xe, !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) ||
732 		  exec_queue_banned(q) || exec_queue_suspended(q));
733 
734 	trace_xe_sched_job_run(job);
735 
736 	if (!exec_queue_killed_or_banned(q) && !xe_sched_job_is_error(job)) {
737 		if (!exec_queue_registered(q))
738 			register_engine(q);
739 		if (!lr)	/* LR jobs are emitted in the exec IOCTL */
740 			q->ring_ops->emit_job(job);
741 		submit_exec_queue(q);
742 	}
743 
744 	if (lr) {
745 		xe_sched_job_set_error(job, -EOPNOTSUPP);
746 		return NULL;
747 	} else if (test_and_set_bit(JOB_FLAG_SUBMIT, &job->fence->flags)) {
748 		return job->fence;
749 	} else {
750 		return dma_fence_get(job->fence);
751 	}
752 }
753 
754 static void guc_exec_queue_free_job(struct drm_sched_job *drm_job)
755 {
756 	struct xe_sched_job *job = to_xe_sched_job(drm_job);
757 
758 	trace_xe_sched_job_free(job);
759 	xe_sched_job_put(job);
760 }
761 
762 static int guc_read_stopped(struct xe_guc *guc)
763 {
764 	return atomic_read(&guc->submission_state.stopped);
765 }
766 
767 #define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable)			\
768 	u32 action[] = {						\
769 		XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET,			\
770 		q->guc->id,						\
771 		GUC_CONTEXT_##enable_disable,				\
772 	}
773 
774 static void disable_scheduling_deregister(struct xe_guc *guc,
775 					  struct xe_exec_queue *q)
776 {
777 	MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
778 	struct xe_device *xe = guc_to_xe(guc);
779 	int ret;
780 
781 	set_min_preemption_timeout(guc, q);
782 	smp_rmb();
783 	ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) ||
784 				 guc_read_stopped(guc), HZ * 5);
785 	if (!ret) {
786 		struct xe_gpu_scheduler *sched = &q->guc->sched;
787 
788 		drm_warn(&xe->drm, "Pending enable failed to respond");
789 		xe_sched_submission_start(sched);
790 		xe_gt_reset_async(q->gt);
791 		xe_sched_tdr_queue_imm(sched);
792 		return;
793 	}
794 
795 	clear_exec_queue_enabled(q);
796 	set_exec_queue_pending_disable(q);
797 	set_exec_queue_destroyed(q);
798 	trace_xe_exec_queue_scheduling_disable(q);
799 
800 	/*
801 	 * Reserve space for both G2H here as the 2nd G2H is sent from a G2H
802 	 * handler and we are not allowed to reserved G2H space in handlers.
803 	 */
804 	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
805 		       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +
806 		       G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
807 }
808 
809 static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p);
810 
811 #if IS_ENABLED(CONFIG_DRM_XE_SIMPLE_ERROR_CAPTURE)
812 static void simple_error_capture(struct xe_exec_queue *q)
813 {
814 	struct xe_guc *guc = exec_queue_to_guc(q);
815 	struct xe_device *xe = guc_to_xe(guc);
816 	struct drm_printer p = drm_err_printer(&xe->drm, NULL);
817 	struct xe_hw_engine *hwe;
818 	enum xe_hw_engine_id id;
819 	u32 adj_logical_mask = q->logical_mask;
820 	u32 width_mask = (0x1 << q->width) - 1;
821 	int i;
822 	bool cookie;
823 
824 	if (q->vm && !q->vm->error_capture.capture_once) {
825 		q->vm->error_capture.capture_once = true;
826 		cookie = dma_fence_begin_signalling();
827 		for (i = 0; q->width > 1 && i < XE_HW_ENGINE_MAX_INSTANCE;) {
828 			if (adj_logical_mask & BIT(i)) {
829 				adj_logical_mask |= width_mask << i;
830 				i += q->width;
831 			} else {
832 				++i;
833 			}
834 		}
835 
836 		xe_force_wake_get(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL);
837 		xe_guc_ct_print(&guc->ct, &p, true);
838 		guc_exec_queue_print(q, &p);
839 		for_each_hw_engine(hwe, guc_to_gt(guc), id) {
840 			if (hwe->class != q->hwe->class ||
841 			    !(BIT(hwe->logical_instance) & adj_logical_mask))
842 				continue;
843 			xe_hw_engine_print(hwe, &p);
844 		}
845 		xe_analyze_vm(&p, q->vm, q->gt->info.id);
846 		xe_force_wake_put(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL);
847 		dma_fence_end_signalling(cookie);
848 	}
849 }
850 #else
851 static void simple_error_capture(struct xe_exec_queue *q)
852 {
853 }
854 #endif
855 
856 static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
857 {
858 	struct xe_guc *guc = exec_queue_to_guc(q);
859 	struct xe_device *xe = guc_to_xe(guc);
860 
861 	/** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
862 	wake_up_all(&xe->ufence_wq);
863 
864 	if (xe_exec_queue_is_lr(q))
865 		queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);
866 	else
867 		xe_sched_tdr_queue_imm(&q->guc->sched);
868 }
869 
870 static void xe_guc_exec_queue_lr_cleanup(struct work_struct *w)
871 {
872 	struct xe_guc_exec_queue *ge =
873 		container_of(w, struct xe_guc_exec_queue, lr_tdr);
874 	struct xe_exec_queue *q = ge->q;
875 	struct xe_guc *guc = exec_queue_to_guc(q);
876 	struct xe_device *xe = guc_to_xe(guc);
877 	struct xe_gpu_scheduler *sched = &ge->sched;
878 
879 	xe_assert(xe, xe_exec_queue_is_lr(q));
880 	trace_xe_exec_queue_lr_cleanup(q);
881 
882 	/* Kill the run_job / process_msg entry points */
883 	xe_sched_submission_stop(sched);
884 
885 	/*
886 	 * Engine state now mostly stable, disable scheduling / deregister if
887 	 * needed. This cleanup routine might be called multiple times, where
888 	 * the actual async engine deregister drops the final engine ref.
889 	 * Calling disable_scheduling_deregister will mark the engine as
890 	 * destroyed and fire off the CT requests to disable scheduling /
891 	 * deregister, which we only want to do once. We also don't want to mark
892 	 * the engine as pending_disable again as this may race with the
893 	 * xe_guc_deregister_done_handler() which treats it as an unexpected
894 	 * state.
895 	 */
896 	if (exec_queue_registered(q) && !exec_queue_destroyed(q)) {
897 		struct xe_guc *guc = exec_queue_to_guc(q);
898 		int ret;
899 
900 		set_exec_queue_banned(q);
901 		disable_scheduling_deregister(guc, q);
902 
903 		/*
904 		 * Must wait for scheduling to be disabled before signalling
905 		 * any fences, if GT broken the GT reset code should signal us.
906 		 */
907 		ret = wait_event_timeout(guc->ct.wq,
908 					 !exec_queue_pending_disable(q) ||
909 					 guc_read_stopped(guc), HZ * 5);
910 		if (!ret) {
911 			drm_warn(&xe->drm, "Schedule disable failed to respond");
912 			xe_sched_submission_start(sched);
913 			xe_gt_reset_async(q->gt);
914 			return;
915 		}
916 	}
917 
918 	xe_sched_submission_start(sched);
919 }
920 
921 static enum drm_gpu_sched_stat
922 guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
923 {
924 	struct xe_sched_job *job = to_xe_sched_job(drm_job);
925 	struct xe_sched_job *tmp_job;
926 	struct xe_exec_queue *q = job->q;
927 	struct xe_gpu_scheduler *sched = &q->guc->sched;
928 	struct xe_device *xe = guc_to_xe(exec_queue_to_guc(q));
929 	int err = -ETIME;
930 	int i = 0;
931 
932 	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) {
933 		drm_notice(&xe->drm, "Timedout job: seqno=%u, guc_id=%d, flags=0x%lx",
934 			   xe_sched_job_seqno(job), q->guc->id, q->flags);
935 		xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL,
936 			   "Kernel-submitted job timed out\n");
937 		xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q),
938 			   "VM job timed out on non-killed execqueue\n");
939 
940 		simple_error_capture(q);
941 		xe_devcoredump(job);
942 	} else {
943 		drm_dbg(&xe->drm, "Timedout signaled job: seqno=%u, guc_id=%d, flags=0x%lx",
944 			 xe_sched_job_seqno(job), q->guc->id, q->flags);
945 	}
946 	trace_xe_sched_job_timedout(job);
947 
948 	/* Kill the run_job entry point */
949 	xe_sched_submission_stop(sched);
950 
951 	/*
952 	 * Kernel jobs should never fail, nor should VM jobs if they do
953 	 * somethings has gone wrong and the GT needs a reset
954 	 */
955 	if (q->flags & EXEC_QUEUE_FLAG_KERNEL ||
956 	    (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q))) {
957 		if (!xe_sched_invalidate_job(job, 2)) {
958 			xe_sched_add_pending_job(sched, job);
959 			xe_sched_submission_start(sched);
960 			xe_gt_reset_async(q->gt);
961 			goto out;
962 		}
963 	}
964 
965 	/* Engine state now stable, disable scheduling if needed */
966 	if (exec_queue_registered(q)) {
967 		struct xe_guc *guc = exec_queue_to_guc(q);
968 		int ret;
969 
970 		if (exec_queue_reset(q))
971 			err = -EIO;
972 		set_exec_queue_banned(q);
973 		if (!exec_queue_destroyed(q)) {
974 			xe_exec_queue_get(q);
975 			disable_scheduling_deregister(guc, q);
976 		}
977 
978 		/*
979 		 * Must wait for scheduling to be disabled before signalling
980 		 * any fences, if GT broken the GT reset code should signal us.
981 		 *
982 		 * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault
983 		 * error) messages which can cause the schedule disable to get
984 		 * lost. If this occurs, trigger a GT reset to recover.
985 		 */
986 		smp_rmb();
987 		ret = wait_event_timeout(guc->ct.wq,
988 					 !exec_queue_pending_disable(q) ||
989 					 guc_read_stopped(guc), HZ * 5);
990 		if (!ret || guc_read_stopped(guc)) {
991 			drm_warn(&xe->drm, "Schedule disable failed to respond");
992 			xe_sched_add_pending_job(sched, job);
993 			xe_sched_submission_start(sched);
994 			xe_gt_reset_async(q->gt);
995 			xe_sched_tdr_queue_imm(sched);
996 			goto out;
997 		}
998 	}
999 
1000 	/* Stop fence signaling */
1001 	xe_hw_fence_irq_stop(q->fence_irq);
1002 
1003 	/*
1004 	 * Fence state now stable, stop / start scheduler which cleans up any
1005 	 * fences that are complete
1006 	 */
1007 	xe_sched_add_pending_job(sched, job);
1008 	xe_sched_submission_start(sched);
1009 	xe_guc_exec_queue_trigger_cleanup(q);
1010 
1011 	/* Mark all outstanding jobs as bad, thus completing them */
1012 	spin_lock(&sched->base.job_list_lock);
1013 	list_for_each_entry(tmp_job, &sched->base.pending_list, drm.list)
1014 		xe_sched_job_set_error(tmp_job, !i++ ? err : -ECANCELED);
1015 	spin_unlock(&sched->base.job_list_lock);
1016 
1017 	/* Start fence signaling */
1018 	xe_hw_fence_irq_start(q->fence_irq);
1019 
1020 out:
1021 	return DRM_GPU_SCHED_STAT_NOMINAL;
1022 }
1023 
1024 static void __guc_exec_queue_fini_async(struct work_struct *w)
1025 {
1026 	struct xe_guc_exec_queue *ge =
1027 		container_of(w, struct xe_guc_exec_queue, fini_async);
1028 	struct xe_exec_queue *q = ge->q;
1029 	struct xe_guc *guc = exec_queue_to_guc(q);
1030 
1031 	trace_xe_exec_queue_destroy(q);
1032 
1033 	if (xe_exec_queue_is_lr(q))
1034 		cancel_work_sync(&ge->lr_tdr);
1035 	release_guc_id(guc, q);
1036 	xe_sched_entity_fini(&ge->entity);
1037 	xe_sched_fini(&ge->sched);
1038 
1039 	kfree(ge);
1040 	xe_exec_queue_fini(q);
1041 }
1042 
1043 static void guc_exec_queue_fini_async(struct xe_exec_queue *q)
1044 {
1045 	INIT_WORK(&q->guc->fini_async, __guc_exec_queue_fini_async);
1046 
1047 	/* We must block on kernel engines so slabs are empty on driver unload */
1048 	if (q->flags & EXEC_QUEUE_FLAG_PERMANENT)
1049 		__guc_exec_queue_fini_async(&q->guc->fini_async);
1050 	else
1051 		queue_work(system_wq, &q->guc->fini_async);
1052 }
1053 
1054 static void __guc_exec_queue_fini(struct xe_guc *guc, struct xe_exec_queue *q)
1055 {
1056 	/*
1057 	 * Might be done from within the GPU scheduler, need to do async as we
1058 	 * fini the scheduler when the engine is fini'd, the scheduler can't
1059 	 * complete fini within itself (circular dependency). Async resolves
1060 	 * this we and don't really care when everything is fini'd, just that it
1061 	 * is.
1062 	 */
1063 	guc_exec_queue_fini_async(q);
1064 }
1065 
1066 static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg)
1067 {
1068 	struct xe_exec_queue *q = msg->private_data;
1069 	struct xe_guc *guc = exec_queue_to_guc(q);
1070 	struct xe_device *xe = guc_to_xe(guc);
1071 
1072 	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT));
1073 	trace_xe_exec_queue_cleanup_entity(q);
1074 
1075 	if (exec_queue_registered(q))
1076 		disable_scheduling_deregister(guc, q);
1077 	else
1078 		__guc_exec_queue_fini(guc, q);
1079 }
1080 
1081 static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q)
1082 {
1083 	return !exec_queue_killed_or_banned(q) && exec_queue_registered(q);
1084 }
1085 
1086 static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg)
1087 {
1088 	struct xe_exec_queue *q = msg->private_data;
1089 	struct xe_guc *guc = exec_queue_to_guc(q);
1090 
1091 	if (guc_exec_queue_allowed_to_change_state(q))
1092 		init_policies(guc, q);
1093 	kfree(msg);
1094 }
1095 
1096 static void suspend_fence_signal(struct xe_exec_queue *q)
1097 {
1098 	struct xe_guc *guc = exec_queue_to_guc(q);
1099 	struct xe_device *xe = guc_to_xe(guc);
1100 
1101 	xe_assert(xe, exec_queue_suspended(q) || exec_queue_killed(q) ||
1102 		  guc_read_stopped(guc));
1103 	xe_assert(xe, q->guc->suspend_pending);
1104 
1105 	q->guc->suspend_pending = false;
1106 	smp_wmb();
1107 	wake_up(&q->guc->suspend_wait);
1108 }
1109 
1110 static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)
1111 {
1112 	struct xe_exec_queue *q = msg->private_data;
1113 	struct xe_guc *guc = exec_queue_to_guc(q);
1114 
1115 	if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) &&
1116 	    exec_queue_enabled(q)) {
1117 		wait_event(guc->ct.wq, q->guc->resume_time != RESUME_PENDING ||
1118 			   guc_read_stopped(guc));
1119 
1120 		if (!guc_read_stopped(guc)) {
1121 			MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
1122 			s64 since_resume_ms =
1123 				ktime_ms_delta(ktime_get(),
1124 					       q->guc->resume_time);
1125 			s64 wait_ms = q->vm->preempt.min_run_period_ms -
1126 				since_resume_ms;
1127 
1128 			if (wait_ms > 0 && q->guc->resume_time)
1129 				msleep(wait_ms);
1130 
1131 			set_exec_queue_suspended(q);
1132 			clear_exec_queue_enabled(q);
1133 			set_exec_queue_pending_disable(q);
1134 			trace_xe_exec_queue_scheduling_disable(q);
1135 
1136 			xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1137 				       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1138 		}
1139 	} else if (q->guc->suspend_pending) {
1140 		set_exec_queue_suspended(q);
1141 		suspend_fence_signal(q);
1142 	}
1143 }
1144 
1145 static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)
1146 {
1147 	struct xe_exec_queue *q = msg->private_data;
1148 	struct xe_guc *guc = exec_queue_to_guc(q);
1149 
1150 	if (guc_exec_queue_allowed_to_change_state(q)) {
1151 		MAKE_SCHED_CONTEXT_ACTION(q, ENABLE);
1152 
1153 		q->guc->resume_time = RESUME_PENDING;
1154 		clear_exec_queue_suspended(q);
1155 		set_exec_queue_pending_enable(q);
1156 		set_exec_queue_enabled(q);
1157 		trace_xe_exec_queue_scheduling_enable(q);
1158 
1159 		xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1160 			       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1161 	} else {
1162 		clear_exec_queue_suspended(q);
1163 	}
1164 }
1165 
1166 #define CLEANUP		1	/* Non-zero values to catch uninitialized msg */
1167 #define SET_SCHED_PROPS	2
1168 #define SUSPEND		3
1169 #define RESUME		4
1170 
1171 static void guc_exec_queue_process_msg(struct xe_sched_msg *msg)
1172 {
1173 	trace_xe_sched_msg_recv(msg);
1174 
1175 	switch (msg->opcode) {
1176 	case CLEANUP:
1177 		__guc_exec_queue_process_msg_cleanup(msg);
1178 		break;
1179 	case SET_SCHED_PROPS:
1180 		__guc_exec_queue_process_msg_set_sched_props(msg);
1181 		break;
1182 	case SUSPEND:
1183 		__guc_exec_queue_process_msg_suspend(msg);
1184 		break;
1185 	case RESUME:
1186 		__guc_exec_queue_process_msg_resume(msg);
1187 		break;
1188 	default:
1189 		XE_WARN_ON("Unknown message type");
1190 	}
1191 }
1192 
1193 static const struct drm_sched_backend_ops drm_sched_ops = {
1194 	.run_job = guc_exec_queue_run_job,
1195 	.free_job = guc_exec_queue_free_job,
1196 	.timedout_job = guc_exec_queue_timedout_job,
1197 };
1198 
1199 static const struct xe_sched_backend_ops xe_sched_ops = {
1200 	.process_msg = guc_exec_queue_process_msg,
1201 };
1202 
1203 static int guc_exec_queue_init(struct xe_exec_queue *q)
1204 {
1205 	struct xe_gpu_scheduler *sched;
1206 	struct xe_guc *guc = exec_queue_to_guc(q);
1207 	struct xe_device *xe = guc_to_xe(guc);
1208 	struct xe_guc_exec_queue *ge;
1209 	long timeout;
1210 	int err;
1211 
1212 	xe_assert(xe, xe_device_uc_enabled(guc_to_xe(guc)));
1213 
1214 	ge = kzalloc(sizeof(*ge), GFP_KERNEL);
1215 	if (!ge)
1216 		return -ENOMEM;
1217 
1218 	q->guc = ge;
1219 	ge->q = q;
1220 	init_waitqueue_head(&ge->suspend_wait);
1221 
1222 	timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT :
1223 		  q->sched_props.job_timeout_ms;
1224 	err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
1225 			    get_submit_wq(guc),
1226 			    q->lrc[0].ring.size / MAX_JOB_SIZE_BYTES, 64,
1227 			    timeout, guc_to_gt(guc)->ordered_wq, NULL,
1228 			    q->name, gt_to_xe(q->gt)->drm.dev);
1229 	if (err)
1230 		goto err_free;
1231 
1232 	sched = &ge->sched;
1233 	err = xe_sched_entity_init(&ge->entity, sched);
1234 	if (err)
1235 		goto err_sched;
1236 
1237 	if (xe_exec_queue_is_lr(q))
1238 		INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup);
1239 
1240 	mutex_lock(&guc->submission_state.lock);
1241 
1242 	err = alloc_guc_id(guc, q);
1243 	if (err)
1244 		goto err_entity;
1245 
1246 	q->entity = &ge->entity;
1247 
1248 	if (guc_read_stopped(guc))
1249 		xe_sched_stop(sched);
1250 
1251 	mutex_unlock(&guc->submission_state.lock);
1252 
1253 	xe_exec_queue_assign_name(q, q->guc->id);
1254 
1255 	trace_xe_exec_queue_create(q);
1256 
1257 	return 0;
1258 
1259 err_entity:
1260 	xe_sched_entity_fini(&ge->entity);
1261 err_sched:
1262 	xe_sched_fini(&ge->sched);
1263 err_free:
1264 	kfree(ge);
1265 
1266 	return err;
1267 }
1268 
1269 static void guc_exec_queue_kill(struct xe_exec_queue *q)
1270 {
1271 	trace_xe_exec_queue_kill(q);
1272 	set_exec_queue_killed(q);
1273 	xe_guc_exec_queue_trigger_cleanup(q);
1274 }
1275 
1276 static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg,
1277 				   u32 opcode)
1278 {
1279 	INIT_LIST_HEAD(&msg->link);
1280 	msg->opcode = opcode;
1281 	msg->private_data = q;
1282 
1283 	trace_xe_sched_msg_add(msg);
1284 	xe_sched_add_msg(&q->guc->sched, msg);
1285 }
1286 
1287 #define STATIC_MSG_CLEANUP	0
1288 #define STATIC_MSG_SUSPEND	1
1289 #define STATIC_MSG_RESUME	2
1290 static void guc_exec_queue_fini(struct xe_exec_queue *q)
1291 {
1292 	struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;
1293 
1294 	if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT))
1295 		guc_exec_queue_add_msg(q, msg, CLEANUP);
1296 	else
1297 		__guc_exec_queue_fini(exec_queue_to_guc(q), q);
1298 }
1299 
1300 static int guc_exec_queue_set_priority(struct xe_exec_queue *q,
1301 				       enum xe_exec_queue_priority priority)
1302 {
1303 	struct xe_sched_msg *msg;
1304 
1305 	if (q->sched_props.priority == priority || exec_queue_killed_or_banned(q))
1306 		return 0;
1307 
1308 	msg = kmalloc(sizeof(*msg), GFP_KERNEL);
1309 	if (!msg)
1310 		return -ENOMEM;
1311 
1312 	q->sched_props.priority = priority;
1313 	guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);
1314 
1315 	return 0;
1316 }
1317 
1318 static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us)
1319 {
1320 	struct xe_sched_msg *msg;
1321 
1322 	if (q->sched_props.timeslice_us == timeslice_us ||
1323 	    exec_queue_killed_or_banned(q))
1324 		return 0;
1325 
1326 	msg = kmalloc(sizeof(*msg), GFP_KERNEL);
1327 	if (!msg)
1328 		return -ENOMEM;
1329 
1330 	q->sched_props.timeslice_us = timeslice_us;
1331 	guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);
1332 
1333 	return 0;
1334 }
1335 
1336 static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q,
1337 					      u32 preempt_timeout_us)
1338 {
1339 	struct xe_sched_msg *msg;
1340 
1341 	if (q->sched_props.preempt_timeout_us == preempt_timeout_us ||
1342 	    exec_queue_killed_or_banned(q))
1343 		return 0;
1344 
1345 	msg = kmalloc(sizeof(*msg), GFP_KERNEL);
1346 	if (!msg)
1347 		return -ENOMEM;
1348 
1349 	q->sched_props.preempt_timeout_us = preempt_timeout_us;
1350 	guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);
1351 
1352 	return 0;
1353 }
1354 
1355 static int guc_exec_queue_suspend(struct xe_exec_queue *q)
1356 {
1357 	struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND;
1358 
1359 	if (exec_queue_killed_or_banned(q) || q->guc->suspend_pending)
1360 		return -EINVAL;
1361 
1362 	q->guc->suspend_pending = true;
1363 	guc_exec_queue_add_msg(q, msg, SUSPEND);
1364 
1365 	return 0;
1366 }
1367 
1368 static void guc_exec_queue_suspend_wait(struct xe_exec_queue *q)
1369 {
1370 	struct xe_guc *guc = exec_queue_to_guc(q);
1371 
1372 	wait_event(q->guc->suspend_wait, !q->guc->suspend_pending ||
1373 		   guc_read_stopped(guc));
1374 }
1375 
1376 static void guc_exec_queue_resume(struct xe_exec_queue *q)
1377 {
1378 	struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME;
1379 	struct xe_guc *guc = exec_queue_to_guc(q);
1380 	struct xe_device *xe = guc_to_xe(guc);
1381 
1382 	xe_assert(xe, !q->guc->suspend_pending);
1383 
1384 	guc_exec_queue_add_msg(q, msg, RESUME);
1385 }
1386 
1387 static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
1388 {
1389 	return exec_queue_reset(q);
1390 }
1391 
1392 /*
1393  * All of these functions are an abstraction layer which other parts of XE can
1394  * use to trap into the GuC backend. All of these functions, aside from init,
1395  * really shouldn't do much other than trap into the DRM scheduler which
1396  * synchronizes these operations.
1397  */
1398 static const struct xe_exec_queue_ops guc_exec_queue_ops = {
1399 	.init = guc_exec_queue_init,
1400 	.kill = guc_exec_queue_kill,
1401 	.fini = guc_exec_queue_fini,
1402 	.set_priority = guc_exec_queue_set_priority,
1403 	.set_timeslice = guc_exec_queue_set_timeslice,
1404 	.set_preempt_timeout = guc_exec_queue_set_preempt_timeout,
1405 	.suspend = guc_exec_queue_suspend,
1406 	.suspend_wait = guc_exec_queue_suspend_wait,
1407 	.resume = guc_exec_queue_resume,
1408 	.reset_status = guc_exec_queue_reset_status,
1409 };
1410 
1411 static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
1412 {
1413 	struct xe_gpu_scheduler *sched = &q->guc->sched;
1414 
1415 	/* Stop scheduling + flush any DRM scheduler operations */
1416 	xe_sched_submission_stop(sched);
1417 
1418 	/* Clean up lost G2H + reset engine state */
1419 	if (exec_queue_registered(q)) {
1420 		if ((exec_queue_banned(q) && exec_queue_destroyed(q)) ||
1421 		    xe_exec_queue_is_lr(q))
1422 			xe_exec_queue_put(q);
1423 		else if (exec_queue_destroyed(q))
1424 			__guc_exec_queue_fini(guc, q);
1425 	}
1426 	if (q->guc->suspend_pending) {
1427 		set_exec_queue_suspended(q);
1428 		suspend_fence_signal(q);
1429 	}
1430 	atomic_and(EXEC_QUEUE_STATE_DESTROYED | ENGINE_STATE_SUSPENDED,
1431 		   &q->guc->state);
1432 	q->guc->resume_time = 0;
1433 	trace_xe_exec_queue_stop(q);
1434 
1435 	/*
1436 	 * Ban any engine (aside from kernel and engines used for VM ops) with a
1437 	 * started but not complete job or if a job has gone through a GT reset
1438 	 * more than twice.
1439 	 */
1440 	if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) {
1441 		struct xe_sched_job *job = xe_sched_first_pending_job(sched);
1442 
1443 		if (job) {
1444 			if ((xe_sched_job_started(job) &&
1445 			    !xe_sched_job_completed(job)) ||
1446 			    xe_sched_invalidate_job(job, 2)) {
1447 				trace_xe_sched_job_ban(job);
1448 				xe_sched_tdr_queue_imm(&q->guc->sched);
1449 				set_exec_queue_banned(q);
1450 			}
1451 		}
1452 	}
1453 }
1454 
1455 int xe_guc_submit_reset_prepare(struct xe_guc *guc)
1456 {
1457 	int ret;
1458 
1459 	/*
1460 	 * Using an atomic here rather than submission_state.lock as this
1461 	 * function can be called while holding the CT lock (engine reset
1462 	 * failure). submission_state.lock needs the CT lock to resubmit jobs.
1463 	 * Atomic is not ideal, but it works to prevent against concurrent reset
1464 	 * and releasing any TDRs waiting on guc->submission_state.stopped.
1465 	 */
1466 	ret = atomic_fetch_or(1, &guc->submission_state.stopped);
1467 	smp_wmb();
1468 	wake_up_all(&guc->ct.wq);
1469 
1470 	return ret;
1471 }
1472 
1473 void xe_guc_submit_reset_wait(struct xe_guc *guc)
1474 {
1475 	wait_event(guc->ct.wq, !guc_read_stopped(guc));
1476 }
1477 
1478 int xe_guc_submit_stop(struct xe_guc *guc)
1479 {
1480 	struct xe_exec_queue *q;
1481 	unsigned long index;
1482 	struct xe_device *xe = guc_to_xe(guc);
1483 
1484 	xe_assert(xe, guc_read_stopped(guc) == 1);
1485 
1486 	mutex_lock(&guc->submission_state.lock);
1487 
1488 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
1489 		guc_exec_queue_stop(guc, q);
1490 
1491 	mutex_unlock(&guc->submission_state.lock);
1492 
1493 	/*
1494 	 * No one can enter the backend at this point, aside from new engine
1495 	 * creation which is protected by guc->submission_state.lock.
1496 	 */
1497 
1498 	return 0;
1499 }
1500 
1501 static void guc_exec_queue_start(struct xe_exec_queue *q)
1502 {
1503 	struct xe_gpu_scheduler *sched = &q->guc->sched;
1504 
1505 	if (!exec_queue_killed_or_banned(q)) {
1506 		int i;
1507 
1508 		trace_xe_exec_queue_resubmit(q);
1509 		for (i = 0; i < q->width; ++i)
1510 			xe_lrc_set_ring_head(q->lrc + i, q->lrc[i].ring.tail);
1511 		xe_sched_resubmit_jobs(sched);
1512 	}
1513 
1514 	xe_sched_submission_start(sched);
1515 }
1516 
1517 int xe_guc_submit_start(struct xe_guc *guc)
1518 {
1519 	struct xe_exec_queue *q;
1520 	unsigned long index;
1521 	struct xe_device *xe = guc_to_xe(guc);
1522 
1523 	xe_assert(xe, guc_read_stopped(guc) == 1);
1524 
1525 	mutex_lock(&guc->submission_state.lock);
1526 	atomic_dec(&guc->submission_state.stopped);
1527 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
1528 		guc_exec_queue_start(q);
1529 	mutex_unlock(&guc->submission_state.lock);
1530 
1531 	wake_up_all(&guc->ct.wq);
1532 
1533 	return 0;
1534 }
1535 
1536 static struct xe_exec_queue *
1537 g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id)
1538 {
1539 	struct xe_device *xe = guc_to_xe(guc);
1540 	struct xe_exec_queue *q;
1541 
1542 	if (unlikely(guc_id >= GUC_ID_MAX)) {
1543 		drm_err(&xe->drm, "Invalid guc_id %u", guc_id);
1544 		return NULL;
1545 	}
1546 
1547 	q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id);
1548 	if (unlikely(!q)) {
1549 		drm_err(&xe->drm, "Not engine present for guc_id %u", guc_id);
1550 		return NULL;
1551 	}
1552 
1553 	xe_assert(xe, guc_id >= q->guc->id);
1554 	xe_assert(xe, guc_id < (q->guc->id + q->width));
1555 
1556 	return q;
1557 }
1558 
1559 static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)
1560 {
1561 	u32 action[] = {
1562 		XE_GUC_ACTION_DEREGISTER_CONTEXT,
1563 		q->guc->id,
1564 	};
1565 
1566 	trace_xe_exec_queue_deregister(q);
1567 
1568 	xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));
1569 }
1570 
1571 int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
1572 {
1573 	struct xe_device *xe = guc_to_xe(guc);
1574 	struct xe_exec_queue *q;
1575 	u32 guc_id = msg[0];
1576 
1577 	if (unlikely(len < 2)) {
1578 		drm_err(&xe->drm, "Invalid length %u", len);
1579 		return -EPROTO;
1580 	}
1581 
1582 	q = g2h_exec_queue_lookup(guc, guc_id);
1583 	if (unlikely(!q))
1584 		return -EPROTO;
1585 
1586 	if (unlikely(!exec_queue_pending_enable(q) &&
1587 		     !exec_queue_pending_disable(q))) {
1588 		drm_err(&xe->drm, "Unexpected engine state 0x%04x",
1589 			atomic_read(&q->guc->state));
1590 		return -EPROTO;
1591 	}
1592 
1593 	trace_xe_exec_queue_scheduling_done(q);
1594 
1595 	if (exec_queue_pending_enable(q)) {
1596 		q->guc->resume_time = ktime_get();
1597 		clear_exec_queue_pending_enable(q);
1598 		smp_wmb();
1599 		wake_up_all(&guc->ct.wq);
1600 	} else {
1601 		clear_exec_queue_pending_disable(q);
1602 		if (q->guc->suspend_pending) {
1603 			suspend_fence_signal(q);
1604 		} else {
1605 			if (exec_queue_banned(q)) {
1606 				smp_wmb();
1607 				wake_up_all(&guc->ct.wq);
1608 			}
1609 			deregister_exec_queue(guc, q);
1610 		}
1611 	}
1612 
1613 	return 0;
1614 }
1615 
1616 int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
1617 {
1618 	struct xe_device *xe = guc_to_xe(guc);
1619 	struct xe_exec_queue *q;
1620 	u32 guc_id = msg[0];
1621 
1622 	if (unlikely(len < 1)) {
1623 		drm_err(&xe->drm, "Invalid length %u", len);
1624 		return -EPROTO;
1625 	}
1626 
1627 	q = g2h_exec_queue_lookup(guc, guc_id);
1628 	if (unlikely(!q))
1629 		return -EPROTO;
1630 
1631 	if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) ||
1632 	    exec_queue_pending_enable(q) || exec_queue_enabled(q)) {
1633 		drm_err(&xe->drm, "Unexpected engine state 0x%04x",
1634 			atomic_read(&q->guc->state));
1635 		return -EPROTO;
1636 	}
1637 
1638 	trace_xe_exec_queue_deregister_done(q);
1639 
1640 	clear_exec_queue_registered(q);
1641 
1642 	if (exec_queue_banned(q) || xe_exec_queue_is_lr(q))
1643 		xe_exec_queue_put(q);
1644 	else
1645 		__guc_exec_queue_fini(guc, q);
1646 
1647 	return 0;
1648 }
1649 
1650 int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len)
1651 {
1652 	struct xe_device *xe = guc_to_xe(guc);
1653 	struct xe_exec_queue *q;
1654 	u32 guc_id = msg[0];
1655 
1656 	if (unlikely(len < 1)) {
1657 		drm_err(&xe->drm, "Invalid length %u", len);
1658 		return -EPROTO;
1659 	}
1660 
1661 	q = g2h_exec_queue_lookup(guc, guc_id);
1662 	if (unlikely(!q))
1663 		return -EPROTO;
1664 
1665 	drm_info(&xe->drm, "Engine reset: guc_id=%d", guc_id);
1666 
1667 	/* FIXME: Do error capture, most likely async */
1668 
1669 	trace_xe_exec_queue_reset(q);
1670 
1671 	/*
1672 	 * A banned engine is a NOP at this point (came from
1673 	 * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel
1674 	 * jobs by setting timeout of the job to the minimum value kicking
1675 	 * guc_exec_queue_timedout_job.
1676 	 */
1677 	set_exec_queue_reset(q);
1678 	if (!exec_queue_banned(q))
1679 		xe_guc_exec_queue_trigger_cleanup(q);
1680 
1681 	return 0;
1682 }
1683 
1684 int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
1685 					       u32 len)
1686 {
1687 	struct xe_device *xe = guc_to_xe(guc);
1688 	struct xe_exec_queue *q;
1689 	u32 guc_id = msg[0];
1690 
1691 	if (unlikely(len < 1)) {
1692 		drm_err(&xe->drm, "Invalid length %u", len);
1693 		return -EPROTO;
1694 	}
1695 
1696 	q = g2h_exec_queue_lookup(guc, guc_id);
1697 	if (unlikely(!q))
1698 		return -EPROTO;
1699 
1700 	drm_dbg(&xe->drm, "Engine memory cat error: guc_id=%d", guc_id);
1701 	trace_xe_exec_queue_memory_cat_error(q);
1702 
1703 	/* Treat the same as engine reset */
1704 	set_exec_queue_reset(q);
1705 	if (!exec_queue_banned(q))
1706 		xe_guc_exec_queue_trigger_cleanup(q);
1707 
1708 	return 0;
1709 }
1710 
1711 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len)
1712 {
1713 	struct xe_device *xe = guc_to_xe(guc);
1714 	u8 guc_class, instance;
1715 	u32 reason;
1716 
1717 	if (unlikely(len != 3)) {
1718 		drm_err(&xe->drm, "Invalid length %u", len);
1719 		return -EPROTO;
1720 	}
1721 
1722 	guc_class = msg[0];
1723 	instance = msg[1];
1724 	reason = msg[2];
1725 
1726 	/* Unexpected failure of a hardware feature, log an actual error */
1727 	drm_err(&xe->drm, "GuC engine reset request failed on %d:%d because 0x%08X",
1728 		guc_class, instance, reason);
1729 
1730 	xe_gt_reset_async(guc_to_gt(guc));
1731 
1732 	return 0;
1733 }
1734 
1735 static void
1736 guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q,
1737 				   struct xe_guc_submit_exec_queue_snapshot *snapshot)
1738 {
1739 	struct xe_guc *guc = exec_queue_to_guc(q);
1740 	struct xe_device *xe = guc_to_xe(guc);
1741 	struct iosys_map map = xe_lrc_parallel_map(q->lrc);
1742 	int i;
1743 
1744 	snapshot->guc.wqi_head = q->guc->wqi_head;
1745 	snapshot->guc.wqi_tail = q->guc->wqi_tail;
1746 	snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head);
1747 	snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail);
1748 	snapshot->parallel.wq_desc.status = parallel_read(xe, map,
1749 							  wq_desc.wq_status);
1750 
1751 	if (snapshot->parallel.wq_desc.head !=
1752 	    snapshot->parallel.wq_desc.tail) {
1753 		for (i = snapshot->parallel.wq_desc.head;
1754 		     i != snapshot->parallel.wq_desc.tail;
1755 		     i = (i + sizeof(u32)) % WQ_SIZE)
1756 			snapshot->parallel.wq[i / sizeof(u32)] =
1757 				parallel_read(xe, map, wq[i / sizeof(u32)]);
1758 	}
1759 }
1760 
1761 static void
1762 guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,
1763 				 struct drm_printer *p)
1764 {
1765 	int i;
1766 
1767 	drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n",
1768 		   snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head);
1769 	drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n",
1770 		   snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail);
1771 	drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status);
1772 
1773 	if (snapshot->parallel.wq_desc.head !=
1774 	    snapshot->parallel.wq_desc.tail) {
1775 		for (i = snapshot->parallel.wq_desc.head;
1776 		     i != snapshot->parallel.wq_desc.tail;
1777 		     i = (i + sizeof(u32)) % WQ_SIZE)
1778 			drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32),
1779 				   snapshot->parallel.wq[i / sizeof(u32)]);
1780 	}
1781 }
1782 
1783 /**
1784  * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine.
1785  * @job: faulty Xe scheduled job.
1786  *
1787  * This can be printed out in a later stage like during dev_coredump
1788  * analysis.
1789  *
1790  * Returns: a GuC Submit Engine snapshot object that must be freed by the
1791  * caller, using `xe_guc_exec_queue_snapshot_free`.
1792  */
1793 struct xe_guc_submit_exec_queue_snapshot *
1794 xe_guc_exec_queue_snapshot_capture(struct xe_sched_job *job)
1795 {
1796 	struct xe_exec_queue *q = job->q;
1797 	struct xe_gpu_scheduler *sched = &q->guc->sched;
1798 	struct xe_guc_submit_exec_queue_snapshot *snapshot;
1799 	int i;
1800 
1801 	snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC);
1802 
1803 	if (!snapshot)
1804 		return NULL;
1805 
1806 	snapshot->guc.id = q->guc->id;
1807 	memcpy(&snapshot->name, &q->name, sizeof(snapshot->name));
1808 	snapshot->class = q->class;
1809 	snapshot->logical_mask = q->logical_mask;
1810 	snapshot->width = q->width;
1811 	snapshot->refcount = kref_read(&q->refcount);
1812 	snapshot->sched_timeout = sched->base.timeout;
1813 	snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us;
1814 	snapshot->sched_props.preempt_timeout_us =
1815 		q->sched_props.preempt_timeout_us;
1816 
1817 	snapshot->lrc = kmalloc_array(q->width, sizeof(struct lrc_snapshot),
1818 				      GFP_ATOMIC);
1819 
1820 	if (snapshot->lrc) {
1821 		for (i = 0; i < q->width; ++i) {
1822 			struct xe_lrc *lrc = q->lrc + i;
1823 
1824 			snapshot->lrc[i].context_desc =
1825 				lower_32_bits(xe_lrc_ggtt_addr(lrc));
1826 			snapshot->lrc[i].head = xe_lrc_ring_head(lrc);
1827 			snapshot->lrc[i].tail.internal = lrc->ring.tail;
1828 			snapshot->lrc[i].tail.memory =
1829 				xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL);
1830 			snapshot->lrc[i].start_seqno = xe_lrc_start_seqno(lrc);
1831 			snapshot->lrc[i].seqno = xe_lrc_seqno(lrc);
1832 		}
1833 	}
1834 
1835 	snapshot->schedule_state = atomic_read(&q->guc->state);
1836 	snapshot->exec_queue_flags = q->flags;
1837 
1838 	snapshot->parallel_execution = xe_exec_queue_is_parallel(q);
1839 	if (snapshot->parallel_execution)
1840 		guc_exec_queue_wq_snapshot_capture(q, snapshot);
1841 
1842 	spin_lock(&sched->base.job_list_lock);
1843 	snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list);
1844 	snapshot->pending_list = kmalloc_array(snapshot->pending_list_size,
1845 					       sizeof(struct pending_list_snapshot),
1846 					       GFP_ATOMIC);
1847 
1848 	if (snapshot->pending_list) {
1849 		struct xe_sched_job *job_iter;
1850 
1851 		i = 0;
1852 		list_for_each_entry(job_iter, &sched->base.pending_list, drm.list) {
1853 			snapshot->pending_list[i].seqno =
1854 				xe_sched_job_seqno(job_iter);
1855 			snapshot->pending_list[i].fence =
1856 				dma_fence_is_signaled(job_iter->fence) ? 1 : 0;
1857 			snapshot->pending_list[i].finished =
1858 				dma_fence_is_signaled(&job_iter->drm.s_fence->finished)
1859 				? 1 : 0;
1860 			i++;
1861 		}
1862 	}
1863 
1864 	spin_unlock(&sched->base.job_list_lock);
1865 
1866 	return snapshot;
1867 }
1868 
1869 /**
1870  * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot.
1871  * @snapshot: GuC Submit Engine snapshot object.
1872  * @p: drm_printer where it will be printed out.
1873  *
1874  * This function prints out a given GuC Submit Engine snapshot object.
1875  */
1876 void
1877 xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,
1878 				 struct drm_printer *p)
1879 {
1880 	int i;
1881 
1882 	if (!snapshot)
1883 		return;
1884 
1885 	drm_printf(p, "\nGuC ID: %d\n", snapshot->guc.id);
1886 	drm_printf(p, "\tName: %s\n", snapshot->name);
1887 	drm_printf(p, "\tClass: %d\n", snapshot->class);
1888 	drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask);
1889 	drm_printf(p, "\tWidth: %d\n", snapshot->width);
1890 	drm_printf(p, "\tRef: %d\n", snapshot->refcount);
1891 	drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout);
1892 	drm_printf(p, "\tTimeslice: %u (us)\n",
1893 		   snapshot->sched_props.timeslice_us);
1894 	drm_printf(p, "\tPreempt timeout: %u (us)\n",
1895 		   snapshot->sched_props.preempt_timeout_us);
1896 
1897 	for (i = 0; snapshot->lrc && i < snapshot->width; ++i) {
1898 		drm_printf(p, "\tHW Context Desc: 0x%08x\n",
1899 			   snapshot->lrc[i].context_desc);
1900 		drm_printf(p, "\tLRC Head: (memory) %u\n",
1901 			   snapshot->lrc[i].head);
1902 		drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n",
1903 			   snapshot->lrc[i].tail.internal,
1904 			   snapshot->lrc[i].tail.memory);
1905 		drm_printf(p, "\tStart seqno: (memory) %d\n",
1906 			   snapshot->lrc[i].start_seqno);
1907 		drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->lrc[i].seqno);
1908 	}
1909 	drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state);
1910 	drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags);
1911 
1912 	if (snapshot->parallel_execution)
1913 		guc_exec_queue_wq_snapshot_print(snapshot, p);
1914 
1915 	for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size;
1916 	     i++)
1917 		drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n",
1918 			   snapshot->pending_list[i].seqno,
1919 			   snapshot->pending_list[i].fence,
1920 			   snapshot->pending_list[i].finished);
1921 }
1922 
1923 /**
1924  * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given
1925  * snapshot.
1926  * @snapshot: GuC Submit Engine snapshot object.
1927  *
1928  * This function free all the memory that needed to be allocated at capture
1929  * time.
1930  */
1931 void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot)
1932 {
1933 	if (!snapshot)
1934 		return;
1935 
1936 	kfree(snapshot->lrc);
1937 	kfree(snapshot->pending_list);
1938 	kfree(snapshot);
1939 }
1940 
1941 static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p)
1942 {
1943 	struct xe_guc_submit_exec_queue_snapshot *snapshot;
1944 	struct xe_gpu_scheduler *sched = &q->guc->sched;
1945 	struct xe_sched_job *job;
1946 	bool found = false;
1947 
1948 	spin_lock(&sched->base.job_list_lock);
1949 	list_for_each_entry(job, &sched->base.pending_list, drm.list) {
1950 		if (job->q == q) {
1951 			xe_sched_job_get(job);
1952 			found = true;
1953 			break;
1954 		}
1955 	}
1956 	spin_unlock(&sched->base.job_list_lock);
1957 
1958 	if (!found)
1959 		return;
1960 
1961 	snapshot = xe_guc_exec_queue_snapshot_capture(job);
1962 	xe_guc_exec_queue_snapshot_print(snapshot, p);
1963 	xe_guc_exec_queue_snapshot_free(snapshot);
1964 
1965 	xe_sched_job_put(job);
1966 }
1967 
1968 /**
1969  * xe_guc_submit_print - GuC Submit Print.
1970  * @guc: GuC.
1971  * @p: drm_printer where it will be printed out.
1972  *
1973  * This function capture and prints snapshots of **all** GuC Engines.
1974  */
1975 void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p)
1976 {
1977 	struct xe_exec_queue *q;
1978 	unsigned long index;
1979 
1980 	if (!xe_device_uc_enabled(guc_to_xe(guc)))
1981 		return;
1982 
1983 	mutex_lock(&guc->submission_state.lock);
1984 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
1985 		guc_exec_queue_print(q, p);
1986 	mutex_unlock(&guc->submission_state.lock);
1987 }
1988