xref: /linux/drivers/gpu/drm/xe/xe_guc_ct.c (revision 3ef7acec975bde28ab9cef92af76be8fc2ce684d)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_guc_ct.h"
7 
8 #include <linux/bitfield.h>
9 #include <linux/circ_buf.h>
10 #include <linux/delay.h>
11 #include <linux/fault-inject.h>
12 
13 #include <kunit/static_stub.h>
14 
15 #include <drm/drm_managed.h>
16 
17 #include "abi/guc_actions_abi.h"
18 #include "abi/guc_actions_sriov_abi.h"
19 #include "abi/guc_klvs_abi.h"
20 #include "xe_bo.h"
21 #include "xe_devcoredump.h"
22 #include "xe_device.h"
23 #include "xe_gt.h"
24 #include "xe_gt_pagefault.h"
25 #include "xe_gt_printk.h"
26 #include "xe_gt_sriov_pf_control.h"
27 #include "xe_gt_sriov_pf_monitor.h"
28 #include "xe_gt_tlb_invalidation.h"
29 #include "xe_guc.h"
30 #include "xe_guc_log.h"
31 #include "xe_guc_relay.h"
32 #include "xe_guc_submit.h"
33 #include "xe_map.h"
34 #include "xe_pm.h"
35 #include "xe_trace_guc.h"
36 
37 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
38 enum {
39 	/* Internal states, not error conditions */
40 	CT_DEAD_STATE_REARM,			/* 0x0001 */
41 	CT_DEAD_STATE_CAPTURE,			/* 0x0002 */
42 
43 	/* Error conditions */
44 	CT_DEAD_SETUP,				/* 0x0004 */
45 	CT_DEAD_H2G_WRITE,			/* 0x0008 */
46 	CT_DEAD_H2G_HAS_ROOM,			/* 0x0010 */
47 	CT_DEAD_G2H_READ,			/* 0x0020 */
48 	CT_DEAD_G2H_RECV,			/* 0x0040 */
49 	CT_DEAD_G2H_RELEASE,			/* 0x0080 */
50 	CT_DEAD_DEADLOCK,			/* 0x0100 */
51 	CT_DEAD_PROCESS_FAILED,			/* 0x0200 */
52 	CT_DEAD_FAST_G2H,			/* 0x0400 */
53 	CT_DEAD_PARSE_G2H_RESPONSE,		/* 0x0800 */
54 	CT_DEAD_PARSE_G2H_UNKNOWN,		/* 0x1000 */
55 	CT_DEAD_PARSE_G2H_ORIGIN,		/* 0x2000 */
56 	CT_DEAD_PARSE_G2H_TYPE,			/* 0x4000 */
57 	CT_DEAD_CRASH,				/* 0x8000 */
58 };
59 
60 static void ct_dead_worker_func(struct work_struct *w);
61 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code);
62 
63 #define CT_DEAD(ct, ctb, reason_code)		ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
64 #else
65 #define CT_DEAD(ct, ctb, reason)			\
66 	do {						\
67 		struct guc_ctb *_ctb = (ctb);		\
68 		if (_ctb)				\
69 			_ctb->info.broken = true;	\
70 	} while (0)
71 #endif
72 
73 /* Used when a CT send wants to block and / or receive data */
74 struct g2h_fence {
75 	u32 *response_buffer;
76 	u32 seqno;
77 	u32 response_data;
78 	u16 response_len;
79 	u16 error;
80 	u16 hint;
81 	u16 reason;
82 	bool retry;
83 	bool fail;
84 	bool done;
85 };
86 
g2h_fence_init(struct g2h_fence * g2h_fence,u32 * response_buffer)87 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer)
88 {
89 	g2h_fence->response_buffer = response_buffer;
90 	g2h_fence->response_data = 0;
91 	g2h_fence->response_len = 0;
92 	g2h_fence->fail = false;
93 	g2h_fence->retry = false;
94 	g2h_fence->done = false;
95 	g2h_fence->seqno = ~0x0;
96 }
97 
g2h_fence_needs_alloc(struct g2h_fence * g2h_fence)98 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
99 {
100 	return g2h_fence->seqno == ~0x0;
101 }
102 
103 static struct xe_guc *
ct_to_guc(struct xe_guc_ct * ct)104 ct_to_guc(struct xe_guc_ct *ct)
105 {
106 	return container_of(ct, struct xe_guc, ct);
107 }
108 
109 static struct xe_gt *
ct_to_gt(struct xe_guc_ct * ct)110 ct_to_gt(struct xe_guc_ct *ct)
111 {
112 	return container_of(ct, struct xe_gt, uc.guc.ct);
113 }
114 
115 static struct xe_device *
ct_to_xe(struct xe_guc_ct * ct)116 ct_to_xe(struct xe_guc_ct *ct)
117 {
118 	return gt_to_xe(ct_to_gt(ct));
119 }
120 
121 /**
122  * DOC: GuC CTB Blob
123  *
124  * We allocate single blob to hold both CTB descriptors and buffers:
125  *
126  *      +--------+-----------------------------------------------+------+
127  *      | offset | contents                                      | size |
128  *      +========+===============================================+======+
129  *      | 0x0000 | H2G CTB Descriptor (send)                     |      |
130  *      +--------+-----------------------------------------------+  4K  |
131  *      | 0x0800 | G2H CTB Descriptor (g2h)                      |      |
132  *      +--------+-----------------------------------------------+------+
133  *      | 0x1000 | H2G CT Buffer (send)                          | n*4K |
134  *      |        |                                               |      |
135  *      +--------+-----------------------------------------------+------+
136  *      | 0x1000 | G2H CT Buffer (g2h)                           | m*4K |
137  *      | + n*4K |                                               |      |
138  *      +--------+-----------------------------------------------+------+
139  *
140  * Size of each ``CT Buffer`` must be multiple of 4K.
141  * We don't expect too many messages in flight at any time, unless we are
142  * using the GuC submission. In that case each request requires a minimum
143  * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
144  * enough space to avoid backpressure on the driver. We increase the size
145  * of the receive buffer (relative to the send) to ensure a G2H response
146  * CTB has a landing spot.
147  *
148  * In addition to submissions, the G2H buffer needs to be able to hold
149  * enough space for recoverable page fault notifications. The number of
150  * page faults is interrupt driven and can be as much as the number of
151  * compute resources available. However, most of the actual work for these
152  * is in a separate page fault worker thread. Therefore we only need to
153  * make sure the queue has enough space to handle all of the submissions
154  * and responses and an extra buffer for incoming page faults.
155  */
156 
157 #define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
158 #define CTB_H2G_BUFFER_SIZE	(SZ_4K)
159 #define CTB_G2H_BUFFER_SIZE	(SZ_128K)
160 #define G2H_ROOM_BUFFER_SIZE	(CTB_G2H_BUFFER_SIZE / 2)
161 
162 /**
163  * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full
164  * CT command queue
165  * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future.
166  *
167  * Observation is that a 4KiB buffer full of commands takes a little over a
168  * second to process. Use that to calculate maximum time to process a full CT
169  * command queue.
170  *
171  * Return: Maximum time to process a full CT queue in jiffies.
172  */
xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct * ct)173 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct)
174 {
175 	BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4));
176 	return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ;
177 }
178 
guc_ct_size(void)179 static size_t guc_ct_size(void)
180 {
181 	return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE +
182 		CTB_G2H_BUFFER_SIZE;
183 }
184 
guc_ct_fini(struct drm_device * drm,void * arg)185 static void guc_ct_fini(struct drm_device *drm, void *arg)
186 {
187 	struct xe_guc_ct *ct = arg;
188 
189 	destroy_workqueue(ct->g2h_wq);
190 	xa_destroy(&ct->fence_lookup);
191 }
192 
193 static void receive_g2h(struct xe_guc_ct *ct);
194 static void g2h_worker_func(struct work_struct *w);
195 static void safe_mode_worker_func(struct work_struct *w);
196 
primelockdep(struct xe_guc_ct * ct)197 static void primelockdep(struct xe_guc_ct *ct)
198 {
199 	if (!IS_ENABLED(CONFIG_LOCKDEP))
200 		return;
201 
202 	fs_reclaim_acquire(GFP_KERNEL);
203 	might_lock(&ct->lock);
204 	fs_reclaim_release(GFP_KERNEL);
205 }
206 
xe_guc_ct_init(struct xe_guc_ct * ct)207 int xe_guc_ct_init(struct xe_guc_ct *ct)
208 {
209 	struct xe_device *xe = ct_to_xe(ct);
210 	struct xe_gt *gt = ct_to_gt(ct);
211 	struct xe_tile *tile = gt_to_tile(gt);
212 	struct xe_bo *bo;
213 	int err;
214 
215 	xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE));
216 
217 	ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM);
218 	if (!ct->g2h_wq)
219 		return -ENOMEM;
220 
221 	spin_lock_init(&ct->fast_lock);
222 	xa_init(&ct->fence_lookup);
223 	INIT_WORK(&ct->g2h_worker, g2h_worker_func);
224 	INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);
225 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
226 	spin_lock_init(&ct->dead.lock);
227 	INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
228 #endif
229 	init_waitqueue_head(&ct->wq);
230 	init_waitqueue_head(&ct->g2h_fence_wq);
231 
232 	err = drmm_mutex_init(&xe->drm, &ct->lock);
233 	if (err)
234 		return err;
235 
236 	primelockdep(ct);
237 
238 	bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
239 					  XE_BO_FLAG_SYSTEM |
240 					  XE_BO_FLAG_GGTT |
241 					  XE_BO_FLAG_GGTT_INVALIDATE);
242 	if (IS_ERR(bo))
243 		return PTR_ERR(bo);
244 
245 	ct->bo = bo;
246 
247 	err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct);
248 	if (err)
249 		return err;
250 
251 	xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED);
252 	ct->state = XE_GUC_CT_STATE_DISABLED;
253 	return 0;
254 }
255 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */
256 
257 #define desc_read(xe_, guc_ctb__, field_)			\
258 	xe_map_rd_field(xe_, &guc_ctb__->desc, 0,		\
259 			struct guc_ct_buffer_desc, field_)
260 
261 #define desc_write(xe_, guc_ctb__, field_, val_)		\
262 	xe_map_wr_field(xe_, &guc_ctb__->desc, 0,		\
263 			struct guc_ct_buffer_desc, field_, val_)
264 
guc_ct_ctb_h2g_init(struct xe_device * xe,struct guc_ctb * h2g,struct iosys_map * map)265 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,
266 				struct iosys_map *map)
267 {
268 	h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);
269 	h2g->info.resv_space = 0;
270 	h2g->info.tail = 0;
271 	h2g->info.head = 0;
272 	h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
273 				     h2g->info.size) -
274 			  h2g->info.resv_space;
275 	h2g->info.broken = false;
276 
277 	h2g->desc = *map;
278 	xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
279 
280 	h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2);
281 }
282 
guc_ct_ctb_g2h_init(struct xe_device * xe,struct guc_ctb * g2h,struct iosys_map * map)283 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,
284 				struct iosys_map *map)
285 {
286 	g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);
287 	g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);
288 	g2h->info.head = 0;
289 	g2h->info.tail = 0;
290 	g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
291 				     g2h->info.size) -
292 			  g2h->info.resv_space;
293 	g2h->info.broken = false;
294 
295 	g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE);
296 	xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
297 
298 	g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 +
299 					    CTB_H2G_BUFFER_SIZE);
300 }
301 
guc_ct_ctb_h2g_register(struct xe_guc_ct * ct)302 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)
303 {
304 	struct xe_guc *guc = ct_to_guc(ct);
305 	u32 desc_addr, ctb_addr, size;
306 	int err;
307 
308 	desc_addr = xe_bo_ggtt_addr(ct->bo);
309 	ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2;
310 	size = ct->ctbs.h2g.info.size * sizeof(u32);
311 
312 	err = xe_guc_self_cfg64(guc,
313 				GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY,
314 				desc_addr);
315 	if (err)
316 		return err;
317 
318 	err = xe_guc_self_cfg64(guc,
319 				GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY,
320 				ctb_addr);
321 	if (err)
322 		return err;
323 
324 	return xe_guc_self_cfg32(guc,
325 				 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY,
326 				 size);
327 }
328 
guc_ct_ctb_g2h_register(struct xe_guc_ct * ct)329 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct)
330 {
331 	struct xe_guc *guc = ct_to_guc(ct);
332 	u32 desc_addr, ctb_addr, size;
333 	int err;
334 
335 	desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE;
336 	ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 +
337 		CTB_H2G_BUFFER_SIZE;
338 	size = ct->ctbs.g2h.info.size * sizeof(u32);
339 
340 	err = xe_guc_self_cfg64(guc,
341 				GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
342 				desc_addr);
343 	if (err)
344 		return err;
345 
346 	err = xe_guc_self_cfg64(guc,
347 				GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
348 				ctb_addr);
349 	if (err)
350 		return err;
351 
352 	return xe_guc_self_cfg32(guc,
353 				 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
354 				 size);
355 }
356 
guc_ct_control_toggle(struct xe_guc_ct * ct,bool enable)357 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable)
358 {
359 	u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
360 		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
361 		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
362 		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
363 			   GUC_ACTION_HOST2GUC_CONTROL_CTB),
364 		FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL,
365 			   enable ? GUC_CTB_CONTROL_ENABLE :
366 			   GUC_CTB_CONTROL_DISABLE),
367 	};
368 	int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request));
369 
370 	return ret > 0 ? -EPROTO : ret;
371 }
372 
xe_guc_ct_set_state(struct xe_guc_ct * ct,enum xe_guc_ct_state state)373 static void xe_guc_ct_set_state(struct xe_guc_ct *ct,
374 				enum xe_guc_ct_state state)
375 {
376 	mutex_lock(&ct->lock);		/* Serialise dequeue_one_g2h() */
377 	spin_lock_irq(&ct->fast_lock);	/* Serialise CT fast-path */
378 
379 	xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 ||
380 		     state == XE_GUC_CT_STATE_STOPPED);
381 
382 	if (ct->g2h_outstanding)
383 		xe_pm_runtime_put(ct_to_xe(ct));
384 	ct->g2h_outstanding = 0;
385 	ct->state = state;
386 
387 	spin_unlock_irq(&ct->fast_lock);
388 
389 	/*
390 	 * Lockdep doesn't like this under the fast lock and he destroy only
391 	 * needs to be serialized with the send path which ct lock provides.
392 	 */
393 	xa_destroy(&ct->fence_lookup);
394 
395 	mutex_unlock(&ct->lock);
396 }
397 
ct_needs_safe_mode(struct xe_guc_ct * ct)398 static bool ct_needs_safe_mode(struct xe_guc_ct *ct)
399 {
400 	return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev));
401 }
402 
ct_restart_safe_mode_worker(struct xe_guc_ct * ct)403 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct)
404 {
405 	if (!ct_needs_safe_mode(ct))
406 		return false;
407 
408 	queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10);
409 	return true;
410 }
411 
safe_mode_worker_func(struct work_struct * w)412 static void safe_mode_worker_func(struct work_struct *w)
413 {
414 	struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work);
415 
416 	receive_g2h(ct);
417 
418 	if (!ct_restart_safe_mode_worker(ct))
419 		xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n");
420 }
421 
ct_enter_safe_mode(struct xe_guc_ct * ct)422 static void ct_enter_safe_mode(struct xe_guc_ct *ct)
423 {
424 	if (ct_restart_safe_mode_worker(ct))
425 		xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n");
426 }
427 
ct_exit_safe_mode(struct xe_guc_ct * ct)428 static void ct_exit_safe_mode(struct xe_guc_ct *ct)
429 {
430 	if (cancel_delayed_work_sync(&ct->safe_mode_worker))
431 		xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n");
432 }
433 
xe_guc_ct_enable(struct xe_guc_ct * ct)434 int xe_guc_ct_enable(struct xe_guc_ct *ct)
435 {
436 	struct xe_device *xe = ct_to_xe(ct);
437 	struct xe_gt *gt = ct_to_gt(ct);
438 	int err;
439 
440 	xe_gt_assert(gt, !xe_guc_ct_enabled(ct));
441 
442 	xe_map_memset(xe, &ct->bo->vmap, 0, 0, ct->bo->size);
443 	guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);
444 	guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);
445 
446 	err = guc_ct_ctb_h2g_register(ct);
447 	if (err)
448 		goto err_out;
449 
450 	err = guc_ct_ctb_g2h_register(ct);
451 	if (err)
452 		goto err_out;
453 
454 	err = guc_ct_control_toggle(ct, true);
455 	if (err)
456 		goto err_out;
457 
458 	xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED);
459 
460 	smp_mb();
461 	wake_up_all(&ct->wq);
462 	xe_gt_dbg(gt, "GuC CT communication channel enabled\n");
463 
464 	if (ct_needs_safe_mode(ct))
465 		ct_enter_safe_mode(ct);
466 
467 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
468 	/*
469 	 * The CT has now been reset so the dumper can be re-armed
470 	 * after any existing dead state has been dumped.
471 	 */
472 	spin_lock_irq(&ct->dead.lock);
473 	if (ct->dead.reason) {
474 		ct->dead.reason |= (1 << CT_DEAD_STATE_REARM);
475 		queue_work(system_unbound_wq, &ct->dead.worker);
476 	}
477 	spin_unlock_irq(&ct->dead.lock);
478 #endif
479 
480 	return 0;
481 
482 err_out:
483 	xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err));
484 	CT_DEAD(ct, NULL, SETUP);
485 
486 	return err;
487 }
488 
stop_g2h_handler(struct xe_guc_ct * ct)489 static void stop_g2h_handler(struct xe_guc_ct *ct)
490 {
491 	cancel_work_sync(&ct->g2h_worker);
492 }
493 
494 /**
495  * xe_guc_ct_disable - Set GuC to disabled state
496  * @ct: the &xe_guc_ct
497  *
498  * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected
499  * in this transition.
500  */
xe_guc_ct_disable(struct xe_guc_ct * ct)501 void xe_guc_ct_disable(struct xe_guc_ct *ct)
502 {
503 	xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED);
504 	ct_exit_safe_mode(ct);
505 	stop_g2h_handler(ct);
506 }
507 
508 /**
509  * xe_guc_ct_stop - Set GuC to stopped state
510  * @ct: the &xe_guc_ct
511  *
512  * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h
513  */
xe_guc_ct_stop(struct xe_guc_ct * ct)514 void xe_guc_ct_stop(struct xe_guc_ct *ct)
515 {
516 	xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED);
517 	stop_g2h_handler(ct);
518 }
519 
h2g_has_room(struct xe_guc_ct * ct,u32 cmd_len)520 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
521 {
522 	struct guc_ctb *h2g = &ct->ctbs.h2g;
523 
524 	lockdep_assert_held(&ct->lock);
525 
526 	if (cmd_len > h2g->info.space) {
527 		h2g->info.head = desc_read(ct_to_xe(ct), h2g, head);
528 
529 		if (h2g->info.head > h2g->info.size) {
530 			struct xe_device *xe = ct_to_xe(ct);
531 			u32 desc_status = desc_read(xe, h2g, status);
532 
533 			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
534 
535 			xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n",
536 				  h2g->info.head, h2g->info.size);
537 			CT_DEAD(ct, h2g, H2G_HAS_ROOM);
538 			return false;
539 		}
540 
541 		h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
542 					     h2g->info.size) -
543 				  h2g->info.resv_space;
544 		if (cmd_len > h2g->info.space)
545 			return false;
546 	}
547 
548 	return true;
549 }
550 
g2h_has_room(struct xe_guc_ct * ct,u32 g2h_len)551 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len)
552 {
553 	if (!g2h_len)
554 		return true;
555 
556 	lockdep_assert_held(&ct->fast_lock);
557 
558 	return ct->ctbs.g2h.info.space > g2h_len;
559 }
560 
has_room(struct xe_guc_ct * ct,u32 cmd_len,u32 g2h_len)561 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len)
562 {
563 	lockdep_assert_held(&ct->lock);
564 
565 	if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len))
566 		return -EBUSY;
567 
568 	return 0;
569 }
570 
h2g_reserve_space(struct xe_guc_ct * ct,u32 cmd_len)571 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)
572 {
573 	lockdep_assert_held(&ct->lock);
574 	ct->ctbs.h2g.info.space -= cmd_len;
575 }
576 
__g2h_reserve_space(struct xe_guc_ct * ct,u32 g2h_len,u32 num_g2h)577 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
578 {
579 	xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space);
580 	xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) ||
581 		     (g2h_len && num_g2h));
582 
583 	if (g2h_len) {
584 		lockdep_assert_held(&ct->fast_lock);
585 
586 		if (!ct->g2h_outstanding)
587 			xe_pm_runtime_get_noresume(ct_to_xe(ct));
588 
589 		ct->ctbs.g2h.info.space -= g2h_len;
590 		ct->g2h_outstanding += num_g2h;
591 	}
592 }
593 
__g2h_release_space(struct xe_guc_ct * ct,u32 g2h_len)594 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
595 {
596 	bool bad = false;
597 
598 	lockdep_assert_held(&ct->fast_lock);
599 
600 	bad = ct->ctbs.g2h.info.space + g2h_len >
601 		     ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space;
602 	bad |= !ct->g2h_outstanding;
603 
604 	if (bad) {
605 		xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n",
606 			  ct->ctbs.g2h.info.space, g2h_len,
607 			  ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space,
608 			  ct->ctbs.g2h.info.space + g2h_len,
609 			  ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space,
610 			  ct->g2h_outstanding);
611 		CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE);
612 		return;
613 	}
614 
615 	ct->ctbs.g2h.info.space += g2h_len;
616 	if (!--ct->g2h_outstanding)
617 		xe_pm_runtime_put(ct_to_xe(ct));
618 }
619 
g2h_release_space(struct xe_guc_ct * ct,u32 g2h_len)620 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
621 {
622 	spin_lock_irq(&ct->fast_lock);
623 	__g2h_release_space(ct, g2h_len);
624 	spin_unlock_irq(&ct->fast_lock);
625 }
626 
627 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */
628 
h2g_write(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 ct_fence_value,bool want_response)629 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
630 		     u32 ct_fence_value, bool want_response)
631 {
632 	struct xe_device *xe = ct_to_xe(ct);
633 	struct xe_gt *gt = ct_to_gt(ct);
634 	struct guc_ctb *h2g = &ct->ctbs.h2g;
635 	u32 cmd[H2G_CT_HEADERS];
636 	u32 tail = h2g->info.tail;
637 	u32 full_len;
638 	struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds,
639 							 tail * sizeof(u32));
640 	u32 desc_status;
641 
642 	full_len = len + GUC_CTB_HDR_LEN;
643 
644 	lockdep_assert_held(&ct->lock);
645 	xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);
646 
647 	desc_status = desc_read(xe, h2g, status);
648 	if (desc_status) {
649 		xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status);
650 		goto corrupted;
651 	}
652 
653 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
654 		u32 desc_tail = desc_read(xe, h2g, tail);
655 		u32 desc_head = desc_read(xe, h2g, head);
656 
657 		if (tail != desc_tail) {
658 			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH);
659 			xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail);
660 			goto corrupted;
661 		}
662 
663 		if (tail > h2g->info.size) {
664 			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
665 			xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n",
666 				  tail, h2g->info.size);
667 			goto corrupted;
668 		}
669 
670 		if (desc_head >= h2g->info.size) {
671 			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
672 			xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n",
673 				  desc_head, h2g->info.size);
674 			goto corrupted;
675 		}
676 	}
677 
678 	/* Command will wrap, zero fill (NOPs), return and check credits again */
679 	if (tail + full_len > h2g->info.size) {
680 		xe_map_memset(xe, &map, 0, 0,
681 			      (h2g->info.size - tail) * sizeof(u32));
682 		h2g_reserve_space(ct, (h2g->info.size - tail));
683 		h2g->info.tail = 0;
684 		desc_write(xe, h2g, tail, h2g->info.tail);
685 
686 		return -EAGAIN;
687 	}
688 
689 	/*
690 	 * dw0: CT header (including fence)
691 	 * dw1: HXG header (including action code)
692 	 * dw2+: action data
693 	 */
694 	cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
695 		FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
696 		FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value);
697 	if (want_response) {
698 		cmd[1] =
699 			FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
700 			FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
701 				   GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
702 	} else {
703 		cmd[1] =
704 			FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) |
705 			FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
706 				   GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
707 	}
708 
709 	/* H2G header in cmd[1] replaces action[0] so: */
710 	--len;
711 	++action;
712 
713 	/* Write H2G ensuring visible before descriptor update */
714 	xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32));
715 	xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32));
716 	xe_device_wmb(xe);
717 
718 	/* Update local copies */
719 	h2g->info.tail = (tail + full_len) % h2g->info.size;
720 	h2g_reserve_space(ct, full_len);
721 
722 	/* Update descriptor */
723 	desc_write(xe, h2g, tail, h2g->info.tail);
724 
725 	trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len,
726 			     desc_read(xe, h2g, head), h2g->info.tail);
727 
728 	return 0;
729 
730 corrupted:
731 	CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE);
732 	return -EPIPE;
733 }
734 
735 /*
736  * The CT protocol accepts a 16 bits fence. This field is fully owned by the
737  * driver, the GuC will just copy it to the reply message. Since we need to
738  * be able to distinguish between replies to REQUEST and FAST_REQUEST messages,
739  * we use one bit of the seqno as an indicator for that and a rolling counter
740  * for the remaining 15 bits.
741  */
742 #define CT_SEQNO_MASK GENMASK(14, 0)
743 #define CT_SEQNO_UNTRACKED BIT(15)
next_ct_seqno(struct xe_guc_ct * ct,bool is_g2h_fence)744 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence)
745 {
746 	u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK;
747 
748 	if (!is_g2h_fence)
749 		seqno |= CT_SEQNO_UNTRACKED;
750 
751 	return seqno;
752 }
753 
__guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)754 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
755 				u32 len, u32 g2h_len, u32 num_g2h,
756 				struct g2h_fence *g2h_fence)
757 {
758 	struct xe_gt *gt __maybe_unused = ct_to_gt(ct);
759 	u16 seqno;
760 	int ret;
761 
762 	xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
763 	xe_gt_assert(gt, !g2h_len || !g2h_fence);
764 	xe_gt_assert(gt, !num_g2h || !g2h_fence);
765 	xe_gt_assert(gt, !g2h_len || num_g2h);
766 	xe_gt_assert(gt, g2h_len || !num_g2h);
767 	lockdep_assert_held(&ct->lock);
768 
769 	if (unlikely(ct->ctbs.h2g.info.broken)) {
770 		ret = -EPIPE;
771 		goto out;
772 	}
773 
774 	if (ct->state == XE_GUC_CT_STATE_DISABLED) {
775 		ret = -ENODEV;
776 		goto out;
777 	}
778 
779 	if (ct->state == XE_GUC_CT_STATE_STOPPED) {
780 		ret = -ECANCELED;
781 		goto out;
782 	}
783 
784 	xe_gt_assert(gt, xe_guc_ct_enabled(ct));
785 
786 	if (g2h_fence) {
787 		g2h_len = GUC_CTB_HXG_MSG_MAX_LEN;
788 		num_g2h = 1;
789 
790 		if (g2h_fence_needs_alloc(g2h_fence)) {
791 			g2h_fence->seqno = next_ct_seqno(ct, true);
792 			ret = xa_err(xa_store(&ct->fence_lookup,
793 					      g2h_fence->seqno, g2h_fence,
794 					      GFP_ATOMIC));
795 			if (ret)
796 				goto out;
797 		}
798 
799 		seqno = g2h_fence->seqno;
800 	} else {
801 		seqno = next_ct_seqno(ct, false);
802 	}
803 
804 	if (g2h_len)
805 		spin_lock_irq(&ct->fast_lock);
806 retry:
807 	ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len);
808 	if (unlikely(ret))
809 		goto out_unlock;
810 
811 	ret = h2g_write(ct, action, len, seqno, !!g2h_fence);
812 	if (unlikely(ret)) {
813 		if (ret == -EAGAIN)
814 			goto retry;
815 		goto out_unlock;
816 	}
817 
818 	__g2h_reserve_space(ct, g2h_len, num_g2h);
819 	xe_guc_notify(ct_to_guc(ct));
820 out_unlock:
821 	if (g2h_len)
822 		spin_unlock_irq(&ct->fast_lock);
823 out:
824 	return ret;
825 }
826 
kick_reset(struct xe_guc_ct * ct)827 static void kick_reset(struct xe_guc_ct *ct)
828 {
829 	xe_gt_reset_async(ct_to_gt(ct));
830 }
831 
832 static int dequeue_one_g2h(struct xe_guc_ct *ct);
833 
guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)834 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
835 			      u32 g2h_len, u32 num_g2h,
836 			      struct g2h_fence *g2h_fence)
837 {
838 	struct xe_device *xe = ct_to_xe(ct);
839 	struct xe_gt *gt = ct_to_gt(ct);
840 	unsigned int sleep_period_ms = 1;
841 	int ret;
842 
843 	xe_gt_assert(gt, !g2h_len || !g2h_fence);
844 	lockdep_assert_held(&ct->lock);
845 	xe_device_assert_mem_access(ct_to_xe(ct));
846 
847 try_again:
848 	ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h,
849 				   g2h_fence);
850 
851 	/*
852 	 * We wait to try to restore credits for about 1 second before bailing.
853 	 * In the case of H2G credits we have no choice but just to wait for the
854 	 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In
855 	 * the case of G2H we process any G2H in the channel, hopefully freeing
856 	 * credits as we consume the G2H messages.
857 	 */
858 	if (unlikely(ret == -EBUSY &&
859 		     !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) {
860 		struct guc_ctb *h2g = &ct->ctbs.h2g;
861 
862 		if (sleep_period_ms == 1024)
863 			goto broken;
864 
865 		trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail,
866 						 h2g->info.size,
867 						 h2g->info.space,
868 						 len + GUC_CTB_HDR_LEN);
869 		msleep(sleep_period_ms);
870 		sleep_period_ms <<= 1;
871 
872 		goto try_again;
873 	} else if (unlikely(ret == -EBUSY)) {
874 		struct xe_device *xe = ct_to_xe(ct);
875 		struct guc_ctb *g2h = &ct->ctbs.g2h;
876 
877 		trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head,
878 						 desc_read(xe, g2h, tail),
879 						 g2h->info.size,
880 						 g2h->info.space,
881 						 g2h_fence ?
882 						 GUC_CTB_HXG_MSG_MAX_LEN :
883 						 g2h_len);
884 
885 #define g2h_avail(ct)	\
886 	(desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head)
887 		if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding ||
888 					g2h_avail(ct), HZ))
889 			goto broken;
890 #undef g2h_avail
891 
892 		ret = dequeue_one_g2h(ct);
893 		if (ret < 0) {
894 			if (ret != -ECANCELED)
895 				xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)",
896 					  ERR_PTR(ret));
897 			goto broken;
898 		}
899 
900 		goto try_again;
901 	}
902 
903 	return ret;
904 
905 broken:
906 	xe_gt_err(gt, "No forward process on H2G, reset required\n");
907 	CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK);
908 
909 	return -EDEADLK;
910 }
911 
guc_ct_send(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)912 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
913 		       u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence)
914 {
915 	int ret;
916 
917 	xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence);
918 
919 	mutex_lock(&ct->lock);
920 	ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);
921 	mutex_unlock(&ct->lock);
922 
923 	return ret;
924 }
925 
xe_guc_ct_send(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h)926 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
927 		   u32 g2h_len, u32 num_g2h)
928 {
929 	int ret;
930 
931 	ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL);
932 	if (ret == -EDEADLK)
933 		kick_reset(ct);
934 
935 	return ret;
936 }
937 
xe_guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h)938 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
939 			  u32 g2h_len, u32 num_g2h)
940 {
941 	int ret;
942 
943 	ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL);
944 	if (ret == -EDEADLK)
945 		kick_reset(ct);
946 
947 	return ret;
948 }
949 
xe_guc_ct_send_g2h_handler(struct xe_guc_ct * ct,const u32 * action,u32 len)950 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len)
951 {
952 	int ret;
953 
954 	lockdep_assert_held(&ct->lock);
955 
956 	ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL);
957 	if (ret == -EDEADLK)
958 		kick_reset(ct);
959 
960 	return ret;
961 }
962 
963 /*
964  * Check if a GT reset is in progress or will occur and if GT reset brought the
965  * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset.
966  */
retry_failure(struct xe_guc_ct * ct,int ret)967 static bool retry_failure(struct xe_guc_ct *ct, int ret)
968 {
969 	if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV))
970 		return false;
971 
972 #define ct_alive(ct)	\
973 	(xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \
974 	 !ct->ctbs.g2h.info.broken)
975 	if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5))
976 		return false;
977 #undef ct_alive
978 
979 	return true;
980 }
981 
guc_ct_send_recv(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer,bool no_fail)982 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
983 			    u32 *response_buffer, bool no_fail)
984 {
985 	struct xe_gt *gt = ct_to_gt(ct);
986 	struct g2h_fence g2h_fence;
987 	int ret = 0;
988 
989 	/*
990 	 * We use a fence to implement blocking sends / receiving response data.
991 	 * The seqno of the fence is sent in the H2G, returned in the G2H, and
992 	 * an xarray is used as storage media with the seqno being to key.
993 	 * Fields in the fence hold success, failure, retry status and the
994 	 * response data. Safe to allocate on the stack as the xarray is the
995 	 * only reference and it cannot be present after this function exits.
996 	 */
997 retry:
998 	g2h_fence_init(&g2h_fence, response_buffer);
999 retry_same_fence:
1000 	ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence);
1001 	if (unlikely(ret == -ENOMEM)) {
1002 		/* Retry allocation /w GFP_KERNEL */
1003 		ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno,
1004 				      &g2h_fence, GFP_KERNEL));
1005 		if (ret)
1006 			return ret;
1007 
1008 		goto retry_same_fence;
1009 	} else if (unlikely(ret)) {
1010 		if (ret == -EDEADLK)
1011 			kick_reset(ct);
1012 
1013 		if (no_fail && retry_failure(ct, ret))
1014 			goto retry_same_fence;
1015 
1016 		if (!g2h_fence_needs_alloc(&g2h_fence))
1017 			xa_erase(&ct->fence_lookup, g2h_fence.seqno);
1018 
1019 		return ret;
1020 	}
1021 
1022 	ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ);
1023 	if (!ret) {
1024 		LNL_FLUSH_WORK(&ct->g2h_worker);
1025 		if (g2h_fence.done) {
1026 			xe_gt_warn(gt, "G2H fence %u, action %04x, done\n",
1027 				   g2h_fence.seqno, action[0]);
1028 			ret = 1;
1029 		}
1030 	}
1031 
1032 	/*
1033 	 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on
1034 	 * the stack, since we have no clue if it will fire after the timeout before we can erase
1035 	 * from the xa. Also we have some dependent loads and stores below for which we need the
1036 	 * correct ordering, and we lack the needed barriers.
1037 	 */
1038 	mutex_lock(&ct->lock);
1039 	if (!ret) {
1040 		xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s",
1041 			  g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done));
1042 		xa_erase(&ct->fence_lookup, g2h_fence.seqno);
1043 		mutex_unlock(&ct->lock);
1044 		return -ETIME;
1045 	}
1046 
1047 	if (g2h_fence.retry) {
1048 		xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n",
1049 			  action[0], g2h_fence.reason);
1050 		mutex_unlock(&ct->lock);
1051 		goto retry;
1052 	}
1053 	if (g2h_fence.fail) {
1054 		xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n",
1055 			  action[0], g2h_fence.error, g2h_fence.hint);
1056 		ret = -EIO;
1057 	}
1058 
1059 	if (ret > 0)
1060 		ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data;
1061 
1062 	mutex_unlock(&ct->lock);
1063 
1064 	return ret;
1065 }
1066 
1067 /**
1068  * xe_guc_ct_send_recv - Send and receive HXG to the GuC
1069  * @ct: the &xe_guc_ct
1070  * @action: the dword array with `HXG Request`_ message (can't be NULL)
1071  * @len: length of the `HXG Request`_ message (in dwords, can't be 0)
1072  * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL)
1073  *
1074  * Send a `HXG Request`_ message to the GuC over CT communication channel and
1075  * blocks until GuC replies with a `HXG Response`_ message.
1076  *
1077  * For non-blocking communication with GuC use xe_guc_ct_send().
1078  *
1079  * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_.
1080  *
1081  * Return: response length (in dwords) if &response_buffer was not NULL, or
1082  *         DATA0 from `HXG Response`_ if &response_buffer was NULL, or
1083  *         a negative error code on failure.
1084  */
xe_guc_ct_send_recv(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer)1085 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
1086 			u32 *response_buffer)
1087 {
1088 	KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer);
1089 	return guc_ct_send_recv(ct, action, len, response_buffer, false);
1090 }
1091 
xe_guc_ct_send_recv_no_fail(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer)1092 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action,
1093 				u32 len, u32 *response_buffer)
1094 {
1095 	return guc_ct_send_recv(ct, action, len, response_buffer, true);
1096 }
1097 
msg_to_hxg(u32 * msg)1098 static u32 *msg_to_hxg(u32 *msg)
1099 {
1100 	return msg + GUC_CTB_MSG_MIN_LEN;
1101 }
1102 
msg_len_to_hxg_len(u32 len)1103 static u32 msg_len_to_hxg_len(u32 len)
1104 {
1105 	return len - GUC_CTB_MSG_MIN_LEN;
1106 }
1107 
parse_g2h_event(struct xe_guc_ct * ct,u32 * msg,u32 len)1108 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)
1109 {
1110 	u32 *hxg = msg_to_hxg(msg);
1111 	u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1112 
1113 	lockdep_assert_held(&ct->lock);
1114 
1115 	switch (action) {
1116 	case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1117 	case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1118 	case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1119 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1120 		g2h_release_space(ct, len);
1121 	}
1122 
1123 	return 0;
1124 }
1125 
guc_crash_process_msg(struct xe_guc_ct * ct,u32 action)1126 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action)
1127 {
1128 	struct xe_gt *gt = ct_to_gt(ct);
1129 
1130 	if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED)
1131 		xe_gt_err(gt, "GuC Crash dump notification\n");
1132 	else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION)
1133 		xe_gt_err(gt, "GuC Exception notification\n");
1134 	else
1135 		xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action);
1136 
1137 	CT_DEAD(ct, NULL, CRASH);
1138 
1139 	kick_reset(ct);
1140 
1141 	return 0;
1142 }
1143 
parse_g2h_response(struct xe_guc_ct * ct,u32 * msg,u32 len)1144 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
1145 {
1146 	struct xe_gt *gt =  ct_to_gt(ct);
1147 	u32 *hxg = msg_to_hxg(msg);
1148 	u32 hxg_len = msg_len_to_hxg_len(len);
1149 	u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]);
1150 	u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1151 	struct g2h_fence *g2h_fence;
1152 
1153 	lockdep_assert_held(&ct->lock);
1154 
1155 	/*
1156 	 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup.
1157 	 * Those messages should never fail, so if we do get an error back it
1158 	 * means we're likely doing an illegal operation and the GuC is
1159 	 * rejecting it. We have no way to inform the code that submitted the
1160 	 * H2G that the message was rejected, so we need to escalate the
1161 	 * failure to trigger a reset.
1162 	 */
1163 	if (fence & CT_SEQNO_UNTRACKED) {
1164 		if (type == GUC_HXG_TYPE_RESPONSE_FAILURE)
1165 			xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n",
1166 				  fence,
1167 				  FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]),
1168 				  FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]));
1169 		else
1170 			xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n",
1171 				  type, fence);
1172 		CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE);
1173 
1174 		return -EPROTO;
1175 	}
1176 
1177 	g2h_fence = xa_erase(&ct->fence_lookup, fence);
1178 	if (unlikely(!g2h_fence)) {
1179 		/* Don't tear down channel, as send could've timed out */
1180 		/* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */
1181 		xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence);
1182 		g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1183 		return 0;
1184 	}
1185 
1186 	xe_gt_assert(gt, fence == g2h_fence->seqno);
1187 
1188 	if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {
1189 		g2h_fence->fail = true;
1190 		g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]);
1191 		g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]);
1192 	} else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1193 		g2h_fence->retry = true;
1194 		g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]);
1195 	} else if (g2h_fence->response_buffer) {
1196 		g2h_fence->response_len = hxg_len;
1197 		memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32));
1198 	} else {
1199 		g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]);
1200 	}
1201 
1202 	g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1203 
1204 	g2h_fence->done = true;
1205 	smp_mb();
1206 
1207 	wake_up_all(&ct->g2h_fence_wq);
1208 
1209 	return 0;
1210 }
1211 
parse_g2h_msg(struct xe_guc_ct * ct,u32 * msg,u32 len)1212 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1213 {
1214 	struct xe_gt *gt = ct_to_gt(ct);
1215 	u32 *hxg = msg_to_hxg(msg);
1216 	u32 origin, type;
1217 	int ret;
1218 
1219 	lockdep_assert_held(&ct->lock);
1220 
1221 	origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1222 	if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1223 		xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n",
1224 			  origin);
1225 		CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN);
1226 
1227 		return -EPROTO;
1228 	}
1229 
1230 	type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1231 	switch (type) {
1232 	case GUC_HXG_TYPE_EVENT:
1233 		ret = parse_g2h_event(ct, msg, len);
1234 		break;
1235 	case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1236 	case GUC_HXG_TYPE_RESPONSE_FAILURE:
1237 	case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1238 		ret = parse_g2h_response(ct, msg, len);
1239 		break;
1240 	default:
1241 		xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n",
1242 			  type);
1243 		CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE);
1244 
1245 		ret = -EOPNOTSUPP;
1246 	}
1247 
1248 	return ret;
1249 }
1250 
process_g2h_msg(struct xe_guc_ct * ct,u32 * msg,u32 len)1251 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1252 {
1253 	struct xe_guc *guc = ct_to_guc(ct);
1254 	struct xe_gt *gt = ct_to_gt(ct);
1255 	u32 hxg_len = msg_len_to_hxg_len(len);
1256 	u32 *hxg = msg_to_hxg(msg);
1257 	u32 action, adj_len;
1258 	u32 *payload;
1259 	int ret = 0;
1260 
1261 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1262 		return 0;
1263 
1264 	action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1265 	payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN;
1266 	adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN;
1267 
1268 	switch (action) {
1269 	case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1270 		ret = xe_guc_sched_done_handler(guc, payload, adj_len);
1271 		break;
1272 	case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1273 		ret = xe_guc_deregister_done_handler(guc, payload, adj_len);
1274 		break;
1275 	case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1276 		ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len);
1277 		break;
1278 	case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1279 		ret = xe_guc_exec_queue_reset_failure_handler(guc, payload,
1280 							      adj_len);
1281 		break;
1282 	case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1283 		/* Selftest only at the moment */
1284 		break;
1285 	case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1286 		ret = xe_guc_error_capture_handler(guc, payload, adj_len);
1287 		break;
1288 	case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1289 		/* FIXME: Handle this */
1290 		break;
1291 	case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR:
1292 		ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload,
1293 								 adj_len);
1294 		break;
1295 	case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1296 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1297 		break;
1298 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1299 		ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1300 							   adj_len);
1301 		break;
1302 	case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY:
1303 		ret = xe_guc_access_counter_notify_handler(guc, payload,
1304 							   adj_len);
1305 		break;
1306 	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
1307 		ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);
1308 		break;
1309 	case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF:
1310 		ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len);
1311 		break;
1312 	case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY:
1313 		ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len);
1314 		break;
1315 	case GUC_ACTION_GUC2PF_ADVERSE_EVENT:
1316 		ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len);
1317 		break;
1318 	case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1319 	case XE_GUC_ACTION_NOTIFY_EXCEPTION:
1320 		ret = guc_crash_process_msg(ct, action);
1321 		break;
1322 	default:
1323 		xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
1324 	}
1325 
1326 	if (ret) {
1327 		xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n",
1328 			  action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg);
1329 		CT_DEAD(ct, NULL, PROCESS_FAILED);
1330 	}
1331 
1332 	return 0;
1333 }
1334 
g2h_read(struct xe_guc_ct * ct,u32 * msg,bool fast_path)1335 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
1336 {
1337 	struct xe_device *xe = ct_to_xe(ct);
1338 	struct xe_gt *gt = ct_to_gt(ct);
1339 	struct guc_ctb *g2h = &ct->ctbs.g2h;
1340 	u32 tail, head, len, desc_status;
1341 	s32 avail;
1342 	u32 action;
1343 	u32 *hxg;
1344 
1345 	xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
1346 	lockdep_assert_held(&ct->fast_lock);
1347 
1348 	if (ct->state == XE_GUC_CT_STATE_DISABLED)
1349 		return -ENODEV;
1350 
1351 	if (ct->state == XE_GUC_CT_STATE_STOPPED)
1352 		return -ECANCELED;
1353 
1354 	if (g2h->info.broken)
1355 		return -EPIPE;
1356 
1357 	xe_gt_assert(gt, xe_guc_ct_enabled(ct));
1358 
1359 	desc_status = desc_read(xe, g2h, status);
1360 	if (desc_status) {
1361 		if (desc_status & GUC_CTB_STATUS_DISABLED) {
1362 			/*
1363 			 * Potentially valid if a CLIENT_RESET request resulted in
1364 			 * contexts/engines being reset. But should never happen as
1365 			 * no contexts should be active when CLIENT_RESET is sent.
1366 			 */
1367 			xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n");
1368 			desc_status &= ~GUC_CTB_STATUS_DISABLED;
1369 		}
1370 
1371 		if (desc_status) {
1372 			xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status);
1373 			goto corrupted;
1374 		}
1375 	}
1376 
1377 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
1378 		u32 desc_tail = desc_read(xe, g2h, tail);
1379 		/*
1380 		u32 desc_head = desc_read(xe, g2h, head);
1381 
1382 		 * info.head and desc_head are updated back-to-back at the end of
1383 		 * this function and nowhere else. Hence, they cannot be different
1384 		 * unless two g2h_read calls are running concurrently. Which is not
1385 		 * possible because it is guarded by ct->fast_lock. And yet, some
1386 		 * discrete platforms are regularly hitting this error :(.
1387 		 *
1388 		 * desc_head rolling backwards shouldn't cause any noticeable
1389 		 * problems - just a delay in GuC being allowed to proceed past that
1390 		 * point in the queue. So for now, just disable the error until it
1391 		 * can be root caused.
1392 		 *
1393 		if (g2h->info.head != desc_head) {
1394 			desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH);
1395 			xe_gt_err(gt, "CT read: head was modified %u != %u\n",
1396 				  desc_head, g2h->info.head);
1397 			goto corrupted;
1398 		}
1399 		 */
1400 
1401 		if (g2h->info.head > g2h->info.size) {
1402 			desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
1403 			xe_gt_err(gt, "CT read: head out of range: %u vs %u\n",
1404 				  g2h->info.head, g2h->info.size);
1405 			goto corrupted;
1406 		}
1407 
1408 		if (desc_tail >= g2h->info.size) {
1409 			desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
1410 			xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n",
1411 				  desc_tail, g2h->info.size);
1412 			goto corrupted;
1413 		}
1414 	}
1415 
1416 	/* Calculate DW available to read */
1417 	tail = desc_read(xe, g2h, tail);
1418 	avail = tail - g2h->info.head;
1419 	if (unlikely(avail == 0))
1420 		return 0;
1421 
1422 	if (avail < 0)
1423 		avail += g2h->info.size;
1424 
1425 	/* Read header */
1426 	xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head,
1427 			   sizeof(u32));
1428 	len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN;
1429 	if (len > avail) {
1430 		xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n",
1431 			  avail, len);
1432 		goto corrupted;
1433 	}
1434 
1435 	head = (g2h->info.head + 1) % g2h->info.size;
1436 	avail = len - 1;
1437 
1438 	/* Read G2H message */
1439 	if (avail + head > g2h->info.size) {
1440 		u32 avail_til_wrap = g2h->info.size - head;
1441 
1442 		xe_map_memcpy_from(xe, msg + 1,
1443 				   &g2h->cmds, sizeof(u32) * head,
1444 				   avail_til_wrap * sizeof(u32));
1445 		xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap,
1446 				   &g2h->cmds, 0,
1447 				   (avail - avail_til_wrap) * sizeof(u32));
1448 	} else {
1449 		xe_map_memcpy_from(xe, msg + 1,
1450 				   &g2h->cmds, sizeof(u32) * head,
1451 				   avail * sizeof(u32));
1452 	}
1453 
1454 	hxg = msg_to_hxg(msg);
1455 	action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1456 
1457 	if (fast_path) {
1458 		if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1459 			return 0;
1460 
1461 		switch (action) {
1462 		case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1463 		case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1464 			break;	/* Process these in fast-path */
1465 		default:
1466 			return 0;
1467 		}
1468 	}
1469 
1470 	/* Update local / descriptor header */
1471 	g2h->info.head = (head + avail) % g2h->info.size;
1472 	desc_write(xe, g2h, head, g2h->info.head);
1473 
1474 	trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id,
1475 			     action, len, g2h->info.head, tail);
1476 
1477 	return len;
1478 
1479 corrupted:
1480 	CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ);
1481 	return -EPROTO;
1482 }
1483 
g2h_fast_path(struct xe_guc_ct * ct,u32 * msg,u32 len)1484 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
1485 {
1486 	struct xe_gt *gt = ct_to_gt(ct);
1487 	struct xe_guc *guc = ct_to_guc(ct);
1488 	u32 hxg_len = msg_len_to_hxg_len(len);
1489 	u32 *hxg = msg_to_hxg(msg);
1490 	u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1491 	u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN;
1492 	u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1493 	int ret = 0;
1494 
1495 	switch (action) {
1496 	case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1497 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1498 		break;
1499 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1500 		__g2h_release_space(ct, len);
1501 		ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1502 							   adj_len);
1503 		break;
1504 	default:
1505 		xe_gt_warn(gt, "NOT_POSSIBLE");
1506 	}
1507 
1508 	if (ret) {
1509 		xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
1510 			  action, ERR_PTR(ret));
1511 		CT_DEAD(ct, NULL, FAST_G2H);
1512 	}
1513 }
1514 
1515 /**
1516  * xe_guc_ct_fast_path - process critical G2H in the IRQ handler
1517  * @ct: GuC CT object
1518  *
1519  * Anything related to page faults is critical for performance, process these
1520  * critical G2H in the IRQ. This is safe as these handlers either just wake up
1521  * waiters or queue another worker.
1522  */
xe_guc_ct_fast_path(struct xe_guc_ct * ct)1523 void xe_guc_ct_fast_path(struct xe_guc_ct *ct)
1524 {
1525 	struct xe_device *xe = ct_to_xe(ct);
1526 	bool ongoing;
1527 	int len;
1528 
1529 	ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1530 	if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1531 		return;
1532 
1533 	spin_lock(&ct->fast_lock);
1534 	do {
1535 		len = g2h_read(ct, ct->fast_msg, true);
1536 		if (len > 0)
1537 			g2h_fast_path(ct, ct->fast_msg, len);
1538 	} while (len > 0);
1539 	spin_unlock(&ct->fast_lock);
1540 
1541 	if (ongoing)
1542 		xe_pm_runtime_put(xe);
1543 }
1544 
1545 /* Returns less than zero on error, 0 on done, 1 on more available */
dequeue_one_g2h(struct xe_guc_ct * ct)1546 static int dequeue_one_g2h(struct xe_guc_ct *ct)
1547 {
1548 	int len;
1549 	int ret;
1550 
1551 	lockdep_assert_held(&ct->lock);
1552 
1553 	spin_lock_irq(&ct->fast_lock);
1554 	len = g2h_read(ct, ct->msg, false);
1555 	spin_unlock_irq(&ct->fast_lock);
1556 	if (len <= 0)
1557 		return len;
1558 
1559 	ret = parse_g2h_msg(ct, ct->msg, len);
1560 	if (unlikely(ret < 0))
1561 		return ret;
1562 
1563 	ret = process_g2h_msg(ct, ct->msg, len);
1564 	if (unlikely(ret < 0))
1565 		return ret;
1566 
1567 	return 1;
1568 }
1569 
receive_g2h(struct xe_guc_ct * ct)1570 static void receive_g2h(struct xe_guc_ct *ct)
1571 {
1572 	bool ongoing;
1573 	int ret;
1574 
1575 	/*
1576 	 * Normal users must always hold mem_access.ref around CT calls. However
1577 	 * during the runtime pm callbacks we rely on CT to talk to the GuC, but
1578 	 * at this stage we can't rely on mem_access.ref and even the
1579 	 * callback_task will be different than current.  For such cases we just
1580 	 * need to ensure we always process the responses from any blocking
1581 	 * ct_send requests or where we otherwise expect some response when
1582 	 * initiated from those callbacks (which will need to wait for the below
1583 	 * dequeue_one_g2h()).  The dequeue_one_g2h() will gracefully fail if
1584 	 * the device has suspended to the point that the CT communication has
1585 	 * been disabled.
1586 	 *
1587 	 * If we are inside the runtime pm callback, we can be the only task
1588 	 * still issuing CT requests (since that requires having the
1589 	 * mem_access.ref).  It seems like it might in theory be possible to
1590 	 * receive unsolicited events from the GuC just as we are
1591 	 * suspending-resuming, but those will currently anyway be lost when
1592 	 * eventually exiting from suspend, hence no need to wake up the device
1593 	 * here. If we ever need something stronger than get_if_ongoing() then
1594 	 * we need to be careful with blocking the pm callbacks from getting CT
1595 	 * responses, if the worker here is blocked on those callbacks
1596 	 * completing, creating a deadlock.
1597 	 */
1598 	ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1599 	if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1600 		return;
1601 
1602 	do {
1603 		mutex_lock(&ct->lock);
1604 		ret = dequeue_one_g2h(ct);
1605 		mutex_unlock(&ct->lock);
1606 
1607 		if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {
1608 			xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret);
1609 			CT_DEAD(ct, NULL, G2H_RECV);
1610 			kick_reset(ct);
1611 		}
1612 	} while (ret == 1);
1613 
1614 	if (ongoing)
1615 		xe_pm_runtime_put(ct_to_xe(ct));
1616 }
1617 
g2h_worker_func(struct work_struct * w)1618 static void g2h_worker_func(struct work_struct *w)
1619 {
1620 	struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker);
1621 
1622 	receive_g2h(ct);
1623 }
1624 
guc_ct_snapshot_alloc(struct xe_guc_ct * ct,bool atomic,bool want_ctb)1625 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic,
1626 							bool want_ctb)
1627 {
1628 	struct xe_guc_ct_snapshot *snapshot;
1629 
1630 	snapshot = kzalloc(sizeof(*snapshot), atomic ? GFP_ATOMIC : GFP_KERNEL);
1631 	if (!snapshot)
1632 		return NULL;
1633 
1634 	if (ct->bo && want_ctb) {
1635 		snapshot->ctb_size = ct->bo->size;
1636 		snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL);
1637 	}
1638 
1639 	return snapshot;
1640 }
1641 
guc_ctb_snapshot_capture(struct xe_device * xe,struct guc_ctb * ctb,struct guc_ctb_snapshot * snapshot)1642 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb,
1643 				     struct guc_ctb_snapshot *snapshot)
1644 {
1645 	xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0,
1646 			   sizeof(struct guc_ct_buffer_desc));
1647 	memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info));
1648 }
1649 
guc_ctb_snapshot_print(struct guc_ctb_snapshot * snapshot,struct drm_printer * p)1650 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot,
1651 				   struct drm_printer *p)
1652 {
1653 	drm_printf(p, "\tsize: %d\n", snapshot->info.size);
1654 	drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space);
1655 	drm_printf(p, "\thead: %d\n", snapshot->info.head);
1656 	drm_printf(p, "\ttail: %d\n", snapshot->info.tail);
1657 	drm_printf(p, "\tspace: %d\n", snapshot->info.space);
1658 	drm_printf(p, "\tbroken: %d\n", snapshot->info.broken);
1659 	drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head);
1660 	drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail);
1661 	drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status);
1662 }
1663 
guc_ct_snapshot_capture(struct xe_guc_ct * ct,bool atomic,bool want_ctb)1664 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic,
1665 							  bool want_ctb)
1666 {
1667 	struct xe_device *xe = ct_to_xe(ct);
1668 	struct xe_guc_ct_snapshot *snapshot;
1669 
1670 	snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb);
1671 	if (!snapshot) {
1672 		xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n");
1673 		return NULL;
1674 	}
1675 
1676 	if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) {
1677 		snapshot->ct_enabled = true;
1678 		snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);
1679 		guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g);
1680 		guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h);
1681 	}
1682 
1683 	if (ct->bo && snapshot->ctb)
1684 		xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size);
1685 
1686 	return snapshot;
1687 }
1688 
1689 /**
1690  * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state.
1691  * @ct: GuC CT object.
1692  *
1693  * This can be printed out in a later stage like during dev_coredump
1694  * analysis. This is safe to be called during atomic context.
1695  *
1696  * Returns: a GuC CT snapshot object that must be freed by the caller
1697  * by using `xe_guc_ct_snapshot_free`.
1698  */
xe_guc_ct_snapshot_capture(struct xe_guc_ct * ct)1699 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct)
1700 {
1701 	return guc_ct_snapshot_capture(ct, true, true);
1702 }
1703 
1704 /**
1705  * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot.
1706  * @snapshot: GuC CT snapshot object.
1707  * @p: drm_printer where it will be printed out.
1708  *
1709  * This function prints out a given GuC CT snapshot object.
1710  */
xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot * snapshot,struct drm_printer * p)1711 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot,
1712 			      struct drm_printer *p)
1713 {
1714 	if (!snapshot)
1715 		return;
1716 
1717 	if (snapshot->ct_enabled) {
1718 		drm_puts(p, "H2G CTB (all sizes in DW):\n");
1719 		guc_ctb_snapshot_print(&snapshot->h2g, p);
1720 
1721 		drm_puts(p, "G2H CTB (all sizes in DW):\n");
1722 		guc_ctb_snapshot_print(&snapshot->g2h, p);
1723 		drm_printf(p, "\tg2h outstanding: %d\n",
1724 			   snapshot->g2h_outstanding);
1725 
1726 		if (snapshot->ctb) {
1727 			drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size);
1728 			xe_print_blob_ascii85(p, "[CTB].data", '\n',
1729 					      snapshot->ctb, 0, snapshot->ctb_size);
1730 		}
1731 	} else {
1732 		drm_puts(p, "CT disabled\n");
1733 	}
1734 }
1735 
1736 /**
1737  * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot.
1738  * @snapshot: GuC CT snapshot object.
1739  *
1740  * This function free all the memory that needed to be allocated at capture
1741  * time.
1742  */
xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot * snapshot)1743 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot)
1744 {
1745 	if (!snapshot)
1746 		return;
1747 
1748 	kfree(snapshot->ctb);
1749 	kfree(snapshot);
1750 }
1751 
1752 /**
1753  * xe_guc_ct_print - GuC CT Print.
1754  * @ct: GuC CT.
1755  * @p: drm_printer where it will be printed out.
1756  * @want_ctb: Should the full CTB content be dumped (vs just the headers)
1757  *
1758  * This function will quickly capture a snapshot of the CT state
1759  * and immediately print it out.
1760  */
xe_guc_ct_print(struct xe_guc_ct * ct,struct drm_printer * p,bool want_ctb)1761 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb)
1762 {
1763 	struct xe_guc_ct_snapshot *snapshot;
1764 
1765 	snapshot = guc_ct_snapshot_capture(ct, false, want_ctb);
1766 	xe_guc_ct_snapshot_print(snapshot, p);
1767 	xe_guc_ct_snapshot_free(snapshot);
1768 }
1769 
1770 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
ct_dead_capture(struct xe_guc_ct * ct,struct guc_ctb * ctb,u32 reason_code)1771 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code)
1772 {
1773 	struct xe_guc_log_snapshot *snapshot_log;
1774 	struct xe_guc_ct_snapshot *snapshot_ct;
1775 	struct xe_guc *guc = ct_to_guc(ct);
1776 	unsigned long flags;
1777 	bool have_capture;
1778 
1779 	if (ctb)
1780 		ctb->info.broken = true;
1781 
1782 	/* Ignore further errors after the first dump until a reset */
1783 	if (ct->dead.reported)
1784 		return;
1785 
1786 	spin_lock_irqsave(&ct->dead.lock, flags);
1787 
1788 	/* And only capture one dump at a time */
1789 	have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE);
1790 	ct->dead.reason |= (1 << reason_code) |
1791 			   (1 << CT_DEAD_STATE_CAPTURE);
1792 
1793 	spin_unlock_irqrestore(&ct->dead.lock, flags);
1794 
1795 	if (have_capture)
1796 		return;
1797 
1798 	snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true);
1799 	snapshot_ct = xe_guc_ct_snapshot_capture((ct));
1800 
1801 	spin_lock_irqsave(&ct->dead.lock, flags);
1802 
1803 	if (ct->dead.snapshot_log || ct->dead.snapshot_ct) {
1804 		xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n");
1805 		xe_guc_log_snapshot_free(snapshot_log);
1806 		xe_guc_ct_snapshot_free(snapshot_ct);
1807 	} else {
1808 		ct->dead.snapshot_log = snapshot_log;
1809 		ct->dead.snapshot_ct = snapshot_ct;
1810 	}
1811 
1812 	spin_unlock_irqrestore(&ct->dead.lock, flags);
1813 
1814 	queue_work(system_unbound_wq, &(ct)->dead.worker);
1815 }
1816 
ct_dead_print(struct xe_dead_ct * dead)1817 static void ct_dead_print(struct xe_dead_ct *dead)
1818 {
1819 	struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead);
1820 	struct xe_device *xe = ct_to_xe(ct);
1821 	struct xe_gt *gt = ct_to_gt(ct);
1822 	static int g_count;
1823 	struct drm_printer ip = xe_gt_info_printer(gt);
1824 	struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count);
1825 
1826 	if (!dead->reason) {
1827 		xe_gt_err(gt, "CTB is dead for no reason!?\n");
1828 		return;
1829 	}
1830 
1831 	drm_printf(&lp, "CTB is dead - reason=0x%X\n", dead->reason);
1832 
1833 	/* Can't generate a genuine core dump at this point, so just do the good bits */
1834 	drm_puts(&lp, "**** Xe Device Coredump ****\n");
1835 	xe_device_snapshot_print(xe, &lp);
1836 
1837 	drm_printf(&lp, "**** GT #%d ****\n", gt->info.id);
1838 	drm_printf(&lp, "\tTile: %d\n", gt->tile->id);
1839 
1840 	drm_puts(&lp, "**** GuC Log ****\n");
1841 	xe_guc_log_snapshot_print(dead->snapshot_log, &lp);
1842 
1843 	drm_puts(&lp, "**** GuC CT ****\n");
1844 	xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp);
1845 
1846 	drm_puts(&lp, "Done.\n");
1847 }
1848 
ct_dead_worker_func(struct work_struct * w)1849 static void ct_dead_worker_func(struct work_struct *w)
1850 {
1851 	struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker);
1852 
1853 	if (!ct->dead.reported) {
1854 		ct->dead.reported = true;
1855 		ct_dead_print(&ct->dead);
1856 	}
1857 
1858 	spin_lock_irq(&ct->dead.lock);
1859 
1860 	xe_guc_log_snapshot_free(ct->dead.snapshot_log);
1861 	ct->dead.snapshot_log = NULL;
1862 	xe_guc_ct_snapshot_free(ct->dead.snapshot_ct);
1863 	ct->dead.snapshot_ct = NULL;
1864 
1865 	if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) {
1866 		/* A reset has occurred so re-arm the error reporting */
1867 		ct->dead.reason = 0;
1868 		ct->dead.reported = false;
1869 	}
1870 
1871 	spin_unlock_irq(&ct->dead.lock);
1872 }
1873 #endif
1874