1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc_ct.h"
7
8 #include <linux/bitfield.h>
9 #include <linux/circ_buf.h>
10 #include <linux/delay.h>
11 #include <linux/fault-inject.h>
12
13 #include <kunit/static_stub.h>
14
15 #include <drm/drm_managed.h>
16
17 #include "abi/guc_actions_abi.h"
18 #include "abi/guc_actions_sriov_abi.h"
19 #include "abi/guc_klvs_abi.h"
20 #include "xe_bo.h"
21 #include "xe_devcoredump.h"
22 #include "xe_device.h"
23 #include "xe_gt.h"
24 #include "xe_gt_pagefault.h"
25 #include "xe_gt_printk.h"
26 #include "xe_gt_sriov_pf_control.h"
27 #include "xe_gt_sriov_pf_monitor.h"
28 #include "xe_gt_tlb_invalidation.h"
29 #include "xe_guc.h"
30 #include "xe_guc_log.h"
31 #include "xe_guc_relay.h"
32 #include "xe_guc_submit.h"
33 #include "xe_map.h"
34 #include "xe_pm.h"
35 #include "xe_trace_guc.h"
36
37 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
38 enum {
39 /* Internal states, not error conditions */
40 CT_DEAD_STATE_REARM, /* 0x0001 */
41 CT_DEAD_STATE_CAPTURE, /* 0x0002 */
42
43 /* Error conditions */
44 CT_DEAD_SETUP, /* 0x0004 */
45 CT_DEAD_H2G_WRITE, /* 0x0008 */
46 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */
47 CT_DEAD_G2H_READ, /* 0x0020 */
48 CT_DEAD_G2H_RECV, /* 0x0040 */
49 CT_DEAD_G2H_RELEASE, /* 0x0080 */
50 CT_DEAD_DEADLOCK, /* 0x0100 */
51 CT_DEAD_PROCESS_FAILED, /* 0x0200 */
52 CT_DEAD_FAST_G2H, /* 0x0400 */
53 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */
54 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */
55 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */
56 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */
57 CT_DEAD_CRASH, /* 0x8000 */
58 };
59
60 static void ct_dead_worker_func(struct work_struct *w);
61 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code);
62
63 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
64 #else
65 #define CT_DEAD(ct, ctb, reason) \
66 do { \
67 struct guc_ctb *_ctb = (ctb); \
68 if (_ctb) \
69 _ctb->info.broken = true; \
70 } while (0)
71 #endif
72
73 /* Used when a CT send wants to block and / or receive data */
74 struct g2h_fence {
75 u32 *response_buffer;
76 u32 seqno;
77 u32 response_data;
78 u16 response_len;
79 u16 error;
80 u16 hint;
81 u16 reason;
82 bool retry;
83 bool fail;
84 bool done;
85 };
86
g2h_fence_init(struct g2h_fence * g2h_fence,u32 * response_buffer)87 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer)
88 {
89 g2h_fence->response_buffer = response_buffer;
90 g2h_fence->response_data = 0;
91 g2h_fence->response_len = 0;
92 g2h_fence->fail = false;
93 g2h_fence->retry = false;
94 g2h_fence->done = false;
95 g2h_fence->seqno = ~0x0;
96 }
97
g2h_fence_needs_alloc(struct g2h_fence * g2h_fence)98 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
99 {
100 return g2h_fence->seqno == ~0x0;
101 }
102
103 static struct xe_guc *
ct_to_guc(struct xe_guc_ct * ct)104 ct_to_guc(struct xe_guc_ct *ct)
105 {
106 return container_of(ct, struct xe_guc, ct);
107 }
108
109 static struct xe_gt *
ct_to_gt(struct xe_guc_ct * ct)110 ct_to_gt(struct xe_guc_ct *ct)
111 {
112 return container_of(ct, struct xe_gt, uc.guc.ct);
113 }
114
115 static struct xe_device *
ct_to_xe(struct xe_guc_ct * ct)116 ct_to_xe(struct xe_guc_ct *ct)
117 {
118 return gt_to_xe(ct_to_gt(ct));
119 }
120
121 /**
122 * DOC: GuC CTB Blob
123 *
124 * We allocate single blob to hold both CTB descriptors and buffers:
125 *
126 * +--------+-----------------------------------------------+------+
127 * | offset | contents | size |
128 * +========+===============================================+======+
129 * | 0x0000 | H2G CTB Descriptor (send) | |
130 * +--------+-----------------------------------------------+ 4K |
131 * | 0x0800 | G2H CTB Descriptor (g2h) | |
132 * +--------+-----------------------------------------------+------+
133 * | 0x1000 | H2G CT Buffer (send) | n*4K |
134 * | | | |
135 * +--------+-----------------------------------------------+------+
136 * | 0x1000 | G2H CT Buffer (g2h) | m*4K |
137 * | + n*4K | | |
138 * +--------+-----------------------------------------------+------+
139 *
140 * Size of each ``CT Buffer`` must be multiple of 4K.
141 * We don't expect too many messages in flight at any time, unless we are
142 * using the GuC submission. In that case each request requires a minimum
143 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
144 * enough space to avoid backpressure on the driver. We increase the size
145 * of the receive buffer (relative to the send) to ensure a G2H response
146 * CTB has a landing spot.
147 *
148 * In addition to submissions, the G2H buffer needs to be able to hold
149 * enough space for recoverable page fault notifications. The number of
150 * page faults is interrupt driven and can be as much as the number of
151 * compute resources available. However, most of the actual work for these
152 * is in a separate page fault worker thread. Therefore we only need to
153 * make sure the queue has enough space to handle all of the submissions
154 * and responses and an extra buffer for incoming page faults.
155 */
156
157 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
158 #define CTB_H2G_BUFFER_SIZE (SZ_4K)
159 #define CTB_G2H_BUFFER_SIZE (SZ_128K)
160 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2)
161
162 /**
163 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full
164 * CT command queue
165 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future.
166 *
167 * Observation is that a 4KiB buffer full of commands takes a little over a
168 * second to process. Use that to calculate maximum time to process a full CT
169 * command queue.
170 *
171 * Return: Maximum time to process a full CT queue in jiffies.
172 */
xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct * ct)173 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct)
174 {
175 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4));
176 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ;
177 }
178
guc_ct_size(void)179 static size_t guc_ct_size(void)
180 {
181 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE +
182 CTB_G2H_BUFFER_SIZE;
183 }
184
guc_ct_fini(struct drm_device * drm,void * arg)185 static void guc_ct_fini(struct drm_device *drm, void *arg)
186 {
187 struct xe_guc_ct *ct = arg;
188
189 destroy_workqueue(ct->g2h_wq);
190 xa_destroy(&ct->fence_lookup);
191 }
192
193 static void receive_g2h(struct xe_guc_ct *ct);
194 static void g2h_worker_func(struct work_struct *w);
195 static void safe_mode_worker_func(struct work_struct *w);
196
primelockdep(struct xe_guc_ct * ct)197 static void primelockdep(struct xe_guc_ct *ct)
198 {
199 if (!IS_ENABLED(CONFIG_LOCKDEP))
200 return;
201
202 fs_reclaim_acquire(GFP_KERNEL);
203 might_lock(&ct->lock);
204 fs_reclaim_release(GFP_KERNEL);
205 }
206
xe_guc_ct_init(struct xe_guc_ct * ct)207 int xe_guc_ct_init(struct xe_guc_ct *ct)
208 {
209 struct xe_device *xe = ct_to_xe(ct);
210 struct xe_gt *gt = ct_to_gt(ct);
211 struct xe_tile *tile = gt_to_tile(gt);
212 struct xe_bo *bo;
213 int err;
214
215 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE));
216
217 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM);
218 if (!ct->g2h_wq)
219 return -ENOMEM;
220
221 spin_lock_init(&ct->fast_lock);
222 xa_init(&ct->fence_lookup);
223 INIT_WORK(&ct->g2h_worker, g2h_worker_func);
224 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);
225 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
226 spin_lock_init(&ct->dead.lock);
227 INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
228 #endif
229 init_waitqueue_head(&ct->wq);
230 init_waitqueue_head(&ct->g2h_fence_wq);
231
232 err = drmm_mutex_init(&xe->drm, &ct->lock);
233 if (err)
234 return err;
235
236 primelockdep(ct);
237
238 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
239 XE_BO_FLAG_SYSTEM |
240 XE_BO_FLAG_GGTT |
241 XE_BO_FLAG_GGTT_INVALIDATE);
242 if (IS_ERR(bo))
243 return PTR_ERR(bo);
244
245 ct->bo = bo;
246
247 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct);
248 if (err)
249 return err;
250
251 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED);
252 ct->state = XE_GUC_CT_STATE_DISABLED;
253 return 0;
254 }
255 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */
256
257 #define desc_read(xe_, guc_ctb__, field_) \
258 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \
259 struct guc_ct_buffer_desc, field_)
260
261 #define desc_write(xe_, guc_ctb__, field_, val_) \
262 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \
263 struct guc_ct_buffer_desc, field_, val_)
264
guc_ct_ctb_h2g_init(struct xe_device * xe,struct guc_ctb * h2g,struct iosys_map * map)265 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,
266 struct iosys_map *map)
267 {
268 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);
269 h2g->info.resv_space = 0;
270 h2g->info.tail = 0;
271 h2g->info.head = 0;
272 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
273 h2g->info.size) -
274 h2g->info.resv_space;
275 h2g->info.broken = false;
276
277 h2g->desc = *map;
278 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
279
280 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2);
281 }
282
guc_ct_ctb_g2h_init(struct xe_device * xe,struct guc_ctb * g2h,struct iosys_map * map)283 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,
284 struct iosys_map *map)
285 {
286 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);
287 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);
288 g2h->info.head = 0;
289 g2h->info.tail = 0;
290 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
291 g2h->info.size) -
292 g2h->info.resv_space;
293 g2h->info.broken = false;
294
295 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE);
296 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
297
298 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 +
299 CTB_H2G_BUFFER_SIZE);
300 }
301
guc_ct_ctb_h2g_register(struct xe_guc_ct * ct)302 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)
303 {
304 struct xe_guc *guc = ct_to_guc(ct);
305 u32 desc_addr, ctb_addr, size;
306 int err;
307
308 desc_addr = xe_bo_ggtt_addr(ct->bo);
309 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2;
310 size = ct->ctbs.h2g.info.size * sizeof(u32);
311
312 err = xe_guc_self_cfg64(guc,
313 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY,
314 desc_addr);
315 if (err)
316 return err;
317
318 err = xe_guc_self_cfg64(guc,
319 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY,
320 ctb_addr);
321 if (err)
322 return err;
323
324 return xe_guc_self_cfg32(guc,
325 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY,
326 size);
327 }
328
guc_ct_ctb_g2h_register(struct xe_guc_ct * ct)329 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct)
330 {
331 struct xe_guc *guc = ct_to_guc(ct);
332 u32 desc_addr, ctb_addr, size;
333 int err;
334
335 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE;
336 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 +
337 CTB_H2G_BUFFER_SIZE;
338 size = ct->ctbs.g2h.info.size * sizeof(u32);
339
340 err = xe_guc_self_cfg64(guc,
341 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
342 desc_addr);
343 if (err)
344 return err;
345
346 err = xe_guc_self_cfg64(guc,
347 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
348 ctb_addr);
349 if (err)
350 return err;
351
352 return xe_guc_self_cfg32(guc,
353 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
354 size);
355 }
356
guc_ct_control_toggle(struct xe_guc_ct * ct,bool enable)357 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable)
358 {
359 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
360 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
361 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
362 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
363 GUC_ACTION_HOST2GUC_CONTROL_CTB),
364 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL,
365 enable ? GUC_CTB_CONTROL_ENABLE :
366 GUC_CTB_CONTROL_DISABLE),
367 };
368 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request));
369
370 return ret > 0 ? -EPROTO : ret;
371 }
372
xe_guc_ct_set_state(struct xe_guc_ct * ct,enum xe_guc_ct_state state)373 static void xe_guc_ct_set_state(struct xe_guc_ct *ct,
374 enum xe_guc_ct_state state)
375 {
376 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */
377 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */
378
379 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 ||
380 state == XE_GUC_CT_STATE_STOPPED);
381
382 if (ct->g2h_outstanding)
383 xe_pm_runtime_put(ct_to_xe(ct));
384 ct->g2h_outstanding = 0;
385 ct->state = state;
386
387 spin_unlock_irq(&ct->fast_lock);
388
389 /*
390 * Lockdep doesn't like this under the fast lock and he destroy only
391 * needs to be serialized with the send path which ct lock provides.
392 */
393 xa_destroy(&ct->fence_lookup);
394
395 mutex_unlock(&ct->lock);
396 }
397
ct_needs_safe_mode(struct xe_guc_ct * ct)398 static bool ct_needs_safe_mode(struct xe_guc_ct *ct)
399 {
400 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev));
401 }
402
ct_restart_safe_mode_worker(struct xe_guc_ct * ct)403 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct)
404 {
405 if (!ct_needs_safe_mode(ct))
406 return false;
407
408 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10);
409 return true;
410 }
411
safe_mode_worker_func(struct work_struct * w)412 static void safe_mode_worker_func(struct work_struct *w)
413 {
414 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work);
415
416 receive_g2h(ct);
417
418 if (!ct_restart_safe_mode_worker(ct))
419 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n");
420 }
421
ct_enter_safe_mode(struct xe_guc_ct * ct)422 static void ct_enter_safe_mode(struct xe_guc_ct *ct)
423 {
424 if (ct_restart_safe_mode_worker(ct))
425 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n");
426 }
427
ct_exit_safe_mode(struct xe_guc_ct * ct)428 static void ct_exit_safe_mode(struct xe_guc_ct *ct)
429 {
430 if (cancel_delayed_work_sync(&ct->safe_mode_worker))
431 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n");
432 }
433
xe_guc_ct_enable(struct xe_guc_ct * ct)434 int xe_guc_ct_enable(struct xe_guc_ct *ct)
435 {
436 struct xe_device *xe = ct_to_xe(ct);
437 struct xe_gt *gt = ct_to_gt(ct);
438 int err;
439
440 xe_gt_assert(gt, !xe_guc_ct_enabled(ct));
441
442 xe_map_memset(xe, &ct->bo->vmap, 0, 0, ct->bo->size);
443 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);
444 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);
445
446 err = guc_ct_ctb_h2g_register(ct);
447 if (err)
448 goto err_out;
449
450 err = guc_ct_ctb_g2h_register(ct);
451 if (err)
452 goto err_out;
453
454 err = guc_ct_control_toggle(ct, true);
455 if (err)
456 goto err_out;
457
458 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED);
459
460 smp_mb();
461 wake_up_all(&ct->wq);
462 xe_gt_dbg(gt, "GuC CT communication channel enabled\n");
463
464 if (ct_needs_safe_mode(ct))
465 ct_enter_safe_mode(ct);
466
467 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
468 /*
469 * The CT has now been reset so the dumper can be re-armed
470 * after any existing dead state has been dumped.
471 */
472 spin_lock_irq(&ct->dead.lock);
473 if (ct->dead.reason) {
474 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM);
475 queue_work(system_unbound_wq, &ct->dead.worker);
476 }
477 spin_unlock_irq(&ct->dead.lock);
478 #endif
479
480 return 0;
481
482 err_out:
483 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err));
484 CT_DEAD(ct, NULL, SETUP);
485
486 return err;
487 }
488
stop_g2h_handler(struct xe_guc_ct * ct)489 static void stop_g2h_handler(struct xe_guc_ct *ct)
490 {
491 cancel_work_sync(&ct->g2h_worker);
492 }
493
494 /**
495 * xe_guc_ct_disable - Set GuC to disabled state
496 * @ct: the &xe_guc_ct
497 *
498 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected
499 * in this transition.
500 */
xe_guc_ct_disable(struct xe_guc_ct * ct)501 void xe_guc_ct_disable(struct xe_guc_ct *ct)
502 {
503 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED);
504 ct_exit_safe_mode(ct);
505 stop_g2h_handler(ct);
506 }
507
508 /**
509 * xe_guc_ct_stop - Set GuC to stopped state
510 * @ct: the &xe_guc_ct
511 *
512 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h
513 */
xe_guc_ct_stop(struct xe_guc_ct * ct)514 void xe_guc_ct_stop(struct xe_guc_ct *ct)
515 {
516 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED);
517 stop_g2h_handler(ct);
518 }
519
h2g_has_room(struct xe_guc_ct * ct,u32 cmd_len)520 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
521 {
522 struct guc_ctb *h2g = &ct->ctbs.h2g;
523
524 lockdep_assert_held(&ct->lock);
525
526 if (cmd_len > h2g->info.space) {
527 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head);
528
529 if (h2g->info.head > h2g->info.size) {
530 struct xe_device *xe = ct_to_xe(ct);
531 u32 desc_status = desc_read(xe, h2g, status);
532
533 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
534
535 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n",
536 h2g->info.head, h2g->info.size);
537 CT_DEAD(ct, h2g, H2G_HAS_ROOM);
538 return false;
539 }
540
541 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
542 h2g->info.size) -
543 h2g->info.resv_space;
544 if (cmd_len > h2g->info.space)
545 return false;
546 }
547
548 return true;
549 }
550
g2h_has_room(struct xe_guc_ct * ct,u32 g2h_len)551 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len)
552 {
553 if (!g2h_len)
554 return true;
555
556 lockdep_assert_held(&ct->fast_lock);
557
558 return ct->ctbs.g2h.info.space > g2h_len;
559 }
560
has_room(struct xe_guc_ct * ct,u32 cmd_len,u32 g2h_len)561 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len)
562 {
563 lockdep_assert_held(&ct->lock);
564
565 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len))
566 return -EBUSY;
567
568 return 0;
569 }
570
h2g_reserve_space(struct xe_guc_ct * ct,u32 cmd_len)571 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)
572 {
573 lockdep_assert_held(&ct->lock);
574 ct->ctbs.h2g.info.space -= cmd_len;
575 }
576
__g2h_reserve_space(struct xe_guc_ct * ct,u32 g2h_len,u32 num_g2h)577 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
578 {
579 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space);
580 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) ||
581 (g2h_len && num_g2h));
582
583 if (g2h_len) {
584 lockdep_assert_held(&ct->fast_lock);
585
586 if (!ct->g2h_outstanding)
587 xe_pm_runtime_get_noresume(ct_to_xe(ct));
588
589 ct->ctbs.g2h.info.space -= g2h_len;
590 ct->g2h_outstanding += num_g2h;
591 }
592 }
593
__g2h_release_space(struct xe_guc_ct * ct,u32 g2h_len)594 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
595 {
596 bool bad = false;
597
598 lockdep_assert_held(&ct->fast_lock);
599
600 bad = ct->ctbs.g2h.info.space + g2h_len >
601 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space;
602 bad |= !ct->g2h_outstanding;
603
604 if (bad) {
605 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n",
606 ct->ctbs.g2h.info.space, g2h_len,
607 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space,
608 ct->ctbs.g2h.info.space + g2h_len,
609 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space,
610 ct->g2h_outstanding);
611 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE);
612 return;
613 }
614
615 ct->ctbs.g2h.info.space += g2h_len;
616 if (!--ct->g2h_outstanding)
617 xe_pm_runtime_put(ct_to_xe(ct));
618 }
619
g2h_release_space(struct xe_guc_ct * ct,u32 g2h_len)620 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
621 {
622 spin_lock_irq(&ct->fast_lock);
623 __g2h_release_space(ct, g2h_len);
624 spin_unlock_irq(&ct->fast_lock);
625 }
626
627 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */
628
h2g_write(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 ct_fence_value,bool want_response)629 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
630 u32 ct_fence_value, bool want_response)
631 {
632 struct xe_device *xe = ct_to_xe(ct);
633 struct xe_gt *gt = ct_to_gt(ct);
634 struct guc_ctb *h2g = &ct->ctbs.h2g;
635 u32 cmd[H2G_CT_HEADERS];
636 u32 tail = h2g->info.tail;
637 u32 full_len;
638 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds,
639 tail * sizeof(u32));
640 u32 desc_status;
641
642 full_len = len + GUC_CTB_HDR_LEN;
643
644 lockdep_assert_held(&ct->lock);
645 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);
646
647 desc_status = desc_read(xe, h2g, status);
648 if (desc_status) {
649 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status);
650 goto corrupted;
651 }
652
653 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
654 u32 desc_tail = desc_read(xe, h2g, tail);
655 u32 desc_head = desc_read(xe, h2g, head);
656
657 if (tail != desc_tail) {
658 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH);
659 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail);
660 goto corrupted;
661 }
662
663 if (tail > h2g->info.size) {
664 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
665 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n",
666 tail, h2g->info.size);
667 goto corrupted;
668 }
669
670 if (desc_head >= h2g->info.size) {
671 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
672 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n",
673 desc_head, h2g->info.size);
674 goto corrupted;
675 }
676 }
677
678 /* Command will wrap, zero fill (NOPs), return and check credits again */
679 if (tail + full_len > h2g->info.size) {
680 xe_map_memset(xe, &map, 0, 0,
681 (h2g->info.size - tail) * sizeof(u32));
682 h2g_reserve_space(ct, (h2g->info.size - tail));
683 h2g->info.tail = 0;
684 desc_write(xe, h2g, tail, h2g->info.tail);
685
686 return -EAGAIN;
687 }
688
689 /*
690 * dw0: CT header (including fence)
691 * dw1: HXG header (including action code)
692 * dw2+: action data
693 */
694 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
695 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
696 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value);
697 if (want_response) {
698 cmd[1] =
699 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
700 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
701 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
702 } else {
703 cmd[1] =
704 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) |
705 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
706 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
707 }
708
709 /* H2G header in cmd[1] replaces action[0] so: */
710 --len;
711 ++action;
712
713 /* Write H2G ensuring visible before descriptor update */
714 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32));
715 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32));
716 xe_device_wmb(xe);
717
718 /* Update local copies */
719 h2g->info.tail = (tail + full_len) % h2g->info.size;
720 h2g_reserve_space(ct, full_len);
721
722 /* Update descriptor */
723 desc_write(xe, h2g, tail, h2g->info.tail);
724
725 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len,
726 desc_read(xe, h2g, head), h2g->info.tail);
727
728 return 0;
729
730 corrupted:
731 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE);
732 return -EPIPE;
733 }
734
735 /*
736 * The CT protocol accepts a 16 bits fence. This field is fully owned by the
737 * driver, the GuC will just copy it to the reply message. Since we need to
738 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages,
739 * we use one bit of the seqno as an indicator for that and a rolling counter
740 * for the remaining 15 bits.
741 */
742 #define CT_SEQNO_MASK GENMASK(14, 0)
743 #define CT_SEQNO_UNTRACKED BIT(15)
next_ct_seqno(struct xe_guc_ct * ct,bool is_g2h_fence)744 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence)
745 {
746 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK;
747
748 if (!is_g2h_fence)
749 seqno |= CT_SEQNO_UNTRACKED;
750
751 return seqno;
752 }
753
__guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)754 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
755 u32 len, u32 g2h_len, u32 num_g2h,
756 struct g2h_fence *g2h_fence)
757 {
758 struct xe_gt *gt __maybe_unused = ct_to_gt(ct);
759 u16 seqno;
760 int ret;
761
762 xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
763 xe_gt_assert(gt, !g2h_len || !g2h_fence);
764 xe_gt_assert(gt, !num_g2h || !g2h_fence);
765 xe_gt_assert(gt, !g2h_len || num_g2h);
766 xe_gt_assert(gt, g2h_len || !num_g2h);
767 lockdep_assert_held(&ct->lock);
768
769 if (unlikely(ct->ctbs.h2g.info.broken)) {
770 ret = -EPIPE;
771 goto out;
772 }
773
774 if (ct->state == XE_GUC_CT_STATE_DISABLED) {
775 ret = -ENODEV;
776 goto out;
777 }
778
779 if (ct->state == XE_GUC_CT_STATE_STOPPED) {
780 ret = -ECANCELED;
781 goto out;
782 }
783
784 xe_gt_assert(gt, xe_guc_ct_enabled(ct));
785
786 if (g2h_fence) {
787 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN;
788 num_g2h = 1;
789
790 if (g2h_fence_needs_alloc(g2h_fence)) {
791 g2h_fence->seqno = next_ct_seqno(ct, true);
792 ret = xa_err(xa_store(&ct->fence_lookup,
793 g2h_fence->seqno, g2h_fence,
794 GFP_ATOMIC));
795 if (ret)
796 goto out;
797 }
798
799 seqno = g2h_fence->seqno;
800 } else {
801 seqno = next_ct_seqno(ct, false);
802 }
803
804 if (g2h_len)
805 spin_lock_irq(&ct->fast_lock);
806 retry:
807 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len);
808 if (unlikely(ret))
809 goto out_unlock;
810
811 ret = h2g_write(ct, action, len, seqno, !!g2h_fence);
812 if (unlikely(ret)) {
813 if (ret == -EAGAIN)
814 goto retry;
815 goto out_unlock;
816 }
817
818 __g2h_reserve_space(ct, g2h_len, num_g2h);
819 xe_guc_notify(ct_to_guc(ct));
820 out_unlock:
821 if (g2h_len)
822 spin_unlock_irq(&ct->fast_lock);
823 out:
824 return ret;
825 }
826
kick_reset(struct xe_guc_ct * ct)827 static void kick_reset(struct xe_guc_ct *ct)
828 {
829 xe_gt_reset_async(ct_to_gt(ct));
830 }
831
832 static int dequeue_one_g2h(struct xe_guc_ct *ct);
833
guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)834 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
835 u32 g2h_len, u32 num_g2h,
836 struct g2h_fence *g2h_fence)
837 {
838 struct xe_device *xe = ct_to_xe(ct);
839 struct xe_gt *gt = ct_to_gt(ct);
840 unsigned int sleep_period_ms = 1;
841 int ret;
842
843 xe_gt_assert(gt, !g2h_len || !g2h_fence);
844 lockdep_assert_held(&ct->lock);
845 xe_device_assert_mem_access(ct_to_xe(ct));
846
847 try_again:
848 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h,
849 g2h_fence);
850
851 /*
852 * We wait to try to restore credits for about 1 second before bailing.
853 * In the case of H2G credits we have no choice but just to wait for the
854 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In
855 * the case of G2H we process any G2H in the channel, hopefully freeing
856 * credits as we consume the G2H messages.
857 */
858 if (unlikely(ret == -EBUSY &&
859 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) {
860 struct guc_ctb *h2g = &ct->ctbs.h2g;
861
862 if (sleep_period_ms == 1024)
863 goto broken;
864
865 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail,
866 h2g->info.size,
867 h2g->info.space,
868 len + GUC_CTB_HDR_LEN);
869 msleep(sleep_period_ms);
870 sleep_period_ms <<= 1;
871
872 goto try_again;
873 } else if (unlikely(ret == -EBUSY)) {
874 struct xe_device *xe = ct_to_xe(ct);
875 struct guc_ctb *g2h = &ct->ctbs.g2h;
876
877 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head,
878 desc_read(xe, g2h, tail),
879 g2h->info.size,
880 g2h->info.space,
881 g2h_fence ?
882 GUC_CTB_HXG_MSG_MAX_LEN :
883 g2h_len);
884
885 #define g2h_avail(ct) \
886 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head)
887 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding ||
888 g2h_avail(ct), HZ))
889 goto broken;
890 #undef g2h_avail
891
892 ret = dequeue_one_g2h(ct);
893 if (ret < 0) {
894 if (ret != -ECANCELED)
895 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)",
896 ERR_PTR(ret));
897 goto broken;
898 }
899
900 goto try_again;
901 }
902
903 return ret;
904
905 broken:
906 xe_gt_err(gt, "No forward process on H2G, reset required\n");
907 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK);
908
909 return -EDEADLK;
910 }
911
guc_ct_send(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)912 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
913 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence)
914 {
915 int ret;
916
917 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence);
918
919 mutex_lock(&ct->lock);
920 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);
921 mutex_unlock(&ct->lock);
922
923 return ret;
924 }
925
xe_guc_ct_send(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h)926 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
927 u32 g2h_len, u32 num_g2h)
928 {
929 int ret;
930
931 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL);
932 if (ret == -EDEADLK)
933 kick_reset(ct);
934
935 return ret;
936 }
937
xe_guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h)938 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
939 u32 g2h_len, u32 num_g2h)
940 {
941 int ret;
942
943 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL);
944 if (ret == -EDEADLK)
945 kick_reset(ct);
946
947 return ret;
948 }
949
xe_guc_ct_send_g2h_handler(struct xe_guc_ct * ct,const u32 * action,u32 len)950 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len)
951 {
952 int ret;
953
954 lockdep_assert_held(&ct->lock);
955
956 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL);
957 if (ret == -EDEADLK)
958 kick_reset(ct);
959
960 return ret;
961 }
962
963 /*
964 * Check if a GT reset is in progress or will occur and if GT reset brought the
965 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset.
966 */
retry_failure(struct xe_guc_ct * ct,int ret)967 static bool retry_failure(struct xe_guc_ct *ct, int ret)
968 {
969 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV))
970 return false;
971
972 #define ct_alive(ct) \
973 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \
974 !ct->ctbs.g2h.info.broken)
975 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5))
976 return false;
977 #undef ct_alive
978
979 return true;
980 }
981
guc_ct_send_recv(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer,bool no_fail)982 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
983 u32 *response_buffer, bool no_fail)
984 {
985 struct xe_gt *gt = ct_to_gt(ct);
986 struct g2h_fence g2h_fence;
987 int ret = 0;
988
989 /*
990 * We use a fence to implement blocking sends / receiving response data.
991 * The seqno of the fence is sent in the H2G, returned in the G2H, and
992 * an xarray is used as storage media with the seqno being to key.
993 * Fields in the fence hold success, failure, retry status and the
994 * response data. Safe to allocate on the stack as the xarray is the
995 * only reference and it cannot be present after this function exits.
996 */
997 retry:
998 g2h_fence_init(&g2h_fence, response_buffer);
999 retry_same_fence:
1000 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence);
1001 if (unlikely(ret == -ENOMEM)) {
1002 /* Retry allocation /w GFP_KERNEL */
1003 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno,
1004 &g2h_fence, GFP_KERNEL));
1005 if (ret)
1006 return ret;
1007
1008 goto retry_same_fence;
1009 } else if (unlikely(ret)) {
1010 if (ret == -EDEADLK)
1011 kick_reset(ct);
1012
1013 if (no_fail && retry_failure(ct, ret))
1014 goto retry_same_fence;
1015
1016 if (!g2h_fence_needs_alloc(&g2h_fence))
1017 xa_erase(&ct->fence_lookup, g2h_fence.seqno);
1018
1019 return ret;
1020 }
1021
1022 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ);
1023 if (!ret) {
1024 LNL_FLUSH_WORK(&ct->g2h_worker);
1025 if (g2h_fence.done) {
1026 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n",
1027 g2h_fence.seqno, action[0]);
1028 ret = 1;
1029 }
1030 }
1031
1032 /*
1033 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on
1034 * the stack, since we have no clue if it will fire after the timeout before we can erase
1035 * from the xa. Also we have some dependent loads and stores below for which we need the
1036 * correct ordering, and we lack the needed barriers.
1037 */
1038 mutex_lock(&ct->lock);
1039 if (!ret) {
1040 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s",
1041 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done));
1042 xa_erase(&ct->fence_lookup, g2h_fence.seqno);
1043 mutex_unlock(&ct->lock);
1044 return -ETIME;
1045 }
1046
1047 if (g2h_fence.retry) {
1048 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n",
1049 action[0], g2h_fence.reason);
1050 mutex_unlock(&ct->lock);
1051 goto retry;
1052 }
1053 if (g2h_fence.fail) {
1054 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n",
1055 action[0], g2h_fence.error, g2h_fence.hint);
1056 ret = -EIO;
1057 }
1058
1059 if (ret > 0)
1060 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data;
1061
1062 mutex_unlock(&ct->lock);
1063
1064 return ret;
1065 }
1066
1067 /**
1068 * xe_guc_ct_send_recv - Send and receive HXG to the GuC
1069 * @ct: the &xe_guc_ct
1070 * @action: the dword array with `HXG Request`_ message (can't be NULL)
1071 * @len: length of the `HXG Request`_ message (in dwords, can't be 0)
1072 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL)
1073 *
1074 * Send a `HXG Request`_ message to the GuC over CT communication channel and
1075 * blocks until GuC replies with a `HXG Response`_ message.
1076 *
1077 * For non-blocking communication with GuC use xe_guc_ct_send().
1078 *
1079 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_.
1080 *
1081 * Return: response length (in dwords) if &response_buffer was not NULL, or
1082 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or
1083 * a negative error code on failure.
1084 */
xe_guc_ct_send_recv(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer)1085 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
1086 u32 *response_buffer)
1087 {
1088 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer);
1089 return guc_ct_send_recv(ct, action, len, response_buffer, false);
1090 }
1091
xe_guc_ct_send_recv_no_fail(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer)1092 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action,
1093 u32 len, u32 *response_buffer)
1094 {
1095 return guc_ct_send_recv(ct, action, len, response_buffer, true);
1096 }
1097
msg_to_hxg(u32 * msg)1098 static u32 *msg_to_hxg(u32 *msg)
1099 {
1100 return msg + GUC_CTB_MSG_MIN_LEN;
1101 }
1102
msg_len_to_hxg_len(u32 len)1103 static u32 msg_len_to_hxg_len(u32 len)
1104 {
1105 return len - GUC_CTB_MSG_MIN_LEN;
1106 }
1107
parse_g2h_event(struct xe_guc_ct * ct,u32 * msg,u32 len)1108 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)
1109 {
1110 u32 *hxg = msg_to_hxg(msg);
1111 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1112
1113 lockdep_assert_held(&ct->lock);
1114
1115 switch (action) {
1116 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1117 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1118 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1119 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1120 g2h_release_space(ct, len);
1121 }
1122
1123 return 0;
1124 }
1125
guc_crash_process_msg(struct xe_guc_ct * ct,u32 action)1126 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action)
1127 {
1128 struct xe_gt *gt = ct_to_gt(ct);
1129
1130 if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED)
1131 xe_gt_err(gt, "GuC Crash dump notification\n");
1132 else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION)
1133 xe_gt_err(gt, "GuC Exception notification\n");
1134 else
1135 xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action);
1136
1137 CT_DEAD(ct, NULL, CRASH);
1138
1139 kick_reset(ct);
1140
1141 return 0;
1142 }
1143
parse_g2h_response(struct xe_guc_ct * ct,u32 * msg,u32 len)1144 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
1145 {
1146 struct xe_gt *gt = ct_to_gt(ct);
1147 u32 *hxg = msg_to_hxg(msg);
1148 u32 hxg_len = msg_len_to_hxg_len(len);
1149 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]);
1150 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1151 struct g2h_fence *g2h_fence;
1152
1153 lockdep_assert_held(&ct->lock);
1154
1155 /*
1156 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup.
1157 * Those messages should never fail, so if we do get an error back it
1158 * means we're likely doing an illegal operation and the GuC is
1159 * rejecting it. We have no way to inform the code that submitted the
1160 * H2G that the message was rejected, so we need to escalate the
1161 * failure to trigger a reset.
1162 */
1163 if (fence & CT_SEQNO_UNTRACKED) {
1164 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE)
1165 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n",
1166 fence,
1167 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]),
1168 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]));
1169 else
1170 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n",
1171 type, fence);
1172 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE);
1173
1174 return -EPROTO;
1175 }
1176
1177 g2h_fence = xa_erase(&ct->fence_lookup, fence);
1178 if (unlikely(!g2h_fence)) {
1179 /* Don't tear down channel, as send could've timed out */
1180 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */
1181 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence);
1182 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1183 return 0;
1184 }
1185
1186 xe_gt_assert(gt, fence == g2h_fence->seqno);
1187
1188 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {
1189 g2h_fence->fail = true;
1190 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]);
1191 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]);
1192 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1193 g2h_fence->retry = true;
1194 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]);
1195 } else if (g2h_fence->response_buffer) {
1196 g2h_fence->response_len = hxg_len;
1197 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32));
1198 } else {
1199 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]);
1200 }
1201
1202 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1203
1204 g2h_fence->done = true;
1205 smp_mb();
1206
1207 wake_up_all(&ct->g2h_fence_wq);
1208
1209 return 0;
1210 }
1211
parse_g2h_msg(struct xe_guc_ct * ct,u32 * msg,u32 len)1212 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1213 {
1214 struct xe_gt *gt = ct_to_gt(ct);
1215 u32 *hxg = msg_to_hxg(msg);
1216 u32 origin, type;
1217 int ret;
1218
1219 lockdep_assert_held(&ct->lock);
1220
1221 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1222 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1223 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n",
1224 origin);
1225 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN);
1226
1227 return -EPROTO;
1228 }
1229
1230 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1231 switch (type) {
1232 case GUC_HXG_TYPE_EVENT:
1233 ret = parse_g2h_event(ct, msg, len);
1234 break;
1235 case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1236 case GUC_HXG_TYPE_RESPONSE_FAILURE:
1237 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1238 ret = parse_g2h_response(ct, msg, len);
1239 break;
1240 default:
1241 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n",
1242 type);
1243 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE);
1244
1245 ret = -EOPNOTSUPP;
1246 }
1247
1248 return ret;
1249 }
1250
process_g2h_msg(struct xe_guc_ct * ct,u32 * msg,u32 len)1251 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1252 {
1253 struct xe_guc *guc = ct_to_guc(ct);
1254 struct xe_gt *gt = ct_to_gt(ct);
1255 u32 hxg_len = msg_len_to_hxg_len(len);
1256 u32 *hxg = msg_to_hxg(msg);
1257 u32 action, adj_len;
1258 u32 *payload;
1259 int ret = 0;
1260
1261 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1262 return 0;
1263
1264 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1265 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN;
1266 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN;
1267
1268 switch (action) {
1269 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1270 ret = xe_guc_sched_done_handler(guc, payload, adj_len);
1271 break;
1272 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1273 ret = xe_guc_deregister_done_handler(guc, payload, adj_len);
1274 break;
1275 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1276 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len);
1277 break;
1278 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1279 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload,
1280 adj_len);
1281 break;
1282 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1283 /* Selftest only at the moment */
1284 break;
1285 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1286 ret = xe_guc_error_capture_handler(guc, payload, adj_len);
1287 break;
1288 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1289 /* FIXME: Handle this */
1290 break;
1291 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR:
1292 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload,
1293 adj_len);
1294 break;
1295 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1296 ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1297 break;
1298 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1299 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1300 adj_len);
1301 break;
1302 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY:
1303 ret = xe_guc_access_counter_notify_handler(guc, payload,
1304 adj_len);
1305 break;
1306 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
1307 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);
1308 break;
1309 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF:
1310 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len);
1311 break;
1312 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY:
1313 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len);
1314 break;
1315 case GUC_ACTION_GUC2PF_ADVERSE_EVENT:
1316 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len);
1317 break;
1318 case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1319 case XE_GUC_ACTION_NOTIFY_EXCEPTION:
1320 ret = guc_crash_process_msg(ct, action);
1321 break;
1322 default:
1323 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
1324 }
1325
1326 if (ret) {
1327 xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n",
1328 action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg);
1329 CT_DEAD(ct, NULL, PROCESS_FAILED);
1330 }
1331
1332 return 0;
1333 }
1334
g2h_read(struct xe_guc_ct * ct,u32 * msg,bool fast_path)1335 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
1336 {
1337 struct xe_device *xe = ct_to_xe(ct);
1338 struct xe_gt *gt = ct_to_gt(ct);
1339 struct guc_ctb *g2h = &ct->ctbs.g2h;
1340 u32 tail, head, len, desc_status;
1341 s32 avail;
1342 u32 action;
1343 u32 *hxg;
1344
1345 xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
1346 lockdep_assert_held(&ct->fast_lock);
1347
1348 if (ct->state == XE_GUC_CT_STATE_DISABLED)
1349 return -ENODEV;
1350
1351 if (ct->state == XE_GUC_CT_STATE_STOPPED)
1352 return -ECANCELED;
1353
1354 if (g2h->info.broken)
1355 return -EPIPE;
1356
1357 xe_gt_assert(gt, xe_guc_ct_enabled(ct));
1358
1359 desc_status = desc_read(xe, g2h, status);
1360 if (desc_status) {
1361 if (desc_status & GUC_CTB_STATUS_DISABLED) {
1362 /*
1363 * Potentially valid if a CLIENT_RESET request resulted in
1364 * contexts/engines being reset. But should never happen as
1365 * no contexts should be active when CLIENT_RESET is sent.
1366 */
1367 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n");
1368 desc_status &= ~GUC_CTB_STATUS_DISABLED;
1369 }
1370
1371 if (desc_status) {
1372 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status);
1373 goto corrupted;
1374 }
1375 }
1376
1377 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
1378 u32 desc_tail = desc_read(xe, g2h, tail);
1379 /*
1380 u32 desc_head = desc_read(xe, g2h, head);
1381
1382 * info.head and desc_head are updated back-to-back at the end of
1383 * this function and nowhere else. Hence, they cannot be different
1384 * unless two g2h_read calls are running concurrently. Which is not
1385 * possible because it is guarded by ct->fast_lock. And yet, some
1386 * discrete platforms are regularly hitting this error :(.
1387 *
1388 * desc_head rolling backwards shouldn't cause any noticeable
1389 * problems - just a delay in GuC being allowed to proceed past that
1390 * point in the queue. So for now, just disable the error until it
1391 * can be root caused.
1392 *
1393 if (g2h->info.head != desc_head) {
1394 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH);
1395 xe_gt_err(gt, "CT read: head was modified %u != %u\n",
1396 desc_head, g2h->info.head);
1397 goto corrupted;
1398 }
1399 */
1400
1401 if (g2h->info.head > g2h->info.size) {
1402 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
1403 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n",
1404 g2h->info.head, g2h->info.size);
1405 goto corrupted;
1406 }
1407
1408 if (desc_tail >= g2h->info.size) {
1409 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
1410 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n",
1411 desc_tail, g2h->info.size);
1412 goto corrupted;
1413 }
1414 }
1415
1416 /* Calculate DW available to read */
1417 tail = desc_read(xe, g2h, tail);
1418 avail = tail - g2h->info.head;
1419 if (unlikely(avail == 0))
1420 return 0;
1421
1422 if (avail < 0)
1423 avail += g2h->info.size;
1424
1425 /* Read header */
1426 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head,
1427 sizeof(u32));
1428 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN;
1429 if (len > avail) {
1430 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n",
1431 avail, len);
1432 goto corrupted;
1433 }
1434
1435 head = (g2h->info.head + 1) % g2h->info.size;
1436 avail = len - 1;
1437
1438 /* Read G2H message */
1439 if (avail + head > g2h->info.size) {
1440 u32 avail_til_wrap = g2h->info.size - head;
1441
1442 xe_map_memcpy_from(xe, msg + 1,
1443 &g2h->cmds, sizeof(u32) * head,
1444 avail_til_wrap * sizeof(u32));
1445 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap,
1446 &g2h->cmds, 0,
1447 (avail - avail_til_wrap) * sizeof(u32));
1448 } else {
1449 xe_map_memcpy_from(xe, msg + 1,
1450 &g2h->cmds, sizeof(u32) * head,
1451 avail * sizeof(u32));
1452 }
1453
1454 hxg = msg_to_hxg(msg);
1455 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1456
1457 if (fast_path) {
1458 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1459 return 0;
1460
1461 switch (action) {
1462 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1463 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1464 break; /* Process these in fast-path */
1465 default:
1466 return 0;
1467 }
1468 }
1469
1470 /* Update local / descriptor header */
1471 g2h->info.head = (head + avail) % g2h->info.size;
1472 desc_write(xe, g2h, head, g2h->info.head);
1473
1474 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id,
1475 action, len, g2h->info.head, tail);
1476
1477 return len;
1478
1479 corrupted:
1480 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ);
1481 return -EPROTO;
1482 }
1483
g2h_fast_path(struct xe_guc_ct * ct,u32 * msg,u32 len)1484 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
1485 {
1486 struct xe_gt *gt = ct_to_gt(ct);
1487 struct xe_guc *guc = ct_to_guc(ct);
1488 u32 hxg_len = msg_len_to_hxg_len(len);
1489 u32 *hxg = msg_to_hxg(msg);
1490 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1491 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN;
1492 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1493 int ret = 0;
1494
1495 switch (action) {
1496 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1497 ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1498 break;
1499 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1500 __g2h_release_space(ct, len);
1501 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1502 adj_len);
1503 break;
1504 default:
1505 xe_gt_warn(gt, "NOT_POSSIBLE");
1506 }
1507
1508 if (ret) {
1509 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
1510 action, ERR_PTR(ret));
1511 CT_DEAD(ct, NULL, FAST_G2H);
1512 }
1513 }
1514
1515 /**
1516 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler
1517 * @ct: GuC CT object
1518 *
1519 * Anything related to page faults is critical for performance, process these
1520 * critical G2H in the IRQ. This is safe as these handlers either just wake up
1521 * waiters or queue another worker.
1522 */
xe_guc_ct_fast_path(struct xe_guc_ct * ct)1523 void xe_guc_ct_fast_path(struct xe_guc_ct *ct)
1524 {
1525 struct xe_device *xe = ct_to_xe(ct);
1526 bool ongoing;
1527 int len;
1528
1529 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1530 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1531 return;
1532
1533 spin_lock(&ct->fast_lock);
1534 do {
1535 len = g2h_read(ct, ct->fast_msg, true);
1536 if (len > 0)
1537 g2h_fast_path(ct, ct->fast_msg, len);
1538 } while (len > 0);
1539 spin_unlock(&ct->fast_lock);
1540
1541 if (ongoing)
1542 xe_pm_runtime_put(xe);
1543 }
1544
1545 /* Returns less than zero on error, 0 on done, 1 on more available */
dequeue_one_g2h(struct xe_guc_ct * ct)1546 static int dequeue_one_g2h(struct xe_guc_ct *ct)
1547 {
1548 int len;
1549 int ret;
1550
1551 lockdep_assert_held(&ct->lock);
1552
1553 spin_lock_irq(&ct->fast_lock);
1554 len = g2h_read(ct, ct->msg, false);
1555 spin_unlock_irq(&ct->fast_lock);
1556 if (len <= 0)
1557 return len;
1558
1559 ret = parse_g2h_msg(ct, ct->msg, len);
1560 if (unlikely(ret < 0))
1561 return ret;
1562
1563 ret = process_g2h_msg(ct, ct->msg, len);
1564 if (unlikely(ret < 0))
1565 return ret;
1566
1567 return 1;
1568 }
1569
receive_g2h(struct xe_guc_ct * ct)1570 static void receive_g2h(struct xe_guc_ct *ct)
1571 {
1572 bool ongoing;
1573 int ret;
1574
1575 /*
1576 * Normal users must always hold mem_access.ref around CT calls. However
1577 * during the runtime pm callbacks we rely on CT to talk to the GuC, but
1578 * at this stage we can't rely on mem_access.ref and even the
1579 * callback_task will be different than current. For such cases we just
1580 * need to ensure we always process the responses from any blocking
1581 * ct_send requests or where we otherwise expect some response when
1582 * initiated from those callbacks (which will need to wait for the below
1583 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if
1584 * the device has suspended to the point that the CT communication has
1585 * been disabled.
1586 *
1587 * If we are inside the runtime pm callback, we can be the only task
1588 * still issuing CT requests (since that requires having the
1589 * mem_access.ref). It seems like it might in theory be possible to
1590 * receive unsolicited events from the GuC just as we are
1591 * suspending-resuming, but those will currently anyway be lost when
1592 * eventually exiting from suspend, hence no need to wake up the device
1593 * here. If we ever need something stronger than get_if_ongoing() then
1594 * we need to be careful with blocking the pm callbacks from getting CT
1595 * responses, if the worker here is blocked on those callbacks
1596 * completing, creating a deadlock.
1597 */
1598 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1599 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1600 return;
1601
1602 do {
1603 mutex_lock(&ct->lock);
1604 ret = dequeue_one_g2h(ct);
1605 mutex_unlock(&ct->lock);
1606
1607 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {
1608 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret);
1609 CT_DEAD(ct, NULL, G2H_RECV);
1610 kick_reset(ct);
1611 }
1612 } while (ret == 1);
1613
1614 if (ongoing)
1615 xe_pm_runtime_put(ct_to_xe(ct));
1616 }
1617
g2h_worker_func(struct work_struct * w)1618 static void g2h_worker_func(struct work_struct *w)
1619 {
1620 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker);
1621
1622 receive_g2h(ct);
1623 }
1624
guc_ct_snapshot_alloc(struct xe_guc_ct * ct,bool atomic,bool want_ctb)1625 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic,
1626 bool want_ctb)
1627 {
1628 struct xe_guc_ct_snapshot *snapshot;
1629
1630 snapshot = kzalloc(sizeof(*snapshot), atomic ? GFP_ATOMIC : GFP_KERNEL);
1631 if (!snapshot)
1632 return NULL;
1633
1634 if (ct->bo && want_ctb) {
1635 snapshot->ctb_size = ct->bo->size;
1636 snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL);
1637 }
1638
1639 return snapshot;
1640 }
1641
guc_ctb_snapshot_capture(struct xe_device * xe,struct guc_ctb * ctb,struct guc_ctb_snapshot * snapshot)1642 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb,
1643 struct guc_ctb_snapshot *snapshot)
1644 {
1645 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0,
1646 sizeof(struct guc_ct_buffer_desc));
1647 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info));
1648 }
1649
guc_ctb_snapshot_print(struct guc_ctb_snapshot * snapshot,struct drm_printer * p)1650 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot,
1651 struct drm_printer *p)
1652 {
1653 drm_printf(p, "\tsize: %d\n", snapshot->info.size);
1654 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space);
1655 drm_printf(p, "\thead: %d\n", snapshot->info.head);
1656 drm_printf(p, "\ttail: %d\n", snapshot->info.tail);
1657 drm_printf(p, "\tspace: %d\n", snapshot->info.space);
1658 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken);
1659 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head);
1660 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail);
1661 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status);
1662 }
1663
guc_ct_snapshot_capture(struct xe_guc_ct * ct,bool atomic,bool want_ctb)1664 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic,
1665 bool want_ctb)
1666 {
1667 struct xe_device *xe = ct_to_xe(ct);
1668 struct xe_guc_ct_snapshot *snapshot;
1669
1670 snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb);
1671 if (!snapshot) {
1672 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n");
1673 return NULL;
1674 }
1675
1676 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) {
1677 snapshot->ct_enabled = true;
1678 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);
1679 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g);
1680 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h);
1681 }
1682
1683 if (ct->bo && snapshot->ctb)
1684 xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size);
1685
1686 return snapshot;
1687 }
1688
1689 /**
1690 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state.
1691 * @ct: GuC CT object.
1692 *
1693 * This can be printed out in a later stage like during dev_coredump
1694 * analysis. This is safe to be called during atomic context.
1695 *
1696 * Returns: a GuC CT snapshot object that must be freed by the caller
1697 * by using `xe_guc_ct_snapshot_free`.
1698 */
xe_guc_ct_snapshot_capture(struct xe_guc_ct * ct)1699 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct)
1700 {
1701 return guc_ct_snapshot_capture(ct, true, true);
1702 }
1703
1704 /**
1705 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot.
1706 * @snapshot: GuC CT snapshot object.
1707 * @p: drm_printer where it will be printed out.
1708 *
1709 * This function prints out a given GuC CT snapshot object.
1710 */
xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot * snapshot,struct drm_printer * p)1711 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot,
1712 struct drm_printer *p)
1713 {
1714 if (!snapshot)
1715 return;
1716
1717 if (snapshot->ct_enabled) {
1718 drm_puts(p, "H2G CTB (all sizes in DW):\n");
1719 guc_ctb_snapshot_print(&snapshot->h2g, p);
1720
1721 drm_puts(p, "G2H CTB (all sizes in DW):\n");
1722 guc_ctb_snapshot_print(&snapshot->g2h, p);
1723 drm_printf(p, "\tg2h outstanding: %d\n",
1724 snapshot->g2h_outstanding);
1725
1726 if (snapshot->ctb) {
1727 drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size);
1728 xe_print_blob_ascii85(p, "[CTB].data", '\n',
1729 snapshot->ctb, 0, snapshot->ctb_size);
1730 }
1731 } else {
1732 drm_puts(p, "CT disabled\n");
1733 }
1734 }
1735
1736 /**
1737 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot.
1738 * @snapshot: GuC CT snapshot object.
1739 *
1740 * This function free all the memory that needed to be allocated at capture
1741 * time.
1742 */
xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot * snapshot)1743 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot)
1744 {
1745 if (!snapshot)
1746 return;
1747
1748 kfree(snapshot->ctb);
1749 kfree(snapshot);
1750 }
1751
1752 /**
1753 * xe_guc_ct_print - GuC CT Print.
1754 * @ct: GuC CT.
1755 * @p: drm_printer where it will be printed out.
1756 * @want_ctb: Should the full CTB content be dumped (vs just the headers)
1757 *
1758 * This function will quickly capture a snapshot of the CT state
1759 * and immediately print it out.
1760 */
xe_guc_ct_print(struct xe_guc_ct * ct,struct drm_printer * p,bool want_ctb)1761 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb)
1762 {
1763 struct xe_guc_ct_snapshot *snapshot;
1764
1765 snapshot = guc_ct_snapshot_capture(ct, false, want_ctb);
1766 xe_guc_ct_snapshot_print(snapshot, p);
1767 xe_guc_ct_snapshot_free(snapshot);
1768 }
1769
1770 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
ct_dead_capture(struct xe_guc_ct * ct,struct guc_ctb * ctb,u32 reason_code)1771 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code)
1772 {
1773 struct xe_guc_log_snapshot *snapshot_log;
1774 struct xe_guc_ct_snapshot *snapshot_ct;
1775 struct xe_guc *guc = ct_to_guc(ct);
1776 unsigned long flags;
1777 bool have_capture;
1778
1779 if (ctb)
1780 ctb->info.broken = true;
1781
1782 /* Ignore further errors after the first dump until a reset */
1783 if (ct->dead.reported)
1784 return;
1785
1786 spin_lock_irqsave(&ct->dead.lock, flags);
1787
1788 /* And only capture one dump at a time */
1789 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE);
1790 ct->dead.reason |= (1 << reason_code) |
1791 (1 << CT_DEAD_STATE_CAPTURE);
1792
1793 spin_unlock_irqrestore(&ct->dead.lock, flags);
1794
1795 if (have_capture)
1796 return;
1797
1798 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true);
1799 snapshot_ct = xe_guc_ct_snapshot_capture((ct));
1800
1801 spin_lock_irqsave(&ct->dead.lock, flags);
1802
1803 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) {
1804 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n");
1805 xe_guc_log_snapshot_free(snapshot_log);
1806 xe_guc_ct_snapshot_free(snapshot_ct);
1807 } else {
1808 ct->dead.snapshot_log = snapshot_log;
1809 ct->dead.snapshot_ct = snapshot_ct;
1810 }
1811
1812 spin_unlock_irqrestore(&ct->dead.lock, flags);
1813
1814 queue_work(system_unbound_wq, &(ct)->dead.worker);
1815 }
1816
ct_dead_print(struct xe_dead_ct * dead)1817 static void ct_dead_print(struct xe_dead_ct *dead)
1818 {
1819 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead);
1820 struct xe_device *xe = ct_to_xe(ct);
1821 struct xe_gt *gt = ct_to_gt(ct);
1822 static int g_count;
1823 struct drm_printer ip = xe_gt_info_printer(gt);
1824 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count);
1825
1826 if (!dead->reason) {
1827 xe_gt_err(gt, "CTB is dead for no reason!?\n");
1828 return;
1829 }
1830
1831 drm_printf(&lp, "CTB is dead - reason=0x%X\n", dead->reason);
1832
1833 /* Can't generate a genuine core dump at this point, so just do the good bits */
1834 drm_puts(&lp, "**** Xe Device Coredump ****\n");
1835 xe_device_snapshot_print(xe, &lp);
1836
1837 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id);
1838 drm_printf(&lp, "\tTile: %d\n", gt->tile->id);
1839
1840 drm_puts(&lp, "**** GuC Log ****\n");
1841 xe_guc_log_snapshot_print(dead->snapshot_log, &lp);
1842
1843 drm_puts(&lp, "**** GuC CT ****\n");
1844 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp);
1845
1846 drm_puts(&lp, "Done.\n");
1847 }
1848
ct_dead_worker_func(struct work_struct * w)1849 static void ct_dead_worker_func(struct work_struct *w)
1850 {
1851 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker);
1852
1853 if (!ct->dead.reported) {
1854 ct->dead.reported = true;
1855 ct_dead_print(&ct->dead);
1856 }
1857
1858 spin_lock_irq(&ct->dead.lock);
1859
1860 xe_guc_log_snapshot_free(ct->dead.snapshot_log);
1861 ct->dead.snapshot_log = NULL;
1862 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct);
1863 ct->dead.snapshot_ct = NULL;
1864
1865 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) {
1866 /* A reset has occurred so re-arm the error reporting */
1867 ct->dead.reason = 0;
1868 ct->dead.reported = false;
1869 }
1870
1871 spin_unlock_irq(&ct->dead.lock);
1872 }
1873 #endif
1874