1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc_ct.h"
7
8 #include <linux/bitfield.h>
9 #include <linux/circ_buf.h>
10 #include <linux/delay.h>
11
12 #include <kunit/static_stub.h>
13
14 #include <drm/drm_managed.h>
15
16 #include "abi/guc_actions_abi.h"
17 #include "abi/guc_actions_sriov_abi.h"
18 #include "abi/guc_klvs_abi.h"
19 #include "xe_bo.h"
20 #include "xe_device.h"
21 #include "xe_gt.h"
22 #include "xe_gt_pagefault.h"
23 #include "xe_gt_printk.h"
24 #include "xe_gt_sriov_pf_control.h"
25 #include "xe_gt_sriov_pf_monitor.h"
26 #include "xe_gt_tlb_invalidation.h"
27 #include "xe_guc.h"
28 #include "xe_guc_relay.h"
29 #include "xe_guc_submit.h"
30 #include "xe_map.h"
31 #include "xe_pm.h"
32 #include "xe_trace_guc.h"
33
34 /* Used when a CT send wants to block and / or receive data */
35 struct g2h_fence {
36 u32 *response_buffer;
37 u32 seqno;
38 u32 response_data;
39 u16 response_len;
40 u16 error;
41 u16 hint;
42 u16 reason;
43 bool retry;
44 bool fail;
45 bool done;
46 };
47
g2h_fence_init(struct g2h_fence * g2h_fence,u32 * response_buffer)48 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer)
49 {
50 g2h_fence->response_buffer = response_buffer;
51 g2h_fence->response_data = 0;
52 g2h_fence->response_len = 0;
53 g2h_fence->fail = false;
54 g2h_fence->retry = false;
55 g2h_fence->done = false;
56 g2h_fence->seqno = ~0x0;
57 }
58
g2h_fence_needs_alloc(struct g2h_fence * g2h_fence)59 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
60 {
61 return g2h_fence->seqno == ~0x0;
62 }
63
64 static struct xe_guc *
ct_to_guc(struct xe_guc_ct * ct)65 ct_to_guc(struct xe_guc_ct *ct)
66 {
67 return container_of(ct, struct xe_guc, ct);
68 }
69
70 static struct xe_gt *
ct_to_gt(struct xe_guc_ct * ct)71 ct_to_gt(struct xe_guc_ct *ct)
72 {
73 return container_of(ct, struct xe_gt, uc.guc.ct);
74 }
75
76 static struct xe_device *
ct_to_xe(struct xe_guc_ct * ct)77 ct_to_xe(struct xe_guc_ct *ct)
78 {
79 return gt_to_xe(ct_to_gt(ct));
80 }
81
82 /**
83 * DOC: GuC CTB Blob
84 *
85 * We allocate single blob to hold both CTB descriptors and buffers:
86 *
87 * +--------+-----------------------------------------------+------+
88 * | offset | contents | size |
89 * +========+===============================================+======+
90 * | 0x0000 | H2G CTB Descriptor (send) | |
91 * +--------+-----------------------------------------------+ 4K |
92 * | 0x0800 | G2H CTB Descriptor (g2h) | |
93 * +--------+-----------------------------------------------+------+
94 * | 0x1000 | H2G CT Buffer (send) | n*4K |
95 * | | | |
96 * +--------+-----------------------------------------------+------+
97 * | 0x1000 | G2H CT Buffer (g2h) | m*4K |
98 * | + n*4K | | |
99 * +--------+-----------------------------------------------+------+
100 *
101 * Size of each ``CT Buffer`` must be multiple of 4K.
102 * We don't expect too many messages in flight at any time, unless we are
103 * using the GuC submission. In that case each request requires a minimum
104 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
105 * enough space to avoid backpressure on the driver. We increase the size
106 * of the receive buffer (relative to the send) to ensure a G2H response
107 * CTB has a landing spot.
108 *
109 * In addition to submissions, the G2H buffer needs to be able to hold
110 * enough space for recoverable page fault notifications. The number of
111 * page faults is interrupt driven and can be as much as the number of
112 * compute resources available. However, most of the actual work for these
113 * is in a separate page fault worker thread. Therefore we only need to
114 * make sure the queue has enough space to handle all of the submissions
115 * and responses and an extra buffer for incoming page faults.
116 */
117
118 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
119 #define CTB_H2G_BUFFER_SIZE (SZ_4K)
120 #define CTB_G2H_BUFFER_SIZE (SZ_128K)
121 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2)
122
123 /**
124 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full
125 * CT command queue
126 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future.
127 *
128 * Observation is that a 4KiB buffer full of commands takes a little over a
129 * second to process. Use that to calculate maximum time to process a full CT
130 * command queue.
131 *
132 * Return: Maximum time to process a full CT queue in jiffies.
133 */
xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct * ct)134 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct)
135 {
136 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4));
137 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ;
138 }
139
guc_ct_size(void)140 static size_t guc_ct_size(void)
141 {
142 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE +
143 CTB_G2H_BUFFER_SIZE;
144 }
145
guc_ct_fini(struct drm_device * drm,void * arg)146 static void guc_ct_fini(struct drm_device *drm, void *arg)
147 {
148 struct xe_guc_ct *ct = arg;
149
150 destroy_workqueue(ct->g2h_wq);
151 xa_destroy(&ct->fence_lookup);
152 }
153
154 static void receive_g2h(struct xe_guc_ct *ct);
155 static void g2h_worker_func(struct work_struct *w);
156 static void safe_mode_worker_func(struct work_struct *w);
157
primelockdep(struct xe_guc_ct * ct)158 static void primelockdep(struct xe_guc_ct *ct)
159 {
160 if (!IS_ENABLED(CONFIG_LOCKDEP))
161 return;
162
163 fs_reclaim_acquire(GFP_KERNEL);
164 might_lock(&ct->lock);
165 fs_reclaim_release(GFP_KERNEL);
166 }
167
xe_guc_ct_init(struct xe_guc_ct * ct)168 int xe_guc_ct_init(struct xe_guc_ct *ct)
169 {
170 struct xe_device *xe = ct_to_xe(ct);
171 struct xe_gt *gt = ct_to_gt(ct);
172 struct xe_tile *tile = gt_to_tile(gt);
173 struct xe_bo *bo;
174 int err;
175
176 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE));
177
178 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", 0);
179 if (!ct->g2h_wq)
180 return -ENOMEM;
181
182 spin_lock_init(&ct->fast_lock);
183 xa_init(&ct->fence_lookup);
184 INIT_WORK(&ct->g2h_worker, g2h_worker_func);
185 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);
186 init_waitqueue_head(&ct->wq);
187 init_waitqueue_head(&ct->g2h_fence_wq);
188
189 err = drmm_mutex_init(&xe->drm, &ct->lock);
190 if (err)
191 return err;
192
193 primelockdep(ct);
194
195 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
196 XE_BO_FLAG_SYSTEM |
197 XE_BO_FLAG_GGTT |
198 XE_BO_FLAG_GGTT_INVALIDATE);
199 if (IS_ERR(bo))
200 return PTR_ERR(bo);
201
202 ct->bo = bo;
203
204 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct);
205 if (err)
206 return err;
207
208 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED);
209 ct->state = XE_GUC_CT_STATE_DISABLED;
210 return 0;
211 }
212
213 #define desc_read(xe_, guc_ctb__, field_) \
214 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \
215 struct guc_ct_buffer_desc, field_)
216
217 #define desc_write(xe_, guc_ctb__, field_, val_) \
218 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \
219 struct guc_ct_buffer_desc, field_, val_)
220
guc_ct_ctb_h2g_init(struct xe_device * xe,struct guc_ctb * h2g,struct iosys_map * map)221 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,
222 struct iosys_map *map)
223 {
224 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);
225 h2g->info.resv_space = 0;
226 h2g->info.tail = 0;
227 h2g->info.head = 0;
228 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
229 h2g->info.size) -
230 h2g->info.resv_space;
231 h2g->info.broken = false;
232
233 h2g->desc = *map;
234 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
235
236 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2);
237 }
238
guc_ct_ctb_g2h_init(struct xe_device * xe,struct guc_ctb * g2h,struct iosys_map * map)239 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,
240 struct iosys_map *map)
241 {
242 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);
243 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);
244 g2h->info.head = 0;
245 g2h->info.tail = 0;
246 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
247 g2h->info.size) -
248 g2h->info.resv_space;
249 g2h->info.broken = false;
250
251 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE);
252 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
253
254 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 +
255 CTB_H2G_BUFFER_SIZE);
256 }
257
guc_ct_ctb_h2g_register(struct xe_guc_ct * ct)258 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)
259 {
260 struct xe_guc *guc = ct_to_guc(ct);
261 u32 desc_addr, ctb_addr, size;
262 int err;
263
264 desc_addr = xe_bo_ggtt_addr(ct->bo);
265 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2;
266 size = ct->ctbs.h2g.info.size * sizeof(u32);
267
268 err = xe_guc_self_cfg64(guc,
269 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY,
270 desc_addr);
271 if (err)
272 return err;
273
274 err = xe_guc_self_cfg64(guc,
275 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY,
276 ctb_addr);
277 if (err)
278 return err;
279
280 return xe_guc_self_cfg32(guc,
281 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY,
282 size);
283 }
284
guc_ct_ctb_g2h_register(struct xe_guc_ct * ct)285 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct)
286 {
287 struct xe_guc *guc = ct_to_guc(ct);
288 u32 desc_addr, ctb_addr, size;
289 int err;
290
291 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE;
292 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 +
293 CTB_H2G_BUFFER_SIZE;
294 size = ct->ctbs.g2h.info.size * sizeof(u32);
295
296 err = xe_guc_self_cfg64(guc,
297 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
298 desc_addr);
299 if (err)
300 return err;
301
302 err = xe_guc_self_cfg64(guc,
303 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
304 ctb_addr);
305 if (err)
306 return err;
307
308 return xe_guc_self_cfg32(guc,
309 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
310 size);
311 }
312
guc_ct_control_toggle(struct xe_guc_ct * ct,bool enable)313 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable)
314 {
315 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
316 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
317 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
318 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
319 GUC_ACTION_HOST2GUC_CONTROL_CTB),
320 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL,
321 enable ? GUC_CTB_CONTROL_ENABLE :
322 GUC_CTB_CONTROL_DISABLE),
323 };
324 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request));
325
326 return ret > 0 ? -EPROTO : ret;
327 }
328
xe_guc_ct_set_state(struct xe_guc_ct * ct,enum xe_guc_ct_state state)329 static void xe_guc_ct_set_state(struct xe_guc_ct *ct,
330 enum xe_guc_ct_state state)
331 {
332 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */
333 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */
334
335 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 ||
336 state == XE_GUC_CT_STATE_STOPPED);
337
338 if (ct->g2h_outstanding)
339 xe_pm_runtime_put(ct_to_xe(ct));
340 ct->g2h_outstanding = 0;
341 ct->state = state;
342
343 spin_unlock_irq(&ct->fast_lock);
344
345 /*
346 * Lockdep doesn't like this under the fast lock and he destroy only
347 * needs to be serialized with the send path which ct lock provides.
348 */
349 xa_destroy(&ct->fence_lookup);
350
351 mutex_unlock(&ct->lock);
352 }
353
ct_needs_safe_mode(struct xe_guc_ct * ct)354 static bool ct_needs_safe_mode(struct xe_guc_ct *ct)
355 {
356 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev));
357 }
358
ct_restart_safe_mode_worker(struct xe_guc_ct * ct)359 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct)
360 {
361 if (!ct_needs_safe_mode(ct))
362 return false;
363
364 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10);
365 return true;
366 }
367
safe_mode_worker_func(struct work_struct * w)368 static void safe_mode_worker_func(struct work_struct *w)
369 {
370 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work);
371
372 receive_g2h(ct);
373
374 if (!ct_restart_safe_mode_worker(ct))
375 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n");
376 }
377
ct_enter_safe_mode(struct xe_guc_ct * ct)378 static void ct_enter_safe_mode(struct xe_guc_ct *ct)
379 {
380 if (ct_restart_safe_mode_worker(ct))
381 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n");
382 }
383
ct_exit_safe_mode(struct xe_guc_ct * ct)384 static void ct_exit_safe_mode(struct xe_guc_ct *ct)
385 {
386 if (cancel_delayed_work_sync(&ct->safe_mode_worker))
387 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n");
388 }
389
xe_guc_ct_enable(struct xe_guc_ct * ct)390 int xe_guc_ct_enable(struct xe_guc_ct *ct)
391 {
392 struct xe_device *xe = ct_to_xe(ct);
393 struct xe_gt *gt = ct_to_gt(ct);
394 int err;
395
396 xe_gt_assert(gt, !xe_guc_ct_enabled(ct));
397
398 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);
399 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);
400
401 err = guc_ct_ctb_h2g_register(ct);
402 if (err)
403 goto err_out;
404
405 err = guc_ct_ctb_g2h_register(ct);
406 if (err)
407 goto err_out;
408
409 err = guc_ct_control_toggle(ct, true);
410 if (err)
411 goto err_out;
412
413 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED);
414
415 smp_mb();
416 wake_up_all(&ct->wq);
417 xe_gt_dbg(gt, "GuC CT communication channel enabled\n");
418
419 if (ct_needs_safe_mode(ct))
420 ct_enter_safe_mode(ct);
421
422 return 0;
423
424 err_out:
425 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err));
426
427 return err;
428 }
429
stop_g2h_handler(struct xe_guc_ct * ct)430 static void stop_g2h_handler(struct xe_guc_ct *ct)
431 {
432 cancel_work_sync(&ct->g2h_worker);
433 }
434
435 /**
436 * xe_guc_ct_disable - Set GuC to disabled state
437 * @ct: the &xe_guc_ct
438 *
439 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected
440 * in this transition.
441 */
xe_guc_ct_disable(struct xe_guc_ct * ct)442 void xe_guc_ct_disable(struct xe_guc_ct *ct)
443 {
444 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED);
445 ct_exit_safe_mode(ct);
446 stop_g2h_handler(ct);
447 }
448
449 /**
450 * xe_guc_ct_stop - Set GuC to stopped state
451 * @ct: the &xe_guc_ct
452 *
453 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h
454 */
xe_guc_ct_stop(struct xe_guc_ct * ct)455 void xe_guc_ct_stop(struct xe_guc_ct *ct)
456 {
457 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED);
458 stop_g2h_handler(ct);
459 }
460
h2g_has_room(struct xe_guc_ct * ct,u32 cmd_len)461 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
462 {
463 struct guc_ctb *h2g = &ct->ctbs.h2g;
464
465 lockdep_assert_held(&ct->lock);
466
467 if (cmd_len > h2g->info.space) {
468 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head);
469 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
470 h2g->info.size) -
471 h2g->info.resv_space;
472 if (cmd_len > h2g->info.space)
473 return false;
474 }
475
476 return true;
477 }
478
g2h_has_room(struct xe_guc_ct * ct,u32 g2h_len)479 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len)
480 {
481 if (!g2h_len)
482 return true;
483
484 lockdep_assert_held(&ct->fast_lock);
485
486 return ct->ctbs.g2h.info.space > g2h_len;
487 }
488
has_room(struct xe_guc_ct * ct,u32 cmd_len,u32 g2h_len)489 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len)
490 {
491 lockdep_assert_held(&ct->lock);
492
493 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len))
494 return -EBUSY;
495
496 return 0;
497 }
498
h2g_reserve_space(struct xe_guc_ct * ct,u32 cmd_len)499 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)
500 {
501 lockdep_assert_held(&ct->lock);
502 ct->ctbs.h2g.info.space -= cmd_len;
503 }
504
__g2h_reserve_space(struct xe_guc_ct * ct,u32 g2h_len,u32 num_g2h)505 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
506 {
507 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space);
508 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) ||
509 (g2h_len && num_g2h));
510
511 if (g2h_len) {
512 lockdep_assert_held(&ct->fast_lock);
513
514 if (!ct->g2h_outstanding)
515 xe_pm_runtime_get_noresume(ct_to_xe(ct));
516
517 ct->ctbs.g2h.info.space -= g2h_len;
518 ct->g2h_outstanding += num_g2h;
519 }
520 }
521
__g2h_release_space(struct xe_guc_ct * ct,u32 g2h_len)522 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
523 {
524 lockdep_assert_held(&ct->fast_lock);
525 xe_gt_assert(ct_to_gt(ct), ct->ctbs.g2h.info.space + g2h_len <=
526 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space);
527 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding);
528
529 ct->ctbs.g2h.info.space += g2h_len;
530 if (!--ct->g2h_outstanding)
531 xe_pm_runtime_put(ct_to_xe(ct));
532 }
533
g2h_release_space(struct xe_guc_ct * ct,u32 g2h_len)534 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
535 {
536 spin_lock_irq(&ct->fast_lock);
537 __g2h_release_space(ct, g2h_len);
538 spin_unlock_irq(&ct->fast_lock);
539 }
540
541 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */
542
h2g_write(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 ct_fence_value,bool want_response)543 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
544 u32 ct_fence_value, bool want_response)
545 {
546 struct xe_device *xe = ct_to_xe(ct);
547 struct xe_gt *gt = ct_to_gt(ct);
548 struct guc_ctb *h2g = &ct->ctbs.h2g;
549 u32 cmd[H2G_CT_HEADERS];
550 u32 tail = h2g->info.tail;
551 u32 full_len;
552 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds,
553 tail * sizeof(u32));
554
555 full_len = len + GUC_CTB_HDR_LEN;
556
557 lockdep_assert_held(&ct->lock);
558 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);
559 xe_gt_assert(gt, tail <= h2g->info.size);
560
561 /* Command will wrap, zero fill (NOPs), return and check credits again */
562 if (tail + full_len > h2g->info.size) {
563 xe_map_memset(xe, &map, 0, 0,
564 (h2g->info.size - tail) * sizeof(u32));
565 h2g_reserve_space(ct, (h2g->info.size - tail));
566 h2g->info.tail = 0;
567 desc_write(xe, h2g, tail, h2g->info.tail);
568
569 return -EAGAIN;
570 }
571
572 /*
573 * dw0: CT header (including fence)
574 * dw1: HXG header (including action code)
575 * dw2+: action data
576 */
577 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
578 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
579 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value);
580 if (want_response) {
581 cmd[1] =
582 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
583 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
584 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
585 } else {
586 cmd[1] =
587 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) |
588 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
589 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
590 }
591
592 /* H2G header in cmd[1] replaces action[0] so: */
593 --len;
594 ++action;
595
596 /* Write H2G ensuring visable before descriptor update */
597 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32));
598 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32));
599 xe_device_wmb(xe);
600
601 /* Update local copies */
602 h2g->info.tail = (tail + full_len) % h2g->info.size;
603 h2g_reserve_space(ct, full_len);
604
605 /* Update descriptor */
606 desc_write(xe, h2g, tail, h2g->info.tail);
607
608 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len,
609 desc_read(xe, h2g, head), h2g->info.tail);
610
611 return 0;
612 }
613
614 /*
615 * The CT protocol accepts a 16 bits fence. This field is fully owned by the
616 * driver, the GuC will just copy it to the reply message. Since we need to
617 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages,
618 * we use one bit of the seqno as an indicator for that and a rolling counter
619 * for the remaining 15 bits.
620 */
621 #define CT_SEQNO_MASK GENMASK(14, 0)
622 #define CT_SEQNO_UNTRACKED BIT(15)
next_ct_seqno(struct xe_guc_ct * ct,bool is_g2h_fence)623 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence)
624 {
625 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK;
626
627 if (!is_g2h_fence)
628 seqno |= CT_SEQNO_UNTRACKED;
629
630 return seqno;
631 }
632
__guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)633 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
634 u32 len, u32 g2h_len, u32 num_g2h,
635 struct g2h_fence *g2h_fence)
636 {
637 struct xe_gt *gt __maybe_unused = ct_to_gt(ct);
638 u16 seqno;
639 int ret;
640
641 xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
642 xe_gt_assert(gt, !g2h_len || !g2h_fence);
643 xe_gt_assert(gt, !num_g2h || !g2h_fence);
644 xe_gt_assert(gt, !g2h_len || num_g2h);
645 xe_gt_assert(gt, g2h_len || !num_g2h);
646 lockdep_assert_held(&ct->lock);
647
648 if (unlikely(ct->ctbs.h2g.info.broken)) {
649 ret = -EPIPE;
650 goto out;
651 }
652
653 if (ct->state == XE_GUC_CT_STATE_DISABLED) {
654 ret = -ENODEV;
655 goto out;
656 }
657
658 if (ct->state == XE_GUC_CT_STATE_STOPPED) {
659 ret = -ECANCELED;
660 goto out;
661 }
662
663 xe_gt_assert(gt, xe_guc_ct_enabled(ct));
664
665 if (g2h_fence) {
666 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN;
667 num_g2h = 1;
668
669 if (g2h_fence_needs_alloc(g2h_fence)) {
670 void *ptr;
671
672 g2h_fence->seqno = next_ct_seqno(ct, true);
673 ptr = xa_store(&ct->fence_lookup,
674 g2h_fence->seqno,
675 g2h_fence, GFP_ATOMIC);
676 if (IS_ERR(ptr)) {
677 ret = PTR_ERR(ptr);
678 goto out;
679 }
680 }
681
682 seqno = g2h_fence->seqno;
683 } else {
684 seqno = next_ct_seqno(ct, false);
685 }
686
687 if (g2h_len)
688 spin_lock_irq(&ct->fast_lock);
689 retry:
690 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len);
691 if (unlikely(ret))
692 goto out_unlock;
693
694 ret = h2g_write(ct, action, len, seqno, !!g2h_fence);
695 if (unlikely(ret)) {
696 if (ret == -EAGAIN)
697 goto retry;
698 goto out_unlock;
699 }
700
701 __g2h_reserve_space(ct, g2h_len, num_g2h);
702 xe_guc_notify(ct_to_guc(ct));
703 out_unlock:
704 if (g2h_len)
705 spin_unlock_irq(&ct->fast_lock);
706 out:
707 return ret;
708 }
709
kick_reset(struct xe_guc_ct * ct)710 static void kick_reset(struct xe_guc_ct *ct)
711 {
712 xe_gt_reset_async(ct_to_gt(ct));
713 }
714
715 static int dequeue_one_g2h(struct xe_guc_ct *ct);
716
guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)717 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
718 u32 g2h_len, u32 num_g2h,
719 struct g2h_fence *g2h_fence)
720 {
721 struct xe_device *xe = ct_to_xe(ct);
722 struct xe_gt *gt = ct_to_gt(ct);
723 struct drm_printer p = xe_gt_info_printer(gt);
724 unsigned int sleep_period_ms = 1;
725 int ret;
726
727 xe_gt_assert(gt, !g2h_len || !g2h_fence);
728 lockdep_assert_held(&ct->lock);
729 xe_device_assert_mem_access(ct_to_xe(ct));
730
731 try_again:
732 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h,
733 g2h_fence);
734
735 /*
736 * We wait to try to restore credits for about 1 second before bailing.
737 * In the case of H2G credits we have no choice but just to wait for the
738 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In
739 * the case of G2H we process any G2H in the channel, hopefully freeing
740 * credits as we consume the G2H messages.
741 */
742 if (unlikely(ret == -EBUSY &&
743 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) {
744 struct guc_ctb *h2g = &ct->ctbs.h2g;
745
746 if (sleep_period_ms == 1024)
747 goto broken;
748
749 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail,
750 h2g->info.size,
751 h2g->info.space,
752 len + GUC_CTB_HDR_LEN);
753 msleep(sleep_period_ms);
754 sleep_period_ms <<= 1;
755
756 goto try_again;
757 } else if (unlikely(ret == -EBUSY)) {
758 struct xe_device *xe = ct_to_xe(ct);
759 struct guc_ctb *g2h = &ct->ctbs.g2h;
760
761 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head,
762 desc_read(xe, g2h, tail),
763 g2h->info.size,
764 g2h->info.space,
765 g2h_fence ?
766 GUC_CTB_HXG_MSG_MAX_LEN :
767 g2h_len);
768
769 #define g2h_avail(ct) \
770 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head)
771 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding ||
772 g2h_avail(ct), HZ))
773 goto broken;
774 #undef g2h_avail
775
776 if (dequeue_one_g2h(ct) < 0)
777 goto broken;
778
779 goto try_again;
780 }
781
782 return ret;
783
784 broken:
785 xe_gt_err(gt, "No forward process on H2G, reset required\n");
786 xe_guc_ct_print(ct, &p, true);
787 ct->ctbs.h2g.info.broken = true;
788
789 return -EDEADLK;
790 }
791
guc_ct_send(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h,struct g2h_fence * g2h_fence)792 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
793 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence)
794 {
795 int ret;
796
797 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence);
798
799 mutex_lock(&ct->lock);
800 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);
801 mutex_unlock(&ct->lock);
802
803 return ret;
804 }
805
xe_guc_ct_send(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h)806 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
807 u32 g2h_len, u32 num_g2h)
808 {
809 int ret;
810
811 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL);
812 if (ret == -EDEADLK)
813 kick_reset(ct);
814
815 return ret;
816 }
817
xe_guc_ct_send_locked(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 g2h_len,u32 num_g2h)818 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
819 u32 g2h_len, u32 num_g2h)
820 {
821 int ret;
822
823 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL);
824 if (ret == -EDEADLK)
825 kick_reset(ct);
826
827 return ret;
828 }
829
xe_guc_ct_send_g2h_handler(struct xe_guc_ct * ct,const u32 * action,u32 len)830 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len)
831 {
832 int ret;
833
834 lockdep_assert_held(&ct->lock);
835
836 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL);
837 if (ret == -EDEADLK)
838 kick_reset(ct);
839
840 return ret;
841 }
842
843 /*
844 * Check if a GT reset is in progress or will occur and if GT reset brought the
845 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset.
846 */
retry_failure(struct xe_guc_ct * ct,int ret)847 static bool retry_failure(struct xe_guc_ct *ct, int ret)
848 {
849 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV))
850 return false;
851
852 #define ct_alive(ct) \
853 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \
854 !ct->ctbs.g2h.info.broken)
855 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5))
856 return false;
857 #undef ct_alive
858
859 return true;
860 }
861
guc_ct_send_recv(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer,bool no_fail)862 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
863 u32 *response_buffer, bool no_fail)
864 {
865 struct xe_gt *gt = ct_to_gt(ct);
866 struct g2h_fence g2h_fence;
867 int ret = 0;
868
869 /*
870 * We use a fence to implement blocking sends / receiving response data.
871 * The seqno of the fence is sent in the H2G, returned in the G2H, and
872 * an xarray is used as storage media with the seqno being to key.
873 * Fields in the fence hold success, failure, retry status and the
874 * response data. Safe to allocate on the stack as the xarray is the
875 * only reference and it cannot be present after this function exits.
876 */
877 retry:
878 g2h_fence_init(&g2h_fence, response_buffer);
879 retry_same_fence:
880 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence);
881 if (unlikely(ret == -ENOMEM)) {
882 void *ptr;
883
884 /* Retry allocation /w GFP_KERNEL */
885 ptr = xa_store(&ct->fence_lookup,
886 g2h_fence.seqno,
887 &g2h_fence, GFP_KERNEL);
888 if (IS_ERR(ptr))
889 return PTR_ERR(ptr);
890
891 goto retry_same_fence;
892 } else if (unlikely(ret)) {
893 if (ret == -EDEADLK)
894 kick_reset(ct);
895
896 if (no_fail && retry_failure(ct, ret))
897 goto retry_same_fence;
898
899 if (!g2h_fence_needs_alloc(&g2h_fence))
900 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);
901
902 return ret;
903 }
904
905 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ);
906 if (!ret) {
907 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x",
908 g2h_fence.seqno, action[0]);
909 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);
910 return -ETIME;
911 }
912
913 if (g2h_fence.retry) {
914 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n",
915 action[0], g2h_fence.reason);
916 goto retry;
917 }
918 if (g2h_fence.fail) {
919 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n",
920 action[0], g2h_fence.error, g2h_fence.hint);
921 ret = -EIO;
922 }
923
924 return ret > 0 ? response_buffer ? g2h_fence.response_len : g2h_fence.response_data : ret;
925 }
926
927 /**
928 * xe_guc_ct_send_recv - Send and receive HXG to the GuC
929 * @ct: the &xe_guc_ct
930 * @action: the dword array with `HXG Request`_ message (can't be NULL)
931 * @len: length of the `HXG Request`_ message (in dwords, can't be 0)
932 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL)
933 *
934 * Send a `HXG Request`_ message to the GuC over CT communication channel and
935 * blocks until GuC replies with a `HXG Response`_ message.
936 *
937 * For non-blocking communication with GuC use xe_guc_ct_send().
938 *
939 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_.
940 *
941 * Return: response length (in dwords) if &response_buffer was not NULL, or
942 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or
943 * a negative error code on failure.
944 */
xe_guc_ct_send_recv(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer)945 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
946 u32 *response_buffer)
947 {
948 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer);
949 return guc_ct_send_recv(ct, action, len, response_buffer, false);
950 }
951
xe_guc_ct_send_recv_no_fail(struct xe_guc_ct * ct,const u32 * action,u32 len,u32 * response_buffer)952 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action,
953 u32 len, u32 *response_buffer)
954 {
955 return guc_ct_send_recv(ct, action, len, response_buffer, true);
956 }
957
msg_to_hxg(u32 * msg)958 static u32 *msg_to_hxg(u32 *msg)
959 {
960 return msg + GUC_CTB_MSG_MIN_LEN;
961 }
962
msg_len_to_hxg_len(u32 len)963 static u32 msg_len_to_hxg_len(u32 len)
964 {
965 return len - GUC_CTB_MSG_MIN_LEN;
966 }
967
parse_g2h_event(struct xe_guc_ct * ct,u32 * msg,u32 len)968 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)
969 {
970 u32 *hxg = msg_to_hxg(msg);
971 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
972
973 lockdep_assert_held(&ct->lock);
974
975 switch (action) {
976 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
977 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
978 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
979 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
980 g2h_release_space(ct, len);
981 }
982
983 return 0;
984 }
985
parse_g2h_response(struct xe_guc_ct * ct,u32 * msg,u32 len)986 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
987 {
988 struct xe_gt *gt = ct_to_gt(ct);
989 u32 *hxg = msg_to_hxg(msg);
990 u32 hxg_len = msg_len_to_hxg_len(len);
991 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]);
992 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
993 struct g2h_fence *g2h_fence;
994
995 lockdep_assert_held(&ct->lock);
996
997 /*
998 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup.
999 * Those messages should never fail, so if we do get an error back it
1000 * means we're likely doing an illegal operation and the GuC is
1001 * rejecting it. We have no way to inform the code that submitted the
1002 * H2G that the message was rejected, so we need to escalate the
1003 * failure to trigger a reset.
1004 */
1005 if (fence & CT_SEQNO_UNTRACKED) {
1006 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE)
1007 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n",
1008 fence,
1009 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]),
1010 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]));
1011 else
1012 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n",
1013 type, fence);
1014
1015 return -EPROTO;
1016 }
1017
1018 g2h_fence = xa_erase(&ct->fence_lookup, fence);
1019 if (unlikely(!g2h_fence)) {
1020 /* Don't tear down channel, as send could've timed out */
1021 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence);
1022 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1023 return 0;
1024 }
1025
1026 xe_gt_assert(gt, fence == g2h_fence->seqno);
1027
1028 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {
1029 g2h_fence->fail = true;
1030 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]);
1031 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]);
1032 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1033 g2h_fence->retry = true;
1034 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]);
1035 } else if (g2h_fence->response_buffer) {
1036 g2h_fence->response_len = hxg_len;
1037 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32));
1038 } else {
1039 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]);
1040 }
1041
1042 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1043
1044 g2h_fence->done = true;
1045 smp_mb();
1046
1047 wake_up_all(&ct->g2h_fence_wq);
1048
1049 return 0;
1050 }
1051
parse_g2h_msg(struct xe_guc_ct * ct,u32 * msg,u32 len)1052 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1053 {
1054 struct xe_gt *gt = ct_to_gt(ct);
1055 u32 *hxg = msg_to_hxg(msg);
1056 u32 origin, type;
1057 int ret;
1058
1059 lockdep_assert_held(&ct->lock);
1060
1061 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1062 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1063 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n",
1064 origin);
1065 ct->ctbs.g2h.info.broken = true;
1066
1067 return -EPROTO;
1068 }
1069
1070 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1071 switch (type) {
1072 case GUC_HXG_TYPE_EVENT:
1073 ret = parse_g2h_event(ct, msg, len);
1074 break;
1075 case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1076 case GUC_HXG_TYPE_RESPONSE_FAILURE:
1077 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1078 ret = parse_g2h_response(ct, msg, len);
1079 break;
1080 default:
1081 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n",
1082 type);
1083 ct->ctbs.g2h.info.broken = true;
1084
1085 ret = -EOPNOTSUPP;
1086 }
1087
1088 return ret;
1089 }
1090
process_g2h_msg(struct xe_guc_ct * ct,u32 * msg,u32 len)1091 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1092 {
1093 struct xe_guc *guc = ct_to_guc(ct);
1094 struct xe_gt *gt = ct_to_gt(ct);
1095 u32 hxg_len = msg_len_to_hxg_len(len);
1096 u32 *hxg = msg_to_hxg(msg);
1097 u32 action, adj_len;
1098 u32 *payload;
1099 int ret = 0;
1100
1101 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1102 return 0;
1103
1104 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1105 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN;
1106 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN;
1107
1108 switch (action) {
1109 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1110 ret = xe_guc_sched_done_handler(guc, payload, adj_len);
1111 break;
1112 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1113 ret = xe_guc_deregister_done_handler(guc, payload, adj_len);
1114 break;
1115 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1116 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len);
1117 break;
1118 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1119 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload,
1120 adj_len);
1121 break;
1122 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1123 /* Selftest only at the moment */
1124 break;
1125 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1126 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1127 /* FIXME: Handle this */
1128 break;
1129 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR:
1130 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload,
1131 adj_len);
1132 break;
1133 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1134 ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1135 break;
1136 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1137 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1138 adj_len);
1139 break;
1140 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY:
1141 ret = xe_guc_access_counter_notify_handler(guc, payload,
1142 adj_len);
1143 break;
1144 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
1145 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);
1146 break;
1147 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF:
1148 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len);
1149 break;
1150 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY:
1151 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len);
1152 break;
1153 case GUC_ACTION_GUC2PF_ADVERSE_EVENT:
1154 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len);
1155 break;
1156 default:
1157 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
1158 }
1159
1160 if (ret)
1161 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
1162 action, ERR_PTR(ret));
1163
1164 return 0;
1165 }
1166
g2h_read(struct xe_guc_ct * ct,u32 * msg,bool fast_path)1167 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
1168 {
1169 struct xe_device *xe = ct_to_xe(ct);
1170 struct xe_gt *gt = ct_to_gt(ct);
1171 struct guc_ctb *g2h = &ct->ctbs.g2h;
1172 u32 tail, head, len;
1173 s32 avail;
1174 u32 action;
1175 u32 *hxg;
1176
1177 xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
1178 lockdep_assert_held(&ct->fast_lock);
1179
1180 if (ct->state == XE_GUC_CT_STATE_DISABLED)
1181 return -ENODEV;
1182
1183 if (ct->state == XE_GUC_CT_STATE_STOPPED)
1184 return -ECANCELED;
1185
1186 if (g2h->info.broken)
1187 return -EPIPE;
1188
1189 xe_gt_assert(gt, xe_guc_ct_enabled(ct));
1190
1191 /* Calculate DW available to read */
1192 tail = desc_read(xe, g2h, tail);
1193 avail = tail - g2h->info.head;
1194 if (unlikely(avail == 0))
1195 return 0;
1196
1197 if (avail < 0)
1198 avail += g2h->info.size;
1199
1200 /* Read header */
1201 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head,
1202 sizeof(u32));
1203 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN;
1204 if (len > avail) {
1205 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n",
1206 avail, len);
1207 g2h->info.broken = true;
1208
1209 return -EPROTO;
1210 }
1211
1212 head = (g2h->info.head + 1) % g2h->info.size;
1213 avail = len - 1;
1214
1215 /* Read G2H message */
1216 if (avail + head > g2h->info.size) {
1217 u32 avail_til_wrap = g2h->info.size - head;
1218
1219 xe_map_memcpy_from(xe, msg + 1,
1220 &g2h->cmds, sizeof(u32) * head,
1221 avail_til_wrap * sizeof(u32));
1222 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap,
1223 &g2h->cmds, 0,
1224 (avail - avail_til_wrap) * sizeof(u32));
1225 } else {
1226 xe_map_memcpy_from(xe, msg + 1,
1227 &g2h->cmds, sizeof(u32) * head,
1228 avail * sizeof(u32));
1229 }
1230
1231 hxg = msg_to_hxg(msg);
1232 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1233
1234 if (fast_path) {
1235 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1236 return 0;
1237
1238 switch (action) {
1239 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1240 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1241 break; /* Process these in fast-path */
1242 default:
1243 return 0;
1244 }
1245 }
1246
1247 /* Update local / descriptor header */
1248 g2h->info.head = (head + avail) % g2h->info.size;
1249 desc_write(xe, g2h, head, g2h->info.head);
1250
1251 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id,
1252 action, len, g2h->info.head, tail);
1253
1254 return len;
1255 }
1256
g2h_fast_path(struct xe_guc_ct * ct,u32 * msg,u32 len)1257 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
1258 {
1259 struct xe_gt *gt = ct_to_gt(ct);
1260 struct xe_guc *guc = ct_to_guc(ct);
1261 u32 hxg_len = msg_len_to_hxg_len(len);
1262 u32 *hxg = msg_to_hxg(msg);
1263 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1264 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN;
1265 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1266 int ret = 0;
1267
1268 switch (action) {
1269 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1270 ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1271 break;
1272 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1273 __g2h_release_space(ct, len);
1274 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1275 adj_len);
1276 break;
1277 default:
1278 xe_gt_warn(gt, "NOT_POSSIBLE");
1279 }
1280
1281 if (ret)
1282 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
1283 action, ERR_PTR(ret));
1284 }
1285
1286 /**
1287 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler
1288 * @ct: GuC CT object
1289 *
1290 * Anything related to page faults is critical for performance, process these
1291 * critical G2H in the IRQ. This is safe as these handlers either just wake up
1292 * waiters or queue another worker.
1293 */
xe_guc_ct_fast_path(struct xe_guc_ct * ct)1294 void xe_guc_ct_fast_path(struct xe_guc_ct *ct)
1295 {
1296 struct xe_device *xe = ct_to_xe(ct);
1297 bool ongoing;
1298 int len;
1299
1300 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1301 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1302 return;
1303
1304 spin_lock(&ct->fast_lock);
1305 do {
1306 len = g2h_read(ct, ct->fast_msg, true);
1307 if (len > 0)
1308 g2h_fast_path(ct, ct->fast_msg, len);
1309 } while (len > 0);
1310 spin_unlock(&ct->fast_lock);
1311
1312 if (ongoing)
1313 xe_pm_runtime_put(xe);
1314 }
1315
1316 /* Returns less than zero on error, 0 on done, 1 on more available */
dequeue_one_g2h(struct xe_guc_ct * ct)1317 static int dequeue_one_g2h(struct xe_guc_ct *ct)
1318 {
1319 int len;
1320 int ret;
1321
1322 lockdep_assert_held(&ct->lock);
1323
1324 spin_lock_irq(&ct->fast_lock);
1325 len = g2h_read(ct, ct->msg, false);
1326 spin_unlock_irq(&ct->fast_lock);
1327 if (len <= 0)
1328 return len;
1329
1330 ret = parse_g2h_msg(ct, ct->msg, len);
1331 if (unlikely(ret < 0))
1332 return ret;
1333
1334 ret = process_g2h_msg(ct, ct->msg, len);
1335 if (unlikely(ret < 0))
1336 return ret;
1337
1338 return 1;
1339 }
1340
receive_g2h(struct xe_guc_ct * ct)1341 static void receive_g2h(struct xe_guc_ct *ct)
1342 {
1343 struct xe_gt *gt = ct_to_gt(ct);
1344 bool ongoing;
1345 int ret;
1346
1347 /*
1348 * Normal users must always hold mem_access.ref around CT calls. However
1349 * during the runtime pm callbacks we rely on CT to talk to the GuC, but
1350 * at this stage we can't rely on mem_access.ref and even the
1351 * callback_task will be different than current. For such cases we just
1352 * need to ensure we always process the responses from any blocking
1353 * ct_send requests or where we otherwise expect some response when
1354 * initiated from those callbacks (which will need to wait for the below
1355 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if
1356 * the device has suspended to the point that the CT communication has
1357 * been disabled.
1358 *
1359 * If we are inside the runtime pm callback, we can be the only task
1360 * still issuing CT requests (since that requires having the
1361 * mem_access.ref). It seems like it might in theory be possible to
1362 * receive unsolicited events from the GuC just as we are
1363 * suspending-resuming, but those will currently anyway be lost when
1364 * eventually exiting from suspend, hence no need to wake up the device
1365 * here. If we ever need something stronger than get_if_ongoing() then
1366 * we need to be careful with blocking the pm callbacks from getting CT
1367 * responses, if the worker here is blocked on those callbacks
1368 * completing, creating a deadlock.
1369 */
1370 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1371 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1372 return;
1373
1374 do {
1375 mutex_lock(&ct->lock);
1376 ret = dequeue_one_g2h(ct);
1377 mutex_unlock(&ct->lock);
1378
1379 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {
1380 struct drm_printer p = xe_gt_info_printer(gt);
1381
1382 xe_guc_ct_print(ct, &p, false);
1383 kick_reset(ct);
1384 }
1385 } while (ret == 1);
1386
1387 if (ongoing)
1388 xe_pm_runtime_put(ct_to_xe(ct));
1389 }
1390
g2h_worker_func(struct work_struct * w)1391 static void g2h_worker_func(struct work_struct *w)
1392 {
1393 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker);
1394
1395 receive_g2h(ct);
1396 }
1397
guc_ctb_snapshot_capture(struct xe_device * xe,struct guc_ctb * ctb,struct guc_ctb_snapshot * snapshot,bool atomic)1398 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb,
1399 struct guc_ctb_snapshot *snapshot,
1400 bool atomic)
1401 {
1402 u32 head, tail;
1403
1404 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0,
1405 sizeof(struct guc_ct_buffer_desc));
1406 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info));
1407
1408 snapshot->cmds = kmalloc_array(ctb->info.size, sizeof(u32),
1409 atomic ? GFP_ATOMIC : GFP_KERNEL);
1410
1411 if (!snapshot->cmds) {
1412 drm_err(&xe->drm, "Skipping CTB commands snapshot. Only CTB info will be available.\n");
1413 return;
1414 }
1415
1416 head = snapshot->desc.head;
1417 tail = snapshot->desc.tail;
1418
1419 if (head != tail) {
1420 struct iosys_map map =
1421 IOSYS_MAP_INIT_OFFSET(&ctb->cmds, head * sizeof(u32));
1422
1423 while (head != tail) {
1424 snapshot->cmds[head] = xe_map_rd(xe, &map, 0, u32);
1425 ++head;
1426 if (head == ctb->info.size) {
1427 head = 0;
1428 map = ctb->cmds;
1429 } else {
1430 iosys_map_incr(&map, sizeof(u32));
1431 }
1432 }
1433 }
1434 }
1435
guc_ctb_snapshot_print(struct guc_ctb_snapshot * snapshot,struct drm_printer * p)1436 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot,
1437 struct drm_printer *p)
1438 {
1439 u32 head, tail;
1440
1441 drm_printf(p, "\tsize: %d\n", snapshot->info.size);
1442 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space);
1443 drm_printf(p, "\thead: %d\n", snapshot->info.head);
1444 drm_printf(p, "\ttail: %d\n", snapshot->info.tail);
1445 drm_printf(p, "\tspace: %d\n", snapshot->info.space);
1446 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken);
1447 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head);
1448 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail);
1449 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status);
1450
1451 if (!snapshot->cmds)
1452 return;
1453
1454 head = snapshot->desc.head;
1455 tail = snapshot->desc.tail;
1456
1457 while (head != tail) {
1458 drm_printf(p, "\tcmd[%d]: 0x%08x\n", head,
1459 snapshot->cmds[head]);
1460 ++head;
1461 if (head == snapshot->info.size)
1462 head = 0;
1463 }
1464 }
1465
guc_ctb_snapshot_free(struct guc_ctb_snapshot * snapshot)1466 static void guc_ctb_snapshot_free(struct guc_ctb_snapshot *snapshot)
1467 {
1468 kfree(snapshot->cmds);
1469 }
1470
1471 /**
1472 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state.
1473 * @ct: GuC CT object.
1474 * @atomic: Boolean to indicate if this is called from atomic context like
1475 * reset or CTB handler or from some regular path like debugfs.
1476 *
1477 * This can be printed out in a later stage like during dev_coredump
1478 * analysis.
1479 *
1480 * Returns: a GuC CT snapshot object that must be freed by the caller
1481 * by using `xe_guc_ct_snapshot_free`.
1482 */
xe_guc_ct_snapshot_capture(struct xe_guc_ct * ct,bool atomic)1483 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct,
1484 bool atomic)
1485 {
1486 struct xe_device *xe = ct_to_xe(ct);
1487 struct xe_guc_ct_snapshot *snapshot;
1488
1489 snapshot = kzalloc(sizeof(*snapshot),
1490 atomic ? GFP_ATOMIC : GFP_KERNEL);
1491
1492 if (!snapshot) {
1493 drm_err(&xe->drm, "Skipping CTB snapshot entirely.\n");
1494 return NULL;
1495 }
1496
1497 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) {
1498 snapshot->ct_enabled = true;
1499 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);
1500 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g,
1501 &snapshot->h2g, atomic);
1502 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h,
1503 &snapshot->g2h, atomic);
1504 }
1505
1506 return snapshot;
1507 }
1508
1509 /**
1510 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot.
1511 * @snapshot: GuC CT snapshot object.
1512 * @p: drm_printer where it will be printed out.
1513 *
1514 * This function prints out a given GuC CT snapshot object.
1515 */
xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot * snapshot,struct drm_printer * p)1516 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot,
1517 struct drm_printer *p)
1518 {
1519 if (!snapshot)
1520 return;
1521
1522 if (snapshot->ct_enabled) {
1523 drm_puts(p, "H2G CTB (all sizes in DW):\n");
1524 guc_ctb_snapshot_print(&snapshot->h2g, p);
1525
1526 drm_puts(p, "\nG2H CTB (all sizes in DW):\n");
1527 guc_ctb_snapshot_print(&snapshot->g2h, p);
1528
1529 drm_printf(p, "\tg2h outstanding: %d\n",
1530 snapshot->g2h_outstanding);
1531 } else {
1532 drm_puts(p, "CT disabled\n");
1533 }
1534 }
1535
1536 /**
1537 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot.
1538 * @snapshot: GuC CT snapshot object.
1539 *
1540 * This function free all the memory that needed to be allocated at capture
1541 * time.
1542 */
xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot * snapshot)1543 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot)
1544 {
1545 if (!snapshot)
1546 return;
1547
1548 guc_ctb_snapshot_free(&snapshot->h2g);
1549 guc_ctb_snapshot_free(&snapshot->g2h);
1550 kfree(snapshot);
1551 }
1552
1553 /**
1554 * xe_guc_ct_print - GuC CT Print.
1555 * @ct: GuC CT.
1556 * @p: drm_printer where it will be printed out.
1557 * @atomic: Boolean to indicate if this is called from atomic context like
1558 * reset or CTB handler or from some regular path like debugfs.
1559 *
1560 * This function quickly capture a snapshot and immediately print it out.
1561 */
xe_guc_ct_print(struct xe_guc_ct * ct,struct drm_printer * p,bool atomic)1562 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool atomic)
1563 {
1564 struct xe_guc_ct_snapshot *snapshot;
1565
1566 snapshot = xe_guc_ct_snapshot_capture(ct, atomic);
1567 xe_guc_ct_snapshot_print(snapshot, p);
1568 xe_guc_ct_snapshot_free(snapshot);
1569 }
1570