1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc.h"
7
8 #include <drm/drm_managed.h>
9
10 #include <generated/xe_wa_oob.h>
11
12 #include "abi/guc_actions_abi.h"
13 #include "abi/guc_errors_abi.h"
14 #include "regs/xe_gt_regs.h"
15 #include "regs/xe_gtt_defs.h"
16 #include "regs/xe_guc_regs.h"
17 #include "regs/xe_irq_regs.h"
18 #include "xe_bo.h"
19 #include "xe_device.h"
20 #include "xe_force_wake.h"
21 #include "xe_gt.h"
22 #include "xe_gt_printk.h"
23 #include "xe_gt_sriov_vf.h"
24 #include "xe_gt_throttle.h"
25 #include "xe_guc_ads.h"
26 #include "xe_guc_capture.h"
27 #include "xe_guc_ct.h"
28 #include "xe_guc_db_mgr.h"
29 #include "xe_guc_hwconfig.h"
30 #include "xe_guc_log.h"
31 #include "xe_guc_pc.h"
32 #include "xe_guc_relay.h"
33 #include "xe_guc_submit.h"
34 #include "xe_memirq.h"
35 #include "xe_mmio.h"
36 #include "xe_platform_types.h"
37 #include "xe_sriov.h"
38 #include "xe_uc.h"
39 #include "xe_uc_fw.h"
40 #include "xe_wa.h"
41 #include "xe_wopcm.h"
42
guc_bo_ggtt_addr(struct xe_guc * guc,struct xe_bo * bo)43 static u32 guc_bo_ggtt_addr(struct xe_guc *guc,
44 struct xe_bo *bo)
45 {
46 struct xe_device *xe = guc_to_xe(guc);
47 u32 addr;
48
49 /*
50 * For most BOs, the address on the allocating tile is fine. However for
51 * some, e.g. G2G CTB, the address on a specific tile is required as it
52 * might be different for each tile. So, just always ask for the address
53 * on the target GuC.
54 */
55 addr = __xe_bo_ggtt_addr(bo, gt_to_tile(guc_to_gt(guc))->id);
56
57 /* GuC addresses above GUC_GGTT_TOP don't map through the GTT */
58 xe_assert(xe, addr >= xe_wopcm_size(guc_to_xe(guc)));
59 xe_assert(xe, addr < GUC_GGTT_TOP);
60 xe_assert(xe, bo->size <= GUC_GGTT_TOP - addr);
61
62 return addr;
63 }
64
guc_ctl_debug_flags(struct xe_guc * guc)65 static u32 guc_ctl_debug_flags(struct xe_guc *guc)
66 {
67 u32 level = xe_guc_log_get_level(&guc->log);
68 u32 flags = 0;
69
70 if (!GUC_LOG_LEVEL_IS_VERBOSE(level))
71 flags |= GUC_LOG_DISABLED;
72 else
73 flags |= GUC_LOG_LEVEL_TO_VERBOSITY(level) <<
74 GUC_LOG_VERBOSITY_SHIFT;
75
76 return flags;
77 }
78
guc_ctl_feature_flags(struct xe_guc * guc)79 static u32 guc_ctl_feature_flags(struct xe_guc *guc)
80 {
81 u32 flags = GUC_CTL_ENABLE_LITE_RESTORE;
82
83 if (!guc_to_xe(guc)->info.skip_guc_pc)
84 flags |= GUC_CTL_ENABLE_SLPC;
85
86 return flags;
87 }
88
guc_ctl_log_params_flags(struct xe_guc * guc)89 static u32 guc_ctl_log_params_flags(struct xe_guc *guc)
90 {
91 u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT;
92 u32 flags;
93
94 #if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
95 #define LOG_UNIT SZ_1M
96 #define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
97 #else
98 #define LOG_UNIT SZ_4K
99 #define LOG_FLAG 0
100 #endif
101
102 #if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
103 #define CAPTURE_UNIT SZ_1M
104 #define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
105 #else
106 #define CAPTURE_UNIT SZ_4K
107 #define CAPTURE_FLAG 0
108 #endif
109
110 BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
111 BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
112 BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
113 BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
114 BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
115 BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
116
117 BUILD_BUG_ON((CRASH_BUFFER_SIZE / LOG_UNIT - 1) >
118 (GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT));
119 BUILD_BUG_ON((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) >
120 (GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT));
121 BUILD_BUG_ON((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) >
122 (GUC_LOG_CAPTURE_MASK >> GUC_LOG_CAPTURE_SHIFT));
123
124 flags = GUC_LOG_VALID |
125 GUC_LOG_NOTIFY_ON_HALF_FULL |
126 CAPTURE_FLAG |
127 LOG_FLAG |
128 ((CRASH_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_CRASH_SHIFT) |
129 ((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_DEBUG_SHIFT) |
130 ((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) <<
131 GUC_LOG_CAPTURE_SHIFT) |
132 (offset << GUC_LOG_BUF_ADDR_SHIFT);
133
134 #undef LOG_UNIT
135 #undef LOG_FLAG
136 #undef CAPTURE_UNIT
137 #undef CAPTURE_FLAG
138
139 return flags;
140 }
141
guc_ctl_ads_flags(struct xe_guc * guc)142 static u32 guc_ctl_ads_flags(struct xe_guc *guc)
143 {
144 u32 ads = guc_bo_ggtt_addr(guc, guc->ads.bo) >> PAGE_SHIFT;
145 u32 flags = ads << GUC_ADS_ADDR_SHIFT;
146
147 return flags;
148 }
149
needs_wa_dual_queue(struct xe_gt * gt)150 static bool needs_wa_dual_queue(struct xe_gt *gt)
151 {
152 /*
153 * The DUAL_QUEUE_WA tells the GuC to not allow concurrent submissions
154 * on RCS and CCSes with different address spaces, which on DG2 is
155 * required as a WA for an HW bug.
156 */
157 if (XE_WA(gt, 22011391025))
158 return true;
159
160 /*
161 * On newer platforms, the HW has been updated to not allow parallel
162 * execution of different address spaces, so the RCS/CCS will stall the
163 * context switch if one of the other RCS/CCSes is busy with a different
164 * address space. While functionally correct, having a submission
165 * stalled on the HW limits the GuC ability to shuffle things around and
166 * can cause complications if the non-stalled submission runs for a long
167 * time, because the GuC doesn't know that the stalled submission isn't
168 * actually running and might declare it as hung. Therefore, we enable
169 * the DUAL_QUEUE_WA on all newer platforms on GTs that have CCS engines
170 * to move management back to the GuC.
171 */
172 if (CCS_MASK(gt) && GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
173 return true;
174
175 return false;
176 }
177
guc_ctl_wa_flags(struct xe_guc * guc)178 static u32 guc_ctl_wa_flags(struct xe_guc *guc)
179 {
180 struct xe_device *xe = guc_to_xe(guc);
181 struct xe_gt *gt = guc_to_gt(guc);
182 u32 flags = 0;
183
184 if (XE_WA(gt, 22012773006))
185 flags |= GUC_WA_POLLCS;
186
187 if (XE_WA(gt, 14014475959))
188 flags |= GUC_WA_HOLD_CCS_SWITCHOUT;
189
190 if (needs_wa_dual_queue(gt))
191 flags |= GUC_WA_DUAL_QUEUE;
192
193 /*
194 * Wa_22011802037: FIXME - there's more to be done than simply setting
195 * this flag: make sure each CS is stopped when preparing for GT reset
196 * and wait for pending MI_FW.
197 */
198 if (GRAPHICS_VERx100(xe) < 1270)
199 flags |= GUC_WA_PRE_PARSER;
200
201 if (XE_WA(gt, 22012727170) || XE_WA(gt, 22012727685))
202 flags |= GUC_WA_CONTEXT_ISOLATION;
203
204 if (XE_WA(gt, 18020744125) &&
205 !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER))
206 flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST;
207
208 if (XE_WA(gt, 1509372804))
209 flags |= GUC_WA_RENDER_RST_RC6_EXIT;
210
211 if (XE_WA(gt, 14018913170))
212 flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
213
214 return flags;
215 }
216
guc_ctl_devid(struct xe_guc * guc)217 static u32 guc_ctl_devid(struct xe_guc *guc)
218 {
219 struct xe_device *xe = guc_to_xe(guc);
220
221 return (((u32)xe->info.devid) << 16) | xe->info.revid;
222 }
223
guc_print_params(struct xe_guc * guc)224 static void guc_print_params(struct xe_guc *guc)
225 {
226 struct xe_gt *gt = guc_to_gt(guc);
227 u32 *params = guc->params;
228 int i;
229
230 BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32));
231 BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT);
232
233 for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
234 xe_gt_dbg(gt, "GuC param[%2d] = 0x%08x\n", i, params[i]);
235 }
236
guc_init_params(struct xe_guc * guc)237 static void guc_init_params(struct xe_guc *guc)
238 {
239 u32 *params = guc->params;
240
241 params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
242 params[GUC_CTL_FEATURE] = 0;
243 params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
244 params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
245 params[GUC_CTL_WA] = 0;
246 params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
247
248 guc_print_params(guc);
249 }
250
guc_init_params_post_hwconfig(struct xe_guc * guc)251 static void guc_init_params_post_hwconfig(struct xe_guc *guc)
252 {
253 u32 *params = guc->params;
254
255 params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
256 params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc);
257 params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
258 params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
259 params[GUC_CTL_WA] = guc_ctl_wa_flags(guc);
260 params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
261
262 guc_print_params(guc);
263 }
264
265 /*
266 * Initialize the GuC parameter block before starting the firmware
267 * transfer. These parameters are read by the firmware on startup
268 * and cannot be changed thereafter.
269 */
guc_write_params(struct xe_guc * guc)270 static void guc_write_params(struct xe_guc *guc)
271 {
272 struct xe_gt *gt = guc_to_gt(guc);
273 int i;
274
275 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
276
277 xe_mmio_write32(>->mmio, SOFT_SCRATCH(0), 0);
278
279 for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
280 xe_mmio_write32(>->mmio, SOFT_SCRATCH(1 + i), guc->params[i]);
281 }
282
guc_action_register_g2g_buffer(struct xe_guc * guc,u32 type,u32 dst_tile,u32 dst_dev,u32 desc_addr,u32 buff_addr,u32 size)283 static int guc_action_register_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev,
284 u32 desc_addr, u32 buff_addr, u32 size)
285 {
286 struct xe_gt *gt = guc_to_gt(guc);
287 struct xe_device *xe = gt_to_xe(gt);
288 u32 action[] = {
289 XE_GUC_ACTION_REGISTER_G2G,
290 FIELD_PREP(XE_G2G_REGISTER_SIZE, size / SZ_4K - 1) |
291 FIELD_PREP(XE_G2G_REGISTER_TYPE, type) |
292 FIELD_PREP(XE_G2G_REGISTER_TILE, dst_tile) |
293 FIELD_PREP(XE_G2G_REGISTER_DEVICE, dst_dev),
294 desc_addr,
295 buff_addr,
296 };
297
298 xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
299 xe_assert(xe, !(size % SZ_4K));
300
301 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
302 }
303
guc_action_deregister_g2g_buffer(struct xe_guc * guc,u32 type,u32 dst_tile,u32 dst_dev)304 static int guc_action_deregister_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev)
305 {
306 struct xe_gt *gt = guc_to_gt(guc);
307 struct xe_device *xe = gt_to_xe(gt);
308 u32 action[] = {
309 XE_GUC_ACTION_DEREGISTER_G2G,
310 FIELD_PREP(XE_G2G_DEREGISTER_TYPE, type) |
311 FIELD_PREP(XE_G2G_DEREGISTER_TILE, dst_tile) |
312 FIELD_PREP(XE_G2G_DEREGISTER_DEVICE, dst_dev),
313 };
314
315 xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
316
317 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
318 }
319
320 #define G2G_DEV(gt) (((gt)->info.type == XE_GT_TYPE_MAIN) ? 0 : 1)
321
322 #define G2G_BUFFER_SIZE (SZ_4K)
323 #define G2G_DESC_SIZE (64)
324 #define G2G_DESC_AREA_SIZE (SZ_4K)
325
326 /*
327 * Generate a unique id for each bi-directional CTB for each pair of
328 * near and far tiles/devices. The id can then be used as an index into
329 * a single allocation that is sub-divided into multiple CTBs.
330 *
331 * For example, with two devices per tile and two tiles, the table should
332 * look like:
333 * Far <tile>.<dev>
334 * 0.0 0.1 1.0 1.1
335 * N 0.0 --/-- 00/01 02/03 04/05
336 * e 0.1 01/00 --/-- 06/07 08/09
337 * a 1.0 03/02 07/06 --/-- 10/11
338 * r 1.1 05/04 09/08 11/10 --/--
339 *
340 * Where each entry is Rx/Tx channel id.
341 *
342 * So GuC #3 (tile 1, dev 1) talking to GuC #2 (tile 1, dev 0) would
343 * be reading from channel #11 and writing to channel #10. Whereas,
344 * GuC #2 talking to GuC #3 would be read on #10 and write to #11.
345 */
g2g_slot(u32 near_tile,u32 near_dev,u32 far_tile,u32 far_dev,u32 type,u32 max_inst,bool have_dev)346 static unsigned int g2g_slot(u32 near_tile, u32 near_dev, u32 far_tile, u32 far_dev,
347 u32 type, u32 max_inst, bool have_dev)
348 {
349 u32 near = near_tile, far = far_tile;
350 u32 idx = 0, x, y, direction;
351 int i;
352
353 if (have_dev) {
354 near = (near << 1) | near_dev;
355 far = (far << 1) | far_dev;
356 }
357
358 /* No need to send to one's self */
359 if (far == near)
360 return -1;
361
362 if (far > near) {
363 /* Top right table half */
364 x = far;
365 y = near;
366
367 /* T/R is 'forwards' direction */
368 direction = type;
369 } else {
370 /* Bottom left table half */
371 x = near;
372 y = far;
373
374 /* B/L is 'backwards' direction */
375 direction = (1 - type);
376 }
377
378 /* Count the rows prior to the target */
379 for (i = y; i > 0; i--)
380 idx += max_inst - i;
381
382 /* Count this row up to the target */
383 idx += (x - 1 - y);
384
385 /* Slots are in Rx/Tx pairs */
386 idx *= 2;
387
388 /* Pick Rx/Tx direction */
389 idx += direction;
390
391 return idx;
392 }
393
guc_g2g_register(struct xe_guc * near_guc,struct xe_gt * far_gt,u32 type,bool have_dev)394 static int guc_g2g_register(struct xe_guc *near_guc, struct xe_gt *far_gt, u32 type, bool have_dev)
395 {
396 struct xe_gt *near_gt = guc_to_gt(near_guc);
397 struct xe_device *xe = gt_to_xe(near_gt);
398 struct xe_bo *g2g_bo;
399 u32 near_tile = gt_to_tile(near_gt)->id;
400 u32 near_dev = G2G_DEV(near_gt);
401 u32 far_tile = gt_to_tile(far_gt)->id;
402 u32 far_dev = G2G_DEV(far_gt);
403 u32 max = xe->info.gt_count;
404 u32 base, desc, buf;
405 int slot;
406
407 /* G2G is not allowed between different cards */
408 xe_assert(xe, xe == gt_to_xe(far_gt));
409
410 g2g_bo = near_guc->g2g.bo;
411 xe_assert(xe, g2g_bo);
412
413 slot = g2g_slot(near_tile, near_dev, far_tile, far_dev, type, max, have_dev);
414 xe_assert(xe, slot >= 0);
415
416 base = guc_bo_ggtt_addr(near_guc, g2g_bo);
417 desc = base + slot * G2G_DESC_SIZE;
418 buf = base + G2G_DESC_AREA_SIZE + slot * G2G_BUFFER_SIZE;
419
420 xe_assert(xe, (desc - base + G2G_DESC_SIZE) <= G2G_DESC_AREA_SIZE);
421 xe_assert(xe, (buf - base + G2G_BUFFER_SIZE) <= g2g_bo->size);
422
423 return guc_action_register_g2g_buffer(near_guc, type, far_tile, far_dev,
424 desc, buf, G2G_BUFFER_SIZE);
425 }
426
guc_g2g_deregister(struct xe_guc * guc,u32 far_tile,u32 far_dev,u32 type)427 static void guc_g2g_deregister(struct xe_guc *guc, u32 far_tile, u32 far_dev, u32 type)
428 {
429 guc_action_deregister_g2g_buffer(guc, type, far_tile, far_dev);
430 }
431
guc_g2g_size(struct xe_guc * guc)432 static u32 guc_g2g_size(struct xe_guc *guc)
433 {
434 struct xe_gt *gt = guc_to_gt(guc);
435 struct xe_device *xe = gt_to_xe(gt);
436 unsigned int count = xe->info.gt_count;
437 u32 num_channels = (count * (count - 1)) / 2;
438
439 xe_assert(xe, num_channels * XE_G2G_TYPE_LIMIT * G2G_DESC_SIZE <= G2G_DESC_AREA_SIZE);
440
441 return num_channels * XE_G2G_TYPE_LIMIT * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE;
442 }
443
xe_guc_g2g_wanted(struct xe_device * xe)444 static bool xe_guc_g2g_wanted(struct xe_device *xe)
445 {
446 /* Can't do GuC to GuC communication if there is only one GuC */
447 if (xe->info.gt_count <= 1)
448 return false;
449
450 /* No current user */
451 return false;
452 }
453
guc_g2g_alloc(struct xe_guc * guc)454 static int guc_g2g_alloc(struct xe_guc *guc)
455 {
456 struct xe_gt *gt = guc_to_gt(guc);
457 struct xe_device *xe = gt_to_xe(gt);
458 struct xe_tile *tile = gt_to_tile(gt);
459 struct xe_bo *bo;
460 u32 g2g_size;
461
462 if (guc->g2g.bo)
463 return 0;
464
465 if (gt->info.id != 0) {
466 struct xe_gt *root_gt = xe_device_get_gt(xe, 0);
467 struct xe_guc *root_guc = &root_gt->uc.guc;
468 struct xe_bo *bo;
469
470 bo = xe_bo_get(root_guc->g2g.bo);
471 if (!bo)
472 return -ENODEV;
473
474 guc->g2g.bo = bo;
475 guc->g2g.owned = false;
476 return 0;
477 }
478
479 g2g_size = guc_g2g_size(guc);
480 bo = xe_managed_bo_create_pin_map(xe, tile, g2g_size,
481 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
482 XE_BO_FLAG_GGTT |
483 XE_BO_FLAG_GGTT_ALL |
484 XE_BO_FLAG_GGTT_INVALIDATE);
485 if (IS_ERR(bo))
486 return PTR_ERR(bo);
487
488 xe_map_memset(xe, &bo->vmap, 0, 0, g2g_size);
489 guc->g2g.bo = bo;
490 guc->g2g.owned = true;
491
492 return 0;
493 }
494
guc_g2g_fini(struct xe_guc * guc)495 static void guc_g2g_fini(struct xe_guc *guc)
496 {
497 if (!guc->g2g.bo)
498 return;
499
500 /* Unpinning the owned object is handled by generic shutdown */
501 if (!guc->g2g.owned)
502 xe_bo_put(guc->g2g.bo);
503
504 guc->g2g.bo = NULL;
505 }
506
guc_g2g_start(struct xe_guc * guc)507 static int guc_g2g_start(struct xe_guc *guc)
508 {
509 struct xe_gt *far_gt, *gt = guc_to_gt(guc);
510 struct xe_device *xe = gt_to_xe(gt);
511 unsigned int i, j;
512 int t, err;
513 bool have_dev;
514
515 if (!guc->g2g.bo) {
516 int ret;
517
518 ret = guc_g2g_alloc(guc);
519 if (ret)
520 return ret;
521 }
522
523 /* GuC interface will need extending if more GT device types are ever created. */
524 xe_gt_assert(gt, (gt->info.type == XE_GT_TYPE_MAIN) || (gt->info.type == XE_GT_TYPE_MEDIA));
525
526 /* Channel numbering depends on whether there are multiple GTs per tile */
527 have_dev = xe->info.gt_count > xe->info.tile_count;
528
529 for_each_gt(far_gt, xe, i) {
530 u32 far_tile, far_dev;
531
532 if (far_gt->info.id == gt->info.id)
533 continue;
534
535 far_tile = gt_to_tile(far_gt)->id;
536 far_dev = G2G_DEV(far_gt);
537
538 for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) {
539 err = guc_g2g_register(guc, far_gt, t, have_dev);
540 if (err) {
541 while (--t >= 0)
542 guc_g2g_deregister(guc, far_tile, far_dev, t);
543 goto err_deregister;
544 }
545 }
546 }
547
548 return 0;
549
550 err_deregister:
551 for_each_gt(far_gt, xe, j) {
552 u32 tile, dev;
553
554 if (far_gt->info.id == gt->info.id)
555 continue;
556
557 if (j >= i)
558 break;
559
560 tile = gt_to_tile(far_gt)->id;
561 dev = G2G_DEV(far_gt);
562
563 for (t = 0; t < XE_G2G_TYPE_LIMIT; t++)
564 guc_g2g_deregister(guc, tile, dev, t);
565 }
566
567 return err;
568 }
569
guc_fini_hw(void * arg)570 static void guc_fini_hw(void *arg)
571 {
572 struct xe_guc *guc = arg;
573 struct xe_gt *gt = guc_to_gt(guc);
574 unsigned int fw_ref;
575
576 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
577 xe_uc_fini_hw(&guc_to_gt(guc)->uc);
578 xe_force_wake_put(gt_to_fw(gt), fw_ref);
579
580 guc_g2g_fini(guc);
581 }
582
583 /**
584 * xe_guc_comm_init_early - early initialization of GuC communication
585 * @guc: the &xe_guc to initialize
586 *
587 * Must be called prior to first MMIO communication with GuC firmware.
588 */
xe_guc_comm_init_early(struct xe_guc * guc)589 void xe_guc_comm_init_early(struct xe_guc *guc)
590 {
591 struct xe_gt *gt = guc_to_gt(guc);
592
593 if (xe_gt_is_media_type(gt))
594 guc->notify_reg = MED_GUC_HOST_INTERRUPT;
595 else
596 guc->notify_reg = GUC_HOST_INTERRUPT;
597 }
598
xe_guc_realloc_post_hwconfig(struct xe_guc * guc)599 static int xe_guc_realloc_post_hwconfig(struct xe_guc *guc)
600 {
601 struct xe_tile *tile = gt_to_tile(guc_to_gt(guc));
602 struct xe_device *xe = guc_to_xe(guc);
603 int ret;
604
605 if (!IS_DGFX(guc_to_xe(guc)))
606 return 0;
607
608 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->fw.bo);
609 if (ret)
610 return ret;
611
612 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->log.bo);
613 if (ret)
614 return ret;
615
616 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ads.bo);
617 if (ret)
618 return ret;
619
620 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ct.bo);
621 if (ret)
622 return ret;
623
624 return 0;
625 }
626
vf_guc_init(struct xe_guc * guc)627 static int vf_guc_init(struct xe_guc *guc)
628 {
629 int err;
630
631 xe_guc_comm_init_early(guc);
632
633 err = xe_guc_ct_init(&guc->ct);
634 if (err)
635 return err;
636
637 err = xe_guc_relay_init(&guc->relay);
638 if (err)
639 return err;
640
641 return 0;
642 }
643
xe_guc_init(struct xe_guc * guc)644 int xe_guc_init(struct xe_guc *guc)
645 {
646 struct xe_device *xe = guc_to_xe(guc);
647 struct xe_gt *gt = guc_to_gt(guc);
648 int ret;
649
650 guc->fw.type = XE_UC_FW_TYPE_GUC;
651 ret = xe_uc_fw_init(&guc->fw);
652 if (ret)
653 goto out;
654
655 if (!xe_uc_fw_is_enabled(&guc->fw))
656 return 0;
657
658 if (IS_SRIOV_VF(xe)) {
659 ret = vf_guc_init(guc);
660 if (ret)
661 goto out;
662 return 0;
663 }
664
665 ret = xe_guc_log_init(&guc->log);
666 if (ret)
667 goto out;
668
669 ret = xe_guc_capture_init(guc);
670 if (ret)
671 goto out;
672
673 ret = xe_guc_ads_init(&guc->ads);
674 if (ret)
675 goto out;
676
677 ret = xe_guc_ct_init(&guc->ct);
678 if (ret)
679 goto out;
680
681 ret = xe_guc_relay_init(&guc->relay);
682 if (ret)
683 goto out;
684
685 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE);
686
687 ret = devm_add_action_or_reset(xe->drm.dev, guc_fini_hw, guc);
688 if (ret)
689 goto out;
690
691 guc_init_params(guc);
692
693 xe_guc_comm_init_early(guc);
694
695 return 0;
696
697 out:
698 xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
699 return ret;
700 }
701
vf_guc_init_post_hwconfig(struct xe_guc * guc)702 static int vf_guc_init_post_hwconfig(struct xe_guc *guc)
703 {
704 int err;
705
706 err = xe_guc_submit_init(guc, xe_gt_sriov_vf_guc_ids(guc_to_gt(guc)));
707 if (err)
708 return err;
709
710 /* XXX xe_guc_db_mgr_init not needed for now */
711
712 return 0;
713 }
714
715 /**
716 * xe_guc_init_post_hwconfig - initialize GuC post hwconfig load
717 * @guc: The GuC object
718 *
719 * Return: 0 on success, negative error code on error.
720 */
xe_guc_init_post_hwconfig(struct xe_guc * guc)721 int xe_guc_init_post_hwconfig(struct xe_guc *guc)
722 {
723 int ret;
724
725 if (IS_SRIOV_VF(guc_to_xe(guc)))
726 return vf_guc_init_post_hwconfig(guc);
727
728 ret = xe_guc_realloc_post_hwconfig(guc);
729 if (ret)
730 return ret;
731
732 guc_init_params_post_hwconfig(guc);
733
734 ret = xe_guc_submit_init(guc, ~0);
735 if (ret)
736 return ret;
737
738 ret = xe_guc_db_mgr_init(&guc->dbm, ~0);
739 if (ret)
740 return ret;
741
742 ret = xe_guc_pc_init(&guc->pc);
743 if (ret)
744 return ret;
745
746 return xe_guc_ads_init_post_hwconfig(&guc->ads);
747 }
748
xe_guc_post_load_init(struct xe_guc * guc)749 int xe_guc_post_load_init(struct xe_guc *guc)
750 {
751 int ret;
752
753 xe_guc_ads_populate_post_load(&guc->ads);
754
755 if (xe_guc_g2g_wanted(guc_to_xe(guc))) {
756 ret = guc_g2g_start(guc);
757 if (ret)
758 return ret;
759 }
760
761 guc->submission_state.enabled = true;
762
763 return 0;
764 }
765
xe_guc_reset(struct xe_guc * guc)766 int xe_guc_reset(struct xe_guc *guc)
767 {
768 struct xe_gt *gt = guc_to_gt(guc);
769 struct xe_mmio *mmio = >->mmio;
770 u32 guc_status, gdrst;
771 int ret;
772
773 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
774
775 if (IS_SRIOV_VF(gt_to_xe(gt)))
776 return xe_gt_sriov_vf_bootstrap(gt);
777
778 xe_mmio_write32(mmio, GDRST, GRDOM_GUC);
779
780 ret = xe_mmio_wait32(mmio, GDRST, GRDOM_GUC, 0, 5000, &gdrst, false);
781 if (ret) {
782 xe_gt_err(gt, "GuC reset timed out, GDRST=%#x\n", gdrst);
783 goto err_out;
784 }
785
786 guc_status = xe_mmio_read32(mmio, GUC_STATUS);
787 if (!(guc_status & GS_MIA_IN_RESET)) {
788 xe_gt_err(gt, "GuC status: %#x, MIA core expected to be in reset\n",
789 guc_status);
790 ret = -EIO;
791 goto err_out;
792 }
793
794 return 0;
795
796 err_out:
797
798 return ret;
799 }
800
guc_prepare_xfer(struct xe_guc * guc)801 static void guc_prepare_xfer(struct xe_guc *guc)
802 {
803 struct xe_gt *gt = guc_to_gt(guc);
804 struct xe_mmio *mmio = >->mmio;
805 struct xe_device *xe = guc_to_xe(guc);
806 u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC |
807 GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA |
808 GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA |
809 GUC_ENABLE_MIA_CLOCK_GATING;
810
811 if (GRAPHICS_VERx100(xe) < 1250)
812 shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES |
813 GUC_ENABLE_MIA_CACHING;
814
815 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
816 shim_flags |= REG_FIELD_PREP(GUC_MOCS_INDEX_MASK, gt->mocs.uc_index);
817
818 /* Must program this register before loading the ucode with DMA */
819 xe_mmio_write32(mmio, GUC_SHIM_CONTROL, shim_flags);
820
821 xe_mmio_write32(mmio, GT_PM_CONFIG, GT_DOORBELL_ENABLE);
822
823 /* Make sure GuC receives ARAT interrupts */
824 xe_mmio_rmw32(mmio, PMINTRMSK, ARAT_EXPIRED_INTRMSK, 0);
825 }
826
827 /*
828 * Supporting MMIO & in memory RSA
829 */
guc_xfer_rsa(struct xe_guc * guc)830 static int guc_xfer_rsa(struct xe_guc *guc)
831 {
832 struct xe_gt *gt = guc_to_gt(guc);
833 u32 rsa[UOS_RSA_SCRATCH_COUNT];
834 size_t copied;
835 int i;
836
837 if (guc->fw.rsa_size > 256) {
838 u32 rsa_ggtt_addr = xe_bo_ggtt_addr(guc->fw.bo) +
839 xe_uc_fw_rsa_offset(&guc->fw);
840 xe_mmio_write32(>->mmio, UOS_RSA_SCRATCH(0), rsa_ggtt_addr);
841 return 0;
842 }
843
844 copied = xe_uc_fw_copy_rsa(&guc->fw, rsa, sizeof(rsa));
845 if (copied < sizeof(rsa))
846 return -ENOMEM;
847
848 for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++)
849 xe_mmio_write32(>->mmio, UOS_RSA_SCRATCH(i), rsa[i]);
850
851 return 0;
852 }
853
854 /*
855 * Check a previously read GuC status register (GUC_STATUS) looking for
856 * known terminal states (either completion or failure) of either the
857 * microkernel status field or the boot ROM status field. Returns +1 for
858 * successful completion, -1 for failure and 0 for any intermediate state.
859 */
guc_load_done(u32 status)860 static int guc_load_done(u32 status)
861 {
862 u32 uk_val = REG_FIELD_GET(GS_UKERNEL_MASK, status);
863 u32 br_val = REG_FIELD_GET(GS_BOOTROM_MASK, status);
864
865 switch (uk_val) {
866 case XE_GUC_LOAD_STATUS_READY:
867 return 1;
868
869 case XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH:
870 case XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH:
871 case XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE:
872 case XE_GUC_LOAD_STATUS_HWCONFIG_ERROR:
873 case XE_GUC_LOAD_STATUS_DPC_ERROR:
874 case XE_GUC_LOAD_STATUS_EXCEPTION:
875 case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID:
876 case XE_GUC_LOAD_STATUS_MPU_DATA_INVALID:
877 case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
878 return -1;
879 }
880
881 switch (br_val) {
882 case XE_BOOTROM_STATUS_NO_KEY_FOUND:
883 case XE_BOOTROM_STATUS_RSA_FAILED:
884 case XE_BOOTROM_STATUS_PAVPC_FAILED:
885 case XE_BOOTROM_STATUS_WOPCM_FAILED:
886 case XE_BOOTROM_STATUS_LOADLOC_FAILED:
887 case XE_BOOTROM_STATUS_JUMP_FAILED:
888 case XE_BOOTROM_STATUS_RC6CTXCONFIG_FAILED:
889 case XE_BOOTROM_STATUS_MPUMAP_INCORRECT:
890 case XE_BOOTROM_STATUS_EXCEPTION:
891 case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
892 return -1;
893 }
894
895 return 0;
896 }
897
guc_pc_get_cur_freq(struct xe_guc_pc * guc_pc)898 static s32 guc_pc_get_cur_freq(struct xe_guc_pc *guc_pc)
899 {
900 u32 freq;
901 int ret = xe_guc_pc_get_cur_freq(guc_pc, &freq);
902
903 return ret ? ret : freq;
904 }
905
906 /*
907 * Wait for the GuC to start up.
908 *
909 * Measurements indicate this should take no more than 20ms (assuming the GT
910 * clock is at maximum frequency). However, thermal throttling and other issues
911 * can prevent the clock hitting max and thus making the load take significantly
912 * longer. Allow up to 200ms as a safety margin for real world worst case situations.
913 *
914 * However, bugs anywhere from KMD to GuC to PCODE to fan failure in a CI farm can
915 * lead to even longer times. E.g. if the GT is clamped to minimum frequency then
916 * the load times can be in the seconds range. So the timeout is increased for debug
917 * builds to ensure that problems can be correctly analysed. For release builds, the
918 * timeout is kept short so that users don't wait forever to find out that there is a
919 * problem. In either case, if the load took longer than is reasonable even with some
920 * 'sensible' throttling, then flag a warning because something is not right.
921 *
922 * Note that there is a limit on how long an individual usleep_range() can wait for,
923 * hence longer waits require wrapping a shorter wait in a loop.
924 *
925 * Note that the only reason an end user should hit the shorter timeout is in case of
926 * extreme thermal throttling. And a system that is that hot during boot is probably
927 * dead anyway!
928 */
929 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
930 #define GUC_LOAD_RETRY_LIMIT 20
931 #else
932 #define GUC_LOAD_RETRY_LIMIT 3
933 #endif
934 #define GUC_LOAD_TIME_WARN_MS 200
935
guc_wait_ucode(struct xe_guc * guc)936 static void guc_wait_ucode(struct xe_guc *guc)
937 {
938 struct xe_gt *gt = guc_to_gt(guc);
939 struct xe_mmio *mmio = >->mmio;
940 struct xe_guc_pc *guc_pc = >->uc.guc.pc;
941 ktime_t before, after, delta;
942 int load_done;
943 u32 status = 0;
944 int count = 0;
945 u64 delta_ms;
946 u32 before_freq;
947
948 before_freq = xe_guc_pc_get_act_freq(guc_pc);
949 before = ktime_get();
950 /*
951 * Note, can't use any kind of timing information from the call to xe_mmio_wait.
952 * It could return a thousand intermediate stages at random times. Instead, must
953 * manually track the total time taken and locally implement the timeout.
954 */
955 do {
956 u32 last_status = status & (GS_UKERNEL_MASK | GS_BOOTROM_MASK);
957 int ret;
958
959 /*
960 * Wait for any change (intermediate or terminal) in the status register.
961 * Note, the return value is a don't care. The only failure code is timeout
962 * but the timeouts need to be accumulated over all the intermediate partial
963 * timeouts rather than allowing a huge timeout each time. So basically, need
964 * to treat a timeout no different to a value change.
965 */
966 ret = xe_mmio_wait32_not(mmio, GUC_STATUS, GS_UKERNEL_MASK | GS_BOOTROM_MASK,
967 last_status, 1000 * 1000, &status, false);
968 if (ret < 0)
969 count++;
970 after = ktime_get();
971 delta = ktime_sub(after, before);
972 delta_ms = ktime_to_ms(delta);
973
974 load_done = guc_load_done(status);
975 if (load_done != 0)
976 break;
977
978 if (delta_ms >= (GUC_LOAD_RETRY_LIMIT * 1000))
979 break;
980
981 xe_gt_dbg(gt, "load still in progress, timeouts = %d, freq = %dMHz (req %dMHz), status = 0x%08X [0x%02X/%02X]\n",
982 count, xe_guc_pc_get_act_freq(guc_pc),
983 guc_pc_get_cur_freq(guc_pc), status,
984 REG_FIELD_GET(GS_BOOTROM_MASK, status),
985 REG_FIELD_GET(GS_UKERNEL_MASK, status));
986 } while (1);
987
988 if (load_done != 1) {
989 u32 ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, status);
990 u32 bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, status);
991
992 xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz), done = %d\n",
993 status, delta_ms, xe_guc_pc_get_act_freq(guc_pc),
994 guc_pc_get_cur_freq(guc_pc), load_done);
995 xe_gt_err(gt, "load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n",
996 REG_FIELD_GET(GS_MIA_IN_RESET, status),
997 bootrom, ukernel,
998 REG_FIELD_GET(GS_MIA_MASK, status),
999 REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
1000
1001 switch (bootrom) {
1002 case XE_BOOTROM_STATUS_NO_KEY_FOUND:
1003 xe_gt_err(gt, "invalid key requested, header = 0x%08X\n",
1004 xe_mmio_read32(mmio, GUC_HEADER_INFO));
1005 break;
1006
1007 case XE_BOOTROM_STATUS_RSA_FAILED:
1008 xe_gt_err(gt, "firmware signature verification failed\n");
1009 break;
1010
1011 case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
1012 xe_gt_err(gt, "firmware production part check failure\n");
1013 break;
1014 }
1015
1016 switch (ukernel) {
1017 case XE_GUC_LOAD_STATUS_EXCEPTION:
1018 xe_gt_err(gt, "firmware exception. EIP: %#x\n",
1019 xe_mmio_read32(mmio, SOFT_SCRATCH(13)));
1020 break;
1021
1022 case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
1023 xe_gt_err(gt, "illegal register in save/restore workaround list\n");
1024 break;
1025
1026 case XE_GUC_LOAD_STATUS_HWCONFIG_START:
1027 xe_gt_err(gt, "still extracting hwconfig table.\n");
1028 break;
1029 }
1030
1031 xe_device_declare_wedged(gt_to_xe(gt));
1032 } else if (delta_ms > GUC_LOAD_TIME_WARN_MS) {
1033 xe_gt_warn(gt, "excessive init time: %lldms! [status = 0x%08X, timeouts = %d]\n",
1034 delta_ms, status, count);
1035 xe_gt_warn(gt, "excessive init time: [freq = %dMHz (req = %dMHz), before = %dMHz, perf_limit_reasons = 0x%08X]\n",
1036 xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc),
1037 before_freq, xe_gt_throttle_get_limit_reasons(gt));
1038 } else {
1039 xe_gt_dbg(gt, "init took %lldms, freq = %dMHz (req = %dMHz), before = %dMHz, status = 0x%08X, timeouts = %d\n",
1040 delta_ms, xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc),
1041 before_freq, status, count);
1042 }
1043 }
1044
__xe_guc_upload(struct xe_guc * guc)1045 static int __xe_guc_upload(struct xe_guc *guc)
1046 {
1047 int ret;
1048
1049 /* Raise GT freq to speed up HuC/GuC load */
1050 xe_guc_pc_raise_unslice(&guc->pc);
1051
1052 guc_write_params(guc);
1053 guc_prepare_xfer(guc);
1054
1055 /*
1056 * Note that GuC needs the CSS header plus uKernel code to be copied
1057 * by the DMA engine in one operation, whereas the RSA signature is
1058 * loaded separately, either by copying it to the UOS_RSA_SCRATCH
1059 * register (if key size <= 256) or through a ggtt-pinned vma (if key
1060 * size > 256). The RSA size and therefore the way we provide it to the
1061 * HW is fixed for each platform and hard-coded in the bootrom.
1062 */
1063 ret = guc_xfer_rsa(guc);
1064 if (ret)
1065 goto out;
1066 /*
1067 * Current uCode expects the code to be loaded at 8k; locations below
1068 * this are used for the stack.
1069 */
1070 ret = xe_uc_fw_upload(&guc->fw, 0x2000, UOS_MOVE);
1071 if (ret)
1072 goto out;
1073
1074 /* Wait for authentication */
1075 guc_wait_ucode(guc);
1076
1077 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_RUNNING);
1078 return 0;
1079
1080 out:
1081 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOAD_FAIL);
1082 return 0 /* FIXME: ret, don't want to stop load currently */;
1083 }
1084
vf_guc_min_load_for_hwconfig(struct xe_guc * guc)1085 static int vf_guc_min_load_for_hwconfig(struct xe_guc *guc)
1086 {
1087 struct xe_gt *gt = guc_to_gt(guc);
1088 int ret;
1089
1090 ret = xe_gt_sriov_vf_bootstrap(gt);
1091 if (ret)
1092 return ret;
1093
1094 ret = xe_gt_sriov_vf_query_config(gt);
1095 if (ret)
1096 return ret;
1097
1098 ret = xe_guc_hwconfig_init(guc);
1099 if (ret)
1100 return ret;
1101
1102 ret = xe_guc_enable_communication(guc);
1103 if (ret)
1104 return ret;
1105
1106 ret = xe_gt_sriov_vf_connect(gt);
1107 if (ret)
1108 return ret;
1109
1110 ret = xe_gt_sriov_vf_query_runtime(gt);
1111 if (ret)
1112 return ret;
1113
1114 return 0;
1115 }
1116
1117 /**
1118 * xe_guc_min_load_for_hwconfig - load minimal GuC and read hwconfig table
1119 * @guc: The GuC object
1120 *
1121 * This function uploads a minimal GuC that does not support submissions but
1122 * in a state where the hwconfig table can be read. Next, it reads and parses
1123 * the hwconfig table so it can be used for subsequent steps in the driver load.
1124 * Lastly, it enables CT communication (XXX: this is needed for PFs/VFs only).
1125 *
1126 * Return: 0 on success, negative error code on error.
1127 */
xe_guc_min_load_for_hwconfig(struct xe_guc * guc)1128 int xe_guc_min_load_for_hwconfig(struct xe_guc *guc)
1129 {
1130 int ret;
1131
1132 if (IS_SRIOV_VF(guc_to_xe(guc)))
1133 return vf_guc_min_load_for_hwconfig(guc);
1134
1135 xe_guc_ads_populate_minimal(&guc->ads);
1136
1137 xe_guc_pc_init_early(&guc->pc);
1138
1139 ret = __xe_guc_upload(guc);
1140 if (ret)
1141 return ret;
1142
1143 ret = xe_guc_hwconfig_init(guc);
1144 if (ret)
1145 return ret;
1146
1147 ret = xe_guc_enable_communication(guc);
1148 if (ret)
1149 return ret;
1150
1151 return 0;
1152 }
1153
xe_guc_upload(struct xe_guc * guc)1154 int xe_guc_upload(struct xe_guc *guc)
1155 {
1156 xe_guc_ads_populate(&guc->ads);
1157
1158 return __xe_guc_upload(guc);
1159 }
1160
guc_handle_mmio_msg(struct xe_guc * guc)1161 static void guc_handle_mmio_msg(struct xe_guc *guc)
1162 {
1163 struct xe_gt *gt = guc_to_gt(guc);
1164 u32 msg;
1165
1166 if (IS_SRIOV_VF(guc_to_xe(guc)))
1167 return;
1168
1169 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
1170
1171 msg = xe_mmio_read32(>->mmio, SOFT_SCRATCH(15));
1172 msg &= XE_GUC_RECV_MSG_EXCEPTION |
1173 XE_GUC_RECV_MSG_CRASH_DUMP_POSTED;
1174 xe_mmio_write32(>->mmio, SOFT_SCRATCH(15), 0);
1175
1176 if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED)
1177 xe_gt_err(gt, "Received early GuC crash dump notification!\n");
1178
1179 if (msg & XE_GUC_RECV_MSG_EXCEPTION)
1180 xe_gt_err(gt, "Received early GuC exception notification!\n");
1181 }
1182
guc_enable_irq(struct xe_guc * guc)1183 static void guc_enable_irq(struct xe_guc *guc)
1184 {
1185 struct xe_gt *gt = guc_to_gt(guc);
1186 u32 events = xe_gt_is_media_type(gt) ?
1187 REG_FIELD_PREP(ENGINE0_MASK, GUC_INTR_GUC2HOST) :
1188 REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST);
1189
1190 /* Primary GuC and media GuC share a single enable bit */
1191 xe_mmio_write32(>->mmio, GUC_SG_INTR_ENABLE,
1192 REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST));
1193
1194 /*
1195 * There are separate mask bits for primary and media GuCs, so use
1196 * a RMW operation to avoid clobbering the other GuC's setting.
1197 */
1198 xe_mmio_rmw32(>->mmio, GUC_SG_INTR_MASK, events, 0);
1199 }
1200
xe_guc_enable_communication(struct xe_guc * guc)1201 int xe_guc_enable_communication(struct xe_guc *guc)
1202 {
1203 struct xe_device *xe = guc_to_xe(guc);
1204 int err;
1205
1206 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) {
1207 struct xe_gt *gt = guc_to_gt(guc);
1208 struct xe_tile *tile = gt_to_tile(gt);
1209
1210 err = xe_memirq_init_guc(&tile->memirq, guc);
1211 if (err)
1212 return err;
1213 } else {
1214 guc_enable_irq(guc);
1215 }
1216
1217 err = xe_guc_ct_enable(&guc->ct);
1218 if (err)
1219 return err;
1220
1221 guc_handle_mmio_msg(guc);
1222
1223 return 0;
1224 }
1225
xe_guc_suspend(struct xe_guc * guc)1226 int xe_guc_suspend(struct xe_guc *guc)
1227 {
1228 struct xe_gt *gt = guc_to_gt(guc);
1229 u32 action[] = {
1230 XE_GUC_ACTION_CLIENT_SOFT_RESET,
1231 };
1232 int ret;
1233
1234 ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action));
1235 if (ret) {
1236 xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret));
1237 return ret;
1238 }
1239
1240 xe_guc_sanitize(guc);
1241 return 0;
1242 }
1243
xe_guc_notify(struct xe_guc * guc)1244 void xe_guc_notify(struct xe_guc *guc)
1245 {
1246 struct xe_gt *gt = guc_to_gt(guc);
1247 const u32 default_notify_data = 0;
1248
1249 /*
1250 * Both GUC_HOST_INTERRUPT and MED_GUC_HOST_INTERRUPT can pass
1251 * additional payload data to the GuC but this capability is not
1252 * used by the firmware yet. Use default value in the meantime.
1253 */
1254 xe_mmio_write32(>->mmio, guc->notify_reg, default_notify_data);
1255 }
1256
xe_guc_auth_huc(struct xe_guc * guc,u32 rsa_addr)1257 int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr)
1258 {
1259 u32 action[] = {
1260 XE_GUC_ACTION_AUTHENTICATE_HUC,
1261 rsa_addr
1262 };
1263
1264 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
1265 }
1266
xe_guc_mmio_send_recv(struct xe_guc * guc,const u32 * request,u32 len,u32 * response_buf)1267 int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
1268 u32 len, u32 *response_buf)
1269 {
1270 struct xe_device *xe = guc_to_xe(guc);
1271 struct xe_gt *gt = guc_to_gt(guc);
1272 struct xe_mmio *mmio = >->mmio;
1273 u32 header, reply;
1274 struct xe_reg reply_reg = xe_gt_is_media_type(gt) ?
1275 MED_VF_SW_FLAG(0) : VF_SW_FLAG(0);
1276 const u32 LAST_INDEX = VF_SW_FLAG_COUNT - 1;
1277 int ret;
1278 int i;
1279
1280 BUILD_BUG_ON(VF_SW_FLAG_COUNT != MED_VF_SW_FLAG_COUNT);
1281
1282 xe_assert(xe, len);
1283 xe_assert(xe, len <= VF_SW_FLAG_COUNT);
1284 xe_assert(xe, len <= MED_VF_SW_FLAG_COUNT);
1285 xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) ==
1286 GUC_HXG_ORIGIN_HOST);
1287 xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) ==
1288 GUC_HXG_TYPE_REQUEST);
1289
1290 retry:
1291 /* Not in critical data-path, just do if else for GT type */
1292 if (xe_gt_is_media_type(gt)) {
1293 for (i = 0; i < len; ++i)
1294 xe_mmio_write32(mmio, MED_VF_SW_FLAG(i),
1295 request[i]);
1296 xe_mmio_read32(mmio, MED_VF_SW_FLAG(LAST_INDEX));
1297 } else {
1298 for (i = 0; i < len; ++i)
1299 xe_mmio_write32(mmio, VF_SW_FLAG(i),
1300 request[i]);
1301 xe_mmio_read32(mmio, VF_SW_FLAG(LAST_INDEX));
1302 }
1303
1304 xe_guc_notify(guc);
1305
1306 ret = xe_mmio_wait32(mmio, reply_reg, GUC_HXG_MSG_0_ORIGIN,
1307 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC),
1308 50000, &reply, false);
1309 if (ret) {
1310 timeout:
1311 xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n",
1312 request[0], reply);
1313 return ret;
1314 }
1315
1316 header = xe_mmio_read32(mmio, reply_reg);
1317 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1318 GUC_HXG_TYPE_NO_RESPONSE_BUSY) {
1319 /*
1320 * Once we got a BUSY reply we must wait again for the final
1321 * response but this time we can't use ORIGIN mask anymore.
1322 * To spot a right change in the reply, we take advantage that
1323 * response SUCCESS and FAILURE differ only by the single bit
1324 * and all other bits are set and can be used as a new mask.
1325 */
1326 u32 resp_bits = GUC_HXG_TYPE_RESPONSE_SUCCESS & GUC_HXG_TYPE_RESPONSE_FAILURE;
1327 u32 resp_mask = FIELD_PREP(GUC_HXG_MSG_0_TYPE, resp_bits);
1328
1329 BUILD_BUG_ON(FIELD_MAX(GUC_HXG_MSG_0_TYPE) != GUC_HXG_TYPE_RESPONSE_SUCCESS);
1330 BUILD_BUG_ON((GUC_HXG_TYPE_RESPONSE_SUCCESS ^ GUC_HXG_TYPE_RESPONSE_FAILURE) != 1);
1331
1332 ret = xe_mmio_wait32(mmio, reply_reg, resp_mask, resp_mask,
1333 1000000, &header, false);
1334
1335 if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) !=
1336 GUC_HXG_ORIGIN_GUC))
1337 goto proto;
1338 if (unlikely(ret)) {
1339 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1340 GUC_HXG_TYPE_NO_RESPONSE_BUSY)
1341 goto proto;
1342 goto timeout;
1343 }
1344 }
1345
1346 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1347 GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1348 u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header);
1349
1350 xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n",
1351 request[0], reason);
1352 goto retry;
1353 }
1354
1355 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1356 GUC_HXG_TYPE_RESPONSE_FAILURE) {
1357 u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
1358 u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
1359
1360 xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
1361 request[0], error, hint);
1362 return -ENXIO;
1363 }
1364
1365 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1366 GUC_HXG_TYPE_RESPONSE_SUCCESS) {
1367 proto:
1368 xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n",
1369 request[0], header);
1370 return -EPROTO;
1371 }
1372
1373 /* Just copy entire possible message response */
1374 if (response_buf) {
1375 response_buf[0] = header;
1376
1377 for (i = 1; i < VF_SW_FLAG_COUNT; i++) {
1378 reply_reg.addr += sizeof(u32);
1379 response_buf[i] = xe_mmio_read32(mmio, reply_reg);
1380 }
1381 }
1382
1383 /* Use data from the GuC response as our return value */
1384 return FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header);
1385 }
1386
xe_guc_mmio_send(struct xe_guc * guc,const u32 * request,u32 len)1387 int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len)
1388 {
1389 return xe_guc_mmio_send_recv(guc, request, len, NULL);
1390 }
1391
guc_self_cfg(struct xe_guc * guc,u16 key,u16 len,u64 val)1392 static int guc_self_cfg(struct xe_guc *guc, u16 key, u16 len, u64 val)
1393 {
1394 struct xe_device *xe = guc_to_xe(guc);
1395 u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = {
1396 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
1397 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
1398 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
1399 GUC_ACTION_HOST2GUC_SELF_CFG),
1400 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) |
1401 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len),
1402 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32,
1403 lower_32_bits(val)),
1404 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64,
1405 upper_32_bits(val)),
1406 };
1407 int ret;
1408
1409 xe_assert(xe, len <= 2);
1410 xe_assert(xe, len != 1 || !upper_32_bits(val));
1411
1412 /* Self config must go over MMIO */
1413 ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request));
1414
1415 if (unlikely(ret < 0))
1416 return ret;
1417 if (unlikely(ret > 1))
1418 return -EPROTO;
1419 if (unlikely(!ret))
1420 return -ENOKEY;
1421
1422 return 0;
1423 }
1424
xe_guc_self_cfg32(struct xe_guc * guc,u16 key,u32 val)1425 int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val)
1426 {
1427 return guc_self_cfg(guc, key, 1, val);
1428 }
1429
xe_guc_self_cfg64(struct xe_guc * guc,u16 key,u64 val)1430 int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val)
1431 {
1432 return guc_self_cfg(guc, key, 2, val);
1433 }
1434
xe_guc_sw_0_irq_handler(struct xe_guc * guc)1435 static void xe_guc_sw_0_irq_handler(struct xe_guc *guc)
1436 {
1437 struct xe_gt *gt = guc_to_gt(guc);
1438
1439 if (IS_SRIOV_VF(gt_to_xe(gt)))
1440 xe_gt_sriov_vf_migrated_event_handler(gt);
1441 }
1442
xe_guc_irq_handler(struct xe_guc * guc,const u16 iir)1443 void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir)
1444 {
1445 if (iir & GUC_INTR_GUC2HOST)
1446 xe_guc_ct_irq_handler(&guc->ct);
1447
1448 if (iir & GUC_INTR_SW_INT_0)
1449 xe_guc_sw_0_irq_handler(guc);
1450 }
1451
xe_guc_sanitize(struct xe_guc * guc)1452 void xe_guc_sanitize(struct xe_guc *guc)
1453 {
1454 xe_uc_fw_sanitize(&guc->fw);
1455 xe_guc_ct_disable(&guc->ct);
1456 guc->submission_state.enabled = false;
1457 }
1458
xe_guc_reset_prepare(struct xe_guc * guc)1459 int xe_guc_reset_prepare(struct xe_guc *guc)
1460 {
1461 return xe_guc_submit_reset_prepare(guc);
1462 }
1463
xe_guc_reset_wait(struct xe_guc * guc)1464 void xe_guc_reset_wait(struct xe_guc *guc)
1465 {
1466 xe_guc_submit_reset_wait(guc);
1467 }
1468
xe_guc_stop_prepare(struct xe_guc * guc)1469 void xe_guc_stop_prepare(struct xe_guc *guc)
1470 {
1471 if (!IS_SRIOV_VF(guc_to_xe(guc))) {
1472 int err;
1473
1474 err = xe_guc_pc_stop(&guc->pc);
1475 xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n",
1476 ERR_PTR(err));
1477 }
1478 }
1479
xe_guc_stop(struct xe_guc * guc)1480 void xe_guc_stop(struct xe_guc *guc)
1481 {
1482 xe_guc_ct_stop(&guc->ct);
1483
1484 xe_guc_submit_stop(guc);
1485 }
1486
xe_guc_start(struct xe_guc * guc)1487 int xe_guc_start(struct xe_guc *guc)
1488 {
1489 if (!IS_SRIOV_VF(guc_to_xe(guc))) {
1490 int err;
1491
1492 err = xe_guc_pc_start(&guc->pc);
1493 xe_gt_WARN(guc_to_gt(guc), err, "Failed to start GuC PC: %pe\n",
1494 ERR_PTR(err));
1495 }
1496
1497 return xe_guc_submit_start(guc);
1498 }
1499
xe_guc_print_info(struct xe_guc * guc,struct drm_printer * p)1500 void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p)
1501 {
1502 struct xe_gt *gt = guc_to_gt(guc);
1503 unsigned int fw_ref;
1504 u32 status;
1505 int i;
1506
1507 xe_uc_fw_print(&guc->fw, p);
1508
1509 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1510 if (!fw_ref)
1511 return;
1512
1513 status = xe_mmio_read32(>->mmio, GUC_STATUS);
1514
1515 drm_printf(p, "\nGuC status 0x%08x:\n", status);
1516 drm_printf(p, "\tBootrom status = 0x%x\n",
1517 REG_FIELD_GET(GS_BOOTROM_MASK, status));
1518 drm_printf(p, "\tuKernel status = 0x%x\n",
1519 REG_FIELD_GET(GS_UKERNEL_MASK, status));
1520 drm_printf(p, "\tMIA Core status = 0x%x\n",
1521 REG_FIELD_GET(GS_MIA_MASK, status));
1522 drm_printf(p, "\tLog level = %d\n",
1523 xe_guc_log_get_level(&guc->log));
1524
1525 drm_puts(p, "\nScratch registers:\n");
1526 for (i = 0; i < SOFT_SCRATCH_COUNT; i++) {
1527 drm_printf(p, "\t%2d: \t0x%x\n",
1528 i, xe_mmio_read32(>->mmio, SOFT_SCRATCH(i)));
1529 }
1530
1531 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1532
1533 drm_puts(p, "\n");
1534 xe_guc_ct_print(&guc->ct, p, false);
1535
1536 drm_puts(p, "\n");
1537 xe_guc_submit_print(guc, p);
1538 }
1539
1540 /**
1541 * xe_guc_declare_wedged() - Declare GuC wedged
1542 * @guc: the GuC object
1543 *
1544 * Wedge the GuC which stops all submission, saves desired debug state, and
1545 * cleans up anything which could timeout.
1546 */
xe_guc_declare_wedged(struct xe_guc * guc)1547 void xe_guc_declare_wedged(struct xe_guc *guc)
1548 {
1549 xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);
1550
1551 xe_guc_reset_prepare(guc);
1552 xe_guc_ct_stop(&guc->ct);
1553 xe_guc_submit_wedge(guc);
1554 }
1555