1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc.h"
7
8 #include <drm/drm_managed.h>
9
10 #include <generated/xe_wa_oob.h>
11
12 #include "abi/guc_actions_abi.h"
13 #include "abi/guc_errors_abi.h"
14 #include "regs/xe_gt_regs.h"
15 #include "regs/xe_gtt_defs.h"
16 #include "regs/xe_guc_regs.h"
17 #include "regs/xe_irq_regs.h"
18 #include "xe_bo.h"
19 #include "xe_configfs.h"
20 #include "xe_device.h"
21 #include "xe_force_wake.h"
22 #include "xe_gt.h"
23 #include "xe_gt_printk.h"
24 #include "xe_gt_sriov_vf.h"
25 #include "xe_gt_throttle.h"
26 #include "xe_guc_ads.h"
27 #include "xe_guc_buf.h"
28 #include "xe_guc_capture.h"
29 #include "xe_guc_ct.h"
30 #include "xe_guc_db_mgr.h"
31 #include "xe_guc_engine_activity.h"
32 #include "xe_guc_hwconfig.h"
33 #include "xe_guc_klv_helpers.h"
34 #include "xe_guc_log.h"
35 #include "xe_guc_pc.h"
36 #include "xe_guc_relay.h"
37 #include "xe_guc_submit.h"
38 #include "xe_memirq.h"
39 #include "xe_mmio.h"
40 #include "xe_platform_types.h"
41 #include "xe_sriov.h"
42 #include "xe_uc.h"
43 #include "xe_uc_fw.h"
44 #include "xe_wa.h"
45 #include "xe_wopcm.h"
46
guc_bo_ggtt_addr(struct xe_guc * guc,struct xe_bo * bo)47 static u32 guc_bo_ggtt_addr(struct xe_guc *guc,
48 struct xe_bo *bo)
49 {
50 struct xe_device *xe = guc_to_xe(guc);
51 u32 addr;
52
53 /*
54 * For most BOs, the address on the allocating tile is fine. However for
55 * some, e.g. G2G CTB, the address on a specific tile is required as it
56 * might be different for each tile. So, just always ask for the address
57 * on the target GuC.
58 */
59 addr = __xe_bo_ggtt_addr(bo, gt_to_tile(guc_to_gt(guc))->id);
60
61 /* GuC addresses above GUC_GGTT_TOP don't map through the GTT */
62 xe_assert(xe, addr >= xe_wopcm_size(guc_to_xe(guc)));
63 xe_assert(xe, addr < GUC_GGTT_TOP);
64 xe_assert(xe, xe_bo_size(bo) <= GUC_GGTT_TOP - addr);
65
66 return addr;
67 }
68
guc_ctl_debug_flags(struct xe_guc * guc)69 static u32 guc_ctl_debug_flags(struct xe_guc *guc)
70 {
71 u32 level = xe_guc_log_get_level(&guc->log);
72 u32 flags = 0;
73
74 if (!GUC_LOG_LEVEL_IS_VERBOSE(level))
75 flags |= GUC_LOG_DISABLED;
76 else
77 flags |= FIELD_PREP(GUC_LOG_VERBOSITY, GUC_LOG_LEVEL_TO_VERBOSITY(level));
78
79 return flags;
80 }
81
guc_ctl_feature_flags(struct xe_guc * guc)82 static u32 guc_ctl_feature_flags(struct xe_guc *guc)
83 {
84 struct xe_device *xe = guc_to_xe(guc);
85 u32 flags = GUC_CTL_ENABLE_LITE_RESTORE;
86
87 if (!xe->info.skip_guc_pc)
88 flags |= GUC_CTL_ENABLE_SLPC;
89
90 if (xe_configfs_get_psmi_enabled(to_pci_dev(xe->drm.dev)))
91 flags |= GUC_CTL_ENABLE_PSMI_LOGGING;
92
93 return flags;
94 }
95
guc_ctl_log_params_flags(struct xe_guc * guc)96 static u32 guc_ctl_log_params_flags(struct xe_guc *guc)
97 {
98 u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT;
99 u32 flags;
100
101 #if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
102 #define LOG_UNIT SZ_1M
103 #define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
104 #else
105 #define LOG_UNIT SZ_4K
106 #define LOG_FLAG 0
107 #endif
108
109 #if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
110 #define CAPTURE_UNIT SZ_1M
111 #define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
112 #else
113 #define CAPTURE_UNIT SZ_4K
114 #define CAPTURE_FLAG 0
115 #endif
116
117 BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
118 BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
119 BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
120 BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
121 BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
122 BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
123
124 flags = GUC_LOG_VALID |
125 GUC_LOG_NOTIFY_ON_HALF_FULL |
126 CAPTURE_FLAG |
127 LOG_FLAG |
128 FIELD_PREP(GUC_LOG_CRASH, CRASH_BUFFER_SIZE / LOG_UNIT - 1) |
129 FIELD_PREP(GUC_LOG_DEBUG, DEBUG_BUFFER_SIZE / LOG_UNIT - 1) |
130 FIELD_PREP(GUC_LOG_CAPTURE, CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) |
131 FIELD_PREP(GUC_LOG_BUF_ADDR, offset);
132
133 #undef LOG_UNIT
134 #undef LOG_FLAG
135 #undef CAPTURE_UNIT
136 #undef CAPTURE_FLAG
137
138 return flags;
139 }
140
guc_ctl_ads_flags(struct xe_guc * guc)141 static u32 guc_ctl_ads_flags(struct xe_guc *guc)
142 {
143 u32 ads = guc_bo_ggtt_addr(guc, guc->ads.bo) >> PAGE_SHIFT;
144 u32 flags = FIELD_PREP(GUC_ADS_ADDR, ads);
145
146 return flags;
147 }
148
needs_wa_dual_queue(struct xe_gt * gt)149 static bool needs_wa_dual_queue(struct xe_gt *gt)
150 {
151 /*
152 * The DUAL_QUEUE_WA tells the GuC to not allow concurrent submissions
153 * on RCS and CCSes with different address spaces, which on DG2 is
154 * required as a WA for an HW bug.
155 */
156 if (XE_GT_WA(gt, 22011391025))
157 return true;
158
159 /*
160 * On newer platforms, the HW has been updated to not allow parallel
161 * execution of different address spaces, so the RCS/CCS will stall the
162 * context switch if one of the other RCS/CCSes is busy with a different
163 * address space. While functionally correct, having a submission
164 * stalled on the HW limits the GuC ability to shuffle things around and
165 * can cause complications if the non-stalled submission runs for a long
166 * time, because the GuC doesn't know that the stalled submission isn't
167 * actually running and might declare it as hung. Therefore, we enable
168 * the DUAL_QUEUE_WA on all newer platforms on GTs that have CCS engines
169 * to move management back to the GuC.
170 */
171 if (CCS_MASK(gt) && GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
172 return true;
173
174 return false;
175 }
176
guc_ctl_wa_flags(struct xe_guc * guc)177 static u32 guc_ctl_wa_flags(struct xe_guc *guc)
178 {
179 struct xe_device *xe = guc_to_xe(guc);
180 struct xe_gt *gt = guc_to_gt(guc);
181 u32 flags = 0;
182
183 if (XE_GT_WA(gt, 22012773006))
184 flags |= GUC_WA_POLLCS;
185
186 if (XE_GT_WA(gt, 14014475959))
187 flags |= GUC_WA_HOLD_CCS_SWITCHOUT;
188
189 if (needs_wa_dual_queue(gt))
190 flags |= GUC_WA_DUAL_QUEUE;
191
192 /*
193 * Wa_22011802037: FIXME - there's more to be done than simply setting
194 * this flag: make sure each CS is stopped when preparing for GT reset
195 * and wait for pending MI_FW.
196 */
197 if (GRAPHICS_VERx100(xe) < 1270)
198 flags |= GUC_WA_PRE_PARSER;
199
200 if (XE_GT_WA(gt, 22012727170) || XE_GT_WA(gt, 22012727685))
201 flags |= GUC_WA_CONTEXT_ISOLATION;
202
203 if (XE_GT_WA(gt, 18020744125) &&
204 !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER))
205 flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST;
206
207 if (XE_GT_WA(gt, 1509372804))
208 flags |= GUC_WA_RENDER_RST_RC6_EXIT;
209
210 if (XE_GT_WA(gt, 14018913170))
211 flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
212
213 if (XE_GT_WA(gt, 16023683509))
214 flags |= GUC_WA_SAVE_RESTORE_MCFG_REG_AT_MC6;
215
216 return flags;
217 }
218
guc_ctl_devid(struct xe_guc * guc)219 static u32 guc_ctl_devid(struct xe_guc *guc)
220 {
221 struct xe_device *xe = guc_to_xe(guc);
222
223 return (((u32)xe->info.devid) << 16) | xe->info.revid;
224 }
225
guc_print_params(struct xe_guc * guc)226 static void guc_print_params(struct xe_guc *guc)
227 {
228 struct xe_gt *gt = guc_to_gt(guc);
229 u32 *params = guc->params;
230 int i;
231
232 BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32));
233 BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT);
234
235 for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
236 xe_gt_dbg(gt, "GuC param[%2d] = 0x%08x\n", i, params[i]);
237 }
238
guc_init_params(struct xe_guc * guc)239 static void guc_init_params(struct xe_guc *guc)
240 {
241 u32 *params = guc->params;
242
243 params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
244 params[GUC_CTL_FEATURE] = 0;
245 params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
246 params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
247 params[GUC_CTL_WA] = 0;
248 params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
249
250 guc_print_params(guc);
251 }
252
guc_init_params_post_hwconfig(struct xe_guc * guc)253 static void guc_init_params_post_hwconfig(struct xe_guc *guc)
254 {
255 u32 *params = guc->params;
256
257 params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
258 params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc);
259 params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
260 params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
261 params[GUC_CTL_WA] = guc_ctl_wa_flags(guc);
262 params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
263
264 guc_print_params(guc);
265 }
266
267 /*
268 * Initialize the GuC parameter block before starting the firmware
269 * transfer. These parameters are read by the firmware on startup
270 * and cannot be changed thereafter.
271 */
guc_write_params(struct xe_guc * guc)272 static void guc_write_params(struct xe_guc *guc)
273 {
274 struct xe_gt *gt = guc_to_gt(guc);
275 int i;
276
277 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
278
279 xe_mmio_write32(>->mmio, SOFT_SCRATCH(0), 0);
280
281 for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
282 xe_mmio_write32(>->mmio, SOFT_SCRATCH(1 + i), guc->params[i]);
283 }
284
guc_action_register_g2g_buffer(struct xe_guc * guc,u32 type,u32 dst_tile,u32 dst_dev,u32 desc_addr,u32 buff_addr,u32 size)285 static int guc_action_register_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev,
286 u32 desc_addr, u32 buff_addr, u32 size)
287 {
288 struct xe_gt *gt = guc_to_gt(guc);
289 struct xe_device *xe = gt_to_xe(gt);
290 u32 action[] = {
291 XE_GUC_ACTION_REGISTER_G2G,
292 FIELD_PREP(XE_G2G_REGISTER_SIZE, size / SZ_4K - 1) |
293 FIELD_PREP(XE_G2G_REGISTER_TYPE, type) |
294 FIELD_PREP(XE_G2G_REGISTER_TILE, dst_tile) |
295 FIELD_PREP(XE_G2G_REGISTER_DEVICE, dst_dev),
296 desc_addr,
297 buff_addr,
298 };
299
300 xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
301 xe_assert(xe, !(size % SZ_4K));
302
303 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
304 }
305
guc_action_deregister_g2g_buffer(struct xe_guc * guc,u32 type,u32 dst_tile,u32 dst_dev)306 static int guc_action_deregister_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev)
307 {
308 struct xe_gt *gt = guc_to_gt(guc);
309 struct xe_device *xe = gt_to_xe(gt);
310 u32 action[] = {
311 XE_GUC_ACTION_DEREGISTER_G2G,
312 FIELD_PREP(XE_G2G_DEREGISTER_TYPE, type) |
313 FIELD_PREP(XE_G2G_DEREGISTER_TILE, dst_tile) |
314 FIELD_PREP(XE_G2G_DEREGISTER_DEVICE, dst_dev),
315 };
316
317 xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
318
319 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
320 }
321
322 #define G2G_DEV(gt) (((gt)->info.type == XE_GT_TYPE_MAIN) ? 0 : 1)
323
324 #define G2G_BUFFER_SIZE (SZ_4K)
325 #define G2G_DESC_SIZE (64)
326 #define G2G_DESC_AREA_SIZE (SZ_4K)
327
328 /*
329 * Generate a unique id for each bi-directional CTB for each pair of
330 * near and far tiles/devices. The id can then be used as an index into
331 * a single allocation that is sub-divided into multiple CTBs.
332 *
333 * For example, with two devices per tile and two tiles, the table should
334 * look like:
335 * Far <tile>.<dev>
336 * 0.0 0.1 1.0 1.1
337 * N 0.0 --/-- 00/01 02/03 04/05
338 * e 0.1 01/00 --/-- 06/07 08/09
339 * a 1.0 03/02 07/06 --/-- 10/11
340 * r 1.1 05/04 09/08 11/10 --/--
341 *
342 * Where each entry is Rx/Tx channel id.
343 *
344 * So GuC #3 (tile 1, dev 1) talking to GuC #2 (tile 1, dev 0) would
345 * be reading from channel #11 and writing to channel #10. Whereas,
346 * GuC #2 talking to GuC #3 would be read on #10 and write to #11.
347 */
g2g_slot(u32 near_tile,u32 near_dev,u32 far_tile,u32 far_dev,u32 type,u32 max_inst,bool have_dev)348 static unsigned int g2g_slot(u32 near_tile, u32 near_dev, u32 far_tile, u32 far_dev,
349 u32 type, u32 max_inst, bool have_dev)
350 {
351 u32 near = near_tile, far = far_tile;
352 u32 idx = 0, x, y, direction;
353 int i;
354
355 if (have_dev) {
356 near = (near << 1) | near_dev;
357 far = (far << 1) | far_dev;
358 }
359
360 /* No need to send to one's self */
361 if (far == near)
362 return -1;
363
364 if (far > near) {
365 /* Top right table half */
366 x = far;
367 y = near;
368
369 /* T/R is 'forwards' direction */
370 direction = type;
371 } else {
372 /* Bottom left table half */
373 x = near;
374 y = far;
375
376 /* B/L is 'backwards' direction */
377 direction = (1 - type);
378 }
379
380 /* Count the rows prior to the target */
381 for (i = y; i > 0; i--)
382 idx += max_inst - i;
383
384 /* Count this row up to the target */
385 idx += (x - 1 - y);
386
387 /* Slots are in Rx/Tx pairs */
388 idx *= 2;
389
390 /* Pick Rx/Tx direction */
391 idx += direction;
392
393 return idx;
394 }
395
guc_g2g_register(struct xe_guc * near_guc,struct xe_gt * far_gt,u32 type,bool have_dev)396 static int guc_g2g_register(struct xe_guc *near_guc, struct xe_gt *far_gt, u32 type, bool have_dev)
397 {
398 struct xe_gt *near_gt = guc_to_gt(near_guc);
399 struct xe_device *xe = gt_to_xe(near_gt);
400 struct xe_bo *g2g_bo;
401 u32 near_tile = gt_to_tile(near_gt)->id;
402 u32 near_dev = G2G_DEV(near_gt);
403 u32 far_tile = gt_to_tile(far_gt)->id;
404 u32 far_dev = G2G_DEV(far_gt);
405 u32 max = xe->info.gt_count;
406 u32 base, desc, buf;
407 int slot;
408
409 /* G2G is not allowed between different cards */
410 xe_assert(xe, xe == gt_to_xe(far_gt));
411
412 g2g_bo = near_guc->g2g.bo;
413 xe_assert(xe, g2g_bo);
414
415 slot = g2g_slot(near_tile, near_dev, far_tile, far_dev, type, max, have_dev);
416 xe_assert(xe, slot >= 0);
417
418 base = guc_bo_ggtt_addr(near_guc, g2g_bo);
419 desc = base + slot * G2G_DESC_SIZE;
420 buf = base + G2G_DESC_AREA_SIZE + slot * G2G_BUFFER_SIZE;
421
422 xe_assert(xe, (desc - base + G2G_DESC_SIZE) <= G2G_DESC_AREA_SIZE);
423 xe_assert(xe, (buf - base + G2G_BUFFER_SIZE) <= xe_bo_size(g2g_bo));
424
425 return guc_action_register_g2g_buffer(near_guc, type, far_tile, far_dev,
426 desc, buf, G2G_BUFFER_SIZE);
427 }
428
guc_g2g_deregister(struct xe_guc * guc,u32 far_tile,u32 far_dev,u32 type)429 static void guc_g2g_deregister(struct xe_guc *guc, u32 far_tile, u32 far_dev, u32 type)
430 {
431 guc_action_deregister_g2g_buffer(guc, type, far_tile, far_dev);
432 }
433
guc_g2g_size(struct xe_guc * guc)434 static u32 guc_g2g_size(struct xe_guc *guc)
435 {
436 struct xe_gt *gt = guc_to_gt(guc);
437 struct xe_device *xe = gt_to_xe(gt);
438 unsigned int count = xe->info.gt_count;
439 u32 num_channels = (count * (count - 1)) / 2;
440
441 xe_assert(xe, num_channels * XE_G2G_TYPE_LIMIT * G2G_DESC_SIZE <= G2G_DESC_AREA_SIZE);
442
443 return num_channels * XE_G2G_TYPE_LIMIT * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE;
444 }
445
xe_guc_g2g_wanted(struct xe_device * xe)446 static bool xe_guc_g2g_wanted(struct xe_device *xe)
447 {
448 /* Can't do GuC to GuC communication if there is only one GuC */
449 if (xe->info.gt_count <= 1)
450 return false;
451
452 /* No current user */
453 return false;
454 }
455
guc_g2g_alloc(struct xe_guc * guc)456 static int guc_g2g_alloc(struct xe_guc *guc)
457 {
458 struct xe_gt *gt = guc_to_gt(guc);
459 struct xe_device *xe = gt_to_xe(gt);
460 struct xe_tile *tile = gt_to_tile(gt);
461 struct xe_bo *bo;
462 u32 g2g_size;
463
464 if (guc->g2g.bo)
465 return 0;
466
467 if (gt->info.id != 0) {
468 struct xe_gt *root_gt = xe_device_get_gt(xe, 0);
469 struct xe_guc *root_guc = &root_gt->uc.guc;
470 struct xe_bo *bo;
471
472 bo = xe_bo_get(root_guc->g2g.bo);
473 if (!bo)
474 return -ENODEV;
475
476 guc->g2g.bo = bo;
477 guc->g2g.owned = false;
478 return 0;
479 }
480
481 g2g_size = guc_g2g_size(guc);
482 bo = xe_managed_bo_create_pin_map(xe, tile, g2g_size,
483 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
484 XE_BO_FLAG_GGTT |
485 XE_BO_FLAG_GGTT_ALL |
486 XE_BO_FLAG_GGTT_INVALIDATE |
487 XE_BO_FLAG_PINNED_NORESTORE);
488 if (IS_ERR(bo))
489 return PTR_ERR(bo);
490
491 xe_map_memset(xe, &bo->vmap, 0, 0, g2g_size);
492 guc->g2g.bo = bo;
493 guc->g2g.owned = true;
494
495 return 0;
496 }
497
guc_g2g_fini(struct xe_guc * guc)498 static void guc_g2g_fini(struct xe_guc *guc)
499 {
500 if (!guc->g2g.bo)
501 return;
502
503 /* Unpinning the owned object is handled by generic shutdown */
504 if (!guc->g2g.owned)
505 xe_bo_put(guc->g2g.bo);
506
507 guc->g2g.bo = NULL;
508 }
509
guc_g2g_start(struct xe_guc * guc)510 static int guc_g2g_start(struct xe_guc *guc)
511 {
512 struct xe_gt *far_gt, *gt = guc_to_gt(guc);
513 struct xe_device *xe = gt_to_xe(gt);
514 unsigned int i, j;
515 int t, err;
516 bool have_dev;
517
518 if (!guc->g2g.bo) {
519 int ret;
520
521 ret = guc_g2g_alloc(guc);
522 if (ret)
523 return ret;
524 }
525
526 /* GuC interface will need extending if more GT device types are ever created. */
527 xe_gt_assert(gt, (gt->info.type == XE_GT_TYPE_MAIN) || (gt->info.type == XE_GT_TYPE_MEDIA));
528
529 /* Channel numbering depends on whether there are multiple GTs per tile */
530 have_dev = xe->info.gt_count > xe->info.tile_count;
531
532 for_each_gt(far_gt, xe, i) {
533 u32 far_tile, far_dev;
534
535 if (far_gt->info.id == gt->info.id)
536 continue;
537
538 far_tile = gt_to_tile(far_gt)->id;
539 far_dev = G2G_DEV(far_gt);
540
541 for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) {
542 err = guc_g2g_register(guc, far_gt, t, have_dev);
543 if (err) {
544 while (--t >= 0)
545 guc_g2g_deregister(guc, far_tile, far_dev, t);
546 goto err_deregister;
547 }
548 }
549 }
550
551 return 0;
552
553 err_deregister:
554 for_each_gt(far_gt, xe, j) {
555 u32 tile, dev;
556
557 if (far_gt->info.id == gt->info.id)
558 continue;
559
560 if (j >= i)
561 break;
562
563 tile = gt_to_tile(far_gt)->id;
564 dev = G2G_DEV(far_gt);
565
566 for (t = 0; t < XE_G2G_TYPE_LIMIT; t++)
567 guc_g2g_deregister(guc, tile, dev, t);
568 }
569
570 return err;
571 }
572
__guc_opt_in_features_enable(struct xe_guc * guc,u64 addr,u32 num_dwords)573 static int __guc_opt_in_features_enable(struct xe_guc *guc, u64 addr, u32 num_dwords)
574 {
575 u32 action[] = {
576 XE_GUC_ACTION_OPT_IN_FEATURE_KLV,
577 lower_32_bits(addr),
578 upper_32_bits(addr),
579 num_dwords
580 };
581
582 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
583 }
584
supports_dynamic_ics(struct xe_guc * guc)585 static bool supports_dynamic_ics(struct xe_guc *guc)
586 {
587 struct xe_device *xe = guc_to_xe(guc);
588 struct xe_gt *gt = guc_to_gt(guc);
589
590 /* Dynamic ICS is available for PVC and Xe2 and newer platforms. */
591 if (xe->info.platform != XE_PVC && GRAPHICS_VER(xe) < 20)
592 return false;
593
594 /*
595 * The feature is currently not compatible with multi-lrc, so the GuC
596 * does not support it at all on the media engines (which are the main
597 * users of mlrc). On the primary GT side, to avoid it being used in
598 * conjunction with mlrc, we only enable it if we are in single CCS
599 * mode.
600 */
601 if (xe_gt_is_media_type(gt) || gt->ccs_mode > 1)
602 return false;
603
604 /*
605 * Dynamic ICS requires GuC v70.40.1, which maps to compatibility
606 * version v1.18.4.
607 */
608 return GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 18, 4);
609 }
610
611 #define OPT_IN_MAX_DWORDS 16
xe_guc_opt_in_features_enable(struct xe_guc * guc)612 int xe_guc_opt_in_features_enable(struct xe_guc *guc)
613 {
614 struct xe_device *xe = guc_to_xe(guc);
615 CLASS(xe_guc_buf, buf)(&guc->buf, OPT_IN_MAX_DWORDS);
616 u32 count = 0;
617 u32 *klvs;
618 int ret;
619
620 if (!xe_guc_buf_is_valid(buf))
621 return -ENOBUFS;
622
623 klvs = xe_guc_buf_cpu_ptr(buf);
624
625 /*
626 * The extra CAT error type opt-in was added in GuC v70.17.0, which maps
627 * to compatibility version v1.7.0.
628 * Note that the GuC allows enabling this KLV even on platforms that do
629 * not support the extra type; in such case the returned type variable
630 * will be set to a known invalid value which we can check against.
631 */
632 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 7, 0))
633 klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_EXT_CAT_ERR_TYPE);
634
635 if (supports_dynamic_ics(guc))
636 klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH);
637
638 if (count) {
639 xe_assert(xe, count <= OPT_IN_MAX_DWORDS);
640
641 ret = __guc_opt_in_features_enable(guc, xe_guc_buf_flush(buf), count);
642 if (ret < 0) {
643 xe_gt_err(guc_to_gt(guc),
644 "failed to enable GuC opt-in features: %pe\n",
645 ERR_PTR(ret));
646 return ret;
647 }
648 }
649
650 return 0;
651 }
652
guc_fini_hw(void * arg)653 static void guc_fini_hw(void *arg)
654 {
655 struct xe_guc *guc = arg;
656 struct xe_gt *gt = guc_to_gt(guc);
657 unsigned int fw_ref;
658
659 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
660 xe_uc_sanitize_reset(&guc_to_gt(guc)->uc);
661 xe_force_wake_put(gt_to_fw(gt), fw_ref);
662
663 guc_g2g_fini(guc);
664 }
665
666 /**
667 * xe_guc_comm_init_early - early initialization of GuC communication
668 * @guc: the &xe_guc to initialize
669 *
670 * Must be called prior to first MMIO communication with GuC firmware.
671 */
xe_guc_comm_init_early(struct xe_guc * guc)672 void xe_guc_comm_init_early(struct xe_guc *guc)
673 {
674 struct xe_gt *gt = guc_to_gt(guc);
675
676 if (xe_gt_is_media_type(gt))
677 guc->notify_reg = MED_GUC_HOST_INTERRUPT;
678 else
679 guc->notify_reg = GUC_HOST_INTERRUPT;
680 }
681
xe_guc_realloc_post_hwconfig(struct xe_guc * guc)682 static int xe_guc_realloc_post_hwconfig(struct xe_guc *guc)
683 {
684 struct xe_tile *tile = gt_to_tile(guc_to_gt(guc));
685 struct xe_device *xe = guc_to_xe(guc);
686 int ret;
687
688 if (!IS_DGFX(guc_to_xe(guc)))
689 return 0;
690
691 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->fw.bo);
692 if (ret)
693 return ret;
694
695 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->log.bo);
696 if (ret)
697 return ret;
698
699 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ads.bo);
700 if (ret)
701 return ret;
702
703 return 0;
704 }
705
vf_guc_init_noalloc(struct xe_guc * guc)706 static int vf_guc_init_noalloc(struct xe_guc *guc)
707 {
708 struct xe_gt *gt = guc_to_gt(guc);
709 int err;
710
711 err = xe_gt_sriov_vf_bootstrap(gt);
712 if (err)
713 return err;
714
715 err = xe_gt_sriov_vf_query_config(gt);
716 if (err)
717 return err;
718
719 return 0;
720 }
721
xe_guc_init_noalloc(struct xe_guc * guc)722 int xe_guc_init_noalloc(struct xe_guc *guc)
723 {
724 struct xe_device *xe = guc_to_xe(guc);
725 struct xe_gt *gt = guc_to_gt(guc);
726 int ret;
727
728 xe_guc_comm_init_early(guc);
729
730 ret = xe_guc_ct_init_noalloc(&guc->ct);
731 if (ret)
732 goto out;
733
734 ret = xe_guc_relay_init(&guc->relay);
735 if (ret)
736 goto out;
737
738 if (IS_SRIOV_VF(xe)) {
739 ret = vf_guc_init_noalloc(guc);
740 if (ret)
741 goto out;
742 }
743
744 return 0;
745
746 out:
747 xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
748 return ret;
749 }
750
xe_guc_init(struct xe_guc * guc)751 int xe_guc_init(struct xe_guc *guc)
752 {
753 struct xe_device *xe = guc_to_xe(guc);
754 struct xe_gt *gt = guc_to_gt(guc);
755 int ret;
756
757 guc->fw.type = XE_UC_FW_TYPE_GUC;
758 ret = xe_uc_fw_init(&guc->fw);
759 if (ret)
760 return ret;
761
762 if (!xe_uc_fw_is_enabled(&guc->fw))
763 return 0;
764
765 if (IS_SRIOV_VF(xe)) {
766 ret = xe_guc_ct_init(&guc->ct);
767 if (ret)
768 goto out;
769 return 0;
770 }
771
772 ret = xe_guc_log_init(&guc->log);
773 if (ret)
774 goto out;
775
776 ret = xe_guc_capture_init(guc);
777 if (ret)
778 goto out;
779
780 ret = xe_guc_ads_init(&guc->ads);
781 if (ret)
782 goto out;
783
784 ret = xe_guc_ct_init(&guc->ct);
785 if (ret)
786 goto out;
787
788 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE);
789
790 ret = devm_add_action_or_reset(xe->drm.dev, guc_fini_hw, guc);
791 if (ret)
792 goto out;
793
794 guc_init_params(guc);
795
796 return 0;
797
798 out:
799 xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
800 return ret;
801 }
802
vf_guc_init_post_hwconfig(struct xe_guc * guc)803 static int vf_guc_init_post_hwconfig(struct xe_guc *guc)
804 {
805 int err;
806
807 err = xe_guc_submit_init(guc, xe_gt_sriov_vf_guc_ids(guc_to_gt(guc)));
808 if (err)
809 return err;
810
811 err = xe_guc_buf_cache_init(&guc->buf);
812 if (err)
813 return err;
814
815 /* XXX xe_guc_db_mgr_init not needed for now */
816
817 return 0;
818 }
819
820 /**
821 * xe_guc_init_post_hwconfig - initialize GuC post hwconfig load
822 * @guc: The GuC object
823 *
824 * Return: 0 on success, negative error code on error.
825 */
xe_guc_init_post_hwconfig(struct xe_guc * guc)826 int xe_guc_init_post_hwconfig(struct xe_guc *guc)
827 {
828 int ret;
829
830 if (IS_SRIOV_VF(guc_to_xe(guc)))
831 return vf_guc_init_post_hwconfig(guc);
832
833 ret = xe_guc_realloc_post_hwconfig(guc);
834 if (ret)
835 return ret;
836
837 ret = xe_guc_ct_init_post_hwconfig(&guc->ct);
838 if (ret)
839 return ret;
840
841 guc_init_params_post_hwconfig(guc);
842
843 ret = xe_guc_submit_init(guc, ~0);
844 if (ret)
845 return ret;
846
847 ret = xe_guc_db_mgr_init(&guc->dbm, ~0);
848 if (ret)
849 return ret;
850
851 ret = xe_guc_pc_init(&guc->pc);
852 if (ret)
853 return ret;
854
855 ret = xe_guc_engine_activity_init(guc);
856 if (ret)
857 return ret;
858
859 ret = xe_guc_buf_cache_init(&guc->buf);
860 if (ret)
861 return ret;
862
863 return xe_guc_ads_init_post_hwconfig(&guc->ads);
864 }
865
xe_guc_post_load_init(struct xe_guc * guc)866 int xe_guc_post_load_init(struct xe_guc *guc)
867 {
868 int ret;
869
870 xe_guc_ads_populate_post_load(&guc->ads);
871
872 ret = xe_guc_opt_in_features_enable(guc);
873 if (ret)
874 return ret;
875
876 if (xe_guc_g2g_wanted(guc_to_xe(guc))) {
877 ret = guc_g2g_start(guc);
878 if (ret)
879 return ret;
880 }
881
882 return xe_guc_submit_enable(guc);
883 }
884
xe_guc_reset(struct xe_guc * guc)885 int xe_guc_reset(struct xe_guc *guc)
886 {
887 struct xe_gt *gt = guc_to_gt(guc);
888 struct xe_mmio *mmio = >->mmio;
889 u32 guc_status, gdrst;
890 int ret;
891
892 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
893
894 if (IS_SRIOV_VF(gt_to_xe(gt)))
895 return xe_gt_sriov_vf_bootstrap(gt);
896
897 xe_mmio_write32(mmio, GDRST, GRDOM_GUC);
898
899 ret = xe_mmio_wait32(mmio, GDRST, GRDOM_GUC, 0, 5000, &gdrst, false);
900 if (ret) {
901 xe_gt_err(gt, "GuC reset timed out, GDRST=%#x\n", gdrst);
902 goto err_out;
903 }
904
905 guc_status = xe_mmio_read32(mmio, GUC_STATUS);
906 if (!(guc_status & GS_MIA_IN_RESET)) {
907 xe_gt_err(gt, "GuC status: %#x, MIA core expected to be in reset\n",
908 guc_status);
909 ret = -EIO;
910 goto err_out;
911 }
912
913 return 0;
914
915 err_out:
916
917 return ret;
918 }
919
guc_prepare_xfer(struct xe_guc * guc)920 static void guc_prepare_xfer(struct xe_guc *guc)
921 {
922 struct xe_gt *gt = guc_to_gt(guc);
923 struct xe_mmio *mmio = >->mmio;
924 struct xe_device *xe = guc_to_xe(guc);
925 u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC |
926 GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA |
927 GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA |
928 GUC_ENABLE_MIA_CLOCK_GATING;
929
930 if (GRAPHICS_VERx100(xe) < 1250)
931 shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES |
932 GUC_ENABLE_MIA_CACHING;
933
934 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
935 shim_flags |= REG_FIELD_PREP(GUC_MOCS_INDEX_MASK, gt->mocs.uc_index);
936
937 /* Must program this register before loading the ucode with DMA */
938 xe_mmio_write32(mmio, GUC_SHIM_CONTROL, shim_flags);
939
940 xe_mmio_write32(mmio, GT_PM_CONFIG, GT_DOORBELL_ENABLE);
941
942 /* Make sure GuC receives ARAT interrupts */
943 xe_mmio_rmw32(mmio, PMINTRMSK, ARAT_EXPIRED_INTRMSK, 0);
944 }
945
946 /*
947 * Supporting MMIO & in memory RSA
948 */
guc_xfer_rsa(struct xe_guc * guc)949 static int guc_xfer_rsa(struct xe_guc *guc)
950 {
951 struct xe_gt *gt = guc_to_gt(guc);
952 u32 rsa[UOS_RSA_SCRATCH_COUNT];
953 size_t copied;
954 int i;
955
956 if (guc->fw.rsa_size > 256) {
957 u32 rsa_ggtt_addr = xe_bo_ggtt_addr(guc->fw.bo) +
958 xe_uc_fw_rsa_offset(&guc->fw);
959 xe_mmio_write32(>->mmio, UOS_RSA_SCRATCH(0), rsa_ggtt_addr);
960 return 0;
961 }
962
963 copied = xe_uc_fw_copy_rsa(&guc->fw, rsa, sizeof(rsa));
964 if (copied < sizeof(rsa))
965 return -ENOMEM;
966
967 for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++)
968 xe_mmio_write32(>->mmio, UOS_RSA_SCRATCH(i), rsa[i]);
969
970 return 0;
971 }
972
973 /*
974 * Check a previously read GuC status register (GUC_STATUS) looking for
975 * known terminal states (either completion or failure) of either the
976 * microkernel status field or the boot ROM status field. Returns +1 for
977 * successful completion, -1 for failure and 0 for any intermediate state.
978 */
guc_load_done(u32 status)979 static int guc_load_done(u32 status)
980 {
981 u32 uk_val = REG_FIELD_GET(GS_UKERNEL_MASK, status);
982 u32 br_val = REG_FIELD_GET(GS_BOOTROM_MASK, status);
983
984 switch (uk_val) {
985 case XE_GUC_LOAD_STATUS_READY:
986 return 1;
987
988 case XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH:
989 case XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH:
990 case XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE:
991 case XE_GUC_LOAD_STATUS_HWCONFIG_ERROR:
992 case XE_GUC_LOAD_STATUS_BOOTROM_VERSION_MISMATCH:
993 case XE_GUC_LOAD_STATUS_DPC_ERROR:
994 case XE_GUC_LOAD_STATUS_EXCEPTION:
995 case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID:
996 case XE_GUC_LOAD_STATUS_MPU_DATA_INVALID:
997 case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
998 case XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR:
999 case XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG:
1000 return -1;
1001 }
1002
1003 switch (br_val) {
1004 case XE_BOOTROM_STATUS_NO_KEY_FOUND:
1005 case XE_BOOTROM_STATUS_RSA_FAILED:
1006 case XE_BOOTROM_STATUS_PAVPC_FAILED:
1007 case XE_BOOTROM_STATUS_WOPCM_FAILED:
1008 case XE_BOOTROM_STATUS_LOADLOC_FAILED:
1009 case XE_BOOTROM_STATUS_JUMP_FAILED:
1010 case XE_BOOTROM_STATUS_RC6CTXCONFIG_FAILED:
1011 case XE_BOOTROM_STATUS_MPUMAP_INCORRECT:
1012 case XE_BOOTROM_STATUS_EXCEPTION:
1013 case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
1014 return -1;
1015 }
1016
1017 return 0;
1018 }
1019
guc_pc_get_cur_freq(struct xe_guc_pc * guc_pc)1020 static s32 guc_pc_get_cur_freq(struct xe_guc_pc *guc_pc)
1021 {
1022 u32 freq;
1023 int ret = xe_guc_pc_get_cur_freq(guc_pc, &freq);
1024
1025 return ret ? ret : freq;
1026 }
1027
1028 /*
1029 * Wait for the GuC to start up.
1030 *
1031 * Measurements indicate this should take no more than 20ms (assuming the GT
1032 * clock is at maximum frequency). However, thermal throttling and other issues
1033 * can prevent the clock hitting max and thus making the load take significantly
1034 * longer. Allow up to 200ms as a safety margin for real world worst case situations.
1035 *
1036 * However, bugs anywhere from KMD to GuC to PCODE to fan failure in a CI farm can
1037 * lead to even longer times. E.g. if the GT is clamped to minimum frequency then
1038 * the load times can be in the seconds range. So the timeout is increased for debug
1039 * builds to ensure that problems can be correctly analysed. For release builds, the
1040 * timeout is kept short so that users don't wait forever to find out that there is a
1041 * problem. In either case, if the load took longer than is reasonable even with some
1042 * 'sensible' throttling, then flag a warning because something is not right.
1043 *
1044 * Note that there is a limit on how long an individual usleep_range() can wait for,
1045 * hence longer waits require wrapping a shorter wait in a loop.
1046 *
1047 * Note that the only reason an end user should hit the shorter timeout is in case of
1048 * extreme thermal throttling. And a system that is that hot during boot is probably
1049 * dead anyway!
1050 */
1051 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
1052 #define GUC_LOAD_RETRY_LIMIT 20
1053 #else
1054 #define GUC_LOAD_RETRY_LIMIT 3
1055 #endif
1056 #define GUC_LOAD_TIME_WARN_MS 200
1057
guc_wait_ucode(struct xe_guc * guc)1058 static int guc_wait_ucode(struct xe_guc *guc)
1059 {
1060 struct xe_gt *gt = guc_to_gt(guc);
1061 struct xe_mmio *mmio = >->mmio;
1062 struct xe_guc_pc *guc_pc = >->uc.guc.pc;
1063 ktime_t before, after, delta;
1064 int load_done;
1065 u32 status = 0;
1066 int count = 0;
1067 u64 delta_ms;
1068 u32 before_freq;
1069
1070 before_freq = xe_guc_pc_get_act_freq(guc_pc);
1071 before = ktime_get();
1072 /*
1073 * Note, can't use any kind of timing information from the call to xe_mmio_wait.
1074 * It could return a thousand intermediate stages at random times. Instead, must
1075 * manually track the total time taken and locally implement the timeout.
1076 */
1077 do {
1078 u32 last_status = status & (GS_UKERNEL_MASK | GS_BOOTROM_MASK);
1079 int ret;
1080
1081 /*
1082 * Wait for any change (intermediate or terminal) in the status register.
1083 * Note, the return value is a don't care. The only failure code is timeout
1084 * but the timeouts need to be accumulated over all the intermediate partial
1085 * timeouts rather than allowing a huge timeout each time. So basically, need
1086 * to treat a timeout no different to a value change.
1087 */
1088 ret = xe_mmio_wait32_not(mmio, GUC_STATUS, GS_UKERNEL_MASK | GS_BOOTROM_MASK,
1089 last_status, 1000 * 1000, &status, false);
1090 if (ret < 0)
1091 count++;
1092 after = ktime_get();
1093 delta = ktime_sub(after, before);
1094 delta_ms = ktime_to_ms(delta);
1095
1096 load_done = guc_load_done(status);
1097 if (load_done != 0)
1098 break;
1099
1100 if (delta_ms >= (GUC_LOAD_RETRY_LIMIT * 1000))
1101 break;
1102
1103 xe_gt_dbg(gt, "load still in progress, timeouts = %d, freq = %dMHz (req %dMHz), status = 0x%08X [0x%02X/%02X]\n",
1104 count, xe_guc_pc_get_act_freq(guc_pc),
1105 guc_pc_get_cur_freq(guc_pc), status,
1106 REG_FIELD_GET(GS_BOOTROM_MASK, status),
1107 REG_FIELD_GET(GS_UKERNEL_MASK, status));
1108 } while (1);
1109
1110 if (load_done != 1) {
1111 u32 ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, status);
1112 u32 bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, status);
1113
1114 xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz), done = %d\n",
1115 status, delta_ms, xe_guc_pc_get_act_freq(guc_pc),
1116 guc_pc_get_cur_freq(guc_pc), load_done);
1117 xe_gt_err(gt, "load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n",
1118 REG_FIELD_GET(GS_MIA_IN_RESET, status),
1119 bootrom, ukernel,
1120 REG_FIELD_GET(GS_MIA_MASK, status),
1121 REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
1122
1123 switch (bootrom) {
1124 case XE_BOOTROM_STATUS_NO_KEY_FOUND:
1125 xe_gt_err(gt, "invalid key requested, header = 0x%08X\n",
1126 xe_mmio_read32(mmio, GUC_HEADER_INFO));
1127 break;
1128
1129 case XE_BOOTROM_STATUS_RSA_FAILED:
1130 xe_gt_err(gt, "firmware signature verification failed\n");
1131 break;
1132
1133 case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
1134 xe_gt_err(gt, "firmware production part check failure\n");
1135 break;
1136 }
1137
1138 switch (ukernel) {
1139 case XE_GUC_LOAD_STATUS_HWCONFIG_START:
1140 xe_gt_err(gt, "still extracting hwconfig table.\n");
1141 break;
1142
1143 case XE_GUC_LOAD_STATUS_EXCEPTION:
1144 xe_gt_err(gt, "firmware exception. EIP: %#x\n",
1145 xe_mmio_read32(mmio, SOFT_SCRATCH(13)));
1146 break;
1147
1148 case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID:
1149 xe_gt_err(gt, "illegal init/ADS data\n");
1150 break;
1151
1152 case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
1153 xe_gt_err(gt, "illegal register in save/restore workaround list\n");
1154 break;
1155
1156 case XE_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR:
1157 xe_gt_err(gt, "illegal workaround KLV data\n");
1158 break;
1159
1160 case XE_GUC_LOAD_STATUS_INVALID_FTR_FLAG:
1161 xe_gt_err(gt, "illegal feature flag specified\n");
1162 break;
1163 }
1164
1165 return -EPROTO;
1166 } else if (delta_ms > GUC_LOAD_TIME_WARN_MS) {
1167 xe_gt_warn(gt, "excessive init time: %lldms! [status = 0x%08X, timeouts = %d]\n",
1168 delta_ms, status, count);
1169 xe_gt_warn(gt, "excessive init time: [freq = %dMHz (req = %dMHz), before = %dMHz, perf_limit_reasons = 0x%08X]\n",
1170 xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc),
1171 before_freq, xe_gt_throttle_get_limit_reasons(gt));
1172 } else {
1173 xe_gt_dbg(gt, "init took %lldms, freq = %dMHz (req = %dMHz), before = %dMHz, status = 0x%08X, timeouts = %d\n",
1174 delta_ms, xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc),
1175 before_freq, status, count);
1176 }
1177
1178 return 0;
1179 }
1180 ALLOW_ERROR_INJECTION(guc_wait_ucode, ERRNO);
1181
__xe_guc_upload(struct xe_guc * guc)1182 static int __xe_guc_upload(struct xe_guc *guc)
1183 {
1184 int ret;
1185
1186 /* Raise GT freq to speed up HuC/GuC load */
1187 xe_guc_pc_raise_unslice(&guc->pc);
1188
1189 guc_write_params(guc);
1190 guc_prepare_xfer(guc);
1191
1192 /*
1193 * Note that GuC needs the CSS header plus uKernel code to be copied
1194 * by the DMA engine in one operation, whereas the RSA signature is
1195 * loaded separately, either by copying it to the UOS_RSA_SCRATCH
1196 * register (if key size <= 256) or through a ggtt-pinned vma (if key
1197 * size > 256). The RSA size and therefore the way we provide it to the
1198 * HW is fixed for each platform and hard-coded in the bootrom.
1199 */
1200 ret = guc_xfer_rsa(guc);
1201 if (ret)
1202 goto out;
1203 /*
1204 * Current uCode expects the code to be loaded at 8k; locations below
1205 * this are used for the stack.
1206 */
1207 ret = xe_uc_fw_upload(&guc->fw, 0x2000, UOS_MOVE);
1208 if (ret)
1209 goto out;
1210
1211 /* Wait for authentication */
1212 ret = guc_wait_ucode(guc);
1213 if (ret)
1214 goto out;
1215
1216 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_RUNNING);
1217 return 0;
1218
1219 out:
1220 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOAD_FAIL);
1221 return ret;
1222 }
1223
vf_guc_min_load_for_hwconfig(struct xe_guc * guc)1224 static int vf_guc_min_load_for_hwconfig(struct xe_guc *guc)
1225 {
1226 struct xe_gt *gt = guc_to_gt(guc);
1227 int ret;
1228
1229 ret = xe_guc_hwconfig_init(guc);
1230 if (ret)
1231 return ret;
1232
1233 ret = xe_guc_enable_communication(guc);
1234 if (ret)
1235 return ret;
1236
1237 ret = xe_gt_sriov_vf_connect(gt);
1238 if (ret)
1239 goto err_out;
1240
1241 ret = xe_gt_sriov_vf_query_runtime(gt);
1242 if (ret)
1243 goto err_out;
1244
1245 return 0;
1246
1247 err_out:
1248 xe_guc_sanitize(guc);
1249 return ret;
1250 }
1251
1252 /**
1253 * xe_guc_min_load_for_hwconfig - load minimal GuC and read hwconfig table
1254 * @guc: The GuC object
1255 *
1256 * This function uploads a minimal GuC that does not support submissions but
1257 * in a state where the hwconfig table can be read. Next, it reads and parses
1258 * the hwconfig table so it can be used for subsequent steps in the driver load.
1259 * Lastly, it enables CT communication (XXX: this is needed for PFs/VFs only).
1260 *
1261 * Return: 0 on success, negative error code on error.
1262 */
xe_guc_min_load_for_hwconfig(struct xe_guc * guc)1263 int xe_guc_min_load_for_hwconfig(struct xe_guc *guc)
1264 {
1265 int ret;
1266
1267 if (IS_SRIOV_VF(guc_to_xe(guc)))
1268 return vf_guc_min_load_for_hwconfig(guc);
1269
1270 xe_guc_ads_populate_minimal(&guc->ads);
1271
1272 xe_guc_pc_init_early(&guc->pc);
1273
1274 ret = __xe_guc_upload(guc);
1275 if (ret)
1276 return ret;
1277
1278 ret = xe_guc_hwconfig_init(guc);
1279 if (ret)
1280 return ret;
1281
1282 ret = xe_guc_enable_communication(guc);
1283 if (ret)
1284 return ret;
1285
1286 return 0;
1287 }
1288
xe_guc_upload(struct xe_guc * guc)1289 int xe_guc_upload(struct xe_guc *guc)
1290 {
1291 xe_guc_ads_populate(&guc->ads);
1292
1293 return __xe_guc_upload(guc);
1294 }
1295
guc_handle_mmio_msg(struct xe_guc * guc)1296 static void guc_handle_mmio_msg(struct xe_guc *guc)
1297 {
1298 struct xe_gt *gt = guc_to_gt(guc);
1299 u32 msg;
1300
1301 if (IS_SRIOV_VF(guc_to_xe(guc)))
1302 return;
1303
1304 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
1305
1306 msg = xe_mmio_read32(>->mmio, SOFT_SCRATCH(15));
1307 msg &= XE_GUC_RECV_MSG_EXCEPTION |
1308 XE_GUC_RECV_MSG_CRASH_DUMP_POSTED;
1309 xe_mmio_write32(>->mmio, SOFT_SCRATCH(15), 0);
1310
1311 if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED)
1312 xe_gt_err(gt, "Received early GuC crash dump notification!\n");
1313
1314 if (msg & XE_GUC_RECV_MSG_EXCEPTION)
1315 xe_gt_err(gt, "Received early GuC exception notification!\n");
1316 }
1317
guc_enable_irq(struct xe_guc * guc)1318 static void guc_enable_irq(struct xe_guc *guc)
1319 {
1320 struct xe_gt *gt = guc_to_gt(guc);
1321 u32 events = xe_gt_is_media_type(gt) ?
1322 REG_FIELD_PREP(ENGINE0_MASK, GUC_INTR_GUC2HOST) :
1323 REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST);
1324
1325 /* Primary GuC and media GuC share a single enable bit */
1326 xe_mmio_write32(>->mmio, GUC_SG_INTR_ENABLE,
1327 REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST));
1328
1329 /*
1330 * There are separate mask bits for primary and media GuCs, so use
1331 * a RMW operation to avoid clobbering the other GuC's setting.
1332 */
1333 xe_mmio_rmw32(>->mmio, GUC_SG_INTR_MASK, events, 0);
1334 }
1335
xe_guc_enable_communication(struct xe_guc * guc)1336 int xe_guc_enable_communication(struct xe_guc *guc)
1337 {
1338 struct xe_device *xe = guc_to_xe(guc);
1339 int err;
1340
1341 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) {
1342 struct xe_gt *gt = guc_to_gt(guc);
1343 struct xe_tile *tile = gt_to_tile(gt);
1344
1345 err = xe_memirq_init_guc(&tile->memirq, guc);
1346 if (err)
1347 return err;
1348 } else {
1349 guc_enable_irq(guc);
1350 }
1351
1352 err = xe_guc_ct_enable(&guc->ct);
1353 if (err)
1354 return err;
1355
1356 guc_handle_mmio_msg(guc);
1357
1358 return 0;
1359 }
1360
xe_guc_suspend(struct xe_guc * guc)1361 int xe_guc_suspend(struct xe_guc *guc)
1362 {
1363 struct xe_gt *gt = guc_to_gt(guc);
1364 u32 action[] = {
1365 XE_GUC_ACTION_CLIENT_SOFT_RESET,
1366 };
1367 int ret;
1368
1369 ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action));
1370 if (ret) {
1371 xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret));
1372 return ret;
1373 }
1374
1375 xe_guc_sanitize(guc);
1376 return 0;
1377 }
1378
xe_guc_notify(struct xe_guc * guc)1379 void xe_guc_notify(struct xe_guc *guc)
1380 {
1381 struct xe_gt *gt = guc_to_gt(guc);
1382 const u32 default_notify_data = 0;
1383
1384 /*
1385 * Both GUC_HOST_INTERRUPT and MED_GUC_HOST_INTERRUPT can pass
1386 * additional payload data to the GuC but this capability is not
1387 * used by the firmware yet. Use default value in the meantime.
1388 */
1389 xe_mmio_write32(>->mmio, guc->notify_reg, default_notify_data);
1390 }
1391
xe_guc_auth_huc(struct xe_guc * guc,u32 rsa_addr)1392 int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr)
1393 {
1394 u32 action[] = {
1395 XE_GUC_ACTION_AUTHENTICATE_HUC,
1396 rsa_addr
1397 };
1398
1399 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
1400 }
1401
xe_guc_mmio_send_recv(struct xe_guc * guc,const u32 * request,u32 len,u32 * response_buf)1402 int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
1403 u32 len, u32 *response_buf)
1404 {
1405 struct xe_device *xe = guc_to_xe(guc);
1406 struct xe_gt *gt = guc_to_gt(guc);
1407 struct xe_mmio *mmio = >->mmio;
1408 u32 header, reply;
1409 struct xe_reg reply_reg = xe_gt_is_media_type(gt) ?
1410 MED_VF_SW_FLAG(0) : VF_SW_FLAG(0);
1411 const u32 LAST_INDEX = VF_SW_FLAG_COUNT - 1;
1412 bool lost = false;
1413 int ret;
1414 int i;
1415
1416 BUILD_BUG_ON(VF_SW_FLAG_COUNT != MED_VF_SW_FLAG_COUNT);
1417
1418 xe_assert(xe, len);
1419 xe_assert(xe, len <= VF_SW_FLAG_COUNT);
1420 xe_assert(xe, len <= MED_VF_SW_FLAG_COUNT);
1421 xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) ==
1422 GUC_HXG_ORIGIN_HOST);
1423 xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) ==
1424 GUC_HXG_TYPE_REQUEST);
1425
1426 retry:
1427 /* Not in critical data-path, just do if else for GT type */
1428 if (xe_gt_is_media_type(gt)) {
1429 for (i = 0; i < len; ++i)
1430 xe_mmio_write32(mmio, MED_VF_SW_FLAG(i),
1431 request[i]);
1432 xe_mmio_read32(mmio, MED_VF_SW_FLAG(LAST_INDEX));
1433 } else {
1434 for (i = 0; i < len; ++i)
1435 xe_mmio_write32(mmio, VF_SW_FLAG(i),
1436 request[i]);
1437 xe_mmio_read32(mmio, VF_SW_FLAG(LAST_INDEX));
1438 }
1439
1440 xe_guc_notify(guc);
1441
1442 ret = xe_mmio_wait32(mmio, reply_reg, GUC_HXG_MSG_0_ORIGIN,
1443 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC),
1444 50000, &reply, false);
1445 if (ret) {
1446 /* scratch registers might be cleared during FLR, try once more */
1447 if (!reply && !lost) {
1448 xe_gt_dbg(gt, "GuC mmio request %#x: lost, trying again\n", request[0]);
1449 lost = true;
1450 goto retry;
1451 }
1452 timeout:
1453 xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n",
1454 request[0], reply);
1455 return ret;
1456 }
1457
1458 header = xe_mmio_read32(mmio, reply_reg);
1459 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1460 GUC_HXG_TYPE_NO_RESPONSE_BUSY) {
1461 /*
1462 * Once we got a BUSY reply we must wait again for the final
1463 * response but this time we can't use ORIGIN mask anymore.
1464 * To spot a right change in the reply, we take advantage that
1465 * response SUCCESS and FAILURE differ only by the single bit
1466 * and all other bits are set and can be used as a new mask.
1467 */
1468 u32 resp_bits = GUC_HXG_TYPE_RESPONSE_SUCCESS & GUC_HXG_TYPE_RESPONSE_FAILURE;
1469 u32 resp_mask = FIELD_PREP(GUC_HXG_MSG_0_TYPE, resp_bits);
1470
1471 BUILD_BUG_ON(FIELD_MAX(GUC_HXG_MSG_0_TYPE) != GUC_HXG_TYPE_RESPONSE_SUCCESS);
1472 BUILD_BUG_ON((GUC_HXG_TYPE_RESPONSE_SUCCESS ^ GUC_HXG_TYPE_RESPONSE_FAILURE) != 1);
1473
1474 ret = xe_mmio_wait32(mmio, reply_reg, resp_mask, resp_mask,
1475 1000000, &header, false);
1476
1477 if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) !=
1478 GUC_HXG_ORIGIN_GUC))
1479 goto proto;
1480 if (unlikely(ret)) {
1481 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1482 GUC_HXG_TYPE_NO_RESPONSE_BUSY)
1483 goto proto;
1484 goto timeout;
1485 }
1486 }
1487
1488 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1489 GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1490 u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header);
1491
1492 xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n",
1493 request[0], reason);
1494 goto retry;
1495 }
1496
1497 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1498 GUC_HXG_TYPE_RESPONSE_FAILURE) {
1499 u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
1500 u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
1501
1502 xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
1503 request[0], error, hint);
1504 return -ENXIO;
1505 }
1506
1507 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1508 GUC_HXG_TYPE_RESPONSE_SUCCESS) {
1509 proto:
1510 xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n",
1511 request[0], header);
1512 return -EPROTO;
1513 }
1514
1515 /* Just copy entire possible message response */
1516 if (response_buf) {
1517 response_buf[0] = header;
1518
1519 for (i = 1; i < VF_SW_FLAG_COUNT; i++) {
1520 reply_reg.addr += sizeof(u32);
1521 response_buf[i] = xe_mmio_read32(mmio, reply_reg);
1522 }
1523 }
1524
1525 /* Use data from the GuC response as our return value */
1526 return FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header);
1527 }
1528 ALLOW_ERROR_INJECTION(xe_guc_mmio_send_recv, ERRNO);
1529
xe_guc_mmio_send(struct xe_guc * guc,const u32 * request,u32 len)1530 int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len)
1531 {
1532 return xe_guc_mmio_send_recv(guc, request, len, NULL);
1533 }
1534
guc_self_cfg(struct xe_guc * guc,u16 key,u16 len,u64 val)1535 static int guc_self_cfg(struct xe_guc *guc, u16 key, u16 len, u64 val)
1536 {
1537 struct xe_device *xe = guc_to_xe(guc);
1538 u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = {
1539 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
1540 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
1541 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
1542 GUC_ACTION_HOST2GUC_SELF_CFG),
1543 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) |
1544 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len),
1545 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32,
1546 lower_32_bits(val)),
1547 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64,
1548 upper_32_bits(val)),
1549 };
1550 int ret;
1551
1552 xe_assert(xe, len <= 2);
1553 xe_assert(xe, len != 1 || !upper_32_bits(val));
1554
1555 /* Self config must go over MMIO */
1556 ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request));
1557
1558 if (unlikely(ret < 0))
1559 return ret;
1560 if (unlikely(ret > 1))
1561 return -EPROTO;
1562 if (unlikely(!ret))
1563 return -ENOKEY;
1564
1565 return 0;
1566 }
1567
xe_guc_self_cfg32(struct xe_guc * guc,u16 key,u32 val)1568 int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val)
1569 {
1570 return guc_self_cfg(guc, key, 1, val);
1571 }
1572
xe_guc_self_cfg64(struct xe_guc * guc,u16 key,u64 val)1573 int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val)
1574 {
1575 return guc_self_cfg(guc, key, 2, val);
1576 }
1577
xe_guc_sw_0_irq_handler(struct xe_guc * guc)1578 static void xe_guc_sw_0_irq_handler(struct xe_guc *guc)
1579 {
1580 struct xe_gt *gt = guc_to_gt(guc);
1581
1582 if (IS_SRIOV_VF(gt_to_xe(gt)))
1583 xe_gt_sriov_vf_migrated_event_handler(gt);
1584 }
1585
xe_guc_irq_handler(struct xe_guc * guc,const u16 iir)1586 void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir)
1587 {
1588 if (iir & GUC_INTR_GUC2HOST)
1589 xe_guc_ct_irq_handler(&guc->ct);
1590
1591 if (iir & GUC_INTR_SW_INT_0)
1592 xe_guc_sw_0_irq_handler(guc);
1593 }
1594
xe_guc_sanitize(struct xe_guc * guc)1595 void xe_guc_sanitize(struct xe_guc *guc)
1596 {
1597 xe_uc_fw_sanitize(&guc->fw);
1598 xe_guc_ct_disable(&guc->ct);
1599 xe_guc_submit_disable(guc);
1600 }
1601
xe_guc_reset_prepare(struct xe_guc * guc)1602 int xe_guc_reset_prepare(struct xe_guc *guc)
1603 {
1604 return xe_guc_submit_reset_prepare(guc);
1605 }
1606
xe_guc_reset_wait(struct xe_guc * guc)1607 void xe_guc_reset_wait(struct xe_guc *guc)
1608 {
1609 xe_guc_submit_reset_wait(guc);
1610 }
1611
xe_guc_stop_prepare(struct xe_guc * guc)1612 void xe_guc_stop_prepare(struct xe_guc *guc)
1613 {
1614 if (!IS_SRIOV_VF(guc_to_xe(guc))) {
1615 int err;
1616
1617 err = xe_guc_pc_stop(&guc->pc);
1618 xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n",
1619 ERR_PTR(err));
1620 }
1621 }
1622
xe_guc_stop(struct xe_guc * guc)1623 void xe_guc_stop(struct xe_guc *guc)
1624 {
1625 xe_guc_ct_stop(&guc->ct);
1626
1627 xe_guc_submit_stop(guc);
1628 }
1629
xe_guc_start(struct xe_guc * guc)1630 int xe_guc_start(struct xe_guc *guc)
1631 {
1632 return xe_guc_submit_start(guc);
1633 }
1634
xe_guc_print_info(struct xe_guc * guc,struct drm_printer * p)1635 void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p)
1636 {
1637 struct xe_gt *gt = guc_to_gt(guc);
1638 unsigned int fw_ref;
1639 u32 status;
1640 int i;
1641
1642 xe_uc_fw_print(&guc->fw, p);
1643
1644 if (!IS_SRIOV_VF(gt_to_xe(gt))) {
1645 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1646 if (!fw_ref)
1647 return;
1648
1649 status = xe_mmio_read32(>->mmio, GUC_STATUS);
1650
1651 drm_printf(p, "\nGuC status 0x%08x:\n", status);
1652 drm_printf(p, "\tBootrom status = 0x%x\n",
1653 REG_FIELD_GET(GS_BOOTROM_MASK, status));
1654 drm_printf(p, "\tuKernel status = 0x%x\n",
1655 REG_FIELD_GET(GS_UKERNEL_MASK, status));
1656 drm_printf(p, "\tMIA Core status = 0x%x\n",
1657 REG_FIELD_GET(GS_MIA_MASK, status));
1658 drm_printf(p, "\tLog level = %d\n",
1659 xe_guc_log_get_level(&guc->log));
1660
1661 drm_puts(p, "\nScratch registers:\n");
1662 for (i = 0; i < SOFT_SCRATCH_COUNT; i++) {
1663 drm_printf(p, "\t%2d: \t0x%x\n",
1664 i, xe_mmio_read32(>->mmio, SOFT_SCRATCH(i)));
1665 }
1666
1667 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1668 }
1669
1670 drm_puts(p, "\n");
1671 xe_guc_ct_print(&guc->ct, p, false);
1672
1673 drm_puts(p, "\n");
1674 xe_guc_submit_print(guc, p);
1675 }
1676
1677 /**
1678 * xe_guc_declare_wedged() - Declare GuC wedged
1679 * @guc: the GuC object
1680 *
1681 * Wedge the GuC which stops all submission, saves desired debug state, and
1682 * cleans up anything which could timeout.
1683 */
xe_guc_declare_wedged(struct xe_guc * guc)1684 void xe_guc_declare_wedged(struct xe_guc *guc)
1685 {
1686 xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);
1687
1688 xe_guc_reset_prepare(guc);
1689 xe_guc_ct_stop(&guc->ct);
1690 xe_guc_submit_wedge(guc);
1691 }
1692
1693 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
1694 #include "tests/xe_guc_g2g_test.c"
1695 #endif
1696