xref: /linux/drivers/gpu/drm/xe/xe_guc.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_guc.h"
7 
8 #include <drm/drm_managed.h>
9 
10 #include <generated/xe_wa_oob.h>
11 
12 #include "abi/guc_actions_abi.h"
13 #include "abi/guc_errors_abi.h"
14 #include "regs/xe_gt_regs.h"
15 #include "regs/xe_gtt_defs.h"
16 #include "regs/xe_guc_regs.h"
17 #include "regs/xe_irq_regs.h"
18 #include "xe_bo.h"
19 #include "xe_device.h"
20 #include "xe_force_wake.h"
21 #include "xe_gt.h"
22 #include "xe_gt_printk.h"
23 #include "xe_gt_sriov_vf.h"
24 #include "xe_gt_throttle.h"
25 #include "xe_guc_ads.h"
26 #include "xe_guc_capture.h"
27 #include "xe_guc_ct.h"
28 #include "xe_guc_db_mgr.h"
29 #include "xe_guc_hwconfig.h"
30 #include "xe_guc_log.h"
31 #include "xe_guc_pc.h"
32 #include "xe_guc_relay.h"
33 #include "xe_guc_submit.h"
34 #include "xe_memirq.h"
35 #include "xe_mmio.h"
36 #include "xe_platform_types.h"
37 #include "xe_sriov.h"
38 #include "xe_uc.h"
39 #include "xe_uc_fw.h"
40 #include "xe_wa.h"
41 #include "xe_wopcm.h"
42 
guc_bo_ggtt_addr(struct xe_guc * guc,struct xe_bo * bo)43 static u32 guc_bo_ggtt_addr(struct xe_guc *guc,
44 			    struct xe_bo *bo)
45 {
46 	struct xe_device *xe = guc_to_xe(guc);
47 	u32 addr;
48 
49 	/*
50 	 * For most BOs, the address on the allocating tile is fine. However for
51 	 * some, e.g. G2G CTB, the address on a specific tile is required as it
52 	 * might be different for each tile. So, just always ask for the address
53 	 * on the target GuC.
54 	 */
55 	addr = __xe_bo_ggtt_addr(bo, gt_to_tile(guc_to_gt(guc))->id);
56 
57 	/* GuC addresses above GUC_GGTT_TOP don't map through the GTT */
58 	xe_assert(xe, addr >= xe_wopcm_size(guc_to_xe(guc)));
59 	xe_assert(xe, addr < GUC_GGTT_TOP);
60 	xe_assert(xe, bo->size <= GUC_GGTT_TOP - addr);
61 
62 	return addr;
63 }
64 
guc_ctl_debug_flags(struct xe_guc * guc)65 static u32 guc_ctl_debug_flags(struct xe_guc *guc)
66 {
67 	u32 level = xe_guc_log_get_level(&guc->log);
68 	u32 flags = 0;
69 
70 	if (!GUC_LOG_LEVEL_IS_VERBOSE(level))
71 		flags |= GUC_LOG_DISABLED;
72 	else
73 		flags |= GUC_LOG_LEVEL_TO_VERBOSITY(level) <<
74 			 GUC_LOG_VERBOSITY_SHIFT;
75 
76 	return flags;
77 }
78 
guc_ctl_feature_flags(struct xe_guc * guc)79 static u32 guc_ctl_feature_flags(struct xe_guc *guc)
80 {
81 	u32 flags = GUC_CTL_ENABLE_LITE_RESTORE;
82 
83 	if (!guc_to_xe(guc)->info.skip_guc_pc)
84 		flags |= GUC_CTL_ENABLE_SLPC;
85 
86 	return flags;
87 }
88 
guc_ctl_log_params_flags(struct xe_guc * guc)89 static u32 guc_ctl_log_params_flags(struct xe_guc *guc)
90 {
91 	u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT;
92 	u32 flags;
93 
94 	#if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
95 	#define LOG_UNIT SZ_1M
96 	#define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
97 	#else
98 	#define LOG_UNIT SZ_4K
99 	#define LOG_FLAG 0
100 	#endif
101 
102 	#if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
103 	#define CAPTURE_UNIT SZ_1M
104 	#define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
105 	#else
106 	#define CAPTURE_UNIT SZ_4K
107 	#define CAPTURE_FLAG 0
108 	#endif
109 
110 	BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
111 	BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
112 	BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
113 	BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
114 	BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
115 	BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
116 
117 	BUILD_BUG_ON((CRASH_BUFFER_SIZE / LOG_UNIT - 1) >
118 			(GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT));
119 	BUILD_BUG_ON((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) >
120 			(GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT));
121 	BUILD_BUG_ON((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) >
122 			(GUC_LOG_CAPTURE_MASK >> GUC_LOG_CAPTURE_SHIFT));
123 
124 	flags = GUC_LOG_VALID |
125 		GUC_LOG_NOTIFY_ON_HALF_FULL |
126 		CAPTURE_FLAG |
127 		LOG_FLAG |
128 		((CRASH_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_CRASH_SHIFT) |
129 		((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_DEBUG_SHIFT) |
130 		((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) <<
131 		 GUC_LOG_CAPTURE_SHIFT) |
132 		(offset << GUC_LOG_BUF_ADDR_SHIFT);
133 
134 	#undef LOG_UNIT
135 	#undef LOG_FLAG
136 	#undef CAPTURE_UNIT
137 	#undef CAPTURE_FLAG
138 
139 	return flags;
140 }
141 
guc_ctl_ads_flags(struct xe_guc * guc)142 static u32 guc_ctl_ads_flags(struct xe_guc *guc)
143 {
144 	u32 ads = guc_bo_ggtt_addr(guc, guc->ads.bo) >> PAGE_SHIFT;
145 	u32 flags = ads << GUC_ADS_ADDR_SHIFT;
146 
147 	return flags;
148 }
149 
needs_wa_dual_queue(struct xe_gt * gt)150 static bool needs_wa_dual_queue(struct xe_gt *gt)
151 {
152 	/*
153 	 * The DUAL_QUEUE_WA tells the GuC to not allow concurrent submissions
154 	 * on RCS and CCSes with different address spaces, which on DG2 is
155 	 * required as a WA for an HW bug.
156 	 */
157 	if (XE_WA(gt, 22011391025))
158 		return true;
159 
160 	/*
161 	 * On newer platforms, the HW has been updated to not allow parallel
162 	 * execution of different address spaces, so the RCS/CCS will stall the
163 	 * context switch if one of the other RCS/CCSes is busy with a different
164 	 * address space. While functionally correct, having a submission
165 	 * stalled on the HW limits the GuC ability to shuffle things around and
166 	 * can cause complications if the non-stalled submission runs for a long
167 	 * time, because the GuC doesn't know that the stalled submission isn't
168 	 * actually running and might declare it as hung. Therefore, we enable
169 	 * the DUAL_QUEUE_WA on all newer platforms on GTs that have CCS engines
170 	 * to move management back to the GuC.
171 	 */
172 	if (CCS_MASK(gt) && GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
173 		return true;
174 
175 	return false;
176 }
177 
guc_ctl_wa_flags(struct xe_guc * guc)178 static u32 guc_ctl_wa_flags(struct xe_guc *guc)
179 {
180 	struct xe_device *xe = guc_to_xe(guc);
181 	struct xe_gt *gt = guc_to_gt(guc);
182 	u32 flags = 0;
183 
184 	if (XE_WA(gt, 22012773006))
185 		flags |= GUC_WA_POLLCS;
186 
187 	if (XE_WA(gt, 14014475959))
188 		flags |= GUC_WA_HOLD_CCS_SWITCHOUT;
189 
190 	if (needs_wa_dual_queue(gt))
191 		flags |= GUC_WA_DUAL_QUEUE;
192 
193 	/*
194 	 * Wa_22011802037: FIXME - there's more to be done than simply setting
195 	 * this flag: make sure each CS is stopped when preparing for GT reset
196 	 * and wait for pending MI_FW.
197 	 */
198 	if (GRAPHICS_VERx100(xe) < 1270)
199 		flags |= GUC_WA_PRE_PARSER;
200 
201 	if (XE_WA(gt, 22012727170) || XE_WA(gt, 22012727685))
202 		flags |= GUC_WA_CONTEXT_ISOLATION;
203 
204 	if (XE_WA(gt, 18020744125) &&
205 	    !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER))
206 		flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST;
207 
208 	if (XE_WA(gt, 1509372804))
209 		flags |= GUC_WA_RENDER_RST_RC6_EXIT;
210 
211 	if (XE_WA(gt, 14018913170))
212 		flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
213 
214 	return flags;
215 }
216 
guc_ctl_devid(struct xe_guc * guc)217 static u32 guc_ctl_devid(struct xe_guc *guc)
218 {
219 	struct xe_device *xe = guc_to_xe(guc);
220 
221 	return (((u32)xe->info.devid) << 16) | xe->info.revid;
222 }
223 
guc_print_params(struct xe_guc * guc)224 static void guc_print_params(struct xe_guc *guc)
225 {
226 	struct xe_gt *gt = guc_to_gt(guc);
227 	u32 *params = guc->params;
228 	int i;
229 
230 	BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32));
231 	BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT);
232 
233 	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
234 		xe_gt_dbg(gt, "GuC param[%2d] = 0x%08x\n", i, params[i]);
235 }
236 
guc_init_params(struct xe_guc * guc)237 static void guc_init_params(struct xe_guc *guc)
238 {
239 	u32 *params = guc->params;
240 
241 	params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
242 	params[GUC_CTL_FEATURE] = 0;
243 	params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
244 	params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
245 	params[GUC_CTL_WA] = 0;
246 	params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
247 
248 	guc_print_params(guc);
249 }
250 
guc_init_params_post_hwconfig(struct xe_guc * guc)251 static void guc_init_params_post_hwconfig(struct xe_guc *guc)
252 {
253 	u32 *params = guc->params;
254 
255 	params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
256 	params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc);
257 	params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
258 	params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
259 	params[GUC_CTL_WA] = guc_ctl_wa_flags(guc);
260 	params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
261 
262 	guc_print_params(guc);
263 }
264 
265 /*
266  * Initialize the GuC parameter block before starting the firmware
267  * transfer. These parameters are read by the firmware on startup
268  * and cannot be changed thereafter.
269  */
guc_write_params(struct xe_guc * guc)270 static void guc_write_params(struct xe_guc *guc)
271 {
272 	struct xe_gt *gt = guc_to_gt(guc);
273 	int i;
274 
275 	xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
276 
277 	xe_mmio_write32(&gt->mmio, SOFT_SCRATCH(0), 0);
278 
279 	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
280 		xe_mmio_write32(&gt->mmio, SOFT_SCRATCH(1 + i), guc->params[i]);
281 }
282 
guc_action_register_g2g_buffer(struct xe_guc * guc,u32 type,u32 dst_tile,u32 dst_dev,u32 desc_addr,u32 buff_addr,u32 size)283 static int guc_action_register_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev,
284 					  u32 desc_addr, u32 buff_addr, u32 size)
285 {
286 	struct xe_gt *gt = guc_to_gt(guc);
287 	struct xe_device *xe = gt_to_xe(gt);
288 	u32 action[] = {
289 		XE_GUC_ACTION_REGISTER_G2G,
290 		FIELD_PREP(XE_G2G_REGISTER_SIZE, size / SZ_4K - 1) |
291 		FIELD_PREP(XE_G2G_REGISTER_TYPE, type) |
292 		FIELD_PREP(XE_G2G_REGISTER_TILE, dst_tile) |
293 		FIELD_PREP(XE_G2G_REGISTER_DEVICE, dst_dev),
294 		desc_addr,
295 		buff_addr,
296 	};
297 
298 	xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
299 	xe_assert(xe, !(size % SZ_4K));
300 
301 	return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
302 }
303 
guc_action_deregister_g2g_buffer(struct xe_guc * guc,u32 type,u32 dst_tile,u32 dst_dev)304 static int guc_action_deregister_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev)
305 {
306 	struct xe_gt *gt = guc_to_gt(guc);
307 	struct xe_device *xe = gt_to_xe(gt);
308 	u32 action[] = {
309 		XE_GUC_ACTION_DEREGISTER_G2G,
310 		FIELD_PREP(XE_G2G_DEREGISTER_TYPE, type) |
311 		FIELD_PREP(XE_G2G_DEREGISTER_TILE, dst_tile) |
312 		FIELD_PREP(XE_G2G_DEREGISTER_DEVICE, dst_dev),
313 	};
314 
315 	xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
316 
317 	return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
318 }
319 
320 #define G2G_DEV(gt)	(((gt)->info.type == XE_GT_TYPE_MAIN) ? 0 : 1)
321 
322 #define G2G_BUFFER_SIZE (SZ_4K)
323 #define G2G_DESC_SIZE (64)
324 #define G2G_DESC_AREA_SIZE (SZ_4K)
325 
326 /*
327  * Generate a unique id for each bi-directional CTB for each pair of
328  * near and far tiles/devices. The id can then be used as an index into
329  * a single allocation that is sub-divided into multiple CTBs.
330  *
331  * For example, with two devices per tile and two tiles, the table should
332  * look like:
333  *           Far <tile>.<dev>
334  *         0.0   0.1   1.0   1.1
335  * N 0.0  --/-- 00/01 02/03 04/05
336  * e 0.1  01/00 --/-- 06/07 08/09
337  * a 1.0  03/02 07/06 --/-- 10/11
338  * r 1.1  05/04 09/08 11/10 --/--
339  *
340  * Where each entry is Rx/Tx channel id.
341  *
342  * So GuC #3 (tile 1, dev 1) talking to GuC #2 (tile 1, dev 0) would
343  * be reading from channel #11 and writing to channel #10. Whereas,
344  * GuC #2 talking to GuC #3 would be read on #10 and write to #11.
345  */
g2g_slot(u32 near_tile,u32 near_dev,u32 far_tile,u32 far_dev,u32 type,u32 max_inst,bool have_dev)346 static unsigned int g2g_slot(u32 near_tile, u32 near_dev, u32 far_tile, u32 far_dev,
347 			     u32 type, u32 max_inst, bool have_dev)
348 {
349 	u32 near = near_tile, far = far_tile;
350 	u32 idx = 0, x, y, direction;
351 	int i;
352 
353 	if (have_dev) {
354 		near = (near << 1) | near_dev;
355 		far = (far << 1) | far_dev;
356 	}
357 
358 	/* No need to send to one's self */
359 	if (far == near)
360 		return -1;
361 
362 	if (far > near) {
363 		/* Top right table half */
364 		x = far;
365 		y = near;
366 
367 		/* T/R is 'forwards' direction */
368 		direction = type;
369 	} else {
370 		/* Bottom left table half */
371 		x = near;
372 		y = far;
373 
374 		/* B/L is 'backwards' direction */
375 		direction = (1 - type);
376 	}
377 
378 	/* Count the rows prior to the target */
379 	for (i = y; i > 0; i--)
380 		idx += max_inst - i;
381 
382 	/* Count this row up to the target */
383 	idx += (x - 1 - y);
384 
385 	/* Slots are in Rx/Tx pairs */
386 	idx *= 2;
387 
388 	/* Pick Rx/Tx direction */
389 	idx += direction;
390 
391 	return idx;
392 }
393 
guc_g2g_register(struct xe_guc * near_guc,struct xe_gt * far_gt,u32 type,bool have_dev)394 static int guc_g2g_register(struct xe_guc *near_guc, struct xe_gt *far_gt, u32 type, bool have_dev)
395 {
396 	struct xe_gt *near_gt = guc_to_gt(near_guc);
397 	struct xe_device *xe = gt_to_xe(near_gt);
398 	struct xe_bo *g2g_bo;
399 	u32 near_tile = gt_to_tile(near_gt)->id;
400 	u32 near_dev = G2G_DEV(near_gt);
401 	u32 far_tile = gt_to_tile(far_gt)->id;
402 	u32 far_dev = G2G_DEV(far_gt);
403 	u32 max = xe->info.gt_count;
404 	u32 base, desc, buf;
405 	int slot;
406 
407 	/* G2G is not allowed between different cards */
408 	xe_assert(xe, xe == gt_to_xe(far_gt));
409 
410 	g2g_bo = near_guc->g2g.bo;
411 	xe_assert(xe, g2g_bo);
412 
413 	slot = g2g_slot(near_tile, near_dev, far_tile, far_dev, type, max, have_dev);
414 	xe_assert(xe, slot >= 0);
415 
416 	base = guc_bo_ggtt_addr(near_guc, g2g_bo);
417 	desc = base + slot * G2G_DESC_SIZE;
418 	buf = base + G2G_DESC_AREA_SIZE + slot * G2G_BUFFER_SIZE;
419 
420 	xe_assert(xe, (desc - base + G2G_DESC_SIZE) <= G2G_DESC_AREA_SIZE);
421 	xe_assert(xe, (buf - base + G2G_BUFFER_SIZE) <= g2g_bo->size);
422 
423 	return guc_action_register_g2g_buffer(near_guc, type, far_tile, far_dev,
424 					      desc, buf, G2G_BUFFER_SIZE);
425 }
426 
guc_g2g_deregister(struct xe_guc * guc,u32 far_tile,u32 far_dev,u32 type)427 static void guc_g2g_deregister(struct xe_guc *guc, u32 far_tile, u32 far_dev, u32 type)
428 {
429 	guc_action_deregister_g2g_buffer(guc, type, far_tile, far_dev);
430 }
431 
guc_g2g_size(struct xe_guc * guc)432 static u32 guc_g2g_size(struct xe_guc *guc)
433 {
434 	struct xe_gt *gt = guc_to_gt(guc);
435 	struct xe_device *xe = gt_to_xe(gt);
436 	unsigned int count = xe->info.gt_count;
437 	u32 num_channels = (count * (count - 1)) / 2;
438 
439 	xe_assert(xe, num_channels * XE_G2G_TYPE_LIMIT * G2G_DESC_SIZE <= G2G_DESC_AREA_SIZE);
440 
441 	return num_channels * XE_G2G_TYPE_LIMIT * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE;
442 }
443 
xe_guc_g2g_wanted(struct xe_device * xe)444 static bool xe_guc_g2g_wanted(struct xe_device *xe)
445 {
446 	/* Can't do GuC to GuC communication if there is only one GuC */
447 	if (xe->info.gt_count <= 1)
448 		return false;
449 
450 	/* No current user */
451 	return false;
452 }
453 
guc_g2g_alloc(struct xe_guc * guc)454 static int guc_g2g_alloc(struct xe_guc *guc)
455 {
456 	struct xe_gt *gt = guc_to_gt(guc);
457 	struct xe_device *xe = gt_to_xe(gt);
458 	struct xe_tile *tile = gt_to_tile(gt);
459 	struct xe_bo *bo;
460 	u32 g2g_size;
461 
462 	if (guc->g2g.bo)
463 		return 0;
464 
465 	if (gt->info.id != 0) {
466 		struct xe_gt *root_gt = xe_device_get_gt(xe, 0);
467 		struct xe_guc *root_guc = &root_gt->uc.guc;
468 		struct xe_bo *bo;
469 
470 		bo = xe_bo_get(root_guc->g2g.bo);
471 		if (!bo)
472 			return -ENODEV;
473 
474 		guc->g2g.bo = bo;
475 		guc->g2g.owned = false;
476 		return 0;
477 	}
478 
479 	g2g_size = guc_g2g_size(guc);
480 	bo = xe_managed_bo_create_pin_map(xe, tile, g2g_size,
481 					  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
482 					  XE_BO_FLAG_GGTT |
483 					  XE_BO_FLAG_GGTT_ALL |
484 					  XE_BO_FLAG_GGTT_INVALIDATE);
485 	if (IS_ERR(bo))
486 		return PTR_ERR(bo);
487 
488 	xe_map_memset(xe, &bo->vmap, 0, 0, g2g_size);
489 	guc->g2g.bo = bo;
490 	guc->g2g.owned = true;
491 
492 	return 0;
493 }
494 
guc_g2g_fini(struct xe_guc * guc)495 static void guc_g2g_fini(struct xe_guc *guc)
496 {
497 	if (!guc->g2g.bo)
498 		return;
499 
500 	/* Unpinning the owned object is handled by generic shutdown */
501 	if (!guc->g2g.owned)
502 		xe_bo_put(guc->g2g.bo);
503 
504 	guc->g2g.bo = NULL;
505 }
506 
guc_g2g_start(struct xe_guc * guc)507 static int guc_g2g_start(struct xe_guc *guc)
508 {
509 	struct xe_gt *far_gt, *gt = guc_to_gt(guc);
510 	struct xe_device *xe = gt_to_xe(gt);
511 	unsigned int i, j;
512 	int t, err;
513 	bool have_dev;
514 
515 	if (!guc->g2g.bo) {
516 		int ret;
517 
518 		ret = guc_g2g_alloc(guc);
519 		if (ret)
520 			return ret;
521 	}
522 
523 	/* GuC interface will need extending if more GT device types are ever created. */
524 	xe_gt_assert(gt, (gt->info.type == XE_GT_TYPE_MAIN) || (gt->info.type == XE_GT_TYPE_MEDIA));
525 
526 	/* Channel numbering depends on whether there are multiple GTs per tile */
527 	have_dev = xe->info.gt_count > xe->info.tile_count;
528 
529 	for_each_gt(far_gt, xe, i) {
530 		u32 far_tile, far_dev;
531 
532 		if (far_gt->info.id == gt->info.id)
533 			continue;
534 
535 		far_tile = gt_to_tile(far_gt)->id;
536 		far_dev = G2G_DEV(far_gt);
537 
538 		for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) {
539 			err = guc_g2g_register(guc, far_gt, t, have_dev);
540 			if (err) {
541 				while (--t >= 0)
542 					guc_g2g_deregister(guc, far_tile, far_dev, t);
543 				goto err_deregister;
544 			}
545 		}
546 	}
547 
548 	return 0;
549 
550 err_deregister:
551 	for_each_gt(far_gt, xe, j) {
552 		u32 tile, dev;
553 
554 		if (far_gt->info.id == gt->info.id)
555 			continue;
556 
557 		if (j >= i)
558 			break;
559 
560 		tile = gt_to_tile(far_gt)->id;
561 		dev = G2G_DEV(far_gt);
562 
563 		for (t = 0; t < XE_G2G_TYPE_LIMIT; t++)
564 			guc_g2g_deregister(guc, tile, dev, t);
565 	}
566 
567 	return err;
568 }
569 
guc_fini_hw(void * arg)570 static void guc_fini_hw(void *arg)
571 {
572 	struct xe_guc *guc = arg;
573 	struct xe_gt *gt = guc_to_gt(guc);
574 	unsigned int fw_ref;
575 
576 	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
577 	xe_uc_fini_hw(&guc_to_gt(guc)->uc);
578 	xe_force_wake_put(gt_to_fw(gt), fw_ref);
579 
580 	guc_g2g_fini(guc);
581 }
582 
583 /**
584  * xe_guc_comm_init_early - early initialization of GuC communication
585  * @guc: the &xe_guc to initialize
586  *
587  * Must be called prior to first MMIO communication with GuC firmware.
588  */
xe_guc_comm_init_early(struct xe_guc * guc)589 void xe_guc_comm_init_early(struct xe_guc *guc)
590 {
591 	struct xe_gt *gt = guc_to_gt(guc);
592 
593 	if (xe_gt_is_media_type(gt))
594 		guc->notify_reg = MED_GUC_HOST_INTERRUPT;
595 	else
596 		guc->notify_reg = GUC_HOST_INTERRUPT;
597 }
598 
xe_guc_realloc_post_hwconfig(struct xe_guc * guc)599 static int xe_guc_realloc_post_hwconfig(struct xe_guc *guc)
600 {
601 	struct xe_tile *tile = gt_to_tile(guc_to_gt(guc));
602 	struct xe_device *xe = guc_to_xe(guc);
603 	int ret;
604 
605 	if (!IS_DGFX(guc_to_xe(guc)))
606 		return 0;
607 
608 	ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->fw.bo);
609 	if (ret)
610 		return ret;
611 
612 	ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->log.bo);
613 	if (ret)
614 		return ret;
615 
616 	ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ads.bo);
617 	if (ret)
618 		return ret;
619 
620 	ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ct.bo);
621 	if (ret)
622 		return ret;
623 
624 	return 0;
625 }
626 
vf_guc_init(struct xe_guc * guc)627 static int vf_guc_init(struct xe_guc *guc)
628 {
629 	int err;
630 
631 	xe_guc_comm_init_early(guc);
632 
633 	err = xe_guc_ct_init(&guc->ct);
634 	if (err)
635 		return err;
636 
637 	err = xe_guc_relay_init(&guc->relay);
638 	if (err)
639 		return err;
640 
641 	return 0;
642 }
643 
xe_guc_init(struct xe_guc * guc)644 int xe_guc_init(struct xe_guc *guc)
645 {
646 	struct xe_device *xe = guc_to_xe(guc);
647 	struct xe_gt *gt = guc_to_gt(guc);
648 	int ret;
649 
650 	guc->fw.type = XE_UC_FW_TYPE_GUC;
651 	ret = xe_uc_fw_init(&guc->fw);
652 	if (ret)
653 		goto out;
654 
655 	if (!xe_uc_fw_is_enabled(&guc->fw))
656 		return 0;
657 
658 	if (IS_SRIOV_VF(xe)) {
659 		ret = vf_guc_init(guc);
660 		if (ret)
661 			goto out;
662 		return 0;
663 	}
664 
665 	ret = xe_guc_log_init(&guc->log);
666 	if (ret)
667 		goto out;
668 
669 	ret = xe_guc_capture_init(guc);
670 	if (ret)
671 		goto out;
672 
673 	ret = xe_guc_ads_init(&guc->ads);
674 	if (ret)
675 		goto out;
676 
677 	ret = xe_guc_ct_init(&guc->ct);
678 	if (ret)
679 		goto out;
680 
681 	ret = xe_guc_relay_init(&guc->relay);
682 	if (ret)
683 		goto out;
684 
685 	xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE);
686 
687 	ret = devm_add_action_or_reset(xe->drm.dev, guc_fini_hw, guc);
688 	if (ret)
689 		goto out;
690 
691 	guc_init_params(guc);
692 
693 	xe_guc_comm_init_early(guc);
694 
695 	return 0;
696 
697 out:
698 	xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
699 	return ret;
700 }
701 
vf_guc_init_post_hwconfig(struct xe_guc * guc)702 static int vf_guc_init_post_hwconfig(struct xe_guc *guc)
703 {
704 	int err;
705 
706 	err = xe_guc_submit_init(guc, xe_gt_sriov_vf_guc_ids(guc_to_gt(guc)));
707 	if (err)
708 		return err;
709 
710 	/* XXX xe_guc_db_mgr_init not needed for now */
711 
712 	return 0;
713 }
714 
715 /**
716  * xe_guc_init_post_hwconfig - initialize GuC post hwconfig load
717  * @guc: The GuC object
718  *
719  * Return: 0 on success, negative error code on error.
720  */
xe_guc_init_post_hwconfig(struct xe_guc * guc)721 int xe_guc_init_post_hwconfig(struct xe_guc *guc)
722 {
723 	int ret;
724 
725 	if (IS_SRIOV_VF(guc_to_xe(guc)))
726 		return vf_guc_init_post_hwconfig(guc);
727 
728 	ret = xe_guc_realloc_post_hwconfig(guc);
729 	if (ret)
730 		return ret;
731 
732 	guc_init_params_post_hwconfig(guc);
733 
734 	ret = xe_guc_submit_init(guc, ~0);
735 	if (ret)
736 		return ret;
737 
738 	ret = xe_guc_db_mgr_init(&guc->dbm, ~0);
739 	if (ret)
740 		return ret;
741 
742 	ret = xe_guc_pc_init(&guc->pc);
743 	if (ret)
744 		return ret;
745 
746 	return xe_guc_ads_init_post_hwconfig(&guc->ads);
747 }
748 
xe_guc_post_load_init(struct xe_guc * guc)749 int xe_guc_post_load_init(struct xe_guc *guc)
750 {
751 	int ret;
752 
753 	xe_guc_ads_populate_post_load(&guc->ads);
754 
755 	if (xe_guc_g2g_wanted(guc_to_xe(guc))) {
756 		ret = guc_g2g_start(guc);
757 		if (ret)
758 			return ret;
759 	}
760 
761 	guc->submission_state.enabled = true;
762 
763 	return 0;
764 }
765 
xe_guc_reset(struct xe_guc * guc)766 int xe_guc_reset(struct xe_guc *guc)
767 {
768 	struct xe_gt *gt = guc_to_gt(guc);
769 	struct xe_mmio *mmio = &gt->mmio;
770 	u32 guc_status, gdrst;
771 	int ret;
772 
773 	xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
774 
775 	if (IS_SRIOV_VF(gt_to_xe(gt)))
776 		return xe_gt_sriov_vf_bootstrap(gt);
777 
778 	xe_mmio_write32(mmio, GDRST, GRDOM_GUC);
779 
780 	ret = xe_mmio_wait32(mmio, GDRST, GRDOM_GUC, 0, 5000, &gdrst, false);
781 	if (ret) {
782 		xe_gt_err(gt, "GuC reset timed out, GDRST=%#x\n", gdrst);
783 		goto err_out;
784 	}
785 
786 	guc_status = xe_mmio_read32(mmio, GUC_STATUS);
787 	if (!(guc_status & GS_MIA_IN_RESET)) {
788 		xe_gt_err(gt, "GuC status: %#x, MIA core expected to be in reset\n",
789 			  guc_status);
790 		ret = -EIO;
791 		goto err_out;
792 	}
793 
794 	return 0;
795 
796 err_out:
797 
798 	return ret;
799 }
800 
guc_prepare_xfer(struct xe_guc * guc)801 static void guc_prepare_xfer(struct xe_guc *guc)
802 {
803 	struct xe_gt *gt = guc_to_gt(guc);
804 	struct xe_mmio *mmio = &gt->mmio;
805 	struct xe_device *xe =  guc_to_xe(guc);
806 	u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC |
807 		GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA |
808 		GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA |
809 		GUC_ENABLE_MIA_CLOCK_GATING;
810 
811 	if (GRAPHICS_VERx100(xe) < 1250)
812 		shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES |
813 				GUC_ENABLE_MIA_CACHING;
814 
815 	if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
816 		shim_flags |= REG_FIELD_PREP(GUC_MOCS_INDEX_MASK, gt->mocs.uc_index);
817 
818 	/* Must program this register before loading the ucode with DMA */
819 	xe_mmio_write32(mmio, GUC_SHIM_CONTROL, shim_flags);
820 
821 	xe_mmio_write32(mmio, GT_PM_CONFIG, GT_DOORBELL_ENABLE);
822 
823 	/* Make sure GuC receives ARAT interrupts */
824 	xe_mmio_rmw32(mmio, PMINTRMSK, ARAT_EXPIRED_INTRMSK, 0);
825 }
826 
827 /*
828  * Supporting MMIO & in memory RSA
829  */
guc_xfer_rsa(struct xe_guc * guc)830 static int guc_xfer_rsa(struct xe_guc *guc)
831 {
832 	struct xe_gt *gt = guc_to_gt(guc);
833 	u32 rsa[UOS_RSA_SCRATCH_COUNT];
834 	size_t copied;
835 	int i;
836 
837 	if (guc->fw.rsa_size > 256) {
838 		u32 rsa_ggtt_addr = xe_bo_ggtt_addr(guc->fw.bo) +
839 				    xe_uc_fw_rsa_offset(&guc->fw);
840 		xe_mmio_write32(&gt->mmio, UOS_RSA_SCRATCH(0), rsa_ggtt_addr);
841 		return 0;
842 	}
843 
844 	copied = xe_uc_fw_copy_rsa(&guc->fw, rsa, sizeof(rsa));
845 	if (copied < sizeof(rsa))
846 		return -ENOMEM;
847 
848 	for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++)
849 		xe_mmio_write32(&gt->mmio, UOS_RSA_SCRATCH(i), rsa[i]);
850 
851 	return 0;
852 }
853 
854 /*
855  * Check a previously read GuC status register (GUC_STATUS) looking for
856  * known terminal states (either completion or failure) of either the
857  * microkernel status field or the boot ROM status field. Returns +1 for
858  * successful completion, -1 for failure and 0 for any intermediate state.
859  */
guc_load_done(u32 status)860 static int guc_load_done(u32 status)
861 {
862 	u32 uk_val = REG_FIELD_GET(GS_UKERNEL_MASK, status);
863 	u32 br_val = REG_FIELD_GET(GS_BOOTROM_MASK, status);
864 
865 	switch (uk_val) {
866 	case XE_GUC_LOAD_STATUS_READY:
867 		return 1;
868 
869 	case XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH:
870 	case XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH:
871 	case XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE:
872 	case XE_GUC_LOAD_STATUS_HWCONFIG_ERROR:
873 	case XE_GUC_LOAD_STATUS_DPC_ERROR:
874 	case XE_GUC_LOAD_STATUS_EXCEPTION:
875 	case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID:
876 	case XE_GUC_LOAD_STATUS_MPU_DATA_INVALID:
877 	case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
878 		return -1;
879 	}
880 
881 	switch (br_val) {
882 	case XE_BOOTROM_STATUS_NO_KEY_FOUND:
883 	case XE_BOOTROM_STATUS_RSA_FAILED:
884 	case XE_BOOTROM_STATUS_PAVPC_FAILED:
885 	case XE_BOOTROM_STATUS_WOPCM_FAILED:
886 	case XE_BOOTROM_STATUS_LOADLOC_FAILED:
887 	case XE_BOOTROM_STATUS_JUMP_FAILED:
888 	case XE_BOOTROM_STATUS_RC6CTXCONFIG_FAILED:
889 	case XE_BOOTROM_STATUS_MPUMAP_INCORRECT:
890 	case XE_BOOTROM_STATUS_EXCEPTION:
891 	case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
892 		return -1;
893 	}
894 
895 	return 0;
896 }
897 
guc_pc_get_cur_freq(struct xe_guc_pc * guc_pc)898 static s32 guc_pc_get_cur_freq(struct xe_guc_pc *guc_pc)
899 {
900 	u32 freq;
901 	int ret = xe_guc_pc_get_cur_freq(guc_pc, &freq);
902 
903 	return ret ? ret : freq;
904 }
905 
906 /*
907  * Wait for the GuC to start up.
908  *
909  * Measurements indicate this should take no more than 20ms (assuming the GT
910  * clock is at maximum frequency). However, thermal throttling and other issues
911  * can prevent the clock hitting max and thus making the load take significantly
912  * longer. Allow up to 200ms as a safety margin for real world worst case situations.
913  *
914  * However, bugs anywhere from KMD to GuC to PCODE to fan failure in a CI farm can
915  * lead to even longer times. E.g. if the GT is clamped to minimum frequency then
916  * the load times can be in the seconds range. So the timeout is increased for debug
917  * builds to ensure that problems can be correctly analysed. For release builds, the
918  * timeout is kept short so that users don't wait forever to find out that there is a
919  * problem. In either case, if the load took longer than is reasonable even with some
920  * 'sensible' throttling, then flag a warning because something is not right.
921  *
922  * Note that there is a limit on how long an individual usleep_range() can wait for,
923  * hence longer waits require wrapping a shorter wait in a loop.
924  *
925  * Note that the only reason an end user should hit the shorter timeout is in case of
926  * extreme thermal throttling. And a system that is that hot during boot is probably
927  * dead anyway!
928  */
929 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
930 #define GUC_LOAD_RETRY_LIMIT	20
931 #else
932 #define GUC_LOAD_RETRY_LIMIT	3
933 #endif
934 #define GUC_LOAD_TIME_WARN_MS      200
935 
guc_wait_ucode(struct xe_guc * guc)936 static void guc_wait_ucode(struct xe_guc *guc)
937 {
938 	struct xe_gt *gt = guc_to_gt(guc);
939 	struct xe_mmio *mmio = &gt->mmio;
940 	struct xe_guc_pc *guc_pc = &gt->uc.guc.pc;
941 	ktime_t before, after, delta;
942 	int load_done;
943 	u32 status = 0;
944 	int count = 0;
945 	u64 delta_ms;
946 	u32 before_freq;
947 
948 	before_freq = xe_guc_pc_get_act_freq(guc_pc);
949 	before = ktime_get();
950 	/*
951 	 * Note, can't use any kind of timing information from the call to xe_mmio_wait.
952 	 * It could return a thousand intermediate stages at random times. Instead, must
953 	 * manually track the total time taken and locally implement the timeout.
954 	 */
955 	do {
956 		u32 last_status = status & (GS_UKERNEL_MASK | GS_BOOTROM_MASK);
957 		int ret;
958 
959 		/*
960 		 * Wait for any change (intermediate or terminal) in the status register.
961 		 * Note, the return value is a don't care. The only failure code is timeout
962 		 * but the timeouts need to be accumulated over all the intermediate partial
963 		 * timeouts rather than allowing a huge timeout each time. So basically, need
964 		 * to treat a timeout no different to a value change.
965 		 */
966 		ret = xe_mmio_wait32_not(mmio, GUC_STATUS, GS_UKERNEL_MASK | GS_BOOTROM_MASK,
967 					 last_status, 1000 * 1000, &status, false);
968 		if (ret < 0)
969 			count++;
970 		after = ktime_get();
971 		delta = ktime_sub(after, before);
972 		delta_ms = ktime_to_ms(delta);
973 
974 		load_done = guc_load_done(status);
975 		if (load_done != 0)
976 			break;
977 
978 		if (delta_ms >= (GUC_LOAD_RETRY_LIMIT * 1000))
979 			break;
980 
981 		xe_gt_dbg(gt, "load still in progress, timeouts = %d, freq = %dMHz (req %dMHz), status = 0x%08X [0x%02X/%02X]\n",
982 			  count, xe_guc_pc_get_act_freq(guc_pc),
983 			  guc_pc_get_cur_freq(guc_pc), status,
984 			  REG_FIELD_GET(GS_BOOTROM_MASK, status),
985 			  REG_FIELD_GET(GS_UKERNEL_MASK, status));
986 	} while (1);
987 
988 	if (load_done != 1) {
989 		u32 ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, status);
990 		u32 bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, status);
991 
992 		xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz), done = %d\n",
993 			  status, delta_ms, xe_guc_pc_get_act_freq(guc_pc),
994 			  guc_pc_get_cur_freq(guc_pc), load_done);
995 		xe_gt_err(gt, "load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n",
996 			  REG_FIELD_GET(GS_MIA_IN_RESET, status),
997 			  bootrom, ukernel,
998 			  REG_FIELD_GET(GS_MIA_MASK, status),
999 			  REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
1000 
1001 		switch (bootrom) {
1002 		case XE_BOOTROM_STATUS_NO_KEY_FOUND:
1003 			xe_gt_err(gt, "invalid key requested, header = 0x%08X\n",
1004 				  xe_mmio_read32(mmio, GUC_HEADER_INFO));
1005 			break;
1006 
1007 		case XE_BOOTROM_STATUS_RSA_FAILED:
1008 			xe_gt_err(gt, "firmware signature verification failed\n");
1009 			break;
1010 
1011 		case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
1012 			xe_gt_err(gt, "firmware production part check failure\n");
1013 			break;
1014 		}
1015 
1016 		switch (ukernel) {
1017 		case XE_GUC_LOAD_STATUS_EXCEPTION:
1018 			xe_gt_err(gt, "firmware exception. EIP: %#x\n",
1019 				  xe_mmio_read32(mmio, SOFT_SCRATCH(13)));
1020 			break;
1021 
1022 		case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
1023 			xe_gt_err(gt, "illegal register in save/restore workaround list\n");
1024 			break;
1025 
1026 		case XE_GUC_LOAD_STATUS_HWCONFIG_START:
1027 			xe_gt_err(gt, "still extracting hwconfig table.\n");
1028 			break;
1029 		}
1030 
1031 		xe_device_declare_wedged(gt_to_xe(gt));
1032 	} else if (delta_ms > GUC_LOAD_TIME_WARN_MS) {
1033 		xe_gt_warn(gt, "excessive init time: %lldms! [status = 0x%08X, timeouts = %d]\n",
1034 			   delta_ms, status, count);
1035 		xe_gt_warn(gt, "excessive init time: [freq = %dMHz (req = %dMHz), before = %dMHz, perf_limit_reasons = 0x%08X]\n",
1036 			   xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc),
1037 			   before_freq, xe_gt_throttle_get_limit_reasons(gt));
1038 	} else {
1039 		xe_gt_dbg(gt, "init took %lldms, freq = %dMHz (req = %dMHz), before = %dMHz, status = 0x%08X, timeouts = %d\n",
1040 			  delta_ms, xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc),
1041 			  before_freq, status, count);
1042 	}
1043 }
1044 
__xe_guc_upload(struct xe_guc * guc)1045 static int __xe_guc_upload(struct xe_guc *guc)
1046 {
1047 	int ret;
1048 
1049 	/* Raise GT freq to speed up HuC/GuC load */
1050 	xe_guc_pc_raise_unslice(&guc->pc);
1051 
1052 	guc_write_params(guc);
1053 	guc_prepare_xfer(guc);
1054 
1055 	/*
1056 	 * Note that GuC needs the CSS header plus uKernel code to be copied
1057 	 * by the DMA engine in one operation, whereas the RSA signature is
1058 	 * loaded separately, either by copying it to the UOS_RSA_SCRATCH
1059 	 * register (if key size <= 256) or through a ggtt-pinned vma (if key
1060 	 * size > 256). The RSA size and therefore the way we provide it to the
1061 	 * HW is fixed for each platform and hard-coded in the bootrom.
1062 	 */
1063 	ret = guc_xfer_rsa(guc);
1064 	if (ret)
1065 		goto out;
1066 	/*
1067 	 * Current uCode expects the code to be loaded at 8k; locations below
1068 	 * this are used for the stack.
1069 	 */
1070 	ret = xe_uc_fw_upload(&guc->fw, 0x2000, UOS_MOVE);
1071 	if (ret)
1072 		goto out;
1073 
1074 	/* Wait for authentication */
1075 	guc_wait_ucode(guc);
1076 
1077 	xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_RUNNING);
1078 	return 0;
1079 
1080 out:
1081 	xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOAD_FAIL);
1082 	return 0	/* FIXME: ret, don't want to stop load currently */;
1083 }
1084 
vf_guc_min_load_for_hwconfig(struct xe_guc * guc)1085 static int vf_guc_min_load_for_hwconfig(struct xe_guc *guc)
1086 {
1087 	struct xe_gt *gt = guc_to_gt(guc);
1088 	int ret;
1089 
1090 	ret = xe_gt_sriov_vf_bootstrap(gt);
1091 	if (ret)
1092 		return ret;
1093 
1094 	ret = xe_gt_sriov_vf_query_config(gt);
1095 	if (ret)
1096 		return ret;
1097 
1098 	ret = xe_guc_hwconfig_init(guc);
1099 	if (ret)
1100 		return ret;
1101 
1102 	ret = xe_guc_enable_communication(guc);
1103 	if (ret)
1104 		return ret;
1105 
1106 	ret = xe_gt_sriov_vf_connect(gt);
1107 	if (ret)
1108 		return ret;
1109 
1110 	ret = xe_gt_sriov_vf_query_runtime(gt);
1111 	if (ret)
1112 		return ret;
1113 
1114 	return 0;
1115 }
1116 
1117 /**
1118  * xe_guc_min_load_for_hwconfig - load minimal GuC and read hwconfig table
1119  * @guc: The GuC object
1120  *
1121  * This function uploads a minimal GuC that does not support submissions but
1122  * in a state where the hwconfig table can be read. Next, it reads and parses
1123  * the hwconfig table so it can be used for subsequent steps in the driver load.
1124  * Lastly, it enables CT communication (XXX: this is needed for PFs/VFs only).
1125  *
1126  * Return: 0 on success, negative error code on error.
1127  */
xe_guc_min_load_for_hwconfig(struct xe_guc * guc)1128 int xe_guc_min_load_for_hwconfig(struct xe_guc *guc)
1129 {
1130 	int ret;
1131 
1132 	if (IS_SRIOV_VF(guc_to_xe(guc)))
1133 		return vf_guc_min_load_for_hwconfig(guc);
1134 
1135 	xe_guc_ads_populate_minimal(&guc->ads);
1136 
1137 	xe_guc_pc_init_early(&guc->pc);
1138 
1139 	ret = __xe_guc_upload(guc);
1140 	if (ret)
1141 		return ret;
1142 
1143 	ret = xe_guc_hwconfig_init(guc);
1144 	if (ret)
1145 		return ret;
1146 
1147 	ret = xe_guc_enable_communication(guc);
1148 	if (ret)
1149 		return ret;
1150 
1151 	return 0;
1152 }
1153 
xe_guc_upload(struct xe_guc * guc)1154 int xe_guc_upload(struct xe_guc *guc)
1155 {
1156 	xe_guc_ads_populate(&guc->ads);
1157 
1158 	return __xe_guc_upload(guc);
1159 }
1160 
guc_handle_mmio_msg(struct xe_guc * guc)1161 static void guc_handle_mmio_msg(struct xe_guc *guc)
1162 {
1163 	struct xe_gt *gt = guc_to_gt(guc);
1164 	u32 msg;
1165 
1166 	if (IS_SRIOV_VF(guc_to_xe(guc)))
1167 		return;
1168 
1169 	xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
1170 
1171 	msg = xe_mmio_read32(&gt->mmio, SOFT_SCRATCH(15));
1172 	msg &= XE_GUC_RECV_MSG_EXCEPTION |
1173 		XE_GUC_RECV_MSG_CRASH_DUMP_POSTED;
1174 	xe_mmio_write32(&gt->mmio, SOFT_SCRATCH(15), 0);
1175 
1176 	if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED)
1177 		xe_gt_err(gt, "Received early GuC crash dump notification!\n");
1178 
1179 	if (msg & XE_GUC_RECV_MSG_EXCEPTION)
1180 		xe_gt_err(gt, "Received early GuC exception notification!\n");
1181 }
1182 
guc_enable_irq(struct xe_guc * guc)1183 static void guc_enable_irq(struct xe_guc *guc)
1184 {
1185 	struct xe_gt *gt = guc_to_gt(guc);
1186 	u32 events = xe_gt_is_media_type(gt) ?
1187 		REG_FIELD_PREP(ENGINE0_MASK, GUC_INTR_GUC2HOST)  :
1188 		REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST);
1189 
1190 	/* Primary GuC and media GuC share a single enable bit */
1191 	xe_mmio_write32(&gt->mmio, GUC_SG_INTR_ENABLE,
1192 			REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST));
1193 
1194 	/*
1195 	 * There are separate mask bits for primary and media GuCs, so use
1196 	 * a RMW operation to avoid clobbering the other GuC's setting.
1197 	 */
1198 	xe_mmio_rmw32(&gt->mmio, GUC_SG_INTR_MASK, events, 0);
1199 }
1200 
xe_guc_enable_communication(struct xe_guc * guc)1201 int xe_guc_enable_communication(struct xe_guc *guc)
1202 {
1203 	struct xe_device *xe = guc_to_xe(guc);
1204 	int err;
1205 
1206 	if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) {
1207 		struct xe_gt *gt = guc_to_gt(guc);
1208 		struct xe_tile *tile = gt_to_tile(gt);
1209 
1210 		err = xe_memirq_init_guc(&tile->memirq, guc);
1211 		if (err)
1212 			return err;
1213 	} else {
1214 		guc_enable_irq(guc);
1215 	}
1216 
1217 	err = xe_guc_ct_enable(&guc->ct);
1218 	if (err)
1219 		return err;
1220 
1221 	guc_handle_mmio_msg(guc);
1222 
1223 	return 0;
1224 }
1225 
xe_guc_suspend(struct xe_guc * guc)1226 int xe_guc_suspend(struct xe_guc *guc)
1227 {
1228 	struct xe_gt *gt = guc_to_gt(guc);
1229 	u32 action[] = {
1230 		XE_GUC_ACTION_CLIENT_SOFT_RESET,
1231 	};
1232 	int ret;
1233 
1234 	ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action));
1235 	if (ret) {
1236 		xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret));
1237 		return ret;
1238 	}
1239 
1240 	xe_guc_sanitize(guc);
1241 	return 0;
1242 }
1243 
xe_guc_notify(struct xe_guc * guc)1244 void xe_guc_notify(struct xe_guc *guc)
1245 {
1246 	struct xe_gt *gt = guc_to_gt(guc);
1247 	const u32 default_notify_data = 0;
1248 
1249 	/*
1250 	 * Both GUC_HOST_INTERRUPT and MED_GUC_HOST_INTERRUPT can pass
1251 	 * additional payload data to the GuC but this capability is not
1252 	 * used by the firmware yet. Use default value in the meantime.
1253 	 */
1254 	xe_mmio_write32(&gt->mmio, guc->notify_reg, default_notify_data);
1255 }
1256 
xe_guc_auth_huc(struct xe_guc * guc,u32 rsa_addr)1257 int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr)
1258 {
1259 	u32 action[] = {
1260 		XE_GUC_ACTION_AUTHENTICATE_HUC,
1261 		rsa_addr
1262 	};
1263 
1264 	return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
1265 }
1266 
xe_guc_mmio_send_recv(struct xe_guc * guc,const u32 * request,u32 len,u32 * response_buf)1267 int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
1268 			  u32 len, u32 *response_buf)
1269 {
1270 	struct xe_device *xe = guc_to_xe(guc);
1271 	struct xe_gt *gt = guc_to_gt(guc);
1272 	struct xe_mmio *mmio = &gt->mmio;
1273 	u32 header, reply;
1274 	struct xe_reg reply_reg = xe_gt_is_media_type(gt) ?
1275 		MED_VF_SW_FLAG(0) : VF_SW_FLAG(0);
1276 	const u32 LAST_INDEX = VF_SW_FLAG_COUNT - 1;
1277 	int ret;
1278 	int i;
1279 
1280 	BUILD_BUG_ON(VF_SW_FLAG_COUNT != MED_VF_SW_FLAG_COUNT);
1281 
1282 	xe_assert(xe, len);
1283 	xe_assert(xe, len <= VF_SW_FLAG_COUNT);
1284 	xe_assert(xe, len <= MED_VF_SW_FLAG_COUNT);
1285 	xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) ==
1286 		  GUC_HXG_ORIGIN_HOST);
1287 	xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) ==
1288 		  GUC_HXG_TYPE_REQUEST);
1289 
1290 retry:
1291 	/* Not in critical data-path, just do if else for GT type */
1292 	if (xe_gt_is_media_type(gt)) {
1293 		for (i = 0; i < len; ++i)
1294 			xe_mmio_write32(mmio, MED_VF_SW_FLAG(i),
1295 					request[i]);
1296 		xe_mmio_read32(mmio, MED_VF_SW_FLAG(LAST_INDEX));
1297 	} else {
1298 		for (i = 0; i < len; ++i)
1299 			xe_mmio_write32(mmio, VF_SW_FLAG(i),
1300 					request[i]);
1301 		xe_mmio_read32(mmio, VF_SW_FLAG(LAST_INDEX));
1302 	}
1303 
1304 	xe_guc_notify(guc);
1305 
1306 	ret = xe_mmio_wait32(mmio, reply_reg, GUC_HXG_MSG_0_ORIGIN,
1307 			     FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC),
1308 			     50000, &reply, false);
1309 	if (ret) {
1310 timeout:
1311 		xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n",
1312 			  request[0], reply);
1313 		return ret;
1314 	}
1315 
1316 	header = xe_mmio_read32(mmio, reply_reg);
1317 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1318 	    GUC_HXG_TYPE_NO_RESPONSE_BUSY) {
1319 		/*
1320 		 * Once we got a BUSY reply we must wait again for the final
1321 		 * response but this time we can't use ORIGIN mask anymore.
1322 		 * To spot a right change in the reply, we take advantage that
1323 		 * response SUCCESS and FAILURE differ only by the single bit
1324 		 * and all other bits are set and can be used as a new mask.
1325 		 */
1326 		u32 resp_bits = GUC_HXG_TYPE_RESPONSE_SUCCESS & GUC_HXG_TYPE_RESPONSE_FAILURE;
1327 		u32 resp_mask = FIELD_PREP(GUC_HXG_MSG_0_TYPE, resp_bits);
1328 
1329 		BUILD_BUG_ON(FIELD_MAX(GUC_HXG_MSG_0_TYPE) != GUC_HXG_TYPE_RESPONSE_SUCCESS);
1330 		BUILD_BUG_ON((GUC_HXG_TYPE_RESPONSE_SUCCESS ^ GUC_HXG_TYPE_RESPONSE_FAILURE) != 1);
1331 
1332 		ret = xe_mmio_wait32(mmio, reply_reg, resp_mask, resp_mask,
1333 				     1000000, &header, false);
1334 
1335 		if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) !=
1336 			     GUC_HXG_ORIGIN_GUC))
1337 			goto proto;
1338 		if (unlikely(ret)) {
1339 			if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1340 			    GUC_HXG_TYPE_NO_RESPONSE_BUSY)
1341 				goto proto;
1342 			goto timeout;
1343 		}
1344 	}
1345 
1346 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1347 	    GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1348 		u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header);
1349 
1350 		xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n",
1351 			  request[0], reason);
1352 		goto retry;
1353 	}
1354 
1355 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1356 	    GUC_HXG_TYPE_RESPONSE_FAILURE) {
1357 		u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
1358 		u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
1359 
1360 		xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
1361 			  request[0], error, hint);
1362 		return -ENXIO;
1363 	}
1364 
1365 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1366 	    GUC_HXG_TYPE_RESPONSE_SUCCESS) {
1367 proto:
1368 		xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n",
1369 			  request[0], header);
1370 		return -EPROTO;
1371 	}
1372 
1373 	/* Just copy entire possible message response */
1374 	if (response_buf) {
1375 		response_buf[0] = header;
1376 
1377 		for (i = 1; i < VF_SW_FLAG_COUNT; i++) {
1378 			reply_reg.addr += sizeof(u32);
1379 			response_buf[i] = xe_mmio_read32(mmio, reply_reg);
1380 		}
1381 	}
1382 
1383 	/* Use data from the GuC response as our return value */
1384 	return FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header);
1385 }
1386 
xe_guc_mmio_send(struct xe_guc * guc,const u32 * request,u32 len)1387 int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len)
1388 {
1389 	return xe_guc_mmio_send_recv(guc, request, len, NULL);
1390 }
1391 
guc_self_cfg(struct xe_guc * guc,u16 key,u16 len,u64 val)1392 static int guc_self_cfg(struct xe_guc *guc, u16 key, u16 len, u64 val)
1393 {
1394 	struct xe_device *xe = guc_to_xe(guc);
1395 	u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = {
1396 		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
1397 		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
1398 		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
1399 			   GUC_ACTION_HOST2GUC_SELF_CFG),
1400 		FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) |
1401 		FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len),
1402 		FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32,
1403 			   lower_32_bits(val)),
1404 		FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64,
1405 			   upper_32_bits(val)),
1406 	};
1407 	int ret;
1408 
1409 	xe_assert(xe, len <= 2);
1410 	xe_assert(xe, len != 1 || !upper_32_bits(val));
1411 
1412 	/* Self config must go over MMIO */
1413 	ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request));
1414 
1415 	if (unlikely(ret < 0))
1416 		return ret;
1417 	if (unlikely(ret > 1))
1418 		return -EPROTO;
1419 	if (unlikely(!ret))
1420 		return -ENOKEY;
1421 
1422 	return 0;
1423 }
1424 
xe_guc_self_cfg32(struct xe_guc * guc,u16 key,u32 val)1425 int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val)
1426 {
1427 	return guc_self_cfg(guc, key, 1, val);
1428 }
1429 
xe_guc_self_cfg64(struct xe_guc * guc,u16 key,u64 val)1430 int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val)
1431 {
1432 	return guc_self_cfg(guc, key, 2, val);
1433 }
1434 
xe_guc_sw_0_irq_handler(struct xe_guc * guc)1435 static void xe_guc_sw_0_irq_handler(struct xe_guc *guc)
1436 {
1437 	struct xe_gt *gt = guc_to_gt(guc);
1438 
1439 	if (IS_SRIOV_VF(gt_to_xe(gt)))
1440 		xe_gt_sriov_vf_migrated_event_handler(gt);
1441 }
1442 
xe_guc_irq_handler(struct xe_guc * guc,const u16 iir)1443 void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir)
1444 {
1445 	if (iir & GUC_INTR_GUC2HOST)
1446 		xe_guc_ct_irq_handler(&guc->ct);
1447 
1448 	if (iir & GUC_INTR_SW_INT_0)
1449 		xe_guc_sw_0_irq_handler(guc);
1450 }
1451 
xe_guc_sanitize(struct xe_guc * guc)1452 void xe_guc_sanitize(struct xe_guc *guc)
1453 {
1454 	xe_uc_fw_sanitize(&guc->fw);
1455 	xe_guc_ct_disable(&guc->ct);
1456 	guc->submission_state.enabled = false;
1457 }
1458 
xe_guc_reset_prepare(struct xe_guc * guc)1459 int xe_guc_reset_prepare(struct xe_guc *guc)
1460 {
1461 	return xe_guc_submit_reset_prepare(guc);
1462 }
1463 
xe_guc_reset_wait(struct xe_guc * guc)1464 void xe_guc_reset_wait(struct xe_guc *guc)
1465 {
1466 	xe_guc_submit_reset_wait(guc);
1467 }
1468 
xe_guc_stop_prepare(struct xe_guc * guc)1469 void xe_guc_stop_prepare(struct xe_guc *guc)
1470 {
1471 	if (!IS_SRIOV_VF(guc_to_xe(guc))) {
1472 		int err;
1473 
1474 		err = xe_guc_pc_stop(&guc->pc);
1475 		xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n",
1476 			   ERR_PTR(err));
1477 	}
1478 }
1479 
xe_guc_stop(struct xe_guc * guc)1480 void xe_guc_stop(struct xe_guc *guc)
1481 {
1482 	xe_guc_ct_stop(&guc->ct);
1483 
1484 	xe_guc_submit_stop(guc);
1485 }
1486 
xe_guc_start(struct xe_guc * guc)1487 int xe_guc_start(struct xe_guc *guc)
1488 {
1489 	if (!IS_SRIOV_VF(guc_to_xe(guc))) {
1490 		int err;
1491 
1492 		err = xe_guc_pc_start(&guc->pc);
1493 		xe_gt_WARN(guc_to_gt(guc), err, "Failed to start GuC PC: %pe\n",
1494 			   ERR_PTR(err));
1495 	}
1496 
1497 	return xe_guc_submit_start(guc);
1498 }
1499 
xe_guc_print_info(struct xe_guc * guc,struct drm_printer * p)1500 void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p)
1501 {
1502 	struct xe_gt *gt = guc_to_gt(guc);
1503 	unsigned int fw_ref;
1504 	u32 status;
1505 	int i;
1506 
1507 	xe_uc_fw_print(&guc->fw, p);
1508 
1509 	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1510 	if (!fw_ref)
1511 		return;
1512 
1513 	status = xe_mmio_read32(&gt->mmio, GUC_STATUS);
1514 
1515 	drm_printf(p, "\nGuC status 0x%08x:\n", status);
1516 	drm_printf(p, "\tBootrom status = 0x%x\n",
1517 		   REG_FIELD_GET(GS_BOOTROM_MASK, status));
1518 	drm_printf(p, "\tuKernel status = 0x%x\n",
1519 		   REG_FIELD_GET(GS_UKERNEL_MASK, status));
1520 	drm_printf(p, "\tMIA Core status = 0x%x\n",
1521 		   REG_FIELD_GET(GS_MIA_MASK, status));
1522 	drm_printf(p, "\tLog level = %d\n",
1523 		   xe_guc_log_get_level(&guc->log));
1524 
1525 	drm_puts(p, "\nScratch registers:\n");
1526 	for (i = 0; i < SOFT_SCRATCH_COUNT; i++) {
1527 		drm_printf(p, "\t%2d: \t0x%x\n",
1528 			   i, xe_mmio_read32(&gt->mmio, SOFT_SCRATCH(i)));
1529 	}
1530 
1531 	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1532 
1533 	drm_puts(p, "\n");
1534 	xe_guc_ct_print(&guc->ct, p, false);
1535 
1536 	drm_puts(p, "\n");
1537 	xe_guc_submit_print(guc, p);
1538 }
1539 
1540 /**
1541  * xe_guc_declare_wedged() - Declare GuC wedged
1542  * @guc: the GuC object
1543  *
1544  * Wedge the GuC which stops all submission, saves desired debug state, and
1545  * cleans up anything which could timeout.
1546  */
xe_guc_declare_wedged(struct xe_guc * guc)1547 void xe_guc_declare_wedged(struct xe_guc *guc)
1548 {
1549 	xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);
1550 
1551 	xe_guc_reset_prepare(guc);
1552 	xe_guc_ct_stop(&guc->ct);
1553 	xe_guc_submit_wedge(guc);
1554 }
1555