xref: /linux/drivers/gpu/drm/xe/xe_gt_mcr.c (revision a6021aa24f6417416d93318bbfa022ab229c33c8)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_gt_mcr.h"
7 
8 #include "regs/xe_gt_regs.h"
9 #include "xe_assert.h"
10 #include "xe_gt.h"
11 #include "xe_gt_printk.h"
12 #include "xe_gt_topology.h"
13 #include "xe_gt_types.h"
14 #include "xe_guc_hwconfig.h"
15 #include "xe_mmio.h"
16 #include "xe_sriov.h"
17 
18 /**
19  * DOC: GT Multicast/Replicated (MCR) Register Support
20  *
21  * Some GT registers are designed as "multicast" or "replicated" registers:
22  * multiple instances of the same register share a single MMIO offset.  MCR
23  * registers are generally used when the hardware needs to potentially track
24  * independent values of a register per hardware unit (e.g., per-subslice,
25  * per-L3bank, etc.).  The specific types of replication that exist vary
26  * per-platform.
27  *
28  * MMIO accesses to MCR registers are controlled according to the settings
29  * programmed in the platform's MCR_SELECTOR register(s).  MMIO writes to MCR
30  * registers can be done in either multicast (a single write updates all
31  * instances of the register to the same value) or unicast (a write updates only
32  * one specific instance) form.  Reads of MCR registers always operate in a
33  * unicast manner regardless of how the multicast/unicast bit is set in
34  * MCR_SELECTOR.  Selection of a specific MCR instance for unicast operations is
35  * referred to as "steering."
36  *
37  * If MCR register operations are steered toward a hardware unit that is
38  * fused off or currently powered down due to power gating, the MMIO operation
39  * is "terminated" by the hardware.  Terminated read operations will return a
40  * value of zero and terminated unicast write operations will be silently
41  * ignored. During device initialization, the goal of the various
42  * ``init_steering_*()`` functions is to apply the platform-specific rules for
43  * each MCR register type to identify a steering target that will select a
44  * non-terminated instance.
45  *
46  * MCR registers are not available on Virtual Function (VF).
47  */
48 
49 #define STEER_SEMAPHORE		XE_REG(0xFD0)
50 
51 static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr)
52 {
53 	return reg_mcr.__reg;
54 }
55 
56 enum {
57 	MCR_OP_READ,
58 	MCR_OP_WRITE
59 };
60 
61 static const struct xe_mmio_range xelp_l3bank_steering_table[] = {
62 	{ 0x00B100, 0x00B3FF },
63 	{},
64 };
65 
66 static const struct xe_mmio_range xehp_l3bank_steering_table[] = {
67 	{ 0x008C80, 0x008CFF },
68 	{ 0x00B100, 0x00B3FF },
69 	{},
70 };
71 
72 /*
73  * Although the bspec lists more "MSLICE" ranges than shown here, some of those
74  * are of a "GAM" subclass that has special rules and doesn't need to be
75  * included here.
76  */
77 static const struct xe_mmio_range xehp_mslice_steering_table[] = {
78 	{ 0x00DD00, 0x00DDFF },
79 	{ 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
80 	{},
81 };
82 
83 static const struct xe_mmio_range xehp_lncf_steering_table[] = {
84 	{ 0x00B000, 0x00B0FF },
85 	{ 0x00D880, 0x00D8FF },
86 	{},
87 };
88 
89 /*
90  * We have several types of MCR registers where steering to (0,0) will always
91  * provide us with a non-terminated value.  We'll stick them all in the same
92  * table for simplicity.
93  */
94 static const struct xe_mmio_range xehpc_instance0_steering_table[] = {
95 	{ 0x004000, 0x004AFF },		/* HALF-BSLICE */
96 	{ 0x008800, 0x00887F },		/* CC */
97 	{ 0x008A80, 0x008AFF },		/* TILEPSMI */
98 	{ 0x00B000, 0x00B0FF },		/* HALF-BSLICE */
99 	{ 0x00B100, 0x00B3FF },		/* L3BANK */
100 	{ 0x00C800, 0x00CFFF },		/* HALF-BSLICE */
101 	{ 0x00D800, 0x00D8FF },		/* HALF-BSLICE */
102 	{ 0x00DD00, 0x00DDFF },		/* BSLICE */
103 	{ 0x00E900, 0x00E9FF },		/* HALF-BSLICE */
104 	{ 0x00EC00, 0x00EEFF },		/* HALF-BSLICE */
105 	{ 0x00F000, 0x00FFFF },		/* HALF-BSLICE */
106 	{ 0x024180, 0x0241FF },		/* HALF-BSLICE */
107 	{},
108 };
109 
110 static const struct xe_mmio_range xelpg_instance0_steering_table[] = {
111 	{ 0x000B00, 0x000BFF },         /* SQIDI */
112 	{ 0x001000, 0x001FFF },         /* SQIDI */
113 	{ 0x004000, 0x0048FF },         /* GAM */
114 	{ 0x008700, 0x0087FF },         /* SQIDI */
115 	{ 0x00B000, 0x00B0FF },         /* NODE */
116 	{ 0x00C800, 0x00CFFF },         /* GAM */
117 	{ 0x00D880, 0x00D8FF },         /* NODE */
118 	{ 0x00DD00, 0x00DDFF },         /* OAAL2 */
119 	{},
120 };
121 
122 static const struct xe_mmio_range xelpg_l3bank_steering_table[] = {
123 	{ 0x00B100, 0x00B3FF },
124 	{},
125 };
126 
127 static const struct xe_mmio_range xelp_dss_steering_table[] = {
128 	{ 0x008150, 0x00815F },
129 	{ 0x009520, 0x00955F },
130 	{ 0x00DE80, 0x00E8FF },
131 	{ 0x024A00, 0x024A7F },
132 	{},
133 };
134 
135 /* DSS steering is used for GSLICE ranges as well */
136 static const struct xe_mmio_range xehp_dss_steering_table[] = {
137 	{ 0x005200, 0x0052FF },		/* GSLICE */
138 	{ 0x005400, 0x007FFF },		/* GSLICE */
139 	{ 0x008140, 0x00815F },		/* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
140 	{ 0x008D00, 0x008DFF },		/* DSS */
141 	{ 0x0094D0, 0x00955F },		/* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
142 	{ 0x009680, 0x0096FF },		/* DSS */
143 	{ 0x00D800, 0x00D87F },		/* GSLICE */
144 	{ 0x00DC00, 0x00DCFF },		/* GSLICE */
145 	{ 0x00DE80, 0x00E8FF },		/* DSS (0xE000-0xE0FF reserved ) */
146 	{ 0x017000, 0x017FFF },		/* GSLICE */
147 	{ 0x024A00, 0x024A7F },		/* DSS */
148 	{},
149 };
150 
151 /* DSS steering is used for COMPUTE ranges as well */
152 static const struct xe_mmio_range xehpc_dss_steering_table[] = {
153 	{ 0x008140, 0x00817F },		/* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */
154 	{ 0x0094D0, 0x00955F },		/* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */
155 	{ 0x009680, 0x0096FF },		/* DSS */
156 	{ 0x00DC00, 0x00DCFF },		/* COMPUTE */
157 	{ 0x00DE80, 0x00E7FF },		/* DSS (0xDF00-0xE1FF reserved ) */
158 	{},
159 };
160 
161 /* DSS steering is used for SLICE ranges as well */
162 static const struct xe_mmio_range xelpg_dss_steering_table[] = {
163 	{ 0x005200, 0x0052FF },		/* SLICE */
164 	{ 0x005500, 0x007FFF },		/* SLICE */
165 	{ 0x008140, 0x00815F },		/* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
166 	{ 0x0094D0, 0x00955F },		/* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
167 	{ 0x009680, 0x0096FF },		/* DSS */
168 	{ 0x00D800, 0x00D87F },		/* SLICE */
169 	{ 0x00DC00, 0x00DCFF },		/* SLICE */
170 	{ 0x00DE80, 0x00E8FF },		/* DSS (0xE000-0xE0FF reserved) */
171 	{},
172 };
173 
174 static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = {
175 	{ 0x393200, 0x39323F },
176 	{ 0x393400, 0x3934FF },
177 	{},
178 };
179 
180 static const struct xe_mmio_range dg2_implicit_steering_table[] = {
181 	{ 0x000B00, 0x000BFF },		/* SF (SQIDI replication) */
182 	{ 0x001000, 0x001FFF },		/* SF (SQIDI replication) */
183 	{ 0x004000, 0x004AFF },		/* GAM (MSLICE replication) */
184 	{ 0x008700, 0x0087FF },		/* MCFG (SQIDI replication) */
185 	{ 0x00C800, 0x00CFFF },		/* GAM (MSLICE replication) */
186 	{ 0x00F000, 0x00FFFF },		/* GAM (MSLICE replication) */
187 	{},
188 };
189 
190 static const struct xe_mmio_range xe2lpg_dss_steering_table[] = {
191 	{ 0x005200, 0x0052FF },         /* SLICE */
192 	{ 0x005500, 0x007FFF },         /* SLICE */
193 	{ 0x008140, 0x00815F },         /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
194 	{ 0x0094D0, 0x00955F },         /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
195 	{ 0x009680, 0x0096FF },         /* DSS */
196 	{ 0x00D800, 0x00D87F },         /* SLICE */
197 	{ 0x00DC00, 0x00DCFF },         /* SLICE */
198 	{ 0x00DE80, 0x00E8FF },         /* DSS (0xE000-0xE0FF reserved) */
199 	{ 0x00E980, 0x00E9FF },         /* SLICE */
200 	{ 0x013000, 0x0133FF },         /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */
201 	{},
202 };
203 
204 static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = {
205 	{ 0x000B00, 0x000BFF },
206 	{ 0x001000, 0x001FFF },
207 	{},
208 };
209 
210 static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = {
211 	{ 0x004000, 0x004AFF },         /* GAM, rsvd, GAMWKR */
212 	{ 0x008700, 0x00887F },         /* SQIDI, MEMPIPE */
213 	{ 0x00B000, 0x00B3FF },         /* NODE, L3BANK */
214 	{ 0x00C800, 0x00CFFF },         /* GAM */
215 	{ 0x00D880, 0x00D8FF },         /* NODE */
216 	{ 0x00DD00, 0x00DDFF },         /* MEMPIPE */
217 	{ 0x00E900, 0x00E97F },         /* MEMPIPE */
218 	{ 0x00F000, 0x00FFFF },         /* GAM, GAMWKR */
219 	{ 0x013400, 0x0135FF },         /* MEMPIPE */
220 	{},
221 };
222 
223 static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = {
224 	{ 0x388160, 0x38817F },
225 	{ 0x389480, 0x3894CF },
226 	{},
227 };
228 
229 static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = {
230 	{ 0x384000, 0x3847DF },         /* GAM, rsvd, GAM */
231 	{ 0x384900, 0x384AFF },         /* GAM */
232 	{ 0x389560, 0x3895FF },         /* MEDIAINF */
233 	{ 0x38B600, 0x38B8FF },         /* L3BANK */
234 	{ 0x38C800, 0x38D07F },         /* GAM, MEDIAINF */
235 	{ 0x38F000, 0x38F0FF },         /* GAM */
236 	{ 0x393C00, 0x393C7F },         /* MEDIAINF */
237 	{},
238 };
239 
240 static void init_steering_l3bank(struct xe_gt *gt)
241 {
242 	if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
243 		u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,
244 						xe_mmio_read32(gt, MIRROR_FUSE3));
245 		u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK,
246 					      xe_mmio_read32(gt, XEHP_FUSE4));
247 
248 		/*
249 		 * Group selects mslice, instance selects bank within mslice.
250 		 * Bank 0 is always valid _except_ when the bank mask is 010b.
251 		 */
252 		gt->steering[L3BANK].group_target = __ffs(mslice_mask);
253 		gt->steering[L3BANK].instance_target =
254 			bank_mask & BIT(0) ? 0 : 2;
255 	} else if (gt_to_xe(gt)->info.platform == XE_DG2) {
256 		u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,
257 						xe_mmio_read32(gt, MIRROR_FUSE3));
258 		u32 bank = __ffs(mslice_mask) * 8;
259 
260 		/*
261 		 * Like mslice registers, look for a valid mslice and steer to
262 		 * the first L3BANK of that quad. Access to the Nth L3 bank is
263 		 * split between the first bits of group and instance
264 		 */
265 		gt->steering[L3BANK].group_target = (bank >> 2) & 0x7;
266 		gt->steering[L3BANK].instance_target = bank & 0x3;
267 	} else {
268 		u32 fuse = REG_FIELD_GET(L3BANK_MASK,
269 					 ~xe_mmio_read32(gt, MIRROR_FUSE3));
270 
271 		gt->steering[L3BANK].group_target = 0;	/* unused */
272 		gt->steering[L3BANK].instance_target = __ffs(fuse);
273 	}
274 }
275 
276 static void init_steering_mslice(struct xe_gt *gt)
277 {
278 	u32 mask = REG_FIELD_GET(MEML3_EN_MASK,
279 				 xe_mmio_read32(gt, MIRROR_FUSE3));
280 
281 	/*
282 	 * mslice registers are valid (not terminated) if either the meml3
283 	 * associated with the mslice is present, or at least one DSS associated
284 	 * with the mslice is present.  There will always be at least one meml3
285 	 * so we can just use that to find a non-terminated mslice and ignore
286 	 * the DSS fusing.
287 	 */
288 	gt->steering[MSLICE].group_target = __ffs(mask);
289 	gt->steering[MSLICE].instance_target = 0;	/* unused */
290 
291 	/*
292 	 * LNCF termination is also based on mslice presence, so we'll set
293 	 * it up here.  Either LNCF within a non-terminated mslice will work,
294 	 * so we just always pick LNCF 0 here.
295 	 */
296 	gt->steering[LNCF].group_target = __ffs(mask) << 1;
297 	gt->steering[LNCF].instance_target = 0;		/* unused */
298 }
299 
300 static unsigned int dss_per_group(struct xe_gt *gt)
301 {
302 	struct xe_guc *guc = &gt->uc.guc;
303 	u32 max_slices = 0, max_subslices = 0;
304 	int ret;
305 
306 	/*
307 	 * Try to query the GuC's hwconfig table for the maximum number of
308 	 * slices and subslices.  These don't reflect the platform's actual
309 	 * slice/DSS counts, just the physical layout by which we should
310 	 * determine the steering targets.  On older platforms with older GuC
311 	 * firmware releases it's possible that these attributes may not be
312 	 * included in the table, so we can always fall back to the old
313 	 * hardcoded layouts.
314 	 */
315 #define HWCONFIG_ATTR_MAX_SLICES	1
316 #define HWCONFIG_ATTR_MAX_SUBSLICES	70
317 
318 	ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES,
319 					 &max_slices);
320 	if (ret < 0 || max_slices == 0)
321 		goto fallback;
322 
323 	ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES,
324 					 &max_subslices);
325 	if (ret < 0 || max_subslices == 0)
326 		goto fallback;
327 
328 	return DIV_ROUND_UP(max_subslices, max_slices);
329 
330 fallback:
331 	xe_gt_dbg(gt, "GuC hwconfig cannot provide dss/slice; using typical fallback values\n");
332 	if (gt_to_xe(gt)->info.platform == XE_PVC)
333 		return 8;
334 	else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250)
335 		return 4;
336 	else
337 		return 6;
338 }
339 
340 /**
341  * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS
342  * @gt: GT structure
343  * @dss: DSS ID to obtain steering for
344  * @group: pointer to storage for steering group ID
345  * @instance: pointer to storage for steering instance ID
346  */
347 void xe_gt_mcr_get_dss_steering(struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance)
348 {
349 	xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS);
350 
351 	*group = dss / gt->steering_dss_per_grp;
352 	*instance = dss % gt->steering_dss_per_grp;
353 }
354 
355 static void init_steering_dss(struct xe_gt *gt)
356 {
357 	gt->steering_dss_per_grp = dss_per_group(gt);
358 
359 	xe_gt_mcr_get_dss_steering(gt,
360 				   min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0),
361 				       xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)),
362 				   &gt->steering[DSS].group_target,
363 				   &gt->steering[DSS].instance_target);
364 }
365 
366 static void init_steering_oaddrm(struct xe_gt *gt)
367 {
368 	/*
369 	 * First instance is only terminated if the entire first media slice
370 	 * is absent (i.e., no VCS0 or VECS0).
371 	 */
372 	if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0))
373 		gt->steering[OADDRM].group_target = 0;
374 	else
375 		gt->steering[OADDRM].group_target = 1;
376 
377 	gt->steering[OADDRM].instance_target = 0;	/* unused */
378 }
379 
380 static void init_steering_sqidi_psmi(struct xe_gt *gt)
381 {
382 	u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK,
383 				 xe_mmio_read32(gt, MIRROR_FUSE3));
384 	u32 select = __ffs(mask);
385 
386 	gt->steering[SQIDI_PSMI].group_target = select >> 1;
387 	gt->steering[SQIDI_PSMI].instance_target = select & 0x1;
388 }
389 
390 static void init_steering_inst0(struct xe_gt *gt)
391 {
392 	gt->steering[INSTANCE0].group_target = 0;	/* unused */
393 	gt->steering[INSTANCE0].instance_target = 0;	/* unused */
394 }
395 
396 static const struct {
397 	const char *name;
398 	void (*init)(struct xe_gt *gt);
399 } xe_steering_types[] = {
400 	[L3BANK] =	{ "L3BANK",	init_steering_l3bank },
401 	[MSLICE] =	{ "MSLICE",	init_steering_mslice },
402 	[LNCF] =	{ "LNCF",	NULL }, /* initialized by mslice init */
403 	[DSS] =		{ "DSS",	init_steering_dss },
404 	[OADDRM] =	{ "OADDRM / GPMXMT", init_steering_oaddrm },
405 	[SQIDI_PSMI] =  { "SQIDI_PSMI", init_steering_sqidi_psmi },
406 	[INSTANCE0] =	{ "INSTANCE 0",	init_steering_inst0 },
407 	[IMPLICIT_STEERING] = { "IMPLICIT", NULL },
408 };
409 
410 /**
411  * xe_gt_mcr_init_early - Early initialization of the MCR support
412  * @gt: GT structure
413  *
414  * Perform early software only initialization of the MCR lock to allow
415  * the synchronization on accessing the STEER_SEMAPHORE register and
416  * use the xe_gt_mcr_multicast_write() function.
417  */
418 void xe_gt_mcr_init_early(struct xe_gt *gt)
419 {
420 	BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES);
421 	BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES);
422 
423 	spin_lock_init(&gt->mcr_lock);
424 }
425 
426 /**
427  * xe_gt_mcr_init - Normal initialization of the MCR support
428  * @gt: GT structure
429  *
430  * Perform normal initialization of the MCR for all usages.
431  */
432 void xe_gt_mcr_init(struct xe_gt *gt)
433 {
434 	struct xe_device *xe = gt_to_xe(gt);
435 
436 	if (IS_SRIOV_VF(xe))
437 		return;
438 
439 	if (gt->info.type == XE_GT_TYPE_MEDIA) {
440 		drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13);
441 
442 		if (MEDIA_VERx100(xe) >= 1301) {
443 			gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table;
444 			gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table;
445 		} else {
446 			gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table;
447 		}
448 	} else {
449 		if (GRAPHICS_VER(xe) >= 20) {
450 			gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
451 			gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table;
452 			gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table;
453 		} else if (GRAPHICS_VERx100(xe) >= 1270) {
454 			gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table;
455 			gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
456 			gt->steering[DSS].ranges = xelpg_dss_steering_table;
457 		} else if (xe->info.platform == XE_PVC) {
458 			gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table;
459 			gt->steering[DSS].ranges = xehpc_dss_steering_table;
460 		} else if (xe->info.platform == XE_DG2) {
461 			gt->steering[L3BANK].ranges = xehp_l3bank_steering_table;
462 			gt->steering[MSLICE].ranges = xehp_mslice_steering_table;
463 			gt->steering[LNCF].ranges = xehp_lncf_steering_table;
464 			gt->steering[DSS].ranges = xehp_dss_steering_table;
465 			gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table;
466 		} else {
467 			gt->steering[L3BANK].ranges = xelp_l3bank_steering_table;
468 			gt->steering[DSS].ranges = xelp_dss_steering_table;
469 		}
470 	}
471 
472 	/* Select non-terminated steering target for each type */
473 	for (int i = 0; i < NUM_STEERING_TYPES; i++)
474 		if (gt->steering[i].ranges && xe_steering_types[i].init)
475 			xe_steering_types[i].init(gt);
476 }
477 
478 /**
479  * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers
480  * @gt: GT structure
481  *
482  * Some register ranges don't need to have their steering control registers
483  * changed on each access - it's sufficient to set them once on initialization.
484  * This function sets those registers for each platform *
485  */
486 void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt)
487 {
488 	struct xe_device *xe = gt_to_xe(gt);
489 
490 	if (IS_SRIOV_VF(xe))
491 		return;
492 
493 	if (xe->info.platform == XE_DG2) {
494 		u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) |
495 			REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2);
496 
497 		xe_mmio_write32(gt, MCFG_MCR_SELECTOR, steer_val);
498 		xe_mmio_write32(gt, SF_MCR_SELECTOR, steer_val);
499 		/*
500 		 * For GAM registers, all reads should be directed to instance 1
501 		 * (unicast reads against other instances are not allowed),
502 		 * and instance 1 is already the hardware's default steering
503 		 * target, which we never change
504 		 */
505 	}
506 }
507 
508 /*
509  * xe_gt_mcr_get_nonterminated_steering - find group/instance values that
510  *    will steer a register to a non-terminated instance
511  * @gt: GT structure
512  * @reg: register for which the steering is required
513  * @group: return variable for group steering
514  * @instance: return variable for instance steering
515  *
516  * This function returns a group/instance pair that is guaranteed to work for
517  * read steering of the given register. Note that a value will be returned even
518  * if the register is not replicated and therefore does not actually require
519  * steering.
520  *
521  * Returns true if the caller should steer to the @group/@instance values
522  * returned.  Returns false if the caller need not perform any steering
523  */
524 static bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
525 						 struct xe_reg_mcr reg_mcr,
526 						 u8 *group, u8 *instance)
527 {
528 	const struct xe_reg reg = to_xe_reg(reg_mcr);
529 	const struct xe_mmio_range *implicit_ranges;
530 
531 	for (int type = 0; type < IMPLICIT_STEERING; type++) {
532 		if (!gt->steering[type].ranges)
533 			continue;
534 
535 		for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) {
536 			if (xe_mmio_in_range(gt, &gt->steering[type].ranges[i], reg)) {
537 				*group = gt->steering[type].group_target;
538 				*instance = gt->steering[type].instance_target;
539 				return true;
540 			}
541 		}
542 	}
543 
544 	implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges;
545 	if (implicit_ranges)
546 		for (int i = 0; implicit_ranges[i].end > 0; i++)
547 			if (xe_mmio_in_range(gt, &implicit_ranges[i], reg))
548 				return false;
549 
550 	/*
551 	 * Not found in a steering table and not a register with implicit
552 	 * steering. Just steer to 0/0 as a guess and raise a warning.
553 	 */
554 	drm_WARN(&gt_to_xe(gt)->drm, true,
555 		 "Did not find MCR register %#x in any MCR steering table\n",
556 		 reg.addr);
557 	*group = 0;
558 	*instance = 0;
559 
560 	return true;
561 }
562 
563 /*
564  * Obtain exclusive access to MCR steering.  On MTL and beyond we also need
565  * to synchronize with external clients (e.g., firmware), so a semaphore
566  * register will also need to be taken.
567  */
568 static void mcr_lock(struct xe_gt *gt) __acquires(&gt->mcr_lock)
569 {
570 	struct xe_device *xe = gt_to_xe(gt);
571 	int ret = 0;
572 
573 	spin_lock(&gt->mcr_lock);
574 
575 	/*
576 	 * Starting with MTL we also need to grab a semaphore register
577 	 * to synchronize with external agents (e.g., firmware) that now
578 	 * shares the same steering control register. The semaphore is obtained
579 	 * when a read to the relevant register returns 1.
580 	 */
581 	if (GRAPHICS_VERx100(xe) >= 1270)
582 		ret = xe_mmio_wait32(gt, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL,
583 				     true);
584 
585 	drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT);
586 }
587 
588 static void mcr_unlock(struct xe_gt *gt) __releases(&gt->mcr_lock)
589 {
590 	/* Release hardware semaphore - this is done by writing 1 to the register */
591 	if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
592 		xe_mmio_write32(gt, STEER_SEMAPHORE, 0x1);
593 
594 	spin_unlock(&gt->mcr_lock);
595 }
596 
597 /*
598  * Access a register with specific MCR steering
599  *
600  * Caller needs to make sure the relevant forcewake wells are up.
601  */
602 static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
603 				u8 rw_flag, int group, int instance, u32 value)
604 {
605 	const struct xe_reg reg = to_xe_reg(reg_mcr);
606 	struct xe_reg steer_reg;
607 	u32 steer_val, val = 0;
608 
609 	lockdep_assert_held(&gt->mcr_lock);
610 
611 	if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
612 		steer_reg = MTL_MCR_SELECTOR;
613 		steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
614 			REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance);
615 	} else {
616 		steer_reg = MCR_SELECTOR;
617 		steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) |
618 			REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance);
619 	}
620 
621 	/*
622 	 * Always leave the hardware in multicast mode when doing reads and only
623 	 * change it to unicast mode when doing writes of a specific instance.
624 	 *
625 	 * The setting of the multicast/unicast bit usually wouldn't matter for
626 	 * read operations (which always return the value from a single register
627 	 * instance regardless of how that bit is set), but some platforms may
628 	 * have workarounds requiring us to remain in multicast mode for reads,
629 	 * e.g. Wa_22013088509 on PVC.  There's no real downside to this, so
630 	 * we'll just go ahead and do so on all platforms; we'll only clear the
631 	 * multicast bit from the mask when explicitly doing a write operation.
632 	 *
633 	 * No need to save old steering reg value.
634 	 */
635 	if (rw_flag == MCR_OP_READ)
636 		steer_val |= MCR_MULTICAST;
637 
638 	xe_mmio_write32(gt, steer_reg, steer_val);
639 
640 	if (rw_flag == MCR_OP_READ)
641 		val = xe_mmio_read32(gt, reg);
642 	else
643 		xe_mmio_write32(gt, reg, value);
644 
645 	/*
646 	 * If we turned off the multicast bit (during a write) we're required
647 	 * to turn it back on before finishing.  The group and instance values
648 	 * don't matter since they'll be re-programmed on the next MCR
649 	 * operation.
650 	 */
651 	if (rw_flag == MCR_OP_WRITE)
652 		xe_mmio_write32(gt, steer_reg, MCR_MULTICAST);
653 
654 	return val;
655 }
656 
657 /**
658  * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register
659  * @gt: GT structure
660  * @reg_mcr: register to read
661  *
662  * Reads a GT MCR register.  The read will be steered to a non-terminated
663  * instance (i.e., one that isn't fused off or powered down by power gating).
664  * This function assumes the caller is already holding any necessary forcewake
665  * domains.
666  *
667  * Returns the value from a non-terminated instance of @reg.
668  */
669 u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr)
670 {
671 	const struct xe_reg reg = to_xe_reg(reg_mcr);
672 	u8 group, instance;
673 	u32 val;
674 	bool steer;
675 
676 	xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
677 
678 	steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr,
679 						     &group, &instance);
680 
681 	if (steer) {
682 		mcr_lock(gt);
683 		val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ,
684 					   group, instance, 0);
685 		mcr_unlock(gt);
686 	} else {
687 		val = xe_mmio_read32(gt, reg);
688 	}
689 
690 	return val;
691 }
692 
693 /**
694  * xe_gt_mcr_unicast_read - read a specific instance of an MCR register
695  * @gt: GT structure
696  * @reg_mcr: the MCR register to read
697  * @group: the MCR group
698  * @instance: the MCR instance
699  *
700  * Returns the value read from an MCR register after steering toward a specific
701  * group/instance.
702  */
703 u32 xe_gt_mcr_unicast_read(struct xe_gt *gt,
704 			   struct xe_reg_mcr reg_mcr,
705 			   int group, int instance)
706 {
707 	u32 val;
708 
709 	xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
710 
711 	mcr_lock(gt);
712 	val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0);
713 	mcr_unlock(gt);
714 
715 	return val;
716 }
717 
718 /**
719  * xe_gt_mcr_unicast_write - write a specific instance of an MCR register
720  * @gt: GT structure
721  * @reg_mcr: the MCR register to write
722  * @value: value to write
723  * @group: the MCR group
724  * @instance: the MCR instance
725  *
726  * Write an MCR register in unicast mode after steering toward a specific
727  * group/instance.
728  */
729 void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
730 			     u32 value, int group, int instance)
731 {
732 	xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
733 
734 	mcr_lock(gt);
735 	rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value);
736 	mcr_unlock(gt);
737 }
738 
739 /**
740  * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register
741  * @gt: GT structure
742  * @reg_mcr: the MCR register to write
743  * @value: value to write
744  *
745  * Write an MCR register in multicast mode to update all instances.
746  */
747 void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
748 			       u32 value)
749 {
750 	struct xe_reg reg = to_xe_reg(reg_mcr);
751 
752 	xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
753 
754 	/*
755 	 * Synchronize with any unicast operations.  Once we have exclusive
756 	 * access, the MULTICAST bit should already be set, so there's no need
757 	 * to touch the steering register.
758 	 */
759 	mcr_lock(gt);
760 	xe_mmio_write32(gt, reg, value);
761 	mcr_unlock(gt);
762 }
763 
764 void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p)
765 {
766 	for (int i = 0; i < NUM_STEERING_TYPES; i++) {
767 		if (gt->steering[i].ranges) {
768 			drm_printf(p, "%s steering: group=%#x, instance=%#x\n",
769 				   xe_steering_types[i].name,
770 				   gt->steering[i].group_target,
771 				   gt->steering[i].instance_target);
772 			for (int j = 0; gt->steering[i].ranges[j].end; j++)
773 				drm_printf(p, "\t0x%06x - 0x%06x\n",
774 					   gt->steering[i].ranges[j].start,
775 					   gt->steering[i].ranges[j].end);
776 		}
777 	}
778 }
779