1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_gt_mcr.h"
7
8 #include "regs/xe_gt_regs.h"
9 #include "xe_assert.h"
10 #include "xe_gt.h"
11 #include "xe_gt_printk.h"
12 #include "xe_gt_topology.h"
13 #include "xe_gt_types.h"
14 #include "xe_guc_hwconfig.h"
15 #include "xe_mmio.h"
16 #include "xe_sriov.h"
17
18 /**
19 * DOC: GT Multicast/Replicated (MCR) Register Support
20 *
21 * Some GT registers are designed as "multicast" or "replicated" registers:
22 * multiple instances of the same register share a single MMIO offset. MCR
23 * registers are generally used when the hardware needs to potentially track
24 * independent values of a register per hardware unit (e.g., per-subslice,
25 * per-L3bank, etc.). The specific types of replication that exist vary
26 * per-platform.
27 *
28 * MMIO accesses to MCR registers are controlled according to the settings
29 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR
30 * registers can be done in either multicast (a single write updates all
31 * instances of the register to the same value) or unicast (a write updates only
32 * one specific instance) form. Reads of MCR registers always operate in a
33 * unicast manner regardless of how the multicast/unicast bit is set in
34 * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is
35 * referred to as "steering."
36 *
37 * If MCR register operations are steered toward a hardware unit that is
38 * fused off or currently powered down due to power gating, the MMIO operation
39 * is "terminated" by the hardware. Terminated read operations will return a
40 * value of zero and terminated unicast write operations will be silently
41 * ignored. During device initialization, the goal of the various
42 * ``init_steering_*()`` functions is to apply the platform-specific rules for
43 * each MCR register type to identify a steering target that will select a
44 * non-terminated instance.
45 *
46 * MCR registers are not available on Virtual Function (VF).
47 */
48
49 #define STEER_SEMAPHORE XE_REG(0xFD0)
50
to_xe_reg(struct xe_reg_mcr reg_mcr)51 static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr)
52 {
53 return reg_mcr.__reg;
54 }
55
56 enum {
57 MCR_OP_READ,
58 MCR_OP_WRITE
59 };
60
61 static const struct xe_mmio_range xelp_l3bank_steering_table[] = {
62 { 0x00B100, 0x00B3FF },
63 {},
64 };
65
66 static const struct xe_mmio_range xehp_l3bank_steering_table[] = {
67 { 0x008C80, 0x008CFF },
68 { 0x00B100, 0x00B3FF },
69 {},
70 };
71
72 /*
73 * Although the bspec lists more "MSLICE" ranges than shown here, some of those
74 * are of a "GAM" subclass that has special rules and doesn't need to be
75 * included here.
76 */
77 static const struct xe_mmio_range xehp_mslice_steering_table[] = {
78 { 0x00DD00, 0x00DDFF },
79 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
80 {},
81 };
82
83 static const struct xe_mmio_range xehp_lncf_steering_table[] = {
84 { 0x00B000, 0x00B0FF },
85 { 0x00D880, 0x00D8FF },
86 {},
87 };
88
89 /*
90 * We have several types of MCR registers where steering to (0,0) will always
91 * provide us with a non-terminated value. We'll stick them all in the same
92 * table for simplicity.
93 */
94 static const struct xe_mmio_range xehpc_instance0_steering_table[] = {
95 { 0x004000, 0x004AFF }, /* HALF-BSLICE */
96 { 0x008800, 0x00887F }, /* CC */
97 { 0x008A80, 0x008AFF }, /* TILEPSMI */
98 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */
99 { 0x00B100, 0x00B3FF }, /* L3BANK */
100 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */
101 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */
102 { 0x00DD00, 0x00DDFF }, /* BSLICE */
103 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */
104 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */
105 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */
106 { 0x024180, 0x0241FF }, /* HALF-BSLICE */
107 {},
108 };
109
110 static const struct xe_mmio_range xelpg_instance0_steering_table[] = {
111 { 0x000B00, 0x000BFF }, /* SQIDI */
112 { 0x001000, 0x001FFF }, /* SQIDI */
113 { 0x004000, 0x0048FF }, /* GAM */
114 { 0x008700, 0x0087FF }, /* SQIDI */
115 { 0x00B000, 0x00B0FF }, /* NODE */
116 { 0x00C800, 0x00CFFF }, /* GAM */
117 { 0x00D880, 0x00D8FF }, /* NODE */
118 { 0x00DD00, 0x00DDFF }, /* OAAL2 */
119 {},
120 };
121
122 static const struct xe_mmio_range xelpg_l3bank_steering_table[] = {
123 { 0x00B100, 0x00B3FF },
124 {},
125 };
126
127 static const struct xe_mmio_range xelp_dss_steering_table[] = {
128 { 0x008150, 0x00815F },
129 { 0x009520, 0x00955F },
130 { 0x00DE80, 0x00E8FF },
131 { 0x024A00, 0x024A7F },
132 {},
133 };
134
135 /* DSS steering is used for GSLICE ranges as well */
136 static const struct xe_mmio_range xehp_dss_steering_table[] = {
137 { 0x005200, 0x0052FF }, /* GSLICE */
138 { 0x005400, 0x007FFF }, /* GSLICE */
139 { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
140 { 0x008D00, 0x008DFF }, /* DSS */
141 { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
142 { 0x009680, 0x0096FF }, /* DSS */
143 { 0x00D800, 0x00D87F }, /* GSLICE */
144 { 0x00DC00, 0x00DCFF }, /* GSLICE */
145 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */
146 { 0x017000, 0x017FFF }, /* GSLICE */
147 { 0x024A00, 0x024A7F }, /* DSS */
148 {},
149 };
150
151 /* DSS steering is used for COMPUTE ranges as well */
152 static const struct xe_mmio_range xehpc_dss_steering_table[] = {
153 { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */
154 { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */
155 { 0x009680, 0x0096FF }, /* DSS */
156 { 0x00DC00, 0x00DCFF }, /* COMPUTE */
157 { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */
158 {},
159 };
160
161 /* DSS steering is used for SLICE ranges as well */
162 static const struct xe_mmio_range xelpg_dss_steering_table[] = {
163 { 0x005200, 0x0052FF }, /* SLICE */
164 { 0x005500, 0x007FFF }, /* SLICE */
165 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
166 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
167 { 0x009680, 0x0096FF }, /* DSS */
168 { 0x00D800, 0x00D87F }, /* SLICE */
169 { 0x00DC00, 0x00DCFF }, /* SLICE */
170 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
171 {},
172 };
173
174 static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = {
175 { 0x393200, 0x39323F },
176 { 0x393400, 0x3934FF },
177 {},
178 };
179
180 static const struct xe_mmio_range dg2_implicit_steering_table[] = {
181 { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */
182 { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */
183 { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */
184 { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */
185 { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */
186 { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */
187 {},
188 };
189
190 static const struct xe_mmio_range xe2lpg_dss_steering_table[] = {
191 { 0x005200, 0x0052FF }, /* SLICE */
192 { 0x005500, 0x007FFF }, /* SLICE */
193 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
194 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
195 { 0x009680, 0x0096FF }, /* DSS */
196 { 0x00D800, 0x00D87F }, /* SLICE */
197 { 0x00DC00, 0x00DCFF }, /* SLICE */
198 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
199 { 0x00E980, 0x00E9FF }, /* SLICE */
200 { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */
201 {},
202 };
203
204 static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = {
205 { 0x000B00, 0x000BFF },
206 { 0x001000, 0x001FFF },
207 {},
208 };
209
210 static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = {
211 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */
212 { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */
213 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */
214 { 0x00C800, 0x00CFFF }, /* GAM */
215 { 0x00D880, 0x00D8FF }, /* NODE */
216 { 0x00DD00, 0x00DDFF }, /* MEMPIPE */
217 { 0x00E900, 0x00E97F }, /* MEMPIPE */
218 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */
219 { 0x013400, 0x0135FF }, /* MEMPIPE */
220 {},
221 };
222
223 static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = {
224 { 0x388160, 0x38817F },
225 { 0x389480, 0x3894CF },
226 {},
227 };
228
229 static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = {
230 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */
231 { 0x384900, 0x384AFF }, /* GAM */
232 { 0x389560, 0x3895FF }, /* MEDIAINF */
233 { 0x38B600, 0x38B8FF }, /* L3BANK */
234 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */
235 { 0x38F000, 0x38F0FF }, /* GAM */
236 { 0x393C00, 0x393C7F }, /* MEDIAINF */
237 {},
238 };
239
240 static const struct xe_mmio_range xe3lpm_instance0_steering_table[] = {
241 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */
242 { 0x384900, 0x384AFF }, /* GAM */
243 { 0x389560, 0x3895FF }, /* MEDIAINF */
244 { 0x38B600, 0x38B8FF }, /* L3BANK */
245 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */
246 { 0x38D0D0, 0x38F0FF }, /* MEDIAINF, GAM */
247 { 0x393C00, 0x393C7F }, /* MEDIAINF */
248 {},
249 };
250
init_steering_l3bank(struct xe_gt * gt)251 static void init_steering_l3bank(struct xe_gt *gt)
252 {
253 struct xe_mmio *mmio = >->mmio;
254
255 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
256 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,
257 xe_mmio_read32(mmio, MIRROR_FUSE3));
258 u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK,
259 xe_mmio_read32(mmio, XEHP_FUSE4));
260
261 /*
262 * Group selects mslice, instance selects bank within mslice.
263 * Bank 0 is always valid _except_ when the bank mask is 010b.
264 */
265 gt->steering[L3BANK].group_target = __ffs(mslice_mask);
266 gt->steering[L3BANK].instance_target =
267 bank_mask & BIT(0) ? 0 : 2;
268 } else if (gt_to_xe(gt)->info.platform == XE_DG2) {
269 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,
270 xe_mmio_read32(mmio, MIRROR_FUSE3));
271 u32 bank = __ffs(mslice_mask) * 8;
272
273 /*
274 * Like mslice registers, look for a valid mslice and steer to
275 * the first L3BANK of that quad. Access to the Nth L3 bank is
276 * split between the first bits of group and instance
277 */
278 gt->steering[L3BANK].group_target = (bank >> 2) & 0x7;
279 gt->steering[L3BANK].instance_target = bank & 0x3;
280 } else {
281 u32 fuse = REG_FIELD_GET(L3BANK_MASK,
282 ~xe_mmio_read32(mmio, MIRROR_FUSE3));
283
284 gt->steering[L3BANK].group_target = 0; /* unused */
285 gt->steering[L3BANK].instance_target = __ffs(fuse);
286 }
287 }
288
init_steering_mslice(struct xe_gt * gt)289 static void init_steering_mslice(struct xe_gt *gt)
290 {
291 u32 mask = REG_FIELD_GET(MEML3_EN_MASK,
292 xe_mmio_read32(>->mmio, MIRROR_FUSE3));
293
294 /*
295 * mslice registers are valid (not terminated) if either the meml3
296 * associated with the mslice is present, or at least one DSS associated
297 * with the mslice is present. There will always be at least one meml3
298 * so we can just use that to find a non-terminated mslice and ignore
299 * the DSS fusing.
300 */
301 gt->steering[MSLICE].group_target = __ffs(mask);
302 gt->steering[MSLICE].instance_target = 0; /* unused */
303
304 /*
305 * LNCF termination is also based on mslice presence, so we'll set
306 * it up here. Either LNCF within a non-terminated mslice will work,
307 * so we just always pick LNCF 0 here.
308 */
309 gt->steering[LNCF].group_target = __ffs(mask) << 1;
310 gt->steering[LNCF].instance_target = 0; /* unused */
311 }
312
dss_per_group(struct xe_gt * gt)313 static unsigned int dss_per_group(struct xe_gt *gt)
314 {
315 struct xe_guc *guc = >->uc.guc;
316 u32 max_slices = 0, max_subslices = 0;
317 int ret;
318
319 /*
320 * Try to query the GuC's hwconfig table for the maximum number of
321 * slices and subslices. These don't reflect the platform's actual
322 * slice/DSS counts, just the physical layout by which we should
323 * determine the steering targets. On older platforms with older GuC
324 * firmware releases it's possible that these attributes may not be
325 * included in the table, so we can always fall back to the old
326 * hardcoded layouts.
327 */
328 #define HWCONFIG_ATTR_MAX_SLICES 1
329 #define HWCONFIG_ATTR_MAX_SUBSLICES 70
330
331 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES,
332 &max_slices);
333 if (ret < 0 || max_slices == 0)
334 goto fallback;
335
336 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES,
337 &max_subslices);
338 if (ret < 0 || max_subslices == 0)
339 goto fallback;
340
341 return DIV_ROUND_UP(max_subslices, max_slices);
342
343 fallback:
344 /*
345 * Some older platforms don't have tables or don't have complete tables.
346 * Newer platforms should always have the required info.
347 */
348 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000 &&
349 !gt_to_xe(gt)->info.force_execlist)
350 xe_gt_err(gt, "Slice/Subslice counts missing from hwconfig table; using typical fallback values\n");
351
352 if (gt_to_xe(gt)->info.platform == XE_PVC)
353 return 8;
354 else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250)
355 return 4;
356 else
357 return 6;
358 }
359
360 /**
361 * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS
362 * @gt: GT structure
363 * @dss: DSS ID to obtain steering for
364 * @group: pointer to storage for steering group ID
365 * @instance: pointer to storage for steering instance ID
366 */
xe_gt_mcr_get_dss_steering(struct xe_gt * gt,unsigned int dss,u16 * group,u16 * instance)367 void xe_gt_mcr_get_dss_steering(struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance)
368 {
369 xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS);
370
371 *group = dss / gt->steering_dss_per_grp;
372 *instance = dss % gt->steering_dss_per_grp;
373 }
374
375 /**
376 * xe_gt_mcr_steering_info_to_dss_id - Get DSS ID from group/instance steering
377 * @gt: GT structure
378 * @group: steering group ID
379 * @instance: steering instance ID
380 *
381 * Return: the converted DSS id.
382 */
xe_gt_mcr_steering_info_to_dss_id(struct xe_gt * gt,u16 group,u16 instance)383 u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance)
384 {
385 return group * dss_per_group(gt) + instance;
386 }
387
init_steering_dss(struct xe_gt * gt)388 static void init_steering_dss(struct xe_gt *gt)
389 {
390 gt->steering_dss_per_grp = dss_per_group(gt);
391
392 xe_gt_mcr_get_dss_steering(gt,
393 min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0),
394 xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)),
395 >->steering[DSS].group_target,
396 >->steering[DSS].instance_target);
397 }
398
init_steering_oaddrm(struct xe_gt * gt)399 static void init_steering_oaddrm(struct xe_gt *gt)
400 {
401 /*
402 * First instance is only terminated if the entire first media slice
403 * is absent (i.e., no VCS0 or VECS0).
404 */
405 if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0))
406 gt->steering[OADDRM].group_target = 0;
407 else
408 gt->steering[OADDRM].group_target = 1;
409
410 gt->steering[OADDRM].instance_target = 0; /* unused */
411 }
412
init_steering_sqidi_psmi(struct xe_gt * gt)413 static void init_steering_sqidi_psmi(struct xe_gt *gt)
414 {
415 u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK,
416 xe_mmio_read32(>->mmio, MIRROR_FUSE3));
417 u32 select = __ffs(mask);
418
419 gt->steering[SQIDI_PSMI].group_target = select >> 1;
420 gt->steering[SQIDI_PSMI].instance_target = select & 0x1;
421 }
422
423 static const struct {
424 const char *name;
425 void (*init)(struct xe_gt *gt);
426 } xe_steering_types[] = {
427 [L3BANK] = { "L3BANK", init_steering_l3bank },
428 [MSLICE] = { "MSLICE", init_steering_mslice },
429 [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */
430 [DSS] = { "DSS", init_steering_dss },
431 [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm },
432 [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi },
433 [INSTANCE0] = { "INSTANCE 0", NULL },
434 [IMPLICIT_STEERING] = { "IMPLICIT", NULL },
435 };
436
437 /**
438 * xe_gt_mcr_init_early - Early initialization of the MCR support
439 * @gt: GT structure
440 *
441 * Perform early software only initialization of the MCR lock to allow
442 * the synchronization on accessing the STEER_SEMAPHORE register and
443 * use the xe_gt_mcr_multicast_write() function, plus the minimum
444 * safe MCR registers required for VRAM/CCS probing.
445 */
xe_gt_mcr_init_early(struct xe_gt * gt)446 void xe_gt_mcr_init_early(struct xe_gt *gt)
447 {
448 struct xe_device *xe = gt_to_xe(gt);
449
450 BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES);
451 BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES);
452
453 spin_lock_init(>->mcr_lock);
454
455 if (IS_SRIOV_VF(xe))
456 return;
457
458 if (gt->info.type == XE_GT_TYPE_MEDIA) {
459 drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13);
460
461 if (MEDIA_VER(xe) >= 30) {
462 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table;
463 gt->steering[INSTANCE0].ranges = xe3lpm_instance0_steering_table;
464 } else if (MEDIA_VERx100(xe) >= 1301) {
465 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table;
466 gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table;
467 } else {
468 gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table;
469 }
470 } else {
471 if (GRAPHICS_VER(xe) >= 20) {
472 gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
473 gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table;
474 gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table;
475 } else if (GRAPHICS_VERx100(xe) >= 1270) {
476 gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table;
477 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
478 gt->steering[DSS].ranges = xelpg_dss_steering_table;
479 } else if (xe->info.platform == XE_PVC) {
480 gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table;
481 gt->steering[DSS].ranges = xehpc_dss_steering_table;
482 } else if (xe->info.platform == XE_DG2) {
483 gt->steering[L3BANK].ranges = xehp_l3bank_steering_table;
484 gt->steering[MSLICE].ranges = xehp_mslice_steering_table;
485 gt->steering[LNCF].ranges = xehp_lncf_steering_table;
486 gt->steering[DSS].ranges = xehp_dss_steering_table;
487 gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table;
488 } else {
489 gt->steering[L3BANK].ranges = xelp_l3bank_steering_table;
490 gt->steering[DSS].ranges = xelp_dss_steering_table;
491 }
492 }
493
494 /* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */
495 gt->steering[INSTANCE0].initialized = true;
496 }
497
498 /**
499 * xe_gt_mcr_init - Normal initialization of the MCR support
500 * @gt: GT structure
501 *
502 * Perform normal initialization of the MCR for all usages.
503 */
xe_gt_mcr_init(struct xe_gt * gt)504 void xe_gt_mcr_init(struct xe_gt *gt)
505 {
506 if (IS_SRIOV_VF(gt_to_xe(gt)))
507 return;
508
509 /* Select non-terminated steering target for each type */
510 for (int i = 0; i < NUM_STEERING_TYPES; i++) {
511 gt->steering[i].initialized = true;
512 if (gt->steering[i].ranges && xe_steering_types[i].init)
513 xe_steering_types[i].init(gt);
514 }
515 }
516
517 /**
518 * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers
519 * @gt: GT structure
520 *
521 * Some register ranges don't need to have their steering control registers
522 * changed on each access - it's sufficient to set them once on initialization.
523 * This function sets those registers for each platform *
524 */
xe_gt_mcr_set_implicit_defaults(struct xe_gt * gt)525 void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt)
526 {
527 struct xe_device *xe = gt_to_xe(gt);
528
529 if (IS_SRIOV_VF(xe))
530 return;
531
532 if (xe->info.platform == XE_DG2) {
533 u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) |
534 REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2);
535
536 xe_mmio_write32(>->mmio, MCFG_MCR_SELECTOR, steer_val);
537 xe_mmio_write32(>->mmio, SF_MCR_SELECTOR, steer_val);
538 /*
539 * For GAM registers, all reads should be directed to instance 1
540 * (unicast reads against other instances are not allowed),
541 * and instance 1 is already the hardware's default steering
542 * target, which we never change
543 */
544 }
545 }
546
547 /*
548 * xe_gt_mcr_get_nonterminated_steering - find group/instance values that
549 * will steer a register to a non-terminated instance
550 * @gt: GT structure
551 * @reg: register for which the steering is required
552 * @group: return variable for group steering
553 * @instance: return variable for instance steering
554 *
555 * This function returns a group/instance pair that is guaranteed to work for
556 * read steering of the given register. Note that a value will be returned even
557 * if the register is not replicated and therefore does not actually require
558 * steering.
559 *
560 * Returns true if the caller should steer to the @group/@instance values
561 * returned. Returns false if the caller need not perform any steering
562 */
xe_gt_mcr_get_nonterminated_steering(struct xe_gt * gt,struct xe_reg_mcr reg_mcr,u8 * group,u8 * instance)563 bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
564 struct xe_reg_mcr reg_mcr,
565 u8 *group, u8 *instance)
566 {
567 const struct xe_reg reg = to_xe_reg(reg_mcr);
568 const struct xe_mmio_range *implicit_ranges;
569
570 for (int type = 0; type < IMPLICIT_STEERING; type++) {
571 if (!gt->steering[type].ranges)
572 continue;
573
574 for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) {
575 if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) {
576 drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized,
577 "Uninitialized usage of MCR register %s/%#x\n",
578 xe_steering_types[type].name, reg.addr);
579
580 *group = gt->steering[type].group_target;
581 *instance = gt->steering[type].instance_target;
582 return true;
583 }
584 }
585 }
586
587 implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges;
588 if (implicit_ranges)
589 for (int i = 0; implicit_ranges[i].end > 0; i++)
590 if (xe_mmio_in_range(>->mmio, &implicit_ranges[i], reg))
591 return false;
592
593 /*
594 * Not found in a steering table and not a register with implicit
595 * steering. Just steer to 0/0 as a guess and raise a warning.
596 */
597 drm_WARN(>_to_xe(gt)->drm, true,
598 "Did not find MCR register %#x in any MCR steering table\n",
599 reg.addr);
600 *group = 0;
601 *instance = 0;
602
603 return true;
604 }
605
606 /*
607 * Obtain exclusive access to MCR steering. On MTL and beyond we also need
608 * to synchronize with external clients (e.g., firmware), so a semaphore
609 * register will also need to be taken.
610 */
mcr_lock(struct xe_gt * gt)611 static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock)
612 {
613 struct xe_device *xe = gt_to_xe(gt);
614 int ret = 0;
615
616 spin_lock(>->mcr_lock);
617
618 /*
619 * Starting with MTL we also need to grab a semaphore register
620 * to synchronize with external agents (e.g., firmware) that now
621 * shares the same steering control register. The semaphore is obtained
622 * when a read to the relevant register returns 1.
623 */
624 if (GRAPHICS_VERx100(xe) >= 1270)
625 ret = xe_mmio_wait32(>->mmio, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL,
626 true);
627
628 drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT);
629 }
630
mcr_unlock(struct xe_gt * gt)631 static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock)
632 {
633 /* Release hardware semaphore - this is done by writing 1 to the register */
634 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
635 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, 0x1);
636
637 spin_unlock(>->mcr_lock);
638 }
639
640 /*
641 * Access a register with specific MCR steering
642 *
643 * Caller needs to make sure the relevant forcewake wells are up.
644 */
rw_with_mcr_steering(struct xe_gt * gt,struct xe_reg_mcr reg_mcr,u8 rw_flag,int group,int instance,u32 value)645 static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
646 u8 rw_flag, int group, int instance, u32 value)
647 {
648 const struct xe_reg reg = to_xe_reg(reg_mcr);
649 struct xe_mmio *mmio = >->mmio;
650 struct xe_reg steer_reg;
651 u32 steer_val, val = 0;
652
653 lockdep_assert_held(>->mcr_lock);
654
655 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
656 steer_reg = MTL_MCR_SELECTOR;
657 steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
658 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance);
659 } else {
660 steer_reg = MCR_SELECTOR;
661 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) |
662 REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance);
663 }
664
665 /*
666 * Always leave the hardware in multicast mode when doing reads and only
667 * change it to unicast mode when doing writes of a specific instance.
668 *
669 * The setting of the multicast/unicast bit usually wouldn't matter for
670 * read operations (which always return the value from a single register
671 * instance regardless of how that bit is set), but some platforms may
672 * have workarounds requiring us to remain in multicast mode for reads,
673 * e.g. Wa_22013088509 on PVC. There's no real downside to this, so
674 * we'll just go ahead and do so on all platforms; we'll only clear the
675 * multicast bit from the mask when explicitly doing a write operation.
676 *
677 * No need to save old steering reg value.
678 */
679 if (rw_flag == MCR_OP_READ)
680 steer_val |= MCR_MULTICAST;
681
682 xe_mmio_write32(mmio, steer_reg, steer_val);
683
684 if (rw_flag == MCR_OP_READ)
685 val = xe_mmio_read32(mmio, reg);
686 else
687 xe_mmio_write32(mmio, reg, value);
688
689 /*
690 * If we turned off the multicast bit (during a write) we're required
691 * to turn it back on before finishing. The group and instance values
692 * don't matter since they'll be re-programmed on the next MCR
693 * operation.
694 */
695 if (rw_flag == MCR_OP_WRITE)
696 xe_mmio_write32(mmio, steer_reg, MCR_MULTICAST);
697
698 return val;
699 }
700
701 /**
702 * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register
703 * @gt: GT structure
704 * @reg_mcr: register to read
705 *
706 * Reads a GT MCR register. The read will be steered to a non-terminated
707 * instance (i.e., one that isn't fused off or powered down by power gating).
708 * This function assumes the caller is already holding any necessary forcewake
709 * domains.
710 *
711 * Returns the value from a non-terminated instance of @reg.
712 */
xe_gt_mcr_unicast_read_any(struct xe_gt * gt,struct xe_reg_mcr reg_mcr)713 u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr)
714 {
715 const struct xe_reg reg = to_xe_reg(reg_mcr);
716 u8 group, instance;
717 u32 val;
718 bool steer;
719
720 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
721
722 steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr,
723 &group, &instance);
724
725 if (steer) {
726 mcr_lock(gt);
727 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ,
728 group, instance, 0);
729 mcr_unlock(gt);
730 } else {
731 val = xe_mmio_read32(>->mmio, reg);
732 }
733
734 return val;
735 }
736
737 /**
738 * xe_gt_mcr_unicast_read - read a specific instance of an MCR register
739 * @gt: GT structure
740 * @reg_mcr: the MCR register to read
741 * @group: the MCR group
742 * @instance: the MCR instance
743 *
744 * Returns the value read from an MCR register after steering toward a specific
745 * group/instance.
746 */
xe_gt_mcr_unicast_read(struct xe_gt * gt,struct xe_reg_mcr reg_mcr,int group,int instance)747 u32 xe_gt_mcr_unicast_read(struct xe_gt *gt,
748 struct xe_reg_mcr reg_mcr,
749 int group, int instance)
750 {
751 u32 val;
752
753 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
754
755 mcr_lock(gt);
756 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0);
757 mcr_unlock(gt);
758
759 return val;
760 }
761
762 /**
763 * xe_gt_mcr_unicast_write - write a specific instance of an MCR register
764 * @gt: GT structure
765 * @reg_mcr: the MCR register to write
766 * @value: value to write
767 * @group: the MCR group
768 * @instance: the MCR instance
769 *
770 * Write an MCR register in unicast mode after steering toward a specific
771 * group/instance.
772 */
xe_gt_mcr_unicast_write(struct xe_gt * gt,struct xe_reg_mcr reg_mcr,u32 value,int group,int instance)773 void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
774 u32 value, int group, int instance)
775 {
776 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
777
778 mcr_lock(gt);
779 rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value);
780 mcr_unlock(gt);
781 }
782
783 /**
784 * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register
785 * @gt: GT structure
786 * @reg_mcr: the MCR register to write
787 * @value: value to write
788 *
789 * Write an MCR register in multicast mode to update all instances.
790 */
xe_gt_mcr_multicast_write(struct xe_gt * gt,struct xe_reg_mcr reg_mcr,u32 value)791 void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
792 u32 value)
793 {
794 struct xe_reg reg = to_xe_reg(reg_mcr);
795
796 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
797
798 /*
799 * Synchronize with any unicast operations. Once we have exclusive
800 * access, the MULTICAST bit should already be set, so there's no need
801 * to touch the steering register.
802 */
803 mcr_lock(gt);
804 xe_mmio_write32(>->mmio, reg, value);
805 mcr_unlock(gt);
806 }
807
xe_gt_mcr_steering_dump(struct xe_gt * gt,struct drm_printer * p)808 void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p)
809 {
810 for (int i = 0; i < NUM_STEERING_TYPES; i++) {
811 if (gt->steering[i].ranges) {
812 drm_printf(p, "%s steering: group=%#x, instance=%#x\n",
813 xe_steering_types[i].name,
814 gt->steering[i].group_target,
815 gt->steering[i].instance_target);
816 for (int j = 0; gt->steering[i].ranges[j].end; j++)
817 drm_printf(p, "\t0x%06x - 0x%06x\n",
818 gt->steering[i].ranges[j].start,
819 gt->steering[i].ranges[j].end);
820 }
821 }
822 }
823