1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include <kunit/static_stub.h> 7 #include <kunit/visibility.h> 8 9 #include "xe_gt_mcr.h" 10 11 #include "regs/xe_gt_regs.h" 12 #include "xe_assert.h" 13 #include "xe_gt_printk.h" 14 #include "xe_gt_topology.h" 15 #include "xe_gt_types.h" 16 #include "xe_guc_hwconfig.h" 17 #include "xe_mmio.h" 18 #include "xe_sriov.h" 19 20 /** 21 * DOC: GT Multicast/Replicated (MCR) Register Support 22 * 23 * Some GT registers are designed as "multicast" or "replicated" registers: 24 * multiple instances of the same register share a single MMIO offset. MCR 25 * registers are generally used when the hardware needs to potentially track 26 * independent values of a register per hardware unit (e.g., per-subslice, 27 * per-L3bank, etc.). The specific types of replication that exist vary 28 * per-platform. 29 * 30 * MMIO accesses to MCR registers are controlled according to the settings 31 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR 32 * registers can be done in either multicast (a single write updates all 33 * instances of the register to the same value) or unicast (a write updates only 34 * one specific instance) form. Reads of MCR registers always operate in a 35 * unicast manner regardless of how the multicast/unicast bit is set in 36 * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is 37 * referred to as "steering." 38 * 39 * If MCR register operations are steered toward a hardware unit that is 40 * fused off or currently powered down due to power gating, the MMIO operation 41 * is "terminated" by the hardware. Terminated read operations will return a 42 * value of zero and terminated unicast write operations will be silently 43 * ignored. During device initialization, the goal of the various 44 * ``init_steering_*()`` functions is to apply the platform-specific rules for 45 * each MCR register type to identify a steering target that will select a 46 * non-terminated instance. 47 * 48 * MCR registers are not available on Virtual Function (VF). 49 */ 50 51 static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr) 52 { 53 return reg_mcr.__reg; 54 } 55 56 enum { 57 MCR_OP_READ, 58 MCR_OP_WRITE 59 }; 60 61 static const struct xe_mmio_range xelp_l3bank_steering_table[] = { 62 { 0x00B100, 0x00B3FF }, 63 {}, 64 }; 65 66 static const struct xe_mmio_range xehp_l3bank_steering_table[] = { 67 { 0x008C80, 0x008CFF }, 68 { 0x00B100, 0x00B3FF }, 69 {}, 70 }; 71 72 /* 73 * Although the bspec lists more "MSLICE" ranges than shown here, some of those 74 * are of a "GAM" subclass that has special rules and doesn't need to be 75 * included here. 76 */ 77 static const struct xe_mmio_range xehp_mslice_steering_table[] = { 78 { 0x00DD00, 0x00DDFF }, 79 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ 80 {}, 81 }; 82 83 static const struct xe_mmio_range xehp_lncf_steering_table[] = { 84 { 0x00B000, 0x00B0FF }, 85 { 0x00D880, 0x00D8FF }, 86 {}, 87 }; 88 89 /* 90 * We have several types of MCR registers where steering to (0,0) will always 91 * provide us with a non-terminated value. We'll stick them all in the same 92 * table for simplicity. 93 */ 94 static const struct xe_mmio_range xehpc_instance0_steering_table[] = { 95 { 0x004000, 0x004AFF }, /* HALF-BSLICE */ 96 { 0x008800, 0x00887F }, /* CC */ 97 { 0x008A80, 0x008AFF }, /* TILEPSMI */ 98 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ 99 { 0x00B100, 0x00B3FF }, /* L3BANK */ 100 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ 101 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ 102 { 0x00DD00, 0x00DDFF }, /* BSLICE */ 103 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ 104 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ 105 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ 106 { 0x024180, 0x0241FF }, /* HALF-BSLICE */ 107 {}, 108 }; 109 110 static const struct xe_mmio_range xelpg_instance0_steering_table[] = { 111 { 0x000B00, 0x000BFF }, /* SQIDI */ 112 { 0x001000, 0x001FFF }, /* SQIDI */ 113 { 0x004000, 0x0048FF }, /* GAM */ 114 { 0x008700, 0x0087FF }, /* SQIDI */ 115 { 0x00B000, 0x00B0FF }, /* NODE */ 116 { 0x00C800, 0x00CFFF }, /* GAM */ 117 { 0x00D880, 0x00D8FF }, /* NODE */ 118 { 0x00DD00, 0x00DDFF }, /* OAAL2 */ 119 {}, 120 }; 121 122 static const struct xe_mmio_range xelpg_l3bank_steering_table[] = { 123 { 0x00B100, 0x00B3FF }, 124 {}, 125 }; 126 127 static const struct xe_mmio_range xelp_dss_steering_table[] = { 128 { 0x008150, 0x00815F }, 129 { 0x009520, 0x00955F }, 130 { 0x00DE80, 0x00E8FF }, 131 { 0x024A00, 0x024A7F }, 132 {}, 133 }; 134 135 /* DSS steering is used for GSLICE ranges as well */ 136 static const struct xe_mmio_range xehp_dss_steering_table[] = { 137 { 0x005200, 0x0052FF }, /* GSLICE */ 138 { 0x005400, 0x007FFF }, /* GSLICE */ 139 { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 140 { 0x008D00, 0x008DFF }, /* DSS */ 141 { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 142 { 0x009680, 0x0096FF }, /* DSS */ 143 { 0x00D800, 0x00D87F }, /* GSLICE */ 144 { 0x00DC00, 0x00DCFF }, /* GSLICE */ 145 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */ 146 { 0x017000, 0x017FFF }, /* GSLICE */ 147 { 0x024A00, 0x024A7F }, /* DSS */ 148 {}, 149 }; 150 151 /* DSS steering is used for COMPUTE ranges as well */ 152 static const struct xe_mmio_range xehpc_dss_steering_table[] = { 153 { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */ 154 { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 155 { 0x009680, 0x0096FF }, /* DSS */ 156 { 0x00DC00, 0x00DCFF }, /* COMPUTE */ 157 { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */ 158 {}, 159 }; 160 161 /* DSS steering is used for SLICE ranges as well */ 162 static const struct xe_mmio_range xelpg_dss_steering_table[] = { 163 { 0x005200, 0x0052FF }, /* SLICE */ 164 { 0x005500, 0x007FFF }, /* SLICE */ 165 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 166 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 167 { 0x009680, 0x0096FF }, /* DSS */ 168 { 0x00D800, 0x00D87F }, /* SLICE */ 169 { 0x00DC00, 0x00DCFF }, /* SLICE */ 170 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 171 {}, 172 }; 173 174 static const struct xe_mmio_range xe3p_xpc_xecore_steering_table[] = { 175 { 0x008140, 0x00817F }, /* SLICE, XeCore, SLICE */ 176 { 0x009480, 0x00955F }, /* SLICE, XeCore */ 177 { 0x00D800, 0x00D87F }, /* SLICE */ 178 { 0x00DC00, 0x00E9FF }, /* SLICE, rsvd, XeCore, rsvd, XeCore, rsvd, XeCore */ 179 { 0x013000, 0x0135FF }, /* XeCore, SLICE */ 180 {}, 181 }; 182 183 static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = { 184 { 0x393200, 0x39323F }, 185 { 0x393400, 0x3934FF }, 186 {}, 187 }; 188 189 static const struct xe_mmio_range dg2_implicit_steering_table[] = { 190 { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */ 191 { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */ 192 { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */ 193 { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */ 194 { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */ 195 { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */ 196 {}, 197 }; 198 199 static const struct xe_mmio_range xe2lpg_dss_steering_table[] = { 200 { 0x005200, 0x0052FF }, /* SLICE */ 201 { 0x005500, 0x007FFF }, /* SLICE */ 202 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 203 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 204 { 0x009680, 0x0096FF }, /* DSS */ 205 { 0x00D800, 0x00D87F }, /* SLICE */ 206 { 0x00DC00, 0x00DCFF }, /* SLICE */ 207 { 0x00DE00, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 208 { 0x00E980, 0x00E9FF }, /* SLICE */ 209 { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */ 210 {}, 211 }; 212 213 static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = { 214 { 0x000B00, 0x000BFF }, 215 { 0x001000, 0x001FFF }, 216 {}, 217 }; 218 219 static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = { 220 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ 221 { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */ 222 { 0x00C800, 0x00CFFF }, /* GAM */ 223 { 0x00DD00, 0x00DDFF }, /* MEMPIPE */ 224 { 0x00E900, 0x00E97F }, /* MEMPIPE */ 225 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ 226 { 0x013400, 0x0135FF }, /* MEMPIPE */ 227 {}, 228 }; 229 230 static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = { 231 { 0x388160, 0x38817F }, 232 { 0x389480, 0x3894CF }, 233 {}, 234 }; 235 236 static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = { 237 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */ 238 { 0x384900, 0x384AFF }, /* GAM */ 239 { 0x389560, 0x3895FF }, /* MEDIAINF */ 240 { 0x38B600, 0x38B8FF }, /* L3BANK */ 241 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ 242 { 0x38F000, 0x38F0FF }, /* GAM */ 243 { 0x393C00, 0x393C7F }, /* MEDIAINF */ 244 {}, 245 }; 246 247 static const struct xe_mmio_range xe3lpm_instance0_steering_table[] = { 248 { 0x384000, 0x3841FF }, /* GAM */ 249 { 0x384400, 0x3847DF }, /* GAM */ 250 { 0x384900, 0x384AFF }, /* GAM */ 251 { 0x389560, 0x3895FF }, /* MEDIAINF */ 252 { 0x38B600, 0x38B8FF }, /* L3BANK */ 253 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ 254 { 0x38D0D0, 0x38F0FF }, /* MEDIAINF, rsvd, GAM */ 255 { 0x393C00, 0x393C7F }, /* MEDIAINF */ 256 {}, 257 }; 258 259 /* 260 * Different "GAM" ranges have different rules; GAMWKRS, STLB, and GAMREQSTRM 261 * range subtypes need to be steered to (1,0), while all other GAM subtypes 262 * are steered to (0,0) and are included in the "INSTANCE0" table farther 263 * down. 264 */ 265 static const struct xe_mmio_range xe3p_xpc_gam_grp1_steering_table[] = { 266 { 0x004000, 0x004AFF }, /* GAMREQSTRM, rsvd, STLB, GAMWKRS, GAMREQSTRM */ 267 { 0x00F100, 0x00FFFF }, /* GAMWKRS */ 268 {}, 269 }; 270 271 static const struct xe_mmio_range xe2_node_steering_table[] = { 272 { 0x00B000, 0x00B0FF }, 273 { 0x00D880, 0x00D8FF }, 274 {}, 275 }; 276 277 static const struct xe_mmio_range xe3p_xpc_instance0_steering_table[] = { 278 { 0x00B500, 0x00B6FF }, /* PSMI */ 279 { 0x00C800, 0x00CFFF }, /* GAMCTRL */ 280 { 0x00F000, 0x00F0FF }, /* GAMCTRL */ 281 {}, 282 }; 283 284 static const struct xe_mmio_range xe3p_lpg_instance0_steering_table[] = { 285 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ 286 { 0x008700, 0x00887F }, /* NODE */ 287 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */ 288 { 0x00B500, 0x00B6FF }, /* PSMI */ 289 { 0x00C800, 0x00CFFF }, /* GAM */ 290 { 0x00D880, 0x00D8FF }, /* NODE */ 291 { 0x00DD00, 0x00DD7F }, /* MEMPIPE */ 292 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ 293 { 0x013400, 0x0135FF }, /* MEMPIPE */ 294 {}, 295 }; 296 297 static void init_steering_l3bank(struct xe_gt *gt) 298 { 299 struct xe_device *xe = gt_to_xe(gt); 300 struct xe_mmio *mmio = >->mmio; 301 302 if (GRAPHICS_VER(xe) >= 20) { 303 unsigned int first_bank = xe_l3_bank_mask_ffs(gt->fuse_topo.l3_bank_mask); 304 const int banks_per_node = 4; 305 unsigned int node = first_bank / banks_per_node; 306 307 /* L3BANK ranges place node in grpID, bank in instanceid */ 308 gt->steering[L3BANK].group_target = node; 309 gt->steering[L3BANK].instance_target = first_bank % banks_per_node; 310 311 /* NODE ranges split the node across grpid and instanceid */ 312 gt->steering[NODE].group_target = node >> 1; 313 gt->steering[NODE].instance_target = node & 1; 314 } else if (GRAPHICS_VERx100(xe) >= 1270) { 315 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, 316 xe_mmio_read32(mmio, MIRROR_FUSE3)); 317 u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK, 318 xe_mmio_read32(mmio, XEHP_FUSE4)); 319 320 /* 321 * Group selects mslice, instance selects bank within mslice. 322 * Bank 0 is always valid _except_ when the bank mask is 010b. 323 */ 324 gt->steering[L3BANK].group_target = __ffs(mslice_mask); 325 gt->steering[L3BANK].instance_target = 326 bank_mask & BIT(0) ? 0 : 2; 327 } else if (xe->info.platform == XE_DG2) { 328 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, 329 xe_mmio_read32(mmio, MIRROR_FUSE3)); 330 u32 bank = __ffs(mslice_mask) * 8; 331 332 /* 333 * Like mslice registers, look for a valid mslice and steer to 334 * the first L3BANK of that quad. Access to the Nth L3 bank is 335 * split between the first bits of group and instance 336 */ 337 gt->steering[L3BANK].group_target = (bank >> 2) & 0x7; 338 gt->steering[L3BANK].instance_target = bank & 0x3; 339 } else { 340 u32 fuse = REG_FIELD_GET(L3BANK_MASK, 341 ~xe_mmio_read32(mmio, MIRROR_FUSE3)); 342 343 gt->steering[L3BANK].group_target = 0; /* unused */ 344 gt->steering[L3BANK].instance_target = __ffs(fuse); 345 } 346 } 347 348 static void init_steering_mslice(struct xe_gt *gt) 349 { 350 u32 mask = REG_FIELD_GET(MEML3_EN_MASK, 351 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 352 353 /* 354 * mslice registers are valid (not terminated) if either the meml3 355 * associated with the mslice is present, or at least one DSS associated 356 * with the mslice is present. There will always be at least one meml3 357 * so we can just use that to find a non-terminated mslice and ignore 358 * the DSS fusing. 359 */ 360 gt->steering[MSLICE].group_target = __ffs(mask); 361 gt->steering[MSLICE].instance_target = 0; /* unused */ 362 363 /* 364 * LNCF termination is also based on mslice presence, so we'll set 365 * it up here. Either LNCF within a non-terminated mslice will work, 366 * so we just always pick LNCF 0 here. 367 */ 368 gt->steering[LNCF].group_target = __ffs(mask) << 1; 369 gt->steering[LNCF].instance_target = 0; /* unused */ 370 } 371 372 static unsigned int dss_per_group(struct xe_gt *gt) 373 { 374 struct xe_guc *guc = >->uc.guc; 375 u32 max_slices = 0, max_subslices = 0; 376 int ret; 377 378 /* 379 * Try to query the GuC's hwconfig table for the maximum number of 380 * slices and subslices. These don't reflect the platform's actual 381 * slice/DSS counts, just the physical layout by which we should 382 * determine the steering targets. On older platforms with older GuC 383 * firmware releases it's possible that these attributes may not be 384 * included in the table, so we can always fall back to the old 385 * hardcoded layouts. 386 */ 387 #define HWCONFIG_ATTR_MAX_SLICES 1 388 #define HWCONFIG_ATTR_MAX_SUBSLICES 70 389 390 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES, 391 &max_slices); 392 if (ret < 0 || max_slices == 0) 393 goto fallback; 394 395 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES, 396 &max_subslices); 397 if (ret < 0 || max_subslices == 0) 398 goto fallback; 399 400 return DIV_ROUND_UP(max_subslices, max_slices); 401 402 fallback: 403 /* 404 * Some older platforms don't have tables or don't have complete tables. 405 * Newer platforms should always have the required info. 406 */ 407 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000 && 408 !gt_to_xe(gt)->info.force_execlist) 409 xe_gt_err(gt, "Slice/Subslice counts missing from hwconfig table; using typical fallback values\n"); 410 411 if (gt_to_xe(gt)->info.platform == XE_PVC) 412 return 8; 413 else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250) 414 return 4; 415 else 416 return 6; 417 } 418 419 /** 420 * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS 421 * @gt: GT structure 422 * @dss: DSS ID to obtain steering for 423 * @group: pointer to storage for steering group ID 424 * @instance: pointer to storage for steering instance ID 425 */ 426 void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance) 427 { 428 xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS); 429 430 *group = dss / gt->steering_dss_per_grp; 431 *instance = dss % gt->steering_dss_per_grp; 432 } 433 434 static void init_steering_dss(struct xe_gt *gt) 435 { 436 gt->steering_dss_per_grp = dss_per_group(gt); 437 438 xe_gt_mcr_get_dss_steering(gt, 439 min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0), 440 xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)), 441 >->steering[DSS].group_target, 442 >->steering[DSS].instance_target); 443 } 444 445 static void init_steering_oaddrm(struct xe_gt *gt) 446 { 447 /* 448 * First instance is only terminated if the entire first media slice 449 * is absent (i.e., no VCS0 or VECS0). 450 */ 451 if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0)) 452 gt->steering[OADDRM].group_target = 0; 453 else 454 gt->steering[OADDRM].group_target = 1; 455 456 gt->steering[OADDRM].instance_target = 0; /* unused */ 457 } 458 459 static void init_steering_sqidi_psmi(struct xe_gt *gt) 460 { 461 u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, 462 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 463 u32 select = __ffs(mask); 464 465 gt->steering[SQIDI_PSMI].group_target = select >> 1; 466 gt->steering[SQIDI_PSMI].instance_target = select & 0x1; 467 } 468 469 static void init_steering_gam1(struct xe_gt *gt) 470 { 471 gt->steering[GAM1].group_target = 1; 472 gt->steering[GAM1].instance_target = 0; 473 } 474 475 static const struct { 476 const char *name; 477 void (*init)(struct xe_gt *gt); 478 } xe_steering_types[] = { 479 [L3BANK] = { "L3BANK", init_steering_l3bank }, 480 [NODE] = { "NODE", NULL }, /* initialized by l3bank init */ 481 [MSLICE] = { "MSLICE", init_steering_mslice }, 482 [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */ 483 [DSS] = { "DSS / XeCore", init_steering_dss }, 484 [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm }, 485 [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi }, 486 [GAM1] = { "GAMWKRS / STLB / GAMREQSTRM", init_steering_gam1 }, 487 [INSTANCE0] = { "INSTANCE 0", NULL }, 488 [IMPLICIT_STEERING] = { "IMPLICIT", NULL }, 489 }; 490 491 /** 492 * xe_gt_mcr_init_early - Early initialization of the MCR support 493 * @gt: GT structure 494 * 495 * Perform early software only initialization of the MCR lock to allow 496 * the synchronization on accessing the STEER_SEMAPHORE register and 497 * use the xe_gt_mcr_multicast_write() function, plus the minimum 498 * safe MCR registers required for VRAM/CCS probing. 499 */ 500 void xe_gt_mcr_init_early(struct xe_gt *gt) 501 { 502 struct xe_device *xe = gt_to_xe(gt); 503 504 BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES); 505 BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES); 506 507 spin_lock_init(>->mcr_lock); 508 509 if (gt->info.type == XE_GT_TYPE_MEDIA) { 510 drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13); 511 512 if (MEDIA_VER(xe) >= 30) { 513 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; 514 gt->steering[INSTANCE0].ranges = xe3lpm_instance0_steering_table; 515 } else if (MEDIA_VERx100(xe) >= 1301) { 516 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; 517 gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table; 518 } else { 519 gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table; 520 } 521 } else { 522 if (GRAPHICS_VERx100(xe) == 3511) { 523 gt->steering[DSS].ranges = xe3p_xpc_xecore_steering_table; 524 gt->steering[GAM1].ranges = xe3p_xpc_gam_grp1_steering_table; 525 gt->steering[INSTANCE0].ranges = xe3p_xpc_instance0_steering_table; 526 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; 527 gt->steering[NODE].ranges = xe2_node_steering_table; 528 } else if (GRAPHICS_VERx100(xe) >= 3510) { 529 gt->steering[DSS].ranges = xe2lpg_dss_steering_table; 530 gt->steering[INSTANCE0].ranges = xe3p_lpg_instance0_steering_table; 531 } else if (GRAPHICS_VER(xe) >= 20) { 532 gt->steering[DSS].ranges = xe2lpg_dss_steering_table; 533 gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table; 534 gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table; 535 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; 536 gt->steering[NODE].ranges = xe2_node_steering_table; 537 } else if (GRAPHICS_VERx100(xe) >= 1270) { 538 gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; 539 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; 540 gt->steering[DSS].ranges = xelpg_dss_steering_table; 541 } else if (xe->info.platform == XE_PVC) { 542 gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table; 543 gt->steering[DSS].ranges = xehpc_dss_steering_table; 544 } else if (xe->info.platform == XE_DG2) { 545 gt->steering[L3BANK].ranges = xehp_l3bank_steering_table; 546 gt->steering[MSLICE].ranges = xehp_mslice_steering_table; 547 gt->steering[LNCF].ranges = xehp_lncf_steering_table; 548 gt->steering[DSS].ranges = xehp_dss_steering_table; 549 gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table; 550 } else { 551 gt->steering[L3BANK].ranges = xelp_l3bank_steering_table; 552 gt->steering[DSS].ranges = xelp_dss_steering_table; 553 } 554 } 555 556 /* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */ 557 gt->steering[INSTANCE0].initialized = true; 558 } 559 EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_init_early); 560 561 /** 562 * xe_gt_mcr_init - Normal initialization of the MCR support 563 * @gt: GT structure 564 * 565 * Perform normal initialization of the MCR for all usages. 566 */ 567 void xe_gt_mcr_init(struct xe_gt *gt) 568 { 569 /* Select non-terminated steering target for each type */ 570 for (int i = 0; i < NUM_STEERING_TYPES; i++) { 571 gt->steering[i].initialized = true; 572 if (gt->steering[i].ranges && xe_steering_types[i].init) 573 xe_steering_types[i].init(gt); 574 } 575 } 576 577 /** 578 * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers 579 * @gt: GT structure 580 * 581 * Some register ranges don't need to have their steering control registers 582 * changed on each access - it's sufficient to set them once on initialization. 583 * This function sets those registers for each platform * 584 */ 585 void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) 586 { 587 struct xe_device *xe = gt_to_xe(gt); 588 589 if (IS_SRIOV_VF(xe)) 590 return; 591 592 if (xe->info.platform == XE_DG2) { 593 u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) | 594 REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2); 595 596 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, steer_val); 597 xe_mmio_write32(>->mmio, SF_MCR_SELECTOR, steer_val); 598 /* 599 * For GAM registers, all reads should be directed to instance 1 600 * (unicast reads against other instances are not allowed), 601 * and instance 1 is already the hardware's default steering 602 * target, which we never change 603 */ 604 } 605 } 606 607 static bool reg_in_steering_type_ranges(struct xe_gt *gt, 608 struct xe_reg reg, 609 int type) 610 { 611 if (!gt->steering[type].ranges) 612 return false; 613 614 for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) 615 if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) 616 return true; 617 618 return false; 619 } 620 621 /* 622 * xe_gt_mcr_check_reg - check if a register is recognized by this GT as MCR 623 * @gt: GT structure 624 * @reg: The register to check 625 * 626 * Returns true if the register offset falls within one of the MMIO ranges 627 * classified as MCR for the GT. 628 */ 629 bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg) 630 { 631 KUNIT_STATIC_STUB_REDIRECT(xe_gt_mcr_check_reg, gt, reg); 632 633 for (int type = 0; type <= IMPLICIT_STEERING; type++) 634 if (reg_in_steering_type_ranges(gt, reg, type)) 635 return true; 636 637 return false; 638 } 639 EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_check_reg); 640 641 /* 642 * xe_gt_mcr_get_nonterminated_steering - find group/instance values that 643 * will steer a register to a non-terminated instance 644 * @gt: GT structure 645 * @reg: register for which the steering is required 646 * @group: return variable for group steering 647 * @instance: return variable for instance steering 648 * 649 * This function returns a group/instance pair that is guaranteed to work for 650 * read steering of the given register. Note that a value will be returned even 651 * if the register is not replicated and therefore does not actually require 652 * steering. 653 * 654 * Returns true if the caller should steer to the @group/@instance values 655 * returned. Returns false if the caller need not perform any steering 656 */ 657 bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, 658 struct xe_reg_mcr reg_mcr, 659 u8 *group, u8 *instance) 660 { 661 const struct xe_reg reg = to_xe_reg(reg_mcr); 662 663 for (int type = 0; type < IMPLICIT_STEERING; type++) { 664 if (reg_in_steering_type_ranges(gt, reg, type)) { 665 drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized, 666 "Uninitialized usage of MCR register %s/%#x\n", 667 xe_steering_types[type].name, reg.addr); 668 669 *group = gt->steering[type].group_target; 670 *instance = gt->steering[type].instance_target; 671 return true; 672 } 673 } 674 675 if (reg_in_steering_type_ranges(gt, reg, IMPLICIT_STEERING)) 676 return false; 677 678 /* 679 * Not found in a steering table and not a register with implicit 680 * steering. Just steer to 0/0 as a guess and raise a warning. 681 */ 682 drm_WARN(>_to_xe(gt)->drm, true, 683 "Did not find MCR register %#x in any MCR steering table\n", 684 reg.addr); 685 *group = 0; 686 *instance = 0; 687 688 return true; 689 } 690 691 /* 692 * Obtain exclusive access to MCR steering. On MTL and beyond we also need 693 * to synchronize with external clients (e.g., firmware), so a semaphore 694 * register will also need to be taken. 695 */ 696 static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock) 697 { 698 struct xe_device *xe = gt_to_xe(gt); 699 int ret = 0; 700 701 spin_lock(>->mcr_lock); 702 703 /* 704 * Starting with MTL we also need to grab a semaphore register 705 * to synchronize with external agents (e.g., firmware) that now 706 * shares the same steering control register. The semaphore is obtained 707 * when a read to the relevant register returns 1. 708 */ 709 if (GRAPHICS_VERx100(xe) >= 1270) 710 ret = xe_mmio_wait32(>->mmio, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL, 711 true); 712 713 drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT); 714 } 715 716 static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock) 717 { 718 /* Release hardware semaphore - this is done by writing 1 to the register */ 719 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) 720 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, 0x1); 721 722 spin_unlock(>->mcr_lock); 723 } 724 725 /* 726 * Access a register with specific MCR steering 727 * 728 * Caller needs to make sure the relevant forcewake wells are up. 729 */ 730 static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 731 u8 rw_flag, int group, int instance, u32 value) 732 { 733 const struct xe_reg reg = to_xe_reg(reg_mcr); 734 struct xe_mmio *mmio = >->mmio; 735 struct xe_reg steer_reg; 736 u32 steer_val, val = 0; 737 738 lockdep_assert_held(>->mcr_lock); 739 740 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { 741 steer_reg = MTL_MCR_SELECTOR; 742 steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) | 743 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance); 744 } else { 745 steer_reg = MCR_SELECTOR; 746 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) | 747 REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance); 748 } 749 750 /* 751 * Always leave the hardware in multicast mode when doing reads and only 752 * change it to unicast mode when doing writes of a specific instance. 753 * 754 * The setting of the multicast/unicast bit usually wouldn't matter for 755 * read operations (which always return the value from a single register 756 * instance regardless of how that bit is set), but some platforms may 757 * have workarounds requiring us to remain in multicast mode for reads, 758 * e.g. Wa_22013088509 on PVC. There's no real downside to this, so 759 * we'll just go ahead and do so on all platforms; we'll only clear the 760 * multicast bit from the mask when explicitly doing a write operation. 761 * 762 * No need to save old steering reg value. 763 */ 764 if (rw_flag == MCR_OP_READ) 765 steer_val |= MCR_MULTICAST; 766 767 xe_mmio_write32(mmio, steer_reg, steer_val); 768 769 if (rw_flag == MCR_OP_READ) 770 val = xe_mmio_read32(mmio, reg); 771 else 772 xe_mmio_write32(mmio, reg, value); 773 774 /* 775 * If we turned off the multicast bit (during a write) we're required 776 * to turn it back on before finishing. The group and instance values 777 * don't matter since they'll be re-programmed on the next MCR 778 * operation. 779 */ 780 if (rw_flag == MCR_OP_WRITE) 781 xe_mmio_write32(mmio, steer_reg, MCR_MULTICAST); 782 783 return val; 784 } 785 786 /** 787 * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register 788 * @gt: GT structure 789 * @reg_mcr: register to read 790 * 791 * Reads a GT MCR register. The read will be steered to a non-terminated 792 * instance (i.e., one that isn't fused off or powered down by power gating). 793 * This function assumes the caller is already holding any necessary forcewake 794 * domains. 795 * 796 * Returns the value from a non-terminated instance of @reg. 797 */ 798 u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr) 799 { 800 const struct xe_reg reg = to_xe_reg(reg_mcr); 801 u8 group, instance; 802 u32 val; 803 bool steer; 804 805 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 806 807 steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr, 808 &group, &instance); 809 810 if (steer) { 811 mcr_lock(gt); 812 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, 813 group, instance, 0); 814 mcr_unlock(gt); 815 } else { 816 val = xe_mmio_read32(>->mmio, reg); 817 } 818 819 return val; 820 } 821 822 /** 823 * xe_gt_mcr_unicast_read - read a specific instance of an MCR register 824 * @gt: GT structure 825 * @reg_mcr: the MCR register to read 826 * @group: the MCR group 827 * @instance: the MCR instance 828 * 829 * Returns the value read from an MCR register after steering toward a specific 830 * group/instance. 831 */ 832 u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, 833 struct xe_reg_mcr reg_mcr, 834 int group, int instance) 835 { 836 u32 val; 837 838 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 839 840 mcr_lock(gt); 841 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0); 842 mcr_unlock(gt); 843 844 return val; 845 } 846 847 /** 848 * xe_gt_mcr_unicast_write - write a specific instance of an MCR register 849 * @gt: GT structure 850 * @reg_mcr: the MCR register to write 851 * @value: value to write 852 * @group: the MCR group 853 * @instance: the MCR instance 854 * 855 * Write an MCR register in unicast mode after steering toward a specific 856 * group/instance. 857 */ 858 void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 859 u32 value, int group, int instance) 860 { 861 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 862 863 mcr_lock(gt); 864 rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value); 865 mcr_unlock(gt); 866 } 867 868 /** 869 * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register 870 * @gt: GT structure 871 * @reg_mcr: the MCR register to write 872 * @value: value to write 873 * 874 * Write an MCR register in multicast mode to update all instances. 875 */ 876 void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 877 u32 value) 878 { 879 struct xe_reg reg = to_xe_reg(reg_mcr); 880 881 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 882 883 /* 884 * Synchronize with any unicast operations. Once we have exclusive 885 * access, the MULTICAST bit should already be set, so there's no need 886 * to touch the steering register. 887 */ 888 mcr_lock(gt); 889 xe_mmio_write32(>->mmio, reg, value); 890 mcr_unlock(gt); 891 } 892 893 void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p) 894 { 895 for (int i = 0; i < NUM_STEERING_TYPES; i++) { 896 if (gt->steering[i].ranges) { 897 drm_printf(p, "%s steering: group=%#x, instance=%#x\n", 898 xe_steering_types[i].name, 899 gt->steering[i].group_target, 900 gt->steering[i].instance_target); 901 for (int j = 0; gt->steering[i].ranges[j].end; j++) 902 drm_printf(p, "\t0x%06x - 0x%06x\n", 903 gt->steering[i].ranges[j].start, 904 gt->steering[i].ranges[j].end); 905 } 906 } 907 } 908