1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_gt_mcr.h" 7 8 #include "regs/xe_gt_regs.h" 9 #include "xe_assert.h" 10 #include "xe_gt_printk.h" 11 #include "xe_gt_topology.h" 12 #include "xe_gt_types.h" 13 #include "xe_guc_hwconfig.h" 14 #include "xe_mmio.h" 15 #include "xe_sriov.h" 16 17 /** 18 * DOC: GT Multicast/Replicated (MCR) Register Support 19 * 20 * Some GT registers are designed as "multicast" or "replicated" registers: 21 * multiple instances of the same register share a single MMIO offset. MCR 22 * registers are generally used when the hardware needs to potentially track 23 * independent values of a register per hardware unit (e.g., per-subslice, 24 * per-L3bank, etc.). The specific types of replication that exist vary 25 * per-platform. 26 * 27 * MMIO accesses to MCR registers are controlled according to the settings 28 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR 29 * registers can be done in either multicast (a single write updates all 30 * instances of the register to the same value) or unicast (a write updates only 31 * one specific instance) form. Reads of MCR registers always operate in a 32 * unicast manner regardless of how the multicast/unicast bit is set in 33 * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is 34 * referred to as "steering." 35 * 36 * If MCR register operations are steered toward a hardware unit that is 37 * fused off or currently powered down due to power gating, the MMIO operation 38 * is "terminated" by the hardware. Terminated read operations will return a 39 * value of zero and terminated unicast write operations will be silently 40 * ignored. During device initialization, the goal of the various 41 * ``init_steering_*()`` functions is to apply the platform-specific rules for 42 * each MCR register type to identify a steering target that will select a 43 * non-terminated instance. 44 * 45 * MCR registers are not available on Virtual Function (VF). 46 */ 47 48 static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr) 49 { 50 return reg_mcr.__reg; 51 } 52 53 enum { 54 MCR_OP_READ, 55 MCR_OP_WRITE 56 }; 57 58 static const struct xe_mmio_range xelp_l3bank_steering_table[] = { 59 { 0x00B100, 0x00B3FF }, 60 {}, 61 }; 62 63 static const struct xe_mmio_range xehp_l3bank_steering_table[] = { 64 { 0x008C80, 0x008CFF }, 65 { 0x00B100, 0x00B3FF }, 66 {}, 67 }; 68 69 /* 70 * Although the bspec lists more "MSLICE" ranges than shown here, some of those 71 * are of a "GAM" subclass that has special rules and doesn't need to be 72 * included here. 73 */ 74 static const struct xe_mmio_range xehp_mslice_steering_table[] = { 75 { 0x00DD00, 0x00DDFF }, 76 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ 77 {}, 78 }; 79 80 static const struct xe_mmio_range xehp_lncf_steering_table[] = { 81 { 0x00B000, 0x00B0FF }, 82 { 0x00D880, 0x00D8FF }, 83 {}, 84 }; 85 86 /* 87 * We have several types of MCR registers where steering to (0,0) will always 88 * provide us with a non-terminated value. We'll stick them all in the same 89 * table for simplicity. 90 */ 91 static const struct xe_mmio_range xehpc_instance0_steering_table[] = { 92 { 0x004000, 0x004AFF }, /* HALF-BSLICE */ 93 { 0x008800, 0x00887F }, /* CC */ 94 { 0x008A80, 0x008AFF }, /* TILEPSMI */ 95 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ 96 { 0x00B100, 0x00B3FF }, /* L3BANK */ 97 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ 98 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ 99 { 0x00DD00, 0x00DDFF }, /* BSLICE */ 100 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ 101 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ 102 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ 103 { 0x024180, 0x0241FF }, /* HALF-BSLICE */ 104 {}, 105 }; 106 107 static const struct xe_mmio_range xelpg_instance0_steering_table[] = { 108 { 0x000B00, 0x000BFF }, /* SQIDI */ 109 { 0x001000, 0x001FFF }, /* SQIDI */ 110 { 0x004000, 0x0048FF }, /* GAM */ 111 { 0x008700, 0x0087FF }, /* SQIDI */ 112 { 0x00B000, 0x00B0FF }, /* NODE */ 113 { 0x00C800, 0x00CFFF }, /* GAM */ 114 { 0x00D880, 0x00D8FF }, /* NODE */ 115 { 0x00DD00, 0x00DDFF }, /* OAAL2 */ 116 {}, 117 }; 118 119 static const struct xe_mmio_range xelpg_l3bank_steering_table[] = { 120 { 0x00B100, 0x00B3FF }, 121 {}, 122 }; 123 124 static const struct xe_mmio_range xelp_dss_steering_table[] = { 125 { 0x008150, 0x00815F }, 126 { 0x009520, 0x00955F }, 127 { 0x00DE80, 0x00E8FF }, 128 { 0x024A00, 0x024A7F }, 129 {}, 130 }; 131 132 /* DSS steering is used for GSLICE ranges as well */ 133 static const struct xe_mmio_range xehp_dss_steering_table[] = { 134 { 0x005200, 0x0052FF }, /* GSLICE */ 135 { 0x005400, 0x007FFF }, /* GSLICE */ 136 { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 137 { 0x008D00, 0x008DFF }, /* DSS */ 138 { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 139 { 0x009680, 0x0096FF }, /* DSS */ 140 { 0x00D800, 0x00D87F }, /* GSLICE */ 141 { 0x00DC00, 0x00DCFF }, /* GSLICE */ 142 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */ 143 { 0x017000, 0x017FFF }, /* GSLICE */ 144 { 0x024A00, 0x024A7F }, /* DSS */ 145 {}, 146 }; 147 148 /* DSS steering is used for COMPUTE ranges as well */ 149 static const struct xe_mmio_range xehpc_dss_steering_table[] = { 150 { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */ 151 { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 152 { 0x009680, 0x0096FF }, /* DSS */ 153 { 0x00DC00, 0x00DCFF }, /* COMPUTE */ 154 { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */ 155 {}, 156 }; 157 158 /* DSS steering is used for SLICE ranges as well */ 159 static const struct xe_mmio_range xelpg_dss_steering_table[] = { 160 { 0x005200, 0x0052FF }, /* SLICE */ 161 { 0x005500, 0x007FFF }, /* SLICE */ 162 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 163 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 164 { 0x009680, 0x0096FF }, /* DSS */ 165 { 0x00D800, 0x00D87F }, /* SLICE */ 166 { 0x00DC00, 0x00DCFF }, /* SLICE */ 167 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 168 {}, 169 }; 170 171 static const struct xe_mmio_range xe3p_xpc_xecore_steering_table[] = { 172 { 0x008140, 0x00817F }, /* SLICE, XeCore, SLICE */ 173 { 0x009480, 0x00955F }, /* SLICE, XeCore */ 174 { 0x00D800, 0x00D87F }, /* SLICE */ 175 { 0x00DC00, 0x00E9FF }, /* SLICE, rsvd, XeCore, rsvd, XeCore, rsvd, XeCore */ 176 { 0x013000, 0x0135FF }, /* XeCore, SLICE */ 177 {}, 178 }; 179 180 static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = { 181 { 0x393200, 0x39323F }, 182 { 0x393400, 0x3934FF }, 183 {}, 184 }; 185 186 static const struct xe_mmio_range dg2_implicit_steering_table[] = { 187 { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */ 188 { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */ 189 { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */ 190 { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */ 191 { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */ 192 { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */ 193 {}, 194 }; 195 196 static const struct xe_mmio_range xe2lpg_dss_steering_table[] = { 197 { 0x005200, 0x0052FF }, /* SLICE */ 198 { 0x005500, 0x007FFF }, /* SLICE */ 199 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 200 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 201 { 0x009680, 0x0096FF }, /* DSS */ 202 { 0x00D800, 0x00D87F }, /* SLICE */ 203 { 0x00DC00, 0x00DCFF }, /* SLICE */ 204 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 205 { 0x00E980, 0x00E9FF }, /* SLICE */ 206 { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */ 207 {}, 208 }; 209 210 static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = { 211 { 0x000B00, 0x000BFF }, 212 { 0x001000, 0x001FFF }, 213 {}, 214 }; 215 216 static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = { 217 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ 218 { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */ 219 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */ 220 { 0x00C800, 0x00CFFF }, /* GAM */ 221 { 0x00D880, 0x00D8FF }, /* NODE */ 222 { 0x00DD00, 0x00DDFF }, /* MEMPIPE */ 223 { 0x00E900, 0x00E97F }, /* MEMPIPE */ 224 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ 225 { 0x013400, 0x0135FF }, /* MEMPIPE */ 226 {}, 227 }; 228 229 static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = { 230 { 0x388160, 0x38817F }, 231 { 0x389480, 0x3894CF }, 232 {}, 233 }; 234 235 static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = { 236 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */ 237 { 0x384900, 0x384AFF }, /* GAM */ 238 { 0x389560, 0x3895FF }, /* MEDIAINF */ 239 { 0x38B600, 0x38B8FF }, /* L3BANK */ 240 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ 241 { 0x38F000, 0x38F0FF }, /* GAM */ 242 { 0x393C00, 0x393C7F }, /* MEDIAINF */ 243 {}, 244 }; 245 246 static const struct xe_mmio_range xe3lpm_instance0_steering_table[] = { 247 { 0x384000, 0x3841FF }, /* GAM */ 248 { 0x384400, 0x3847DF }, /* GAM */ 249 { 0x384900, 0x384AFF }, /* GAM */ 250 { 0x389560, 0x3895FF }, /* MEDIAINF */ 251 { 0x38B600, 0x38B8FF }, /* L3BANK */ 252 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ 253 { 0x38D0D0, 0x38F0FF }, /* MEDIAINF, rsvd, GAM */ 254 { 0x393C00, 0x393C7F }, /* MEDIAINF */ 255 {}, 256 }; 257 258 /* 259 * Different "GAM" ranges have different rules; GAMWKRS, STLB, and GAMREQSTRM 260 * range subtypes need to be steered to (1,0), while all other GAM subtypes 261 * are steered to (0,0) and are included in the "INSTANCE0" table farther 262 * down. 263 */ 264 static const struct xe_mmio_range xe3p_xpc_gam_grp1_steering_table[] = { 265 { 0x004000, 0x004AFF }, /* GAMREQSTRM, rsvd, STLB, GAMWKRS, GAMREQSTRM */ 266 { 0x00F100, 0x00FFFF }, /* GAMWKRS */ 267 {}, 268 }; 269 270 static const struct xe_mmio_range xe3p_xpc_node_steering_table[] = { 271 { 0x00B000, 0x00B0FF }, 272 { 0x00D880, 0x00D8FF }, 273 {}, 274 }; 275 276 static const struct xe_mmio_range xe3p_xpc_instance0_steering_table[] = { 277 { 0x00B500, 0x00B6FF }, /* PSMI */ 278 { 0x00C800, 0x00CFFF }, /* GAMCTRL */ 279 { 0x00F000, 0x00F0FF }, /* GAMCTRL */ 280 {}, 281 }; 282 283 static void init_steering_l3bank(struct xe_gt *gt) 284 { 285 struct xe_device *xe = gt_to_xe(gt); 286 struct xe_mmio *mmio = >->mmio; 287 288 if (GRAPHICS_VER(xe) >= 35) { 289 unsigned int first_bank = xe_l3_bank_mask_ffs(gt->fuse_topo.l3_bank_mask); 290 const int banks_per_node = 4; 291 unsigned int node = first_bank / banks_per_node; 292 293 /* L3BANK ranges place node in grpID, bank in instanceid */ 294 gt->steering[L3BANK].group_target = node; 295 gt->steering[L3BANK].instance_target = first_bank % banks_per_node; 296 297 /* NODE ranges split the node across grpid and instanceid */ 298 gt->steering[NODE].group_target = node >> 1; 299 gt->steering[NODE].instance_target = node & 1; 300 } else if (GRAPHICS_VERx100(xe) >= 1270) { 301 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, 302 xe_mmio_read32(mmio, MIRROR_FUSE3)); 303 u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK, 304 xe_mmio_read32(mmio, XEHP_FUSE4)); 305 306 /* 307 * Group selects mslice, instance selects bank within mslice. 308 * Bank 0 is always valid _except_ when the bank mask is 010b. 309 */ 310 gt->steering[L3BANK].group_target = __ffs(mslice_mask); 311 gt->steering[L3BANK].instance_target = 312 bank_mask & BIT(0) ? 0 : 2; 313 } else if (xe->info.platform == XE_DG2) { 314 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, 315 xe_mmio_read32(mmio, MIRROR_FUSE3)); 316 u32 bank = __ffs(mslice_mask) * 8; 317 318 /* 319 * Like mslice registers, look for a valid mslice and steer to 320 * the first L3BANK of that quad. Access to the Nth L3 bank is 321 * split between the first bits of group and instance 322 */ 323 gt->steering[L3BANK].group_target = (bank >> 2) & 0x7; 324 gt->steering[L3BANK].instance_target = bank & 0x3; 325 } else { 326 u32 fuse = REG_FIELD_GET(L3BANK_MASK, 327 ~xe_mmio_read32(mmio, MIRROR_FUSE3)); 328 329 gt->steering[L3BANK].group_target = 0; /* unused */ 330 gt->steering[L3BANK].instance_target = __ffs(fuse); 331 } 332 } 333 334 static void init_steering_mslice(struct xe_gt *gt) 335 { 336 u32 mask = REG_FIELD_GET(MEML3_EN_MASK, 337 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 338 339 /* 340 * mslice registers are valid (not terminated) if either the meml3 341 * associated with the mslice is present, or at least one DSS associated 342 * with the mslice is present. There will always be at least one meml3 343 * so we can just use that to find a non-terminated mslice and ignore 344 * the DSS fusing. 345 */ 346 gt->steering[MSLICE].group_target = __ffs(mask); 347 gt->steering[MSLICE].instance_target = 0; /* unused */ 348 349 /* 350 * LNCF termination is also based on mslice presence, so we'll set 351 * it up here. Either LNCF within a non-terminated mslice will work, 352 * so we just always pick LNCF 0 here. 353 */ 354 gt->steering[LNCF].group_target = __ffs(mask) << 1; 355 gt->steering[LNCF].instance_target = 0; /* unused */ 356 } 357 358 static unsigned int dss_per_group(struct xe_gt *gt) 359 { 360 struct xe_guc *guc = >->uc.guc; 361 u32 max_slices = 0, max_subslices = 0; 362 int ret; 363 364 /* 365 * Try to query the GuC's hwconfig table for the maximum number of 366 * slices and subslices. These don't reflect the platform's actual 367 * slice/DSS counts, just the physical layout by which we should 368 * determine the steering targets. On older platforms with older GuC 369 * firmware releases it's possible that these attributes may not be 370 * included in the table, so we can always fall back to the old 371 * hardcoded layouts. 372 */ 373 #define HWCONFIG_ATTR_MAX_SLICES 1 374 #define HWCONFIG_ATTR_MAX_SUBSLICES 70 375 376 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES, 377 &max_slices); 378 if (ret < 0 || max_slices == 0) 379 goto fallback; 380 381 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES, 382 &max_subslices); 383 if (ret < 0 || max_subslices == 0) 384 goto fallback; 385 386 return DIV_ROUND_UP(max_subslices, max_slices); 387 388 fallback: 389 /* 390 * Some older platforms don't have tables or don't have complete tables. 391 * Newer platforms should always have the required info. 392 */ 393 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000 && 394 !gt_to_xe(gt)->info.force_execlist) 395 xe_gt_err(gt, "Slice/Subslice counts missing from hwconfig table; using typical fallback values\n"); 396 397 if (gt_to_xe(gt)->info.platform == XE_PVC) 398 return 8; 399 else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250) 400 return 4; 401 else 402 return 6; 403 } 404 405 /** 406 * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS 407 * @gt: GT structure 408 * @dss: DSS ID to obtain steering for 409 * @group: pointer to storage for steering group ID 410 * @instance: pointer to storage for steering instance ID 411 */ 412 void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance) 413 { 414 xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS); 415 416 *group = dss / gt->steering_dss_per_grp; 417 *instance = dss % gt->steering_dss_per_grp; 418 } 419 420 /** 421 * xe_gt_mcr_steering_info_to_dss_id - Get DSS ID from group/instance steering 422 * @gt: GT structure 423 * @group: steering group ID 424 * @instance: steering instance ID 425 * 426 * Return: the converted DSS id. 427 */ 428 u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance) 429 { 430 return group * dss_per_group(gt) + instance; 431 } 432 433 static void init_steering_dss(struct xe_gt *gt) 434 { 435 gt->steering_dss_per_grp = dss_per_group(gt); 436 437 xe_gt_mcr_get_dss_steering(gt, 438 min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0), 439 xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)), 440 >->steering[DSS].group_target, 441 >->steering[DSS].instance_target); 442 } 443 444 static void init_steering_oaddrm(struct xe_gt *gt) 445 { 446 /* 447 * First instance is only terminated if the entire first media slice 448 * is absent (i.e., no VCS0 or VECS0). 449 */ 450 if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0)) 451 gt->steering[OADDRM].group_target = 0; 452 else 453 gt->steering[OADDRM].group_target = 1; 454 455 gt->steering[OADDRM].instance_target = 0; /* unused */ 456 } 457 458 static void init_steering_sqidi_psmi(struct xe_gt *gt) 459 { 460 u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, 461 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 462 u32 select = __ffs(mask); 463 464 gt->steering[SQIDI_PSMI].group_target = select >> 1; 465 gt->steering[SQIDI_PSMI].instance_target = select & 0x1; 466 } 467 468 static void init_steering_gam1(struct xe_gt *gt) 469 { 470 gt->steering[GAM1].group_target = 1; 471 gt->steering[GAM1].instance_target = 0; 472 } 473 474 static const struct { 475 const char *name; 476 void (*init)(struct xe_gt *gt); 477 } xe_steering_types[] = { 478 [L3BANK] = { "L3BANK", init_steering_l3bank }, 479 [NODE] = { "NODE", NULL }, /* initialized by l3bank init */ 480 [MSLICE] = { "MSLICE", init_steering_mslice }, 481 [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */ 482 [DSS] = { "DSS / XeCore", init_steering_dss }, 483 [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm }, 484 [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi }, 485 [GAM1] = { "GAMWKRS / STLB / GAMREQSTRM", init_steering_gam1 }, 486 [INSTANCE0] = { "INSTANCE 0", NULL }, 487 [IMPLICIT_STEERING] = { "IMPLICIT", NULL }, 488 }; 489 490 /** 491 * xe_gt_mcr_init_early - Early initialization of the MCR support 492 * @gt: GT structure 493 * 494 * Perform early software only initialization of the MCR lock to allow 495 * the synchronization on accessing the STEER_SEMAPHORE register and 496 * use the xe_gt_mcr_multicast_write() function, plus the minimum 497 * safe MCR registers required for VRAM/CCS probing. 498 */ 499 void xe_gt_mcr_init_early(struct xe_gt *gt) 500 { 501 struct xe_device *xe = gt_to_xe(gt); 502 503 BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES); 504 BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES); 505 506 spin_lock_init(>->mcr_lock); 507 508 if (IS_SRIOV_VF(xe)) 509 return; 510 511 if (gt->info.type == XE_GT_TYPE_MEDIA) { 512 drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13); 513 514 if (MEDIA_VER(xe) >= 30) { 515 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; 516 gt->steering[INSTANCE0].ranges = xe3lpm_instance0_steering_table; 517 } else if (MEDIA_VERx100(xe) >= 1301) { 518 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; 519 gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table; 520 } else { 521 gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table; 522 } 523 } else { 524 if (GRAPHICS_VERx100(xe) == 3511) { 525 /* 526 * TODO: there are some ranges in bspec with missing 527 * termination: [0x00B000, 0x00B0FF] and 528 * [0x00D880, 0x00D8FF] (NODE); [0x00B100, 0x00B3FF] 529 * (L3BANK). Update them here once bspec is updated. 530 */ 531 gt->steering[DSS].ranges = xe3p_xpc_xecore_steering_table; 532 gt->steering[GAM1].ranges = xe3p_xpc_gam_grp1_steering_table; 533 gt->steering[INSTANCE0].ranges = xe3p_xpc_instance0_steering_table; 534 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; 535 gt->steering[NODE].ranges = xe3p_xpc_node_steering_table; 536 } else if (GRAPHICS_VER(xe) >= 20) { 537 gt->steering[DSS].ranges = xe2lpg_dss_steering_table; 538 gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table; 539 gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table; 540 } else if (GRAPHICS_VERx100(xe) >= 1270) { 541 gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; 542 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; 543 gt->steering[DSS].ranges = xelpg_dss_steering_table; 544 } else if (xe->info.platform == XE_PVC) { 545 gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table; 546 gt->steering[DSS].ranges = xehpc_dss_steering_table; 547 } else if (xe->info.platform == XE_DG2) { 548 gt->steering[L3BANK].ranges = xehp_l3bank_steering_table; 549 gt->steering[MSLICE].ranges = xehp_mslice_steering_table; 550 gt->steering[LNCF].ranges = xehp_lncf_steering_table; 551 gt->steering[DSS].ranges = xehp_dss_steering_table; 552 gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table; 553 } else { 554 gt->steering[L3BANK].ranges = xelp_l3bank_steering_table; 555 gt->steering[DSS].ranges = xelp_dss_steering_table; 556 } 557 } 558 559 /* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */ 560 gt->steering[INSTANCE0].initialized = true; 561 } 562 563 /** 564 * xe_gt_mcr_init - Normal initialization of the MCR support 565 * @gt: GT structure 566 * 567 * Perform normal initialization of the MCR for all usages. 568 */ 569 void xe_gt_mcr_init(struct xe_gt *gt) 570 { 571 if (IS_SRIOV_VF(gt_to_xe(gt))) 572 return; 573 574 /* Select non-terminated steering target for each type */ 575 for (int i = 0; i < NUM_STEERING_TYPES; i++) { 576 gt->steering[i].initialized = true; 577 if (gt->steering[i].ranges && xe_steering_types[i].init) 578 xe_steering_types[i].init(gt); 579 } 580 } 581 582 /** 583 * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers 584 * @gt: GT structure 585 * 586 * Some register ranges don't need to have their steering control registers 587 * changed on each access - it's sufficient to set them once on initialization. 588 * This function sets those registers for each platform * 589 */ 590 void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) 591 { 592 struct xe_device *xe = gt_to_xe(gt); 593 594 if (IS_SRIOV_VF(xe)) 595 return; 596 597 if (xe->info.platform == XE_DG2) { 598 u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) | 599 REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2); 600 601 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, steer_val); 602 xe_mmio_write32(>->mmio, SF_MCR_SELECTOR, steer_val); 603 /* 604 * For GAM registers, all reads should be directed to instance 1 605 * (unicast reads against other instances are not allowed), 606 * and instance 1 is already the hardware's default steering 607 * target, which we never change 608 */ 609 } 610 } 611 612 /* 613 * xe_gt_mcr_get_nonterminated_steering - find group/instance values that 614 * will steer a register to a non-terminated instance 615 * @gt: GT structure 616 * @reg: register for which the steering is required 617 * @group: return variable for group steering 618 * @instance: return variable for instance steering 619 * 620 * This function returns a group/instance pair that is guaranteed to work for 621 * read steering of the given register. Note that a value will be returned even 622 * if the register is not replicated and therefore does not actually require 623 * steering. 624 * 625 * Returns true if the caller should steer to the @group/@instance values 626 * returned. Returns false if the caller need not perform any steering 627 */ 628 bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, 629 struct xe_reg_mcr reg_mcr, 630 u8 *group, u8 *instance) 631 { 632 const struct xe_reg reg = to_xe_reg(reg_mcr); 633 const struct xe_mmio_range *implicit_ranges; 634 635 for (int type = 0; type < IMPLICIT_STEERING; type++) { 636 if (!gt->steering[type].ranges) 637 continue; 638 639 for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) { 640 if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) { 641 drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized, 642 "Uninitialized usage of MCR register %s/%#x\n", 643 xe_steering_types[type].name, reg.addr); 644 645 *group = gt->steering[type].group_target; 646 *instance = gt->steering[type].instance_target; 647 return true; 648 } 649 } 650 } 651 652 implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges; 653 if (implicit_ranges) 654 for (int i = 0; implicit_ranges[i].end > 0; i++) 655 if (xe_mmio_in_range(>->mmio, &implicit_ranges[i], reg)) 656 return false; 657 658 /* 659 * Not found in a steering table and not a register with implicit 660 * steering. Just steer to 0/0 as a guess and raise a warning. 661 */ 662 drm_WARN(>_to_xe(gt)->drm, true, 663 "Did not find MCR register %#x in any MCR steering table\n", 664 reg.addr); 665 *group = 0; 666 *instance = 0; 667 668 return true; 669 } 670 671 /* 672 * Obtain exclusive access to MCR steering. On MTL and beyond we also need 673 * to synchronize with external clients (e.g., firmware), so a semaphore 674 * register will also need to be taken. 675 */ 676 static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock) 677 { 678 struct xe_device *xe = gt_to_xe(gt); 679 int ret = 0; 680 681 spin_lock(>->mcr_lock); 682 683 /* 684 * Starting with MTL we also need to grab a semaphore register 685 * to synchronize with external agents (e.g., firmware) that now 686 * shares the same steering control register. The semaphore is obtained 687 * when a read to the relevant register returns 1. 688 */ 689 if (GRAPHICS_VERx100(xe) >= 1270) 690 ret = xe_mmio_wait32(>->mmio, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL, 691 true); 692 693 drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT); 694 } 695 696 static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock) 697 { 698 /* Release hardware semaphore - this is done by writing 1 to the register */ 699 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) 700 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, 0x1); 701 702 spin_unlock(>->mcr_lock); 703 } 704 705 /* 706 * Access a register with specific MCR steering 707 * 708 * Caller needs to make sure the relevant forcewake wells are up. 709 */ 710 static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 711 u8 rw_flag, int group, int instance, u32 value) 712 { 713 const struct xe_reg reg = to_xe_reg(reg_mcr); 714 struct xe_mmio *mmio = >->mmio; 715 struct xe_reg steer_reg; 716 u32 steer_val, val = 0; 717 718 lockdep_assert_held(>->mcr_lock); 719 720 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { 721 steer_reg = MTL_MCR_SELECTOR; 722 steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) | 723 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance); 724 } else { 725 steer_reg = MCR_SELECTOR; 726 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) | 727 REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance); 728 } 729 730 /* 731 * Always leave the hardware in multicast mode when doing reads and only 732 * change it to unicast mode when doing writes of a specific instance. 733 * 734 * The setting of the multicast/unicast bit usually wouldn't matter for 735 * read operations (which always return the value from a single register 736 * instance regardless of how that bit is set), but some platforms may 737 * have workarounds requiring us to remain in multicast mode for reads, 738 * e.g. Wa_22013088509 on PVC. There's no real downside to this, so 739 * we'll just go ahead and do so on all platforms; we'll only clear the 740 * multicast bit from the mask when explicitly doing a write operation. 741 * 742 * No need to save old steering reg value. 743 */ 744 if (rw_flag == MCR_OP_READ) 745 steer_val |= MCR_MULTICAST; 746 747 xe_mmio_write32(mmio, steer_reg, steer_val); 748 749 if (rw_flag == MCR_OP_READ) 750 val = xe_mmio_read32(mmio, reg); 751 else 752 xe_mmio_write32(mmio, reg, value); 753 754 /* 755 * If we turned off the multicast bit (during a write) we're required 756 * to turn it back on before finishing. The group and instance values 757 * don't matter since they'll be re-programmed on the next MCR 758 * operation. 759 */ 760 if (rw_flag == MCR_OP_WRITE) 761 xe_mmio_write32(mmio, steer_reg, MCR_MULTICAST); 762 763 return val; 764 } 765 766 /** 767 * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register 768 * @gt: GT structure 769 * @reg_mcr: register to read 770 * 771 * Reads a GT MCR register. The read will be steered to a non-terminated 772 * instance (i.e., one that isn't fused off or powered down by power gating). 773 * This function assumes the caller is already holding any necessary forcewake 774 * domains. 775 * 776 * Returns the value from a non-terminated instance of @reg. 777 */ 778 u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr) 779 { 780 const struct xe_reg reg = to_xe_reg(reg_mcr); 781 u8 group, instance; 782 u32 val; 783 bool steer; 784 785 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 786 787 steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr, 788 &group, &instance); 789 790 if (steer) { 791 mcr_lock(gt); 792 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, 793 group, instance, 0); 794 mcr_unlock(gt); 795 } else { 796 val = xe_mmio_read32(>->mmio, reg); 797 } 798 799 return val; 800 } 801 802 /** 803 * xe_gt_mcr_unicast_read - read a specific instance of an MCR register 804 * @gt: GT structure 805 * @reg_mcr: the MCR register to read 806 * @group: the MCR group 807 * @instance: the MCR instance 808 * 809 * Returns the value read from an MCR register after steering toward a specific 810 * group/instance. 811 */ 812 u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, 813 struct xe_reg_mcr reg_mcr, 814 int group, int instance) 815 { 816 u32 val; 817 818 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 819 820 mcr_lock(gt); 821 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0); 822 mcr_unlock(gt); 823 824 return val; 825 } 826 827 /** 828 * xe_gt_mcr_unicast_write - write a specific instance of an MCR register 829 * @gt: GT structure 830 * @reg_mcr: the MCR register to write 831 * @value: value to write 832 * @group: the MCR group 833 * @instance: the MCR instance 834 * 835 * Write an MCR register in unicast mode after steering toward a specific 836 * group/instance. 837 */ 838 void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 839 u32 value, int group, int instance) 840 { 841 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 842 843 mcr_lock(gt); 844 rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value); 845 mcr_unlock(gt); 846 } 847 848 /** 849 * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register 850 * @gt: GT structure 851 * @reg_mcr: the MCR register to write 852 * @value: value to write 853 * 854 * Write an MCR register in multicast mode to update all instances. 855 */ 856 void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 857 u32 value) 858 { 859 struct xe_reg reg = to_xe_reg(reg_mcr); 860 861 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 862 863 /* 864 * Synchronize with any unicast operations. Once we have exclusive 865 * access, the MULTICAST bit should already be set, so there's no need 866 * to touch the steering register. 867 */ 868 mcr_lock(gt); 869 xe_mmio_write32(>->mmio, reg, value); 870 mcr_unlock(gt); 871 } 872 873 void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p) 874 { 875 for (int i = 0; i < NUM_STEERING_TYPES; i++) { 876 if (gt->steering[i].ranges) { 877 drm_printf(p, "%s steering: group=%#x, instance=%#x\n", 878 xe_steering_types[i].name, 879 gt->steering[i].group_target, 880 gt->steering[i].instance_target); 881 for (int j = 0; gt->steering[i].ranges[j].end; j++) 882 drm_printf(p, "\t0x%06x - 0x%06x\n", 883 gt->steering[i].ranges[j].start, 884 gt->steering[i].ranges[j].end); 885 } 886 } 887 } 888