1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_gt_mcr.h" 7 8 #include "regs/xe_gt_regs.h" 9 #include "xe_assert.h" 10 #include "xe_gt.h" 11 #include "xe_gt_printk.h" 12 #include "xe_gt_topology.h" 13 #include "xe_gt_types.h" 14 #include "xe_guc_hwconfig.h" 15 #include "xe_mmio.h" 16 #include "xe_sriov.h" 17 18 /** 19 * DOC: GT Multicast/Replicated (MCR) Register Support 20 * 21 * Some GT registers are designed as "multicast" or "replicated" registers: 22 * multiple instances of the same register share a single MMIO offset. MCR 23 * registers are generally used when the hardware needs to potentially track 24 * independent values of a register per hardware unit (e.g., per-subslice, 25 * per-L3bank, etc.). The specific types of replication that exist vary 26 * per-platform. 27 * 28 * MMIO accesses to MCR registers are controlled according to the settings 29 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR 30 * registers can be done in either multicast (a single write updates all 31 * instances of the register to the same value) or unicast (a write updates only 32 * one specific instance) form. Reads of MCR registers always operate in a 33 * unicast manner regardless of how the multicast/unicast bit is set in 34 * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is 35 * referred to as "steering." 36 * 37 * If MCR register operations are steered toward a hardware unit that is 38 * fused off or currently powered down due to power gating, the MMIO operation 39 * is "terminated" by the hardware. Terminated read operations will return a 40 * value of zero and terminated unicast write operations will be silently 41 * ignored. During device initialization, the goal of the various 42 * ``init_steering_*()`` functions is to apply the platform-specific rules for 43 * each MCR register type to identify a steering target that will select a 44 * non-terminated instance. 45 * 46 * MCR registers are not available on Virtual Function (VF). 47 */ 48 49 static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr) 50 { 51 return reg_mcr.__reg; 52 } 53 54 enum { 55 MCR_OP_READ, 56 MCR_OP_WRITE 57 }; 58 59 static const struct xe_mmio_range xelp_l3bank_steering_table[] = { 60 { 0x00B100, 0x00B3FF }, 61 {}, 62 }; 63 64 static const struct xe_mmio_range xehp_l3bank_steering_table[] = { 65 { 0x008C80, 0x008CFF }, 66 { 0x00B100, 0x00B3FF }, 67 {}, 68 }; 69 70 /* 71 * Although the bspec lists more "MSLICE" ranges than shown here, some of those 72 * are of a "GAM" subclass that has special rules and doesn't need to be 73 * included here. 74 */ 75 static const struct xe_mmio_range xehp_mslice_steering_table[] = { 76 { 0x00DD00, 0x00DDFF }, 77 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ 78 {}, 79 }; 80 81 static const struct xe_mmio_range xehp_lncf_steering_table[] = { 82 { 0x00B000, 0x00B0FF }, 83 { 0x00D880, 0x00D8FF }, 84 {}, 85 }; 86 87 /* 88 * We have several types of MCR registers where steering to (0,0) will always 89 * provide us with a non-terminated value. We'll stick them all in the same 90 * table for simplicity. 91 */ 92 static const struct xe_mmio_range xehpc_instance0_steering_table[] = { 93 { 0x004000, 0x004AFF }, /* HALF-BSLICE */ 94 { 0x008800, 0x00887F }, /* CC */ 95 { 0x008A80, 0x008AFF }, /* TILEPSMI */ 96 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ 97 { 0x00B100, 0x00B3FF }, /* L3BANK */ 98 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ 99 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ 100 { 0x00DD00, 0x00DDFF }, /* BSLICE */ 101 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ 102 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ 103 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ 104 { 0x024180, 0x0241FF }, /* HALF-BSLICE */ 105 {}, 106 }; 107 108 static const struct xe_mmio_range xelpg_instance0_steering_table[] = { 109 { 0x000B00, 0x000BFF }, /* SQIDI */ 110 { 0x001000, 0x001FFF }, /* SQIDI */ 111 { 0x004000, 0x0048FF }, /* GAM */ 112 { 0x008700, 0x0087FF }, /* SQIDI */ 113 { 0x00B000, 0x00B0FF }, /* NODE */ 114 { 0x00C800, 0x00CFFF }, /* GAM */ 115 { 0x00D880, 0x00D8FF }, /* NODE */ 116 { 0x00DD00, 0x00DDFF }, /* OAAL2 */ 117 {}, 118 }; 119 120 static const struct xe_mmio_range xelpg_l3bank_steering_table[] = { 121 { 0x00B100, 0x00B3FF }, 122 {}, 123 }; 124 125 static const struct xe_mmio_range xelp_dss_steering_table[] = { 126 { 0x008150, 0x00815F }, 127 { 0x009520, 0x00955F }, 128 { 0x00DE80, 0x00E8FF }, 129 { 0x024A00, 0x024A7F }, 130 {}, 131 }; 132 133 /* DSS steering is used for GSLICE ranges as well */ 134 static const struct xe_mmio_range xehp_dss_steering_table[] = { 135 { 0x005200, 0x0052FF }, /* GSLICE */ 136 { 0x005400, 0x007FFF }, /* GSLICE */ 137 { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 138 { 0x008D00, 0x008DFF }, /* DSS */ 139 { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 140 { 0x009680, 0x0096FF }, /* DSS */ 141 { 0x00D800, 0x00D87F }, /* GSLICE */ 142 { 0x00DC00, 0x00DCFF }, /* GSLICE */ 143 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */ 144 { 0x017000, 0x017FFF }, /* GSLICE */ 145 { 0x024A00, 0x024A7F }, /* DSS */ 146 {}, 147 }; 148 149 /* DSS steering is used for COMPUTE ranges as well */ 150 static const struct xe_mmio_range xehpc_dss_steering_table[] = { 151 { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */ 152 { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 153 { 0x009680, 0x0096FF }, /* DSS */ 154 { 0x00DC00, 0x00DCFF }, /* COMPUTE */ 155 { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */ 156 {}, 157 }; 158 159 /* DSS steering is used for SLICE ranges as well */ 160 static const struct xe_mmio_range xelpg_dss_steering_table[] = { 161 { 0x005200, 0x0052FF }, /* SLICE */ 162 { 0x005500, 0x007FFF }, /* SLICE */ 163 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 164 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 165 { 0x009680, 0x0096FF }, /* DSS */ 166 { 0x00D800, 0x00D87F }, /* SLICE */ 167 { 0x00DC00, 0x00DCFF }, /* SLICE */ 168 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 169 {}, 170 }; 171 172 static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = { 173 { 0x393200, 0x39323F }, 174 { 0x393400, 0x3934FF }, 175 {}, 176 }; 177 178 static const struct xe_mmio_range dg2_implicit_steering_table[] = { 179 { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */ 180 { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */ 181 { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */ 182 { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */ 183 { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */ 184 { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */ 185 {}, 186 }; 187 188 static const struct xe_mmio_range xe2lpg_dss_steering_table[] = { 189 { 0x005200, 0x0052FF }, /* SLICE */ 190 { 0x005500, 0x007FFF }, /* SLICE */ 191 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 192 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 193 { 0x009680, 0x0096FF }, /* DSS */ 194 { 0x00D800, 0x00D87F }, /* SLICE */ 195 { 0x00DC00, 0x00DCFF }, /* SLICE */ 196 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 197 { 0x00E980, 0x00E9FF }, /* SLICE */ 198 { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */ 199 {}, 200 }; 201 202 static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = { 203 { 0x000B00, 0x000BFF }, 204 { 0x001000, 0x001FFF }, 205 {}, 206 }; 207 208 static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = { 209 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ 210 { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */ 211 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */ 212 { 0x00C800, 0x00CFFF }, /* GAM */ 213 { 0x00D880, 0x00D8FF }, /* NODE */ 214 { 0x00DD00, 0x00DDFF }, /* MEMPIPE */ 215 { 0x00E900, 0x00E97F }, /* MEMPIPE */ 216 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ 217 { 0x013400, 0x0135FF }, /* MEMPIPE */ 218 {}, 219 }; 220 221 static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = { 222 { 0x388160, 0x38817F }, 223 { 0x389480, 0x3894CF }, 224 {}, 225 }; 226 227 static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = { 228 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */ 229 { 0x384900, 0x384AFF }, /* GAM */ 230 { 0x389560, 0x3895FF }, /* MEDIAINF */ 231 { 0x38B600, 0x38B8FF }, /* L3BANK */ 232 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ 233 { 0x38F000, 0x38F0FF }, /* GAM */ 234 { 0x393C00, 0x393C7F }, /* MEDIAINF */ 235 {}, 236 }; 237 238 static const struct xe_mmio_range xe3lpm_instance0_steering_table[] = { 239 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */ 240 { 0x384900, 0x384AFF }, /* GAM */ 241 { 0x389560, 0x3895FF }, /* MEDIAINF */ 242 { 0x38B600, 0x38B8FF }, /* L3BANK */ 243 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ 244 { 0x38D0D0, 0x38F0FF }, /* MEDIAINF, GAM */ 245 { 0x393C00, 0x393C7F }, /* MEDIAINF */ 246 {}, 247 }; 248 249 static void init_steering_l3bank(struct xe_gt *gt) 250 { 251 struct xe_mmio *mmio = >->mmio; 252 253 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { 254 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, 255 xe_mmio_read32(mmio, MIRROR_FUSE3)); 256 u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK, 257 xe_mmio_read32(mmio, XEHP_FUSE4)); 258 259 /* 260 * Group selects mslice, instance selects bank within mslice. 261 * Bank 0 is always valid _except_ when the bank mask is 010b. 262 */ 263 gt->steering[L3BANK].group_target = __ffs(mslice_mask); 264 gt->steering[L3BANK].instance_target = 265 bank_mask & BIT(0) ? 0 : 2; 266 } else if (gt_to_xe(gt)->info.platform == XE_DG2) { 267 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, 268 xe_mmio_read32(mmio, MIRROR_FUSE3)); 269 u32 bank = __ffs(mslice_mask) * 8; 270 271 /* 272 * Like mslice registers, look for a valid mslice and steer to 273 * the first L3BANK of that quad. Access to the Nth L3 bank is 274 * split between the first bits of group and instance 275 */ 276 gt->steering[L3BANK].group_target = (bank >> 2) & 0x7; 277 gt->steering[L3BANK].instance_target = bank & 0x3; 278 } else { 279 u32 fuse = REG_FIELD_GET(L3BANK_MASK, 280 ~xe_mmio_read32(mmio, MIRROR_FUSE3)); 281 282 gt->steering[L3BANK].group_target = 0; /* unused */ 283 gt->steering[L3BANK].instance_target = __ffs(fuse); 284 } 285 } 286 287 static void init_steering_mslice(struct xe_gt *gt) 288 { 289 u32 mask = REG_FIELD_GET(MEML3_EN_MASK, 290 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 291 292 /* 293 * mslice registers are valid (not terminated) if either the meml3 294 * associated with the mslice is present, or at least one DSS associated 295 * with the mslice is present. There will always be at least one meml3 296 * so we can just use that to find a non-terminated mslice and ignore 297 * the DSS fusing. 298 */ 299 gt->steering[MSLICE].group_target = __ffs(mask); 300 gt->steering[MSLICE].instance_target = 0; /* unused */ 301 302 /* 303 * LNCF termination is also based on mslice presence, so we'll set 304 * it up here. Either LNCF within a non-terminated mslice will work, 305 * so we just always pick LNCF 0 here. 306 */ 307 gt->steering[LNCF].group_target = __ffs(mask) << 1; 308 gt->steering[LNCF].instance_target = 0; /* unused */ 309 } 310 311 static unsigned int dss_per_group(struct xe_gt *gt) 312 { 313 struct xe_guc *guc = >->uc.guc; 314 u32 max_slices = 0, max_subslices = 0; 315 int ret; 316 317 /* 318 * Try to query the GuC's hwconfig table for the maximum number of 319 * slices and subslices. These don't reflect the platform's actual 320 * slice/DSS counts, just the physical layout by which we should 321 * determine the steering targets. On older platforms with older GuC 322 * firmware releases it's possible that these attributes may not be 323 * included in the table, so we can always fall back to the old 324 * hardcoded layouts. 325 */ 326 #define HWCONFIG_ATTR_MAX_SLICES 1 327 #define HWCONFIG_ATTR_MAX_SUBSLICES 70 328 329 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES, 330 &max_slices); 331 if (ret < 0 || max_slices == 0) 332 goto fallback; 333 334 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES, 335 &max_subslices); 336 if (ret < 0 || max_subslices == 0) 337 goto fallback; 338 339 return DIV_ROUND_UP(max_subslices, max_slices); 340 341 fallback: 342 /* 343 * Some older platforms don't have tables or don't have complete tables. 344 * Newer platforms should always have the required info. 345 */ 346 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000 && 347 !gt_to_xe(gt)->info.force_execlist) 348 xe_gt_err(gt, "Slice/Subslice counts missing from hwconfig table; using typical fallback values\n"); 349 350 if (gt_to_xe(gt)->info.platform == XE_PVC) 351 return 8; 352 else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250) 353 return 4; 354 else 355 return 6; 356 } 357 358 /** 359 * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS 360 * @gt: GT structure 361 * @dss: DSS ID to obtain steering for 362 * @group: pointer to storage for steering group ID 363 * @instance: pointer to storage for steering instance ID 364 */ 365 void xe_gt_mcr_get_dss_steering(struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance) 366 { 367 xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS); 368 369 *group = dss / gt->steering_dss_per_grp; 370 *instance = dss % gt->steering_dss_per_grp; 371 } 372 373 /** 374 * xe_gt_mcr_steering_info_to_dss_id - Get DSS ID from group/instance steering 375 * @gt: GT structure 376 * @group: steering group ID 377 * @instance: steering instance ID 378 * 379 * Return: the converted DSS id. 380 */ 381 u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance) 382 { 383 return group * dss_per_group(gt) + instance; 384 } 385 386 static void init_steering_dss(struct xe_gt *gt) 387 { 388 gt->steering_dss_per_grp = dss_per_group(gt); 389 390 xe_gt_mcr_get_dss_steering(gt, 391 min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0), 392 xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)), 393 >->steering[DSS].group_target, 394 >->steering[DSS].instance_target); 395 } 396 397 static void init_steering_oaddrm(struct xe_gt *gt) 398 { 399 /* 400 * First instance is only terminated if the entire first media slice 401 * is absent (i.e., no VCS0 or VECS0). 402 */ 403 if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0)) 404 gt->steering[OADDRM].group_target = 0; 405 else 406 gt->steering[OADDRM].group_target = 1; 407 408 gt->steering[OADDRM].instance_target = 0; /* unused */ 409 } 410 411 static void init_steering_sqidi_psmi(struct xe_gt *gt) 412 { 413 u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, 414 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 415 u32 select = __ffs(mask); 416 417 gt->steering[SQIDI_PSMI].group_target = select >> 1; 418 gt->steering[SQIDI_PSMI].instance_target = select & 0x1; 419 } 420 421 static const struct { 422 const char *name; 423 void (*init)(struct xe_gt *gt); 424 } xe_steering_types[] = { 425 [L3BANK] = { "L3BANK", init_steering_l3bank }, 426 [MSLICE] = { "MSLICE", init_steering_mslice }, 427 [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */ 428 [DSS] = { "DSS", init_steering_dss }, 429 [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm }, 430 [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi }, 431 [INSTANCE0] = { "INSTANCE 0", NULL }, 432 [IMPLICIT_STEERING] = { "IMPLICIT", NULL }, 433 }; 434 435 /** 436 * xe_gt_mcr_init_early - Early initialization of the MCR support 437 * @gt: GT structure 438 * 439 * Perform early software only initialization of the MCR lock to allow 440 * the synchronization on accessing the STEER_SEMAPHORE register and 441 * use the xe_gt_mcr_multicast_write() function, plus the minimum 442 * safe MCR registers required for VRAM/CCS probing. 443 */ 444 void xe_gt_mcr_init_early(struct xe_gt *gt) 445 { 446 struct xe_device *xe = gt_to_xe(gt); 447 448 BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES); 449 BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES); 450 451 spin_lock_init(>->mcr_lock); 452 453 if (IS_SRIOV_VF(xe)) 454 return; 455 456 if (gt->info.type == XE_GT_TYPE_MEDIA) { 457 drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13); 458 459 if (MEDIA_VER(xe) >= 30) { 460 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; 461 gt->steering[INSTANCE0].ranges = xe3lpm_instance0_steering_table; 462 } else if (MEDIA_VERx100(xe) >= 1301) { 463 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; 464 gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table; 465 } else { 466 gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table; 467 } 468 } else { 469 if (GRAPHICS_VER(xe) >= 20) { 470 gt->steering[DSS].ranges = xe2lpg_dss_steering_table; 471 gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table; 472 gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table; 473 } else if (GRAPHICS_VERx100(xe) >= 1270) { 474 gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; 475 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; 476 gt->steering[DSS].ranges = xelpg_dss_steering_table; 477 } else if (xe->info.platform == XE_PVC) { 478 gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table; 479 gt->steering[DSS].ranges = xehpc_dss_steering_table; 480 } else if (xe->info.platform == XE_DG2) { 481 gt->steering[L3BANK].ranges = xehp_l3bank_steering_table; 482 gt->steering[MSLICE].ranges = xehp_mslice_steering_table; 483 gt->steering[LNCF].ranges = xehp_lncf_steering_table; 484 gt->steering[DSS].ranges = xehp_dss_steering_table; 485 gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table; 486 } else { 487 gt->steering[L3BANK].ranges = xelp_l3bank_steering_table; 488 gt->steering[DSS].ranges = xelp_dss_steering_table; 489 } 490 } 491 492 /* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */ 493 gt->steering[INSTANCE0].initialized = true; 494 } 495 496 /** 497 * xe_gt_mcr_init - Normal initialization of the MCR support 498 * @gt: GT structure 499 * 500 * Perform normal initialization of the MCR for all usages. 501 */ 502 void xe_gt_mcr_init(struct xe_gt *gt) 503 { 504 if (IS_SRIOV_VF(gt_to_xe(gt))) 505 return; 506 507 /* Select non-terminated steering target for each type */ 508 for (int i = 0; i < NUM_STEERING_TYPES; i++) { 509 gt->steering[i].initialized = true; 510 if (gt->steering[i].ranges && xe_steering_types[i].init) 511 xe_steering_types[i].init(gt); 512 } 513 } 514 515 /** 516 * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers 517 * @gt: GT structure 518 * 519 * Some register ranges don't need to have their steering control registers 520 * changed on each access - it's sufficient to set them once on initialization. 521 * This function sets those registers for each platform * 522 */ 523 void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) 524 { 525 struct xe_device *xe = gt_to_xe(gt); 526 527 if (IS_SRIOV_VF(xe)) 528 return; 529 530 if (xe->info.platform == XE_DG2) { 531 u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) | 532 REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2); 533 534 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, steer_val); 535 xe_mmio_write32(>->mmio, SF_MCR_SELECTOR, steer_val); 536 /* 537 * For GAM registers, all reads should be directed to instance 1 538 * (unicast reads against other instances are not allowed), 539 * and instance 1 is already the hardware's default steering 540 * target, which we never change 541 */ 542 } 543 } 544 545 /* 546 * xe_gt_mcr_get_nonterminated_steering - find group/instance values that 547 * will steer a register to a non-terminated instance 548 * @gt: GT structure 549 * @reg: register for which the steering is required 550 * @group: return variable for group steering 551 * @instance: return variable for instance steering 552 * 553 * This function returns a group/instance pair that is guaranteed to work for 554 * read steering of the given register. Note that a value will be returned even 555 * if the register is not replicated and therefore does not actually require 556 * steering. 557 * 558 * Returns true if the caller should steer to the @group/@instance values 559 * returned. Returns false if the caller need not perform any steering 560 */ 561 bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, 562 struct xe_reg_mcr reg_mcr, 563 u8 *group, u8 *instance) 564 { 565 const struct xe_reg reg = to_xe_reg(reg_mcr); 566 const struct xe_mmio_range *implicit_ranges; 567 568 for (int type = 0; type < IMPLICIT_STEERING; type++) { 569 if (!gt->steering[type].ranges) 570 continue; 571 572 for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) { 573 if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) { 574 drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized, 575 "Uninitialized usage of MCR register %s/%#x\n", 576 xe_steering_types[type].name, reg.addr); 577 578 *group = gt->steering[type].group_target; 579 *instance = gt->steering[type].instance_target; 580 return true; 581 } 582 } 583 } 584 585 implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges; 586 if (implicit_ranges) 587 for (int i = 0; implicit_ranges[i].end > 0; i++) 588 if (xe_mmio_in_range(>->mmio, &implicit_ranges[i], reg)) 589 return false; 590 591 /* 592 * Not found in a steering table and not a register with implicit 593 * steering. Just steer to 0/0 as a guess and raise a warning. 594 */ 595 drm_WARN(>_to_xe(gt)->drm, true, 596 "Did not find MCR register %#x in any MCR steering table\n", 597 reg.addr); 598 *group = 0; 599 *instance = 0; 600 601 return true; 602 } 603 604 /* 605 * Obtain exclusive access to MCR steering. On MTL and beyond we also need 606 * to synchronize with external clients (e.g., firmware), so a semaphore 607 * register will also need to be taken. 608 */ 609 static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock) 610 { 611 struct xe_device *xe = gt_to_xe(gt); 612 int ret = 0; 613 614 spin_lock(>->mcr_lock); 615 616 /* 617 * Starting with MTL we also need to grab a semaphore register 618 * to synchronize with external agents (e.g., firmware) that now 619 * shares the same steering control register. The semaphore is obtained 620 * when a read to the relevant register returns 1. 621 */ 622 if (GRAPHICS_VERx100(xe) >= 1270) 623 ret = xe_mmio_wait32(>->mmio, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL, 624 true); 625 626 drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT); 627 } 628 629 static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock) 630 { 631 /* Release hardware semaphore - this is done by writing 1 to the register */ 632 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) 633 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, 0x1); 634 635 spin_unlock(>->mcr_lock); 636 } 637 638 /* 639 * Access a register with specific MCR steering 640 * 641 * Caller needs to make sure the relevant forcewake wells are up. 642 */ 643 static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 644 u8 rw_flag, int group, int instance, u32 value) 645 { 646 const struct xe_reg reg = to_xe_reg(reg_mcr); 647 struct xe_mmio *mmio = >->mmio; 648 struct xe_reg steer_reg; 649 u32 steer_val, val = 0; 650 651 lockdep_assert_held(>->mcr_lock); 652 653 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { 654 steer_reg = MTL_MCR_SELECTOR; 655 steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) | 656 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance); 657 } else { 658 steer_reg = MCR_SELECTOR; 659 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) | 660 REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance); 661 } 662 663 /* 664 * Always leave the hardware in multicast mode when doing reads and only 665 * change it to unicast mode when doing writes of a specific instance. 666 * 667 * The setting of the multicast/unicast bit usually wouldn't matter for 668 * read operations (which always return the value from a single register 669 * instance regardless of how that bit is set), but some platforms may 670 * have workarounds requiring us to remain in multicast mode for reads, 671 * e.g. Wa_22013088509 on PVC. There's no real downside to this, so 672 * we'll just go ahead and do so on all platforms; we'll only clear the 673 * multicast bit from the mask when explicitly doing a write operation. 674 * 675 * No need to save old steering reg value. 676 */ 677 if (rw_flag == MCR_OP_READ) 678 steer_val |= MCR_MULTICAST; 679 680 xe_mmio_write32(mmio, steer_reg, steer_val); 681 682 if (rw_flag == MCR_OP_READ) 683 val = xe_mmio_read32(mmio, reg); 684 else 685 xe_mmio_write32(mmio, reg, value); 686 687 /* 688 * If we turned off the multicast bit (during a write) we're required 689 * to turn it back on before finishing. The group and instance values 690 * don't matter since they'll be re-programmed on the next MCR 691 * operation. 692 */ 693 if (rw_flag == MCR_OP_WRITE) 694 xe_mmio_write32(mmio, steer_reg, MCR_MULTICAST); 695 696 return val; 697 } 698 699 /** 700 * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register 701 * @gt: GT structure 702 * @reg_mcr: register to read 703 * 704 * Reads a GT MCR register. The read will be steered to a non-terminated 705 * instance (i.e., one that isn't fused off or powered down by power gating). 706 * This function assumes the caller is already holding any necessary forcewake 707 * domains. 708 * 709 * Returns the value from a non-terminated instance of @reg. 710 */ 711 u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr) 712 { 713 const struct xe_reg reg = to_xe_reg(reg_mcr); 714 u8 group, instance; 715 u32 val; 716 bool steer; 717 718 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 719 720 steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr, 721 &group, &instance); 722 723 if (steer) { 724 mcr_lock(gt); 725 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, 726 group, instance, 0); 727 mcr_unlock(gt); 728 } else { 729 val = xe_mmio_read32(>->mmio, reg); 730 } 731 732 return val; 733 } 734 735 /** 736 * xe_gt_mcr_unicast_read - read a specific instance of an MCR register 737 * @gt: GT structure 738 * @reg_mcr: the MCR register to read 739 * @group: the MCR group 740 * @instance: the MCR instance 741 * 742 * Returns the value read from an MCR register after steering toward a specific 743 * group/instance. 744 */ 745 u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, 746 struct xe_reg_mcr reg_mcr, 747 int group, int instance) 748 { 749 u32 val; 750 751 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 752 753 mcr_lock(gt); 754 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0); 755 mcr_unlock(gt); 756 757 return val; 758 } 759 760 /** 761 * xe_gt_mcr_unicast_write - write a specific instance of an MCR register 762 * @gt: GT structure 763 * @reg_mcr: the MCR register to write 764 * @value: value to write 765 * @group: the MCR group 766 * @instance: the MCR instance 767 * 768 * Write an MCR register in unicast mode after steering toward a specific 769 * group/instance. 770 */ 771 void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 772 u32 value, int group, int instance) 773 { 774 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 775 776 mcr_lock(gt); 777 rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value); 778 mcr_unlock(gt); 779 } 780 781 /** 782 * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register 783 * @gt: GT structure 784 * @reg_mcr: the MCR register to write 785 * @value: value to write 786 * 787 * Write an MCR register in multicast mode to update all instances. 788 */ 789 void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 790 u32 value) 791 { 792 struct xe_reg reg = to_xe_reg(reg_mcr); 793 794 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 795 796 /* 797 * Synchronize with any unicast operations. Once we have exclusive 798 * access, the MULTICAST bit should already be set, so there's no need 799 * to touch the steering register. 800 */ 801 mcr_lock(gt); 802 xe_mmio_write32(>->mmio, reg, value); 803 mcr_unlock(gt); 804 } 805 806 void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p) 807 { 808 for (int i = 0; i < NUM_STEERING_TYPES; i++) { 809 if (gt->steering[i].ranges) { 810 drm_printf(p, "%s steering: group=%#x, instance=%#x\n", 811 xe_steering_types[i].name, 812 gt->steering[i].group_target, 813 gt->steering[i].instance_target); 814 for (int j = 0; gt->steering[i].ranges[j].end; j++) 815 drm_printf(p, "\t0x%06x - 0x%06x\n", 816 gt->steering[i].ranges[j].start, 817 gt->steering[i].ranges[j].end); 818 } 819 } 820 } 821