1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_gt_mcr.h" 7 8 #include "regs/xe_gt_regs.h" 9 #include "xe_assert.h" 10 #include "xe_gt.h" 11 #include "xe_gt_printk.h" 12 #include "xe_gt_topology.h" 13 #include "xe_gt_types.h" 14 #include "xe_guc_hwconfig.h" 15 #include "xe_mmio.h" 16 #include "xe_sriov.h" 17 18 /** 19 * DOC: GT Multicast/Replicated (MCR) Register Support 20 * 21 * Some GT registers are designed as "multicast" or "replicated" registers: 22 * multiple instances of the same register share a single MMIO offset. MCR 23 * registers are generally used when the hardware needs to potentially track 24 * independent values of a register per hardware unit (e.g., per-subslice, 25 * per-L3bank, etc.). The specific types of replication that exist vary 26 * per-platform. 27 * 28 * MMIO accesses to MCR registers are controlled according to the settings 29 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR 30 * registers can be done in either multicast (a single write updates all 31 * instances of the register to the same value) or unicast (a write updates only 32 * one specific instance) form. Reads of MCR registers always operate in a 33 * unicast manner regardless of how the multicast/unicast bit is set in 34 * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is 35 * referred to as "steering." 36 * 37 * If MCR register operations are steered toward a hardware unit that is 38 * fused off or currently powered down due to power gating, the MMIO operation 39 * is "terminated" by the hardware. Terminated read operations will return a 40 * value of zero and terminated unicast write operations will be silently 41 * ignored. During device initialization, the goal of the various 42 * ``init_steering_*()`` functions is to apply the platform-specific rules for 43 * each MCR register type to identify a steering target that will select a 44 * non-terminated instance. 45 * 46 * MCR registers are not available on Virtual Function (VF). 47 */ 48 49 #define STEER_SEMAPHORE XE_REG(0xFD0) 50 51 static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr) 52 { 53 return reg_mcr.__reg; 54 } 55 56 enum { 57 MCR_OP_READ, 58 MCR_OP_WRITE 59 }; 60 61 static const struct xe_mmio_range xelp_l3bank_steering_table[] = { 62 { 0x00B100, 0x00B3FF }, 63 {}, 64 }; 65 66 static const struct xe_mmio_range xehp_l3bank_steering_table[] = { 67 { 0x008C80, 0x008CFF }, 68 { 0x00B100, 0x00B3FF }, 69 {}, 70 }; 71 72 /* 73 * Although the bspec lists more "MSLICE" ranges than shown here, some of those 74 * are of a "GAM" subclass that has special rules and doesn't need to be 75 * included here. 76 */ 77 static const struct xe_mmio_range xehp_mslice_steering_table[] = { 78 { 0x00DD00, 0x00DDFF }, 79 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ 80 {}, 81 }; 82 83 static const struct xe_mmio_range xehp_lncf_steering_table[] = { 84 { 0x00B000, 0x00B0FF }, 85 { 0x00D880, 0x00D8FF }, 86 {}, 87 }; 88 89 /* 90 * We have several types of MCR registers where steering to (0,0) will always 91 * provide us with a non-terminated value. We'll stick them all in the same 92 * table for simplicity. 93 */ 94 static const struct xe_mmio_range xehpc_instance0_steering_table[] = { 95 { 0x004000, 0x004AFF }, /* HALF-BSLICE */ 96 { 0x008800, 0x00887F }, /* CC */ 97 { 0x008A80, 0x008AFF }, /* TILEPSMI */ 98 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ 99 { 0x00B100, 0x00B3FF }, /* L3BANK */ 100 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ 101 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ 102 { 0x00DD00, 0x00DDFF }, /* BSLICE */ 103 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ 104 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ 105 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ 106 { 0x024180, 0x0241FF }, /* HALF-BSLICE */ 107 {}, 108 }; 109 110 static const struct xe_mmio_range xelpg_instance0_steering_table[] = { 111 { 0x000B00, 0x000BFF }, /* SQIDI */ 112 { 0x001000, 0x001FFF }, /* SQIDI */ 113 { 0x004000, 0x0048FF }, /* GAM */ 114 { 0x008700, 0x0087FF }, /* SQIDI */ 115 { 0x00B000, 0x00B0FF }, /* NODE */ 116 { 0x00C800, 0x00CFFF }, /* GAM */ 117 { 0x00D880, 0x00D8FF }, /* NODE */ 118 { 0x00DD00, 0x00DDFF }, /* OAAL2 */ 119 {}, 120 }; 121 122 static const struct xe_mmio_range xelpg_l3bank_steering_table[] = { 123 { 0x00B100, 0x00B3FF }, 124 {}, 125 }; 126 127 static const struct xe_mmio_range xelp_dss_steering_table[] = { 128 { 0x008150, 0x00815F }, 129 { 0x009520, 0x00955F }, 130 { 0x00DE80, 0x00E8FF }, 131 { 0x024A00, 0x024A7F }, 132 {}, 133 }; 134 135 /* DSS steering is used for GSLICE ranges as well */ 136 static const struct xe_mmio_range xehp_dss_steering_table[] = { 137 { 0x005200, 0x0052FF }, /* GSLICE */ 138 { 0x005400, 0x007FFF }, /* GSLICE */ 139 { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 140 { 0x008D00, 0x008DFF }, /* DSS */ 141 { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 142 { 0x009680, 0x0096FF }, /* DSS */ 143 { 0x00D800, 0x00D87F }, /* GSLICE */ 144 { 0x00DC00, 0x00DCFF }, /* GSLICE */ 145 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */ 146 { 0x017000, 0x017FFF }, /* GSLICE */ 147 { 0x024A00, 0x024A7F }, /* DSS */ 148 {}, 149 }; 150 151 /* DSS steering is used for COMPUTE ranges as well */ 152 static const struct xe_mmio_range xehpc_dss_steering_table[] = { 153 { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */ 154 { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 155 { 0x009680, 0x0096FF }, /* DSS */ 156 { 0x00DC00, 0x00DCFF }, /* COMPUTE */ 157 { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */ 158 {}, 159 }; 160 161 /* DSS steering is used for SLICE ranges as well */ 162 static const struct xe_mmio_range xelpg_dss_steering_table[] = { 163 { 0x005200, 0x0052FF }, /* SLICE */ 164 { 0x005500, 0x007FFF }, /* SLICE */ 165 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 166 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 167 { 0x009680, 0x0096FF }, /* DSS */ 168 { 0x00D800, 0x00D87F }, /* SLICE */ 169 { 0x00DC00, 0x00DCFF }, /* SLICE */ 170 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 171 {}, 172 }; 173 174 static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = { 175 { 0x393200, 0x39323F }, 176 { 0x393400, 0x3934FF }, 177 {}, 178 }; 179 180 static const struct xe_mmio_range dg2_implicit_steering_table[] = { 181 { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */ 182 { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */ 183 { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */ 184 { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */ 185 { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */ 186 { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */ 187 {}, 188 }; 189 190 static const struct xe_mmio_range xe2lpg_dss_steering_table[] = { 191 { 0x005200, 0x0052FF }, /* SLICE */ 192 { 0x005500, 0x007FFF }, /* SLICE */ 193 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 194 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 195 { 0x009680, 0x0096FF }, /* DSS */ 196 { 0x00D800, 0x00D87F }, /* SLICE */ 197 { 0x00DC00, 0x00DCFF }, /* SLICE */ 198 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 199 { 0x00E980, 0x00E9FF }, /* SLICE */ 200 { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */ 201 {}, 202 }; 203 204 static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = { 205 { 0x000B00, 0x000BFF }, 206 { 0x001000, 0x001FFF }, 207 {}, 208 }; 209 210 static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = { 211 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ 212 { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */ 213 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */ 214 { 0x00C800, 0x00CFFF }, /* GAM */ 215 { 0x00D880, 0x00D8FF }, /* NODE */ 216 { 0x00DD00, 0x00DDFF }, /* MEMPIPE */ 217 { 0x00E900, 0x00E97F }, /* MEMPIPE */ 218 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ 219 { 0x013400, 0x0135FF }, /* MEMPIPE */ 220 {}, 221 }; 222 223 static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = { 224 { 0x388160, 0x38817F }, 225 { 0x389480, 0x3894CF }, 226 {}, 227 }; 228 229 static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = { 230 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */ 231 { 0x384900, 0x384AFF }, /* GAM */ 232 { 0x389560, 0x3895FF }, /* MEDIAINF */ 233 { 0x38B600, 0x38B8FF }, /* L3BANK */ 234 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ 235 { 0x38F000, 0x38F0FF }, /* GAM */ 236 { 0x393C00, 0x393C7F }, /* MEDIAINF */ 237 {}, 238 }; 239 240 static const struct xe_mmio_range xe3lpm_instance0_steering_table[] = { 241 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */ 242 { 0x384900, 0x384AFF }, /* GAM */ 243 { 0x389560, 0x3895FF }, /* MEDIAINF */ 244 { 0x38B600, 0x38B8FF }, /* L3BANK */ 245 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ 246 { 0x38D0D0, 0x38F0FF }, /* MEDIAINF, GAM */ 247 { 0x393C00, 0x393C7F }, /* MEDIAINF */ 248 {}, 249 }; 250 251 static void init_steering_l3bank(struct xe_gt *gt) 252 { 253 struct xe_mmio *mmio = >->mmio; 254 255 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { 256 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, 257 xe_mmio_read32(mmio, MIRROR_FUSE3)); 258 u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK, 259 xe_mmio_read32(mmio, XEHP_FUSE4)); 260 261 /* 262 * Group selects mslice, instance selects bank within mslice. 263 * Bank 0 is always valid _except_ when the bank mask is 010b. 264 */ 265 gt->steering[L3BANK].group_target = __ffs(mslice_mask); 266 gt->steering[L3BANK].instance_target = 267 bank_mask & BIT(0) ? 0 : 2; 268 } else if (gt_to_xe(gt)->info.platform == XE_DG2) { 269 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, 270 xe_mmio_read32(mmio, MIRROR_FUSE3)); 271 u32 bank = __ffs(mslice_mask) * 8; 272 273 /* 274 * Like mslice registers, look for a valid mslice and steer to 275 * the first L3BANK of that quad. Access to the Nth L3 bank is 276 * split between the first bits of group and instance 277 */ 278 gt->steering[L3BANK].group_target = (bank >> 2) & 0x7; 279 gt->steering[L3BANK].instance_target = bank & 0x3; 280 } else { 281 u32 fuse = REG_FIELD_GET(L3BANK_MASK, 282 ~xe_mmio_read32(mmio, MIRROR_FUSE3)); 283 284 gt->steering[L3BANK].group_target = 0; /* unused */ 285 gt->steering[L3BANK].instance_target = __ffs(fuse); 286 } 287 } 288 289 static void init_steering_mslice(struct xe_gt *gt) 290 { 291 u32 mask = REG_FIELD_GET(MEML3_EN_MASK, 292 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 293 294 /* 295 * mslice registers are valid (not terminated) if either the meml3 296 * associated with the mslice is present, or at least one DSS associated 297 * with the mslice is present. There will always be at least one meml3 298 * so we can just use that to find a non-terminated mslice and ignore 299 * the DSS fusing. 300 */ 301 gt->steering[MSLICE].group_target = __ffs(mask); 302 gt->steering[MSLICE].instance_target = 0; /* unused */ 303 304 /* 305 * LNCF termination is also based on mslice presence, so we'll set 306 * it up here. Either LNCF within a non-terminated mslice will work, 307 * so we just always pick LNCF 0 here. 308 */ 309 gt->steering[LNCF].group_target = __ffs(mask) << 1; 310 gt->steering[LNCF].instance_target = 0; /* unused */ 311 } 312 313 static unsigned int dss_per_group(struct xe_gt *gt) 314 { 315 struct xe_guc *guc = >->uc.guc; 316 u32 max_slices = 0, max_subslices = 0; 317 int ret; 318 319 /* 320 * Try to query the GuC's hwconfig table for the maximum number of 321 * slices and subslices. These don't reflect the platform's actual 322 * slice/DSS counts, just the physical layout by which we should 323 * determine the steering targets. On older platforms with older GuC 324 * firmware releases it's possible that these attributes may not be 325 * included in the table, so we can always fall back to the old 326 * hardcoded layouts. 327 */ 328 #define HWCONFIG_ATTR_MAX_SLICES 1 329 #define HWCONFIG_ATTR_MAX_SUBSLICES 70 330 331 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES, 332 &max_slices); 333 if (ret < 0 || max_slices == 0) 334 goto fallback; 335 336 ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES, 337 &max_subslices); 338 if (ret < 0 || max_subslices == 0) 339 goto fallback; 340 341 return DIV_ROUND_UP(max_subslices, max_slices); 342 343 fallback: 344 /* 345 * Some older platforms don't have tables or don't have complete tables. 346 * Newer platforms should always have the required info. 347 */ 348 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000) 349 xe_gt_err(gt, "Slice/Subslice counts missing from hwconfig table; using typical fallback values\n"); 350 351 if (gt_to_xe(gt)->info.platform == XE_PVC) 352 return 8; 353 else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250) 354 return 4; 355 else 356 return 6; 357 } 358 359 /** 360 * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS 361 * @gt: GT structure 362 * @dss: DSS ID to obtain steering for 363 * @group: pointer to storage for steering group ID 364 * @instance: pointer to storage for steering instance ID 365 */ 366 void xe_gt_mcr_get_dss_steering(struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance) 367 { 368 xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS); 369 370 *group = dss / gt->steering_dss_per_grp; 371 *instance = dss % gt->steering_dss_per_grp; 372 } 373 374 /** 375 * xe_gt_mcr_steering_info_to_dss_id - Get DSS ID from group/instance steering 376 * @gt: GT structure 377 * @group: steering group ID 378 * @instance: steering instance ID 379 * 380 * Return: the converted DSS id. 381 */ 382 u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance) 383 { 384 return group * dss_per_group(gt) + instance; 385 } 386 387 static void init_steering_dss(struct xe_gt *gt) 388 { 389 gt->steering_dss_per_grp = dss_per_group(gt); 390 391 xe_gt_mcr_get_dss_steering(gt, 392 min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0), 393 xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)), 394 >->steering[DSS].group_target, 395 >->steering[DSS].instance_target); 396 } 397 398 static void init_steering_oaddrm(struct xe_gt *gt) 399 { 400 /* 401 * First instance is only terminated if the entire first media slice 402 * is absent (i.e., no VCS0 or VECS0). 403 */ 404 if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0)) 405 gt->steering[OADDRM].group_target = 0; 406 else 407 gt->steering[OADDRM].group_target = 1; 408 409 gt->steering[OADDRM].instance_target = 0; /* unused */ 410 } 411 412 static void init_steering_sqidi_psmi(struct xe_gt *gt) 413 { 414 u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, 415 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 416 u32 select = __ffs(mask); 417 418 gt->steering[SQIDI_PSMI].group_target = select >> 1; 419 gt->steering[SQIDI_PSMI].instance_target = select & 0x1; 420 } 421 422 static void init_steering_inst0(struct xe_gt *gt) 423 { 424 gt->steering[INSTANCE0].group_target = 0; /* unused */ 425 gt->steering[INSTANCE0].instance_target = 0; /* unused */ 426 } 427 428 static const struct { 429 const char *name; 430 void (*init)(struct xe_gt *gt); 431 } xe_steering_types[] = { 432 [L3BANK] = { "L3BANK", init_steering_l3bank }, 433 [MSLICE] = { "MSLICE", init_steering_mslice }, 434 [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */ 435 [DSS] = { "DSS", init_steering_dss }, 436 [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm }, 437 [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi }, 438 [INSTANCE0] = { "INSTANCE 0", init_steering_inst0 }, 439 [IMPLICIT_STEERING] = { "IMPLICIT", NULL }, 440 }; 441 442 /** 443 * xe_gt_mcr_init_early - Early initialization of the MCR support 444 * @gt: GT structure 445 * 446 * Perform early software only initialization of the MCR lock to allow 447 * the synchronization on accessing the STEER_SEMAPHORE register and 448 * use the xe_gt_mcr_multicast_write() function. 449 */ 450 void xe_gt_mcr_init_early(struct xe_gt *gt) 451 { 452 BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES); 453 BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES); 454 455 spin_lock_init(>->mcr_lock); 456 } 457 458 /** 459 * xe_gt_mcr_init - Normal initialization of the MCR support 460 * @gt: GT structure 461 * 462 * Perform normal initialization of the MCR for all usages. 463 */ 464 void xe_gt_mcr_init(struct xe_gt *gt) 465 { 466 struct xe_device *xe = gt_to_xe(gt); 467 468 if (IS_SRIOV_VF(xe)) 469 return; 470 471 if (gt->info.type == XE_GT_TYPE_MEDIA) { 472 drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13); 473 474 if (MEDIA_VER(xe) >= 30) { 475 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; 476 gt->steering[INSTANCE0].ranges = xe3lpm_instance0_steering_table; 477 } else if (MEDIA_VERx100(xe) >= 1301) { 478 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; 479 gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table; 480 } else { 481 gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table; 482 } 483 } else { 484 if (GRAPHICS_VER(xe) >= 20) { 485 gt->steering[DSS].ranges = xe2lpg_dss_steering_table; 486 gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table; 487 gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table; 488 } else if (GRAPHICS_VERx100(xe) >= 1270) { 489 gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; 490 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; 491 gt->steering[DSS].ranges = xelpg_dss_steering_table; 492 } else if (xe->info.platform == XE_PVC) { 493 gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table; 494 gt->steering[DSS].ranges = xehpc_dss_steering_table; 495 } else if (xe->info.platform == XE_DG2) { 496 gt->steering[L3BANK].ranges = xehp_l3bank_steering_table; 497 gt->steering[MSLICE].ranges = xehp_mslice_steering_table; 498 gt->steering[LNCF].ranges = xehp_lncf_steering_table; 499 gt->steering[DSS].ranges = xehp_dss_steering_table; 500 gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table; 501 } else { 502 gt->steering[L3BANK].ranges = xelp_l3bank_steering_table; 503 gt->steering[DSS].ranges = xelp_dss_steering_table; 504 } 505 } 506 507 /* Select non-terminated steering target for each type */ 508 for (int i = 0; i < NUM_STEERING_TYPES; i++) 509 if (gt->steering[i].ranges && xe_steering_types[i].init) 510 xe_steering_types[i].init(gt); 511 } 512 513 /** 514 * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers 515 * @gt: GT structure 516 * 517 * Some register ranges don't need to have their steering control registers 518 * changed on each access - it's sufficient to set them once on initialization. 519 * This function sets those registers for each platform * 520 */ 521 void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) 522 { 523 struct xe_device *xe = gt_to_xe(gt); 524 525 if (IS_SRIOV_VF(xe)) 526 return; 527 528 if (xe->info.platform == XE_DG2) { 529 u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) | 530 REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2); 531 532 xe_mmio_write32(>->mmio, MCFG_MCR_SELECTOR, steer_val); 533 xe_mmio_write32(>->mmio, SF_MCR_SELECTOR, steer_val); 534 /* 535 * For GAM registers, all reads should be directed to instance 1 536 * (unicast reads against other instances are not allowed), 537 * and instance 1 is already the hardware's default steering 538 * target, which we never change 539 */ 540 } 541 } 542 543 /* 544 * xe_gt_mcr_get_nonterminated_steering - find group/instance values that 545 * will steer a register to a non-terminated instance 546 * @gt: GT structure 547 * @reg: register for which the steering is required 548 * @group: return variable for group steering 549 * @instance: return variable for instance steering 550 * 551 * This function returns a group/instance pair that is guaranteed to work for 552 * read steering of the given register. Note that a value will be returned even 553 * if the register is not replicated and therefore does not actually require 554 * steering. 555 * 556 * Returns true if the caller should steer to the @group/@instance values 557 * returned. Returns false if the caller need not perform any steering 558 */ 559 bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, 560 struct xe_reg_mcr reg_mcr, 561 u8 *group, u8 *instance) 562 { 563 const struct xe_reg reg = to_xe_reg(reg_mcr); 564 const struct xe_mmio_range *implicit_ranges; 565 566 for (int type = 0; type < IMPLICIT_STEERING; type++) { 567 if (!gt->steering[type].ranges) 568 continue; 569 570 for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) { 571 if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) { 572 *group = gt->steering[type].group_target; 573 *instance = gt->steering[type].instance_target; 574 return true; 575 } 576 } 577 } 578 579 implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges; 580 if (implicit_ranges) 581 for (int i = 0; implicit_ranges[i].end > 0; i++) 582 if (xe_mmio_in_range(>->mmio, &implicit_ranges[i], reg)) 583 return false; 584 585 /* 586 * Not found in a steering table and not a register with implicit 587 * steering. Just steer to 0/0 as a guess and raise a warning. 588 */ 589 drm_WARN(>_to_xe(gt)->drm, true, 590 "Did not find MCR register %#x in any MCR steering table\n", 591 reg.addr); 592 *group = 0; 593 *instance = 0; 594 595 return true; 596 } 597 598 /* 599 * Obtain exclusive access to MCR steering. On MTL and beyond we also need 600 * to synchronize with external clients (e.g., firmware), so a semaphore 601 * register will also need to be taken. 602 */ 603 static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock) 604 { 605 struct xe_device *xe = gt_to_xe(gt); 606 int ret = 0; 607 608 spin_lock(>->mcr_lock); 609 610 /* 611 * Starting with MTL we also need to grab a semaphore register 612 * to synchronize with external agents (e.g., firmware) that now 613 * shares the same steering control register. The semaphore is obtained 614 * when a read to the relevant register returns 1. 615 */ 616 if (GRAPHICS_VERx100(xe) >= 1270) 617 ret = xe_mmio_wait32(>->mmio, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL, 618 true); 619 620 drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT); 621 } 622 623 static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock) 624 { 625 /* Release hardware semaphore - this is done by writing 1 to the register */ 626 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) 627 xe_mmio_write32(>->mmio, STEER_SEMAPHORE, 0x1); 628 629 spin_unlock(>->mcr_lock); 630 } 631 632 /* 633 * Access a register with specific MCR steering 634 * 635 * Caller needs to make sure the relevant forcewake wells are up. 636 */ 637 static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 638 u8 rw_flag, int group, int instance, u32 value) 639 { 640 const struct xe_reg reg = to_xe_reg(reg_mcr); 641 struct xe_mmio *mmio = >->mmio; 642 struct xe_reg steer_reg; 643 u32 steer_val, val = 0; 644 645 lockdep_assert_held(>->mcr_lock); 646 647 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { 648 steer_reg = MTL_MCR_SELECTOR; 649 steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) | 650 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance); 651 } else { 652 steer_reg = MCR_SELECTOR; 653 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) | 654 REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance); 655 } 656 657 /* 658 * Always leave the hardware in multicast mode when doing reads and only 659 * change it to unicast mode when doing writes of a specific instance. 660 * 661 * The setting of the multicast/unicast bit usually wouldn't matter for 662 * read operations (which always return the value from a single register 663 * instance regardless of how that bit is set), but some platforms may 664 * have workarounds requiring us to remain in multicast mode for reads, 665 * e.g. Wa_22013088509 on PVC. There's no real downside to this, so 666 * we'll just go ahead and do so on all platforms; we'll only clear the 667 * multicast bit from the mask when explicitly doing a write operation. 668 * 669 * No need to save old steering reg value. 670 */ 671 if (rw_flag == MCR_OP_READ) 672 steer_val |= MCR_MULTICAST; 673 674 xe_mmio_write32(mmio, steer_reg, steer_val); 675 676 if (rw_flag == MCR_OP_READ) 677 val = xe_mmio_read32(mmio, reg); 678 else 679 xe_mmio_write32(mmio, reg, value); 680 681 /* 682 * If we turned off the multicast bit (during a write) we're required 683 * to turn it back on before finishing. The group and instance values 684 * don't matter since they'll be re-programmed on the next MCR 685 * operation. 686 */ 687 if (rw_flag == MCR_OP_WRITE) 688 xe_mmio_write32(mmio, steer_reg, MCR_MULTICAST); 689 690 return val; 691 } 692 693 /** 694 * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register 695 * @gt: GT structure 696 * @reg_mcr: register to read 697 * 698 * Reads a GT MCR register. The read will be steered to a non-terminated 699 * instance (i.e., one that isn't fused off or powered down by power gating). 700 * This function assumes the caller is already holding any necessary forcewake 701 * domains. 702 * 703 * Returns the value from a non-terminated instance of @reg. 704 */ 705 u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr) 706 { 707 const struct xe_reg reg = to_xe_reg(reg_mcr); 708 u8 group, instance; 709 u32 val; 710 bool steer; 711 712 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 713 714 steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr, 715 &group, &instance); 716 717 if (steer) { 718 mcr_lock(gt); 719 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, 720 group, instance, 0); 721 mcr_unlock(gt); 722 } else { 723 val = xe_mmio_read32(>->mmio, reg); 724 } 725 726 return val; 727 } 728 729 /** 730 * xe_gt_mcr_unicast_read - read a specific instance of an MCR register 731 * @gt: GT structure 732 * @reg_mcr: the MCR register to read 733 * @group: the MCR group 734 * @instance: the MCR instance 735 * 736 * Returns the value read from an MCR register after steering toward a specific 737 * group/instance. 738 */ 739 u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, 740 struct xe_reg_mcr reg_mcr, 741 int group, int instance) 742 { 743 u32 val; 744 745 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 746 747 mcr_lock(gt); 748 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0); 749 mcr_unlock(gt); 750 751 return val; 752 } 753 754 /** 755 * xe_gt_mcr_unicast_write - write a specific instance of an MCR register 756 * @gt: GT structure 757 * @reg_mcr: the MCR register to write 758 * @value: value to write 759 * @group: the MCR group 760 * @instance: the MCR instance 761 * 762 * Write an MCR register in unicast mode after steering toward a specific 763 * group/instance. 764 */ 765 void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 766 u32 value, int group, int instance) 767 { 768 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 769 770 mcr_lock(gt); 771 rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value); 772 mcr_unlock(gt); 773 } 774 775 /** 776 * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register 777 * @gt: GT structure 778 * @reg_mcr: the MCR register to write 779 * @value: value to write 780 * 781 * Write an MCR register in multicast mode to update all instances. 782 */ 783 void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, 784 u32 value) 785 { 786 struct xe_reg reg = to_xe_reg(reg_mcr); 787 788 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); 789 790 /* 791 * Synchronize with any unicast operations. Once we have exclusive 792 * access, the MULTICAST bit should already be set, so there's no need 793 * to touch the steering register. 794 */ 795 mcr_lock(gt); 796 xe_mmio_write32(>->mmio, reg, value); 797 mcr_unlock(gt); 798 } 799 800 void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p) 801 { 802 for (int i = 0; i < NUM_STEERING_TYPES; i++) { 803 if (gt->steering[i].ranges) { 804 drm_printf(p, "%s steering: group=%#x, instance=%#x\n", 805 xe_steering_types[i].name, 806 gt->steering[i].group_target, 807 gt->steering[i].instance_target); 808 for (int j = 0; gt->steering[i].ranges[j].end; j++) 809 drm_printf(p, "\t0x%06x - 0x%06x\n", 810 gt->steering[i].ranges[j].start, 811 gt->steering[i].ranges[j].end); 812 } 813 } 814 } 815