1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "intel_gt.h" 8 #include "intel_gt_mcr.h" 9 #include "intel_gt_print.h" 10 #include "intel_gt_regs.h" 11 12 /** 13 * DOC: GT Multicast/Replicated (MCR) Register Support 14 * 15 * Some GT registers are designed as "multicast" or "replicated" registers: 16 * multiple instances of the same register share a single MMIO offset. MCR 17 * registers are generally used when the hardware needs to potentially track 18 * independent values of a register per hardware unit (e.g., per-subslice, 19 * per-L3bank, etc.). The specific types of replication that exist vary 20 * per-platform. 21 * 22 * MMIO accesses to MCR registers are controlled according to the settings 23 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR 24 * registers can be done in either a (i.e., a single write updates all 25 * instances of the register to the same value) or unicast (a write updates only 26 * one specific instance). Reads of MCR registers always operate in a unicast 27 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR. 28 * Selection of a specific MCR instance for unicast operations is referred to 29 * as "steering." 30 * 31 * If MCR register operations are steered toward a hardware unit that is 32 * fused off or currently powered down due to power gating, the MMIO operation 33 * is "terminated" by the hardware. Terminated read operations will return a 34 * value of zero and terminated unicast write operations will be silently 35 * ignored. 36 */ 37 38 #define HAS_MSLICE_STEERING(i915) (INTEL_INFO(i915)->has_mslice_steering) 39 40 static const char * const intel_steering_types[] = { 41 "L3BANK", 42 "MSLICE", 43 "LNCF", 44 "GAM", 45 "DSS", 46 "OADDRM", 47 "INSTANCE 0", 48 }; 49 50 static const struct intel_mmio_range icl_l3bank_steering_table[] = { 51 { 0x00B100, 0x00B3FF }, 52 {}, 53 }; 54 55 /* 56 * Although the bspec lists more "MSLICE" ranges than shown here, some of those 57 * are of a "GAM" subclass that has special rules. Thus we use a separate 58 * GAM table farther down for those. 59 */ 60 static const struct intel_mmio_range dg2_mslice_steering_table[] = { 61 { 0x00DD00, 0x00DDFF }, 62 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ 63 {}, 64 }; 65 66 static const struct intel_mmio_range dg2_lncf_steering_table[] = { 67 { 0x00B000, 0x00B0FF }, 68 { 0x00D880, 0x00D8FF }, 69 {}, 70 }; 71 72 static const struct intel_mmio_range xelpg_instance0_steering_table[] = { 73 { 0x000B00, 0x000BFF }, /* SQIDI */ 74 { 0x001000, 0x001FFF }, /* SQIDI */ 75 { 0x004000, 0x0048FF }, /* GAM */ 76 { 0x008700, 0x0087FF }, /* SQIDI */ 77 { 0x00B000, 0x00B0FF }, /* NODE */ 78 { 0x00C800, 0x00CFFF }, /* GAM */ 79 { 0x00D880, 0x00D8FF }, /* NODE */ 80 { 0x00DD00, 0x00DDFF }, /* OAAL2 */ 81 {}, 82 }; 83 84 static const struct intel_mmio_range xelpg_l3bank_steering_table[] = { 85 { 0x00B100, 0x00B3FF }, 86 {}, 87 }; 88 89 /* DSS steering is used for SLICE ranges as well */ 90 static const struct intel_mmio_range xelpg_dss_steering_table[] = { 91 { 0x005200, 0x0052FF }, /* SLICE */ 92 { 0x005500, 0x007FFF }, /* SLICE */ 93 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 94 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 95 { 0x009680, 0x0096FF }, /* DSS */ 96 { 0x00D800, 0x00D87F }, /* SLICE */ 97 { 0x00DC00, 0x00DCFF }, /* SLICE */ 98 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 99 {}, 100 }; 101 102 static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = { 103 { 0x393200, 0x39323F }, 104 { 0x393400, 0x3934FF }, 105 {}, 106 }; 107 108 void intel_gt_mcr_init(struct intel_gt *gt) 109 { 110 struct drm_i915_private *i915 = gt->i915; 111 unsigned long fuse; 112 int i; 113 114 spin_lock_init(>->mcr_lock); 115 116 /* 117 * An mslice is unavailable only if both the meml3 for the slice is 118 * disabled *and* all of the DSS in the slice (quadrant) are disabled. 119 */ 120 if (HAS_MSLICE_STEERING(i915)) { 121 gt->info.mslice_mask = 122 intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask, 123 GEN_DSS_PER_MSLICE); 124 gt->info.mslice_mask |= REG_FIELD_GET(GEN12_MEML3_EN_MASK, 125 intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3)); 126 127 if (!gt->info.mslice_mask) /* should be impossible! */ 128 gt_warn(gt, "mslice mask all zero!\n"); 129 } 130 131 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) { 132 gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table; 133 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 134 /* Wa_14016747170 */ 135 if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) || 136 IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0)) 137 fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK, 138 intel_uncore_read(gt->uncore, 139 MTL_GT_ACTIVITY_FACTOR)); 140 else 141 fuse = REG_FIELD_GET(GT_L3_EXC_MASK, 142 intel_uncore_read(gt->uncore, XEHP_FUSE4)); 143 144 /* 145 * Despite the register field being named "exclude mask" the 146 * bits actually represent enabled banks (two banks per bit). 147 */ 148 for_each_set_bit(i, &fuse, 3) 149 gt->info.l3bank_mask |= 0x3 << 2 * i; 150 151 gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table; 152 gt->steering_table[L3BANK] = xelpg_l3bank_steering_table; 153 gt->steering_table[DSS] = xelpg_dss_steering_table; 154 } else if (IS_DG2(i915)) { 155 gt->steering_table[MSLICE] = dg2_mslice_steering_table; 156 gt->steering_table[LNCF] = dg2_lncf_steering_table; 157 /* 158 * No need to hook up the GAM table since it has a dedicated 159 * steering control register on DG2 and can use implicit 160 * steering. 161 */ 162 } else if (GRAPHICS_VER(i915) >= 11 && 163 GRAPHICS_VER_FULL(i915) < IP_VER(12, 55)) { 164 gt->steering_table[L3BANK] = icl_l3bank_steering_table; 165 gt->info.l3bank_mask = 166 ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & 167 GEN10_L3BANK_MASK; 168 if (!gt->info.l3bank_mask) /* should be impossible! */ 169 gt_warn(gt, "L3 bank mask is all zero!\n"); 170 } else if (GRAPHICS_VER(i915) >= 11) { 171 /* 172 * We expect all modern platforms to have at least some 173 * type of steering that needs to be initialized. 174 */ 175 MISSING_CASE(INTEL_INFO(i915)->platform); 176 } 177 } 178 179 /* 180 * Although the rest of the driver should use MCR-specific functions to 181 * read/write MCR registers, we still use the regular intel_uncore_* functions 182 * internally to implement those, so we need a way for the functions in this 183 * file to "cast" an i915_mcr_reg_t into an i915_reg_t. 184 */ 185 static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr) 186 { 187 i915_reg_t r = { .reg = mcr.reg }; 188 189 return r; 190 } 191 192 /* 193 * rw_with_mcr_steering_fw - Access a register with specific MCR steering 194 * @gt: GT to read register from 195 * @reg: register being accessed 196 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access 197 * @group: group number (documented as "sliceid" on older platforms) 198 * @instance: instance number (documented as "subsliceid" on older platforms) 199 * @value: register value to be written (ignored for read) 200 * 201 * Context: The caller must hold the MCR lock 202 * Return: 0 for write access. register value for read access. 203 * 204 * Caller needs to make sure the relevant forcewake wells are up. 205 */ 206 static u32 rw_with_mcr_steering_fw(struct intel_gt *gt, 207 i915_mcr_reg_t reg, u8 rw_flag, 208 int group, int instance, u32 value) 209 { 210 struct intel_uncore *uncore = gt->uncore; 211 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0; 212 213 lockdep_assert_held(>->mcr_lock); 214 215 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) { 216 /* 217 * Always leave the hardware in multicast mode when doing reads 218 * (see comment about Wa_22013088509 below) and only change it 219 * to unicast mode when doing writes of a specific instance. 220 * 221 * No need to save old steering reg value. 222 */ 223 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, 224 REG_FIELD_PREP(MTL_MCR_GROUPID, group) | 225 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) | 226 (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0)); 227 } else if (GRAPHICS_VER(uncore->i915) >= 11) { 228 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; 229 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance); 230 231 /* 232 * Wa_22013088509 233 * 234 * The setting of the multicast/unicast bit usually wouldn't 235 * matter for read operations (which always return the value 236 * from a single register instance regardless of how that bit 237 * is set), but some platforms have a workaround requiring us 238 * to remain in multicast mode for reads. There's no real 239 * downside to this, so we'll just go ahead and do so on all 240 * platforms; we'll only clear the multicast bit from the mask 241 * when explicitly doing a write operation. 242 */ 243 if (rw_flag == FW_REG_WRITE) 244 mcr_mask |= GEN11_MCR_MULTICAST; 245 246 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 247 old_mcr = mcr; 248 249 mcr &= ~mcr_mask; 250 mcr |= mcr_ss; 251 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 252 } else { 253 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; 254 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance); 255 256 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 257 old_mcr = mcr; 258 259 mcr &= ~mcr_mask; 260 mcr |= mcr_ss; 261 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 262 } 263 264 if (rw_flag == FW_REG_READ) 265 val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg)); 266 else 267 intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value); 268 269 /* 270 * For pre-MTL platforms, we need to restore the old value of the 271 * steering control register to ensure that implicit steering continues 272 * to behave as expected. For MTL and beyond, we need only reinstate 273 * the 'multicast' bit (and only if we did a write that cleared it). 274 */ 275 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE) 276 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 277 else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70)) 278 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr); 279 280 return val; 281 } 282 283 static u32 rw_with_mcr_steering(struct intel_gt *gt, 284 i915_mcr_reg_t reg, u8 rw_flag, 285 int group, int instance, 286 u32 value) 287 { 288 struct intel_uncore *uncore = gt->uncore; 289 enum forcewake_domains fw_domains; 290 unsigned long flags; 291 u32 val; 292 293 fw_domains = intel_uncore_forcewake_for_reg(uncore, mcr_reg_cast(reg), 294 rw_flag); 295 fw_domains |= intel_uncore_forcewake_for_reg(uncore, 296 GEN8_MCR_SELECTOR, 297 FW_REG_READ | FW_REG_WRITE); 298 299 intel_gt_mcr_lock(gt, &flags); 300 spin_lock(&uncore->lock); 301 intel_uncore_forcewake_get__locked(uncore, fw_domains); 302 303 val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value); 304 305 intel_uncore_forcewake_put__locked(uncore, fw_domains); 306 spin_unlock(&uncore->lock); 307 intel_gt_mcr_unlock(gt, flags); 308 309 return val; 310 } 311 312 /** 313 * intel_gt_mcr_lock - Acquire MCR steering lock 314 * @gt: GT structure 315 * @flags: storage to save IRQ flags to 316 * 317 * Performs locking to protect the steering for the duration of an MCR 318 * operation. On MTL and beyond, a hardware lock will also be taken to 319 * serialize access not only for the driver, but also for external hardware and 320 * firmware agents. 321 * 322 * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this 323 * function is called, although it may be acquired after this 324 * function call. 325 */ 326 void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags) 327 __acquires(>->mcr_lock) 328 { 329 unsigned long __flags; 330 int err = 0; 331 332 lockdep_assert_not_held(>->uncore->lock); 333 334 /* 335 * Starting with MTL, we need to coordinate not only with other 336 * driver threads, but also with hardware/firmware agents. A dedicated 337 * locking register is used. 338 */ 339 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { 340 /* 341 * The steering control and semaphore registers are inside an 342 * "always on" power domain with respect to RC6. However there 343 * are some issues if higher-level platform sleep states are 344 * entering/exiting at the same time these registers are 345 * accessed. Grabbing GT forcewake and holding it over the 346 * entire lock/steer/unlock cycle ensures that those sleep 347 * states have been fully exited before we access these 348 * registers. This wakeref will be released in the unlock 349 * routine. 350 * 351 * Wa_22018931422 352 */ 353 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_GT); 354 355 err = wait_for(intel_uncore_read_fw(gt->uncore, 356 MTL_STEER_SEMAPHORE) == 0x1, 100); 357 } 358 359 /* 360 * Even on platforms with a hardware lock, we'll continue to grab 361 * a software spinlock too for lockdep purposes. If the hardware lock 362 * was already acquired, there should never be contention on the 363 * software lock. 364 */ 365 spin_lock_irqsave(>->mcr_lock, __flags); 366 367 *flags = __flags; 368 369 /* 370 * In theory we should never fail to acquire the HW semaphore; this 371 * would indicate some hardware/firmware is misbehaving and not 372 * releasing it properly. 373 */ 374 if (err == -ETIMEDOUT) { 375 gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out"); 376 add_taint_for_CI(gt->i915, TAINT_WARN); /* CI is now unreliable */ 377 } 378 } 379 380 /** 381 * intel_gt_mcr_unlock - Release MCR steering lock 382 * @gt: GT structure 383 * @flags: IRQ flags to restore 384 * 385 * Releases the lock acquired by intel_gt_mcr_lock(). 386 * 387 * Context: Releases gt->mcr_lock 388 */ 389 void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags) 390 __releases(>->mcr_lock) 391 { 392 spin_unlock_irqrestore(>->mcr_lock, flags); 393 394 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { 395 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1); 396 397 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_GT); 398 } 399 } 400 401 /** 402 * intel_gt_mcr_lock_sanitize - Sanitize MCR steering lock 403 * @gt: GT structure 404 * 405 * This will be used to sanitize the initial status of the hardware lock 406 * during driver load and resume since there won't be any concurrent access 407 * from other agents at those times, but it's possible that boot firmware 408 * may have left the lock in a bad state. 409 * 410 */ 411 void intel_gt_mcr_lock_sanitize(struct intel_gt *gt) 412 { 413 /* 414 * This gets called at load/resume time, so we shouldn't be 415 * racing with other driver threads grabbing the mcr lock. 416 */ 417 lockdep_assert_not_held(>->mcr_lock); 418 419 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 420 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1); 421 } 422 423 /** 424 * intel_gt_mcr_read - read a specific instance of an MCR register 425 * @gt: GT structure 426 * @reg: the MCR register to read 427 * @group: the MCR group 428 * @instance: the MCR instance 429 * 430 * Context: Takes and releases gt->mcr_lock 431 * 432 * Returns the value read from an MCR register after steering toward a specific 433 * group/instance. 434 */ 435 u32 intel_gt_mcr_read(struct intel_gt *gt, 436 i915_mcr_reg_t reg, 437 int group, int instance) 438 { 439 return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, 0); 440 } 441 442 /** 443 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register 444 * @gt: GT structure 445 * @reg: the MCR register to write 446 * @value: value to write 447 * @group: the MCR group 448 * @instance: the MCR instance 449 * 450 * Write an MCR register in unicast mode after steering toward a specific 451 * group/instance. 452 * 453 * Context: Calls a function that takes and releases gt->mcr_lock 454 */ 455 void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value, 456 int group, int instance) 457 { 458 rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value); 459 } 460 461 /** 462 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register 463 * @gt: GT structure 464 * @reg: the MCR register to write 465 * @value: value to write 466 * 467 * Write an MCR register in multicast mode to update all instances. 468 * 469 * Context: Takes and releases gt->mcr_lock 470 */ 471 void intel_gt_mcr_multicast_write(struct intel_gt *gt, 472 i915_mcr_reg_t reg, u32 value) 473 { 474 unsigned long flags; 475 476 intel_gt_mcr_lock(gt, &flags); 477 478 /* 479 * Ensure we have multicast behavior, just in case some non-i915 agent 480 * left the hardware in unicast mode. 481 */ 482 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 483 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 484 485 intel_uncore_write(gt->uncore, mcr_reg_cast(reg), value); 486 487 intel_gt_mcr_unlock(gt, flags); 488 } 489 490 /** 491 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register 492 * @gt: GT structure 493 * @reg: the MCR register to write 494 * @value: value to write 495 * 496 * Write an MCR register in multicast mode to update all instances. This 497 * function assumes the caller is already holding any necessary forcewake 498 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should 499 * be obtained automatically. 500 * 501 * Context: The caller must hold gt->mcr_lock. 502 */ 503 void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value) 504 { 505 lockdep_assert_held(>->mcr_lock); 506 507 /* 508 * Ensure we have multicast behavior, just in case some non-i915 agent 509 * left the hardware in unicast mode. 510 */ 511 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 512 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 513 514 intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value); 515 } 516 517 /** 518 * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations 519 * @gt: GT structure 520 * @reg: the MCR register to read and write 521 * @clear: bits to clear during RMW 522 * @set: bits to set during RMW 523 * 524 * Performs a read-modify-write on an MCR register in a multicast manner. 525 * This operation only makes sense on MCR registers where all instances are 526 * expected to have the same value. The read will target any non-terminated 527 * instance and the write will be applied to all instances. 528 * 529 * This function assumes the caller is already holding any necessary forcewake 530 * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should 531 * be obtained automatically. 532 * 533 * Context: Calls functions that take and release gt->mcr_lock 534 * 535 * Returns the old (unmodified) value read. 536 */ 537 u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg, 538 u32 clear, u32 set) 539 { 540 u32 val = intel_gt_mcr_read_any(gt, reg); 541 542 intel_gt_mcr_multicast_write(gt, reg, (val & ~clear) | set); 543 544 return val; 545 } 546 547 /* 548 * reg_needs_read_steering - determine whether a register read requires 549 * explicit steering 550 * @gt: GT structure 551 * @reg: the register to check steering requirements for 552 * @type: type of multicast steering to check 553 * 554 * Determines whether @reg needs explicit steering of a specific type for 555 * reads. 556 * 557 * Returns false if @reg does not belong to a register range of the given 558 * steering type, or if the default (subslice-based) steering IDs are suitable 559 * for @type steering too. 560 */ 561 static bool reg_needs_read_steering(struct intel_gt *gt, 562 i915_mcr_reg_t reg, 563 enum intel_steering_type type) 564 { 565 u32 offset = i915_mmio_reg_offset(reg); 566 const struct intel_mmio_range *entry; 567 568 if (likely(!gt->steering_table[type])) 569 return false; 570 571 if (IS_GSI_REG(offset)) 572 offset += gt->uncore->gsi_offset; 573 574 for (entry = gt->steering_table[type]; entry->end; entry++) { 575 if (offset >= entry->start && offset <= entry->end) 576 return true; 577 } 578 579 return false; 580 } 581 582 /* 583 * get_nonterminated_steering - determines valid IDs for a class of MCR steering 584 * @gt: GT structure 585 * @type: multicast register type 586 * @group: Group ID returned 587 * @instance: Instance ID returned 588 * 589 * Determines group and instance values that will steer reads of the specified 590 * MCR class to a non-terminated instance. 591 */ 592 static void get_nonterminated_steering(struct intel_gt *gt, 593 enum intel_steering_type type, 594 u8 *group, u8 *instance) 595 { 596 u32 dss; 597 598 switch (type) { 599 case L3BANK: 600 *group = 0; /* unused */ 601 *instance = __ffs(gt->info.l3bank_mask); 602 break; 603 case MSLICE: 604 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); 605 *group = __ffs(gt->info.mslice_mask); 606 *instance = 0; /* unused */ 607 break; 608 case LNCF: 609 /* 610 * An LNCF is always present if its mslice is present, so we 611 * can safely just steer to LNCF 0 in all cases. 612 */ 613 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); 614 *group = __ffs(gt->info.mslice_mask) << 1; 615 *instance = 0; /* unused */ 616 break; 617 case GAM: 618 *group = IS_DG2(gt->i915) ? 1 : 0; 619 *instance = 0; 620 break; 621 case DSS: 622 dss = intel_sseu_find_first_xehp_dss(>->info.sseu, 0, 0); 623 *group = dss / GEN_DSS_PER_GSLICE; 624 *instance = dss % GEN_DSS_PER_GSLICE; 625 break; 626 case INSTANCE0: 627 /* 628 * There are a lot of MCR types for which instance (0, 0) 629 * will always provide a non-terminated value. 630 */ 631 *group = 0; 632 *instance = 0; 633 break; 634 case OADDRM: 635 if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0)) 636 *group = 0; 637 else 638 *group = 1; 639 *instance = 0; 640 break; 641 default: 642 MISSING_CASE(type); 643 *group = 0; 644 *instance = 0; 645 } 646 } 647 648 /** 649 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that 650 * will steer a register to a non-terminated instance 651 * @gt: GT structure 652 * @reg: register for which the steering is required 653 * @group: return variable for group steering 654 * @instance: return variable for instance steering 655 * 656 * This function returns a group/instance pair that is guaranteed to work for 657 * read steering of the given register. Note that a value will be returned even 658 * if the register is not replicated and therefore does not actually require 659 * steering. 660 */ 661 void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt, 662 i915_mcr_reg_t reg, 663 u8 *group, u8 *instance) 664 { 665 int type; 666 667 for (type = 0; type < NUM_STEERING_TYPES; type++) { 668 if (reg_needs_read_steering(gt, reg, type)) { 669 get_nonterminated_steering(gt, type, group, instance); 670 return; 671 } 672 } 673 674 *group = gt->default_steering.groupid; 675 *instance = gt->default_steering.instanceid; 676 } 677 678 /** 679 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register 680 * @gt: GT structure 681 * @reg: register to read 682 * 683 * Reads a GT MCR register. The read will be steered to a non-terminated 684 * instance (i.e., one that isn't fused off or powered down by power gating). 685 * This function assumes the caller is already holding any necessary forcewake 686 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be 687 * obtained automatically. 688 * 689 * Context: The caller must hold gt->mcr_lock. 690 * 691 * Returns the value from a non-terminated instance of @reg. 692 */ 693 u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg) 694 { 695 int type; 696 u8 group, instance; 697 698 lockdep_assert_held(>->mcr_lock); 699 700 for (type = 0; type < NUM_STEERING_TYPES; type++) { 701 if (reg_needs_read_steering(gt, reg, type)) { 702 get_nonterminated_steering(gt, type, &group, &instance); 703 return rw_with_mcr_steering_fw(gt, reg, 704 FW_REG_READ, 705 group, instance, 0); 706 } 707 } 708 709 return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg)); 710 } 711 712 /** 713 * intel_gt_mcr_read_any - reads one instance of an MCR register 714 * @gt: GT structure 715 * @reg: register to read 716 * 717 * Reads a GT MCR register. The read will be steered to a non-terminated 718 * instance (i.e., one that isn't fused off or powered down by power gating). 719 * 720 * Context: Calls a function that takes and releases gt->mcr_lock. 721 * 722 * Returns the value from a non-terminated instance of @reg. 723 */ 724 u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg) 725 { 726 int type; 727 u8 group, instance; 728 729 for (type = 0; type < NUM_STEERING_TYPES; type++) { 730 if (reg_needs_read_steering(gt, reg, type)) { 731 get_nonterminated_steering(gt, type, &group, &instance); 732 return rw_with_mcr_steering(gt, reg, 733 FW_REG_READ, 734 group, instance, 0); 735 } 736 } 737 738 return intel_uncore_read(gt->uncore, mcr_reg_cast(reg)); 739 } 740 741 static void report_steering_type(struct drm_printer *p, 742 struct intel_gt *gt, 743 enum intel_steering_type type, 744 bool dump_table) 745 { 746 const struct intel_mmio_range *entry; 747 u8 group, instance; 748 749 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES); 750 751 if (!gt->steering_table[type]) { 752 drm_printf(p, "%s steering: uses default steering\n", 753 intel_steering_types[type]); 754 return; 755 } 756 757 get_nonterminated_steering(gt, type, &group, &instance); 758 drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n", 759 intel_steering_types[type], group, instance); 760 761 if (!dump_table) 762 return; 763 764 for (entry = gt->steering_table[type]; entry->end; entry++) 765 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end); 766 } 767 768 void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt, 769 bool dump_table) 770 { 771 /* 772 * Starting with MTL we no longer have default steering; 773 * all ranges are explicitly steered. 774 */ 775 if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70)) 776 drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n", 777 gt->default_steering.groupid, 778 gt->default_steering.instanceid); 779 780 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { 781 for (int i = 0; i < NUM_STEERING_TYPES; i++) 782 if (gt->steering_table[i]) 783 report_steering_type(p, gt, i, dump_table); 784 } else if (HAS_MSLICE_STEERING(gt->i915)) { 785 report_steering_type(p, gt, MSLICE, dump_table); 786 report_steering_type(p, gt, LNCF, dump_table); 787 } 788 } 789 790 /** 791 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS 792 * @gt: GT structure 793 * @dss: DSS ID to obtain steering for 794 * @group: pointer to storage for steering group ID 795 * @instance: pointer to storage for steering instance ID 796 * 797 * Returns the steering IDs (via the @group and @instance parameters) that 798 * correspond to a specific subslice/DSS ID. 799 */ 800 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss, 801 unsigned int *group, unsigned int *instance) 802 { 803 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55)) { 804 *group = dss / GEN_DSS_PER_GSLICE; 805 *instance = dss % GEN_DSS_PER_GSLICE; 806 } else { 807 *group = dss / GEN_MAX_SS_PER_HSW_SLICE; 808 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE; 809 return; 810 } 811 } 812 813 /** 814 * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state 815 * @gt: GT structure 816 * @reg: the register to read 817 * @mask: mask to apply to register value 818 * @value: value to wait for 819 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 820 * @slow_timeout_ms: slow timeout in millisecond 821 * 822 * This routine waits until the target register @reg contains the expected 823 * @value after applying the @mask, i.e. it waits until :: 824 * 825 * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value 826 * 827 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. 828 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us 829 * must be not larger than 20,0000 microseconds. 830 * 831 * This function is basically an MCR-friendly version of 832 * __intel_wait_for_register_fw(). Generally this function will only be used 833 * on GAM registers which are a bit special --- although they're MCR registers, 834 * reads (e.g., waiting for status updates) are always directed to the primary 835 * instance. 836 * 837 * Note that this routine assumes the caller holds forcewake asserted, it is 838 * not suitable for very long waits. 839 * 840 * Context: Calls a function that takes and releases gt->mcr_lock 841 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT. 842 */ 843 int intel_gt_mcr_wait_for_reg(struct intel_gt *gt, 844 i915_mcr_reg_t reg, 845 u32 mask, 846 u32 value, 847 unsigned int fast_timeout_us, 848 unsigned int slow_timeout_ms) 849 { 850 int ret; 851 852 lockdep_assert_not_held(>->mcr_lock); 853 854 #define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value) 855 856 /* Catch any overuse of this function */ 857 might_sleep_if(slow_timeout_ms); 858 GEM_BUG_ON(fast_timeout_us > 20000); 859 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms); 860 861 ret = -ETIMEDOUT; 862 if (fast_timeout_us && fast_timeout_us <= 20000) 863 ret = _wait_for_atomic(done, fast_timeout_us, 0); 864 if (ret && slow_timeout_ms) 865 ret = wait_for(done, slow_timeout_ms); 866 867 return ret; 868 #undef done 869 } 870