1 /* 2 * Copyright © 2013 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #include <drm/drm_managed.h> 25 #include <linux/pm_runtime.h> 26 27 #include "display/intel_display_core.h" 28 29 #include "gt/intel_gt.h" 30 #include "gt/intel_engine_regs.h" 31 #include "gt/intel_gt_regs.h" 32 33 #include "i915_drv.h" 34 #include "i915_iosf_mbi.h" 35 #include "i915_reg.h" 36 #include "i915_vgpu.h" 37 #include "intel_uncore_trace.h" 38 39 #define FORCEWAKE_ACK_TIMEOUT_MS 50 40 #define GT_FIFO_TIMEOUT_MS 10 41 42 struct intel_uncore *to_intel_uncore(struct drm_device *drm) 43 { 44 return &to_i915(drm)->uncore; 45 } 46 47 #define __raw_posting_read(...) ((void)__raw_uncore_read32(__VA_ARGS__)) 48 49 static void 50 fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains) 51 { 52 uncore->fw_get_funcs->force_wake_get(uncore, fw_domains); 53 } 54 55 void 56 intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915) 57 { 58 spin_lock_init(&i915->mmio_debug.lock); 59 i915->mmio_debug.unclaimed_mmio_check = 1; 60 61 i915->uncore.debug = &i915->mmio_debug; 62 } 63 64 static void mmio_debug_suspend(struct intel_uncore *uncore) 65 { 66 if (!uncore->debug) 67 return; 68 69 spin_lock(&uncore->debug->lock); 70 71 /* Save and disable mmio debugging for the user bypass */ 72 if (!uncore->debug->suspend_count++) { 73 uncore->debug->saved_mmio_check = uncore->debug->unclaimed_mmio_check; 74 uncore->debug->unclaimed_mmio_check = 0; 75 } 76 77 spin_unlock(&uncore->debug->lock); 78 } 79 80 static bool check_for_unclaimed_mmio(struct intel_uncore *uncore); 81 82 static void mmio_debug_resume(struct intel_uncore *uncore) 83 { 84 if (!uncore->debug) 85 return; 86 87 spin_lock(&uncore->debug->lock); 88 89 if (!--uncore->debug->suspend_count) 90 uncore->debug->unclaimed_mmio_check = uncore->debug->saved_mmio_check; 91 92 if (check_for_unclaimed_mmio(uncore)) 93 drm_info(&uncore->i915->drm, 94 "Invalid mmio detected during user access\n"); 95 96 spin_unlock(&uncore->debug->lock); 97 } 98 99 static const char * const forcewake_domain_names[] = { 100 "render", 101 "gt", 102 "media", 103 "vdbox0", 104 "vdbox1", 105 "vdbox2", 106 "vdbox3", 107 "vdbox4", 108 "vdbox5", 109 "vdbox6", 110 "vdbox7", 111 "vebox0", 112 "vebox1", 113 "vebox2", 114 "vebox3", 115 "gsc", 116 }; 117 118 const char * 119 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id) 120 { 121 BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT); 122 123 if (id >= 0 && id < FW_DOMAIN_ID_COUNT) 124 return forcewake_domain_names[id]; 125 126 WARN_ON(id); 127 128 return "unknown"; 129 } 130 131 #define fw_ack(d) readl((d)->reg_ack) 132 #define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set) 133 #define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set) 134 135 static inline void 136 fw_domain_reset(const struct intel_uncore_forcewake_domain *d) 137 { 138 /* 139 * We don't really know if the powerwell for the forcewake domain we are 140 * trying to reset here does exist at this point (engines could be fused 141 * off in ICL+), so no waiting for acks 142 */ 143 /* WaRsClearFWBitsAtReset */ 144 if (GRAPHICS_VER(d->uncore->i915) >= 12) 145 fw_clear(d, 0xefff); 146 else 147 fw_clear(d, 0xffff); 148 } 149 150 static inline void 151 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d) 152 { 153 GEM_BUG_ON(d->uncore->fw_domains_timer & d->mask); 154 d->uncore->fw_domains_timer |= d->mask; 155 d->wake_count++; 156 hrtimer_start_range_ns(&d->timer, 157 NSEC_PER_MSEC, 158 NSEC_PER_MSEC, 159 HRTIMER_MODE_REL); 160 } 161 162 static inline int 163 __wait_for_ack(const struct intel_uncore_forcewake_domain *d, 164 const u32 ack, 165 const u32 value) 166 { 167 return wait_for_atomic((fw_ack(d) & ack) == value, 168 FORCEWAKE_ACK_TIMEOUT_MS); 169 } 170 171 static inline int 172 wait_ack_clear(const struct intel_uncore_forcewake_domain *d, 173 const u32 ack) 174 { 175 return __wait_for_ack(d, ack, 0); 176 } 177 178 static inline int 179 wait_ack_set(const struct intel_uncore_forcewake_domain *d, 180 const u32 ack) 181 { 182 return __wait_for_ack(d, ack, ack); 183 } 184 185 static inline void 186 fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d) 187 { 188 if (!wait_ack_clear(d, FORCEWAKE_KERNEL)) 189 return; 190 191 if (fw_ack(d) == ~0) { 192 drm_err(&d->uncore->i915->drm, 193 "%s: MMIO unreliable (forcewake register returns 0xFFFFFFFF)!\n", 194 intel_uncore_forcewake_domain_to_str(d->id)); 195 intel_gt_set_wedged_async(d->uncore->gt); 196 } else { 197 drm_err(&d->uncore->i915->drm, 198 "%s: timed out waiting for forcewake ack to clear.\n", 199 intel_uncore_forcewake_domain_to_str(d->id)); 200 } 201 202 add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */ 203 } 204 205 enum ack_type { 206 ACK_CLEAR = 0, 207 ACK_SET 208 }; 209 210 static int 211 fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d, 212 const enum ack_type type) 213 { 214 const u32 ack_bit = FORCEWAKE_KERNEL; 215 const u32 value = type == ACK_SET ? ack_bit : 0; 216 unsigned int pass; 217 bool ack_detected; 218 219 /* 220 * There is a possibility of driver's wake request colliding 221 * with hardware's own wake requests and that can cause 222 * hardware to not deliver the driver's ack message. 223 * 224 * Use a fallback bit toggle to kick the gpu state machine 225 * in the hope that the original ack will be delivered along with 226 * the fallback ack. 227 * 228 * This workaround is described in HSDES #1604254524 and it's known as: 229 * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl 230 * although the name is a bit misleading. 231 */ 232 233 pass = 1; 234 do { 235 wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK); 236 237 fw_set(d, FORCEWAKE_KERNEL_FALLBACK); 238 /* Give gt some time to relax before the polling frenzy */ 239 udelay(10 * pass); 240 wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK); 241 242 ack_detected = (fw_ack(d) & ack_bit) == value; 243 244 fw_clear(d, FORCEWAKE_KERNEL_FALLBACK); 245 } while (!ack_detected && pass++ < 10); 246 247 drm_dbg(&d->uncore->i915->drm, 248 "%s had to use fallback to %s ack, 0x%x (passes %u)\n", 249 intel_uncore_forcewake_domain_to_str(d->id), 250 type == ACK_SET ? "set" : "clear", 251 fw_ack(d), 252 pass); 253 254 return ack_detected ? 0 : -ETIMEDOUT; 255 } 256 257 static inline void 258 fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d) 259 { 260 if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL))) 261 return; 262 263 if (fw_domain_wait_ack_with_fallback(d, ACK_CLEAR)) 264 fw_domain_wait_ack_clear(d); 265 } 266 267 static inline void 268 fw_domain_get(const struct intel_uncore_forcewake_domain *d) 269 { 270 fw_set(d, FORCEWAKE_KERNEL); 271 } 272 273 static inline void 274 fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d) 275 { 276 if (wait_ack_set(d, FORCEWAKE_KERNEL)) { 277 drm_err(&d->uncore->i915->drm, 278 "%s: timed out waiting for forcewake ack request.\n", 279 intel_uncore_forcewake_domain_to_str(d->id)); 280 add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */ 281 } 282 } 283 284 static inline void 285 fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d) 286 { 287 if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL))) 288 return; 289 290 if (fw_domain_wait_ack_with_fallback(d, ACK_SET)) 291 fw_domain_wait_ack_set(d); 292 } 293 294 static inline void 295 fw_domain_put(const struct intel_uncore_forcewake_domain *d) 296 { 297 fw_clear(d, FORCEWAKE_KERNEL); 298 } 299 300 static void 301 fw_domains_get_normal(struct intel_uncore *uncore, enum forcewake_domains fw_domains) 302 { 303 struct intel_uncore_forcewake_domain *d; 304 unsigned int tmp; 305 306 GEM_BUG_ON(fw_domains & ~uncore->fw_domains); 307 308 for_each_fw_domain_masked(d, fw_domains, uncore, tmp) { 309 fw_domain_wait_ack_clear(d); 310 fw_domain_get(d); 311 } 312 313 for_each_fw_domain_masked(d, fw_domains, uncore, tmp) 314 fw_domain_wait_ack_set(d); 315 316 uncore->fw_domains_active |= fw_domains; 317 } 318 319 static void 320 fw_domains_get_with_fallback(struct intel_uncore *uncore, 321 enum forcewake_domains fw_domains) 322 { 323 struct intel_uncore_forcewake_domain *d; 324 unsigned int tmp; 325 326 GEM_BUG_ON(fw_domains & ~uncore->fw_domains); 327 328 for_each_fw_domain_masked(d, fw_domains, uncore, tmp) { 329 fw_domain_wait_ack_clear_fallback(d); 330 fw_domain_get(d); 331 } 332 333 for_each_fw_domain_masked(d, fw_domains, uncore, tmp) 334 fw_domain_wait_ack_set_fallback(d); 335 336 uncore->fw_domains_active |= fw_domains; 337 } 338 339 static void 340 fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains) 341 { 342 struct intel_uncore_forcewake_domain *d; 343 unsigned int tmp; 344 345 GEM_BUG_ON(fw_domains & ~uncore->fw_domains); 346 347 for_each_fw_domain_masked(d, fw_domains, uncore, tmp) 348 fw_domain_put(d); 349 350 uncore->fw_domains_active &= ~fw_domains; 351 } 352 353 static void 354 fw_domains_reset(struct intel_uncore *uncore, 355 enum forcewake_domains fw_domains) 356 { 357 struct intel_uncore_forcewake_domain *d; 358 unsigned int tmp; 359 360 if (!fw_domains) 361 return; 362 363 GEM_BUG_ON(fw_domains & ~uncore->fw_domains); 364 365 for_each_fw_domain_masked(d, fw_domains, uncore, tmp) 366 fw_domain_reset(d); 367 } 368 369 static inline u32 gt_thread_status(struct intel_uncore *uncore) 370 { 371 u32 val; 372 373 val = __raw_uncore_read32(uncore, GEN6_GT_THREAD_STATUS_REG); 374 val &= GEN6_GT_THREAD_STATUS_CORE_MASK; 375 376 return val; 377 } 378 379 static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore) 380 { 381 /* 382 * w/a for a sporadic read returning 0 by waiting for the GT 383 * thread to wake up. 384 */ 385 drm_WARN_ONCE(&uncore->i915->drm, 386 wait_for_atomic_us(gt_thread_status(uncore) == 0, 5000), 387 "GT thread status wait timed out\n"); 388 } 389 390 static void fw_domains_get_with_thread_status(struct intel_uncore *uncore, 391 enum forcewake_domains fw_domains) 392 { 393 fw_domains_get_normal(uncore, fw_domains); 394 395 /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */ 396 __gen6_gt_wait_for_thread_c0(uncore); 397 } 398 399 static inline u32 fifo_free_entries(struct intel_uncore *uncore) 400 { 401 u32 count = __raw_uncore_read32(uncore, GTFIFOCTL); 402 403 return count & GT_FIFO_FREE_ENTRIES_MASK; 404 } 405 406 static void __gen6_gt_wait_for_fifo(struct intel_uncore *uncore) 407 { 408 u32 n; 409 410 /* On VLV, FIFO will be shared by both SW and HW. 411 * So, we need to read the FREE_ENTRIES everytime */ 412 if (IS_VALLEYVIEW(uncore->i915)) 413 n = fifo_free_entries(uncore); 414 else 415 n = uncore->fifo_count; 416 417 if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) { 418 if (wait_for_atomic((n = fifo_free_entries(uncore)) > 419 GT_FIFO_NUM_RESERVED_ENTRIES, 420 GT_FIFO_TIMEOUT_MS)) { 421 drm_dbg(&uncore->i915->drm, 422 "GT_FIFO timeout, entries: %u\n", n); 423 return; 424 } 425 } 426 427 uncore->fifo_count = n - 1; 428 } 429 430 static enum hrtimer_restart 431 intel_uncore_fw_release_timer(struct hrtimer *timer) 432 { 433 struct intel_uncore_forcewake_domain *domain = 434 container_of(timer, struct intel_uncore_forcewake_domain, timer); 435 struct intel_uncore *uncore = domain->uncore; 436 unsigned long irqflags; 437 438 assert_rpm_device_not_suspended(uncore->rpm); 439 440 if (xchg(&domain->active, false)) 441 return HRTIMER_RESTART; 442 443 spin_lock_irqsave(&uncore->lock, irqflags); 444 445 uncore->fw_domains_timer &= ~domain->mask; 446 447 GEM_BUG_ON(!domain->wake_count); 448 if (--domain->wake_count == 0) 449 fw_domains_put(uncore, domain->mask); 450 451 spin_unlock_irqrestore(&uncore->lock, irqflags); 452 453 return HRTIMER_NORESTART; 454 } 455 456 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */ 457 static unsigned int 458 intel_uncore_forcewake_reset(struct intel_uncore *uncore) 459 { 460 unsigned long irqflags; 461 struct intel_uncore_forcewake_domain *domain; 462 int retry_count = 100; 463 enum forcewake_domains fw, active_domains; 464 465 iosf_mbi_assert_punit_acquired(); 466 467 /* Hold uncore.lock across reset to prevent any register access 468 * with forcewake not set correctly. Wait until all pending 469 * timers are run before holding. 470 */ 471 while (1) { 472 unsigned int tmp; 473 474 active_domains = 0; 475 476 for_each_fw_domain(domain, uncore, tmp) { 477 smp_store_mb(domain->active, false); 478 if (hrtimer_cancel(&domain->timer) == 0) 479 continue; 480 481 intel_uncore_fw_release_timer(&domain->timer); 482 } 483 484 spin_lock_irqsave(&uncore->lock, irqflags); 485 486 for_each_fw_domain(domain, uncore, tmp) { 487 if (hrtimer_active(&domain->timer)) 488 active_domains |= domain->mask; 489 } 490 491 if (active_domains == 0) 492 break; 493 494 if (--retry_count == 0) { 495 drm_err(&uncore->i915->drm, "Timed out waiting for forcewake timers to finish\n"); 496 break; 497 } 498 499 spin_unlock_irqrestore(&uncore->lock, irqflags); 500 cond_resched(); 501 } 502 503 drm_WARN_ON(&uncore->i915->drm, active_domains); 504 505 fw = uncore->fw_domains_active; 506 if (fw) 507 fw_domains_put(uncore, fw); 508 509 fw_domains_reset(uncore, uncore->fw_domains); 510 assert_forcewakes_inactive(uncore); 511 512 spin_unlock_irqrestore(&uncore->lock, irqflags); 513 514 return fw; /* track the lost user forcewake domains */ 515 } 516 517 static bool 518 fpga_check_for_unclaimed_mmio(struct intel_uncore *uncore) 519 { 520 u32 dbg; 521 522 dbg = __raw_uncore_read32(uncore, FPGA_DBG); 523 if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM))) 524 return false; 525 526 /* 527 * Bugs in PCI programming (or failing hardware) can occasionally cause 528 * us to lose access to the MMIO BAR. When this happens, register 529 * reads will come back with 0xFFFFFFFF for every register and things 530 * go bad very quickly. Let's try to detect that special case and at 531 * least try to print a more informative message about what has 532 * happened. 533 * 534 * During normal operation the FPGA_DBG register has several unused 535 * bits that will always read back as 0's so we can use them as canaries 536 * to recognize when MMIO accesses are just busted. 537 */ 538 if (unlikely(dbg == ~0)) 539 drm_err(&uncore->i915->drm, 540 "Lost access to MMIO BAR; all registers now read back as 0xFFFFFFFF!\n"); 541 542 __raw_uncore_write32(uncore, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 543 544 return true; 545 } 546 547 static bool 548 vlv_check_for_unclaimed_mmio(struct intel_uncore *uncore) 549 { 550 u32 cer; 551 552 cer = __raw_uncore_read32(uncore, CLAIM_ER); 553 if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK)))) 554 return false; 555 556 __raw_uncore_write32(uncore, CLAIM_ER, CLAIM_ER_CLR); 557 558 return true; 559 } 560 561 static bool 562 gen6_check_for_fifo_debug(struct intel_uncore *uncore) 563 { 564 u32 fifodbg; 565 566 fifodbg = __raw_uncore_read32(uncore, GTFIFODBG); 567 568 if (unlikely(fifodbg)) { 569 drm_dbg(&uncore->i915->drm, "GTFIFODBG = 0x08%x\n", fifodbg); 570 __raw_uncore_write32(uncore, GTFIFODBG, fifodbg); 571 } 572 573 return fifodbg; 574 } 575 576 static bool 577 check_for_unclaimed_mmio(struct intel_uncore *uncore) 578 { 579 bool ret = false; 580 581 lockdep_assert_held(&uncore->debug->lock); 582 583 if (uncore->debug->suspend_count) 584 return false; 585 586 if (intel_uncore_has_fpga_dbg_unclaimed(uncore)) 587 ret |= fpga_check_for_unclaimed_mmio(uncore); 588 589 if (intel_uncore_has_dbg_unclaimed(uncore)) 590 ret |= vlv_check_for_unclaimed_mmio(uncore); 591 592 if (intel_uncore_has_fifo(uncore)) 593 ret |= gen6_check_for_fifo_debug(uncore); 594 595 return ret; 596 } 597 598 static void forcewake_early_sanitize(struct intel_uncore *uncore, 599 unsigned int restore_forcewake) 600 { 601 GEM_BUG_ON(!intel_uncore_has_forcewake(uncore)); 602 603 /* WaDisableShadowRegForCpd:chv */ 604 if (IS_CHERRYVIEW(uncore->i915)) { 605 __raw_uncore_write32(uncore, GTFIFOCTL, 606 __raw_uncore_read32(uncore, GTFIFOCTL) | 607 GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL | 608 GT_FIFO_CTL_RC6_POLICY_STALL); 609 } 610 611 iosf_mbi_punit_acquire(); 612 intel_uncore_forcewake_reset(uncore); 613 if (restore_forcewake) { 614 spin_lock_irq(&uncore->lock); 615 fw_domains_get(uncore, restore_forcewake); 616 617 if (intel_uncore_has_fifo(uncore)) 618 uncore->fifo_count = fifo_free_entries(uncore); 619 spin_unlock_irq(&uncore->lock); 620 } 621 iosf_mbi_punit_release(); 622 } 623 624 void intel_uncore_suspend(struct intel_uncore *uncore) 625 { 626 if (!intel_uncore_has_forcewake(uncore)) 627 return; 628 629 iosf_mbi_punit_acquire(); 630 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( 631 &uncore->pmic_bus_access_nb); 632 uncore->fw_domains_saved = intel_uncore_forcewake_reset(uncore); 633 iosf_mbi_punit_release(); 634 } 635 636 void intel_uncore_resume_early(struct intel_uncore *uncore) 637 { 638 unsigned int restore_forcewake; 639 640 if (intel_uncore_unclaimed_mmio(uncore)) 641 drm_dbg(&uncore->i915->drm, "unclaimed mmio detected on resume, clearing\n"); 642 643 if (!intel_uncore_has_forcewake(uncore)) 644 return; 645 646 restore_forcewake = fetch_and_zero(&uncore->fw_domains_saved); 647 forcewake_early_sanitize(uncore, restore_forcewake); 648 649 iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb); 650 } 651 652 void intel_uncore_runtime_resume(struct intel_uncore *uncore) 653 { 654 if (!intel_uncore_has_forcewake(uncore)) 655 return; 656 657 iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb); 658 } 659 660 static void __intel_uncore_forcewake_get(struct intel_uncore *uncore, 661 enum forcewake_domains fw_domains) 662 { 663 struct intel_uncore_forcewake_domain *domain; 664 unsigned int tmp; 665 666 fw_domains &= uncore->fw_domains; 667 668 for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) { 669 if (domain->wake_count++) { 670 fw_domains &= ~domain->mask; 671 domain->active = true; 672 } 673 } 674 675 if (fw_domains) 676 fw_domains_get(uncore, fw_domains); 677 } 678 679 /** 680 * intel_uncore_forcewake_get - grab forcewake domain references 681 * @uncore: the intel_uncore structure 682 * @fw_domains: forcewake domains to get reference on 683 * 684 * This function can be used get GT's forcewake domain references. 685 * Normal register access will handle the forcewake domains automatically. 686 * However if some sequence requires the GT to not power down a particular 687 * forcewake domains this function should be called at the beginning of the 688 * sequence. And subsequently the reference should be dropped by symmetric 689 * call to intel_unforce_forcewake_put(). Usually caller wants all the domains 690 * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL. 691 */ 692 void intel_uncore_forcewake_get(struct intel_uncore *uncore, 693 enum forcewake_domains fw_domains) 694 { 695 unsigned long irqflags; 696 697 if (!uncore->fw_get_funcs) 698 return; 699 700 assert_rpm_wakelock_held(uncore->rpm); 701 702 spin_lock_irqsave(&uncore->lock, irqflags); 703 __intel_uncore_forcewake_get(uncore, fw_domains); 704 spin_unlock_irqrestore(&uncore->lock, irqflags); 705 } 706 707 /** 708 * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace 709 * @uncore: the intel_uncore structure 710 * 711 * This function is a wrapper around intel_uncore_forcewake_get() to acquire 712 * the GT powerwell and in the process disable our debugging for the 713 * duration of userspace's bypass. 714 */ 715 void intel_uncore_forcewake_user_get(struct intel_uncore *uncore) 716 { 717 spin_lock_irq(&uncore->lock); 718 if (!uncore->user_forcewake_count++) { 719 intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL); 720 mmio_debug_suspend(uncore); 721 } 722 spin_unlock_irq(&uncore->lock); 723 } 724 725 /** 726 * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace 727 * @uncore: the intel_uncore structure 728 * 729 * This function complements intel_uncore_forcewake_user_get() and releases 730 * the GT powerwell taken on behalf of the userspace bypass. 731 */ 732 void intel_uncore_forcewake_user_put(struct intel_uncore *uncore) 733 { 734 spin_lock_irq(&uncore->lock); 735 if (!--uncore->user_forcewake_count) { 736 mmio_debug_resume(uncore); 737 intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL); 738 } 739 spin_unlock_irq(&uncore->lock); 740 } 741 742 /** 743 * intel_uncore_forcewake_get__locked - grab forcewake domain references 744 * @uncore: the intel_uncore structure 745 * @fw_domains: forcewake domains to get reference on 746 * 747 * See intel_uncore_forcewake_get(). This variant places the onus 748 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. 749 */ 750 void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore, 751 enum forcewake_domains fw_domains) 752 { 753 lockdep_assert_held(&uncore->lock); 754 755 if (!uncore->fw_get_funcs) 756 return; 757 758 __intel_uncore_forcewake_get(uncore, fw_domains); 759 } 760 761 static void __intel_uncore_forcewake_put(struct intel_uncore *uncore, 762 enum forcewake_domains fw_domains, 763 bool delayed) 764 { 765 struct intel_uncore_forcewake_domain *domain; 766 unsigned int tmp; 767 768 fw_domains &= uncore->fw_domains; 769 770 for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) { 771 GEM_BUG_ON(!domain->wake_count); 772 773 if (--domain->wake_count) { 774 domain->active = true; 775 continue; 776 } 777 778 if (delayed && 779 !(domain->uncore->fw_domains_timer & domain->mask)) 780 fw_domain_arm_timer(domain); 781 else 782 fw_domains_put(uncore, domain->mask); 783 } 784 } 785 786 /** 787 * intel_uncore_forcewake_put - release a forcewake domain reference 788 * @uncore: the intel_uncore structure 789 * @fw_domains: forcewake domains to put references 790 * 791 * This function drops the device-level forcewakes for specified 792 * domains obtained by intel_uncore_forcewake_get(). 793 */ 794 void intel_uncore_forcewake_put(struct intel_uncore *uncore, 795 enum forcewake_domains fw_domains) 796 { 797 unsigned long irqflags; 798 799 if (!uncore->fw_get_funcs) 800 return; 801 802 spin_lock_irqsave(&uncore->lock, irqflags); 803 __intel_uncore_forcewake_put(uncore, fw_domains, false); 804 spin_unlock_irqrestore(&uncore->lock, irqflags); 805 } 806 807 void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore, 808 enum forcewake_domains fw_domains) 809 { 810 unsigned long irqflags; 811 812 if (!uncore->fw_get_funcs) 813 return; 814 815 spin_lock_irqsave(&uncore->lock, irqflags); 816 __intel_uncore_forcewake_put(uncore, fw_domains, true); 817 spin_unlock_irqrestore(&uncore->lock, irqflags); 818 } 819 820 /** 821 * intel_uncore_forcewake_flush - flush the delayed release 822 * @uncore: the intel_uncore structure 823 * @fw_domains: forcewake domains to flush 824 */ 825 void intel_uncore_forcewake_flush(struct intel_uncore *uncore, 826 enum forcewake_domains fw_domains) 827 { 828 struct intel_uncore_forcewake_domain *domain; 829 unsigned int tmp; 830 831 if (!uncore->fw_get_funcs) 832 return; 833 834 fw_domains &= uncore->fw_domains; 835 for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) { 836 WRITE_ONCE(domain->active, false); 837 if (hrtimer_cancel(&domain->timer)) 838 intel_uncore_fw_release_timer(&domain->timer); 839 } 840 } 841 842 /** 843 * intel_uncore_forcewake_put__locked - release forcewake domain references 844 * @uncore: the intel_uncore structure 845 * @fw_domains: forcewake domains to put references 846 * 847 * See intel_uncore_forcewake_put(). This variant places the onus 848 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. 849 */ 850 void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore, 851 enum forcewake_domains fw_domains) 852 { 853 lockdep_assert_held(&uncore->lock); 854 855 if (!uncore->fw_get_funcs) 856 return; 857 858 __intel_uncore_forcewake_put(uncore, fw_domains, false); 859 } 860 861 void assert_forcewakes_inactive(struct intel_uncore *uncore) 862 { 863 if (!uncore->fw_get_funcs) 864 return; 865 866 drm_WARN(&uncore->i915->drm, uncore->fw_domains_active, 867 "Expected all fw_domains to be inactive, but %08x are still on\n", 868 uncore->fw_domains_active); 869 } 870 871 void assert_forcewakes_active(struct intel_uncore *uncore, 872 enum forcewake_domains fw_domains) 873 { 874 struct intel_uncore_forcewake_domain *domain; 875 unsigned int tmp; 876 877 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)) 878 return; 879 880 if (!uncore->fw_get_funcs) 881 return; 882 883 spin_lock_irq(&uncore->lock); 884 885 assert_rpm_wakelock_held(uncore->rpm); 886 887 fw_domains &= uncore->fw_domains; 888 drm_WARN(&uncore->i915->drm, fw_domains & ~uncore->fw_domains_active, 889 "Expected %08x fw_domains to be active, but %08x are off\n", 890 fw_domains, fw_domains & ~uncore->fw_domains_active); 891 892 /* 893 * Check that the caller has an explicit wakeref and we don't mistake 894 * it for the auto wakeref. 895 */ 896 for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) { 897 unsigned int actual = READ_ONCE(domain->wake_count); 898 unsigned int expect = 1; 899 900 if (uncore->fw_domains_timer & domain->mask) 901 expect++; /* pending automatic release */ 902 903 if (drm_WARN(&uncore->i915->drm, actual < expect, 904 "Expected domain %d to be held awake by caller, count=%d\n", 905 domain->id, actual)) 906 break; 907 } 908 909 spin_unlock_irq(&uncore->lock); 910 } 911 912 /* 913 * We give fast paths for the really cool registers. The second range includes 914 * media domains (and the GSC starting from Xe_LPM+) 915 */ 916 #define NEEDS_FORCE_WAKE(reg) ({ \ 917 u32 __reg = (reg); \ 918 __reg < 0x40000 || __reg >= 0x116000; \ 919 }) 920 921 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry) 922 { 923 if (offset < entry->start) 924 return -1; 925 else if (offset > entry->end) 926 return 1; 927 else 928 return 0; 929 } 930 931 /* Copied and "macroized" from lib/bsearch.c */ 932 #define BSEARCH(key, base, num, cmp) ({ \ 933 unsigned int start__ = 0, end__ = (num); \ 934 typeof(base) result__ = NULL; \ 935 while (start__ < end__) { \ 936 unsigned int mid__ = start__ + (end__ - start__) / 2; \ 937 int ret__ = (cmp)((key), (base) + mid__); \ 938 if (ret__ < 0) { \ 939 end__ = mid__; \ 940 } else if (ret__ > 0) { \ 941 start__ = mid__ + 1; \ 942 } else { \ 943 result__ = (base) + mid__; \ 944 break; \ 945 } \ 946 } \ 947 result__; \ 948 }) 949 950 static enum forcewake_domains 951 find_fw_domain(struct intel_uncore *uncore, u32 offset) 952 { 953 const struct intel_forcewake_range *entry; 954 955 if (IS_GSI_REG(offset)) 956 offset += uncore->gsi_offset; 957 958 entry = BSEARCH(offset, 959 uncore->fw_domains_table, 960 uncore->fw_domains_table_entries, 961 fw_range_cmp); 962 963 if (!entry) 964 return 0; 965 966 /* 967 * The list of FW domains depends on the SKU in gen11+ so we 968 * can't determine it statically. We use FORCEWAKE_ALL and 969 * translate it here to the list of available domains. 970 */ 971 if (entry->domains == FORCEWAKE_ALL) 972 return uncore->fw_domains; 973 974 drm_WARN(&uncore->i915->drm, entry->domains & ~uncore->fw_domains, 975 "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n", 976 entry->domains & ~uncore->fw_domains, offset); 977 978 return entry->domains; 979 } 980 981 /* 982 * Shadowed register tables describe special register ranges that i915 is 983 * allowed to write to without acquiring forcewake. If these registers' power 984 * wells are down, the hardware will save values written by i915 to a shadow 985 * copy and automatically transfer them into the real register the next time 986 * the power well is woken up. Shadowing only applies to writes; forcewake 987 * must still be acquired when reading from registers in these ranges. 988 * 989 * The documentation for shadowed registers is somewhat spotty on older 990 * platforms. However missing registers from these lists is non-fatal; it just 991 * means we'll wake up the hardware for some register accesses where we didn't 992 * really need to. 993 * 994 * The ranges listed in these tables must be sorted by offset. 995 * 996 * When adding new tables here, please also add them to 997 * intel_shadow_table_check() in selftests/intel_uncore.c so that they will be 998 * scanned for obvious mistakes or typos by the selftests. 999 */ 1000 1001 static const struct i915_range gen8_shadowed_regs[] = { 1002 { .start = 0x2030, .end = 0x2030 }, 1003 { .start = 0xA008, .end = 0xA00C }, 1004 { .start = 0x12030, .end = 0x12030 }, 1005 { .start = 0x1a030, .end = 0x1a030 }, 1006 { .start = 0x22030, .end = 0x22030 }, 1007 }; 1008 1009 static const struct i915_range gen11_shadowed_regs[] = { 1010 { .start = 0x2030, .end = 0x2030 }, 1011 { .start = 0x2550, .end = 0x2550 }, 1012 { .start = 0xA008, .end = 0xA00C }, 1013 { .start = 0x22030, .end = 0x22030 }, 1014 { .start = 0x22230, .end = 0x22230 }, 1015 { .start = 0x22510, .end = 0x22550 }, 1016 { .start = 0x1C0030, .end = 0x1C0030 }, 1017 { .start = 0x1C0230, .end = 0x1C0230 }, 1018 { .start = 0x1C0510, .end = 0x1C0550 }, 1019 { .start = 0x1C4030, .end = 0x1C4030 }, 1020 { .start = 0x1C4230, .end = 0x1C4230 }, 1021 { .start = 0x1C4510, .end = 0x1C4550 }, 1022 { .start = 0x1C8030, .end = 0x1C8030 }, 1023 { .start = 0x1C8230, .end = 0x1C8230 }, 1024 { .start = 0x1C8510, .end = 0x1C8550 }, 1025 { .start = 0x1D0030, .end = 0x1D0030 }, 1026 { .start = 0x1D0230, .end = 0x1D0230 }, 1027 { .start = 0x1D0510, .end = 0x1D0550 }, 1028 { .start = 0x1D4030, .end = 0x1D4030 }, 1029 { .start = 0x1D4230, .end = 0x1D4230 }, 1030 { .start = 0x1D4510, .end = 0x1D4550 }, 1031 { .start = 0x1D8030, .end = 0x1D8030 }, 1032 { .start = 0x1D8230, .end = 0x1D8230 }, 1033 { .start = 0x1D8510, .end = 0x1D8550 }, 1034 }; 1035 1036 static const struct i915_range gen12_shadowed_regs[] = { 1037 { .start = 0x2030, .end = 0x2030 }, 1038 { .start = 0x2510, .end = 0x2550 }, 1039 { .start = 0xA008, .end = 0xA00C }, 1040 { .start = 0xA188, .end = 0xA188 }, 1041 { .start = 0xA278, .end = 0xA278 }, 1042 { .start = 0xA540, .end = 0xA56C }, 1043 { .start = 0xC4C8, .end = 0xC4C8 }, 1044 { .start = 0xC4D4, .end = 0xC4D4 }, 1045 { .start = 0xC600, .end = 0xC600 }, 1046 { .start = 0x22030, .end = 0x22030 }, 1047 { .start = 0x22510, .end = 0x22550 }, 1048 { .start = 0x1C0030, .end = 0x1C0030 }, 1049 { .start = 0x1C0510, .end = 0x1C0550 }, 1050 { .start = 0x1C4030, .end = 0x1C4030 }, 1051 { .start = 0x1C4510, .end = 0x1C4550 }, 1052 { .start = 0x1C8030, .end = 0x1C8030 }, 1053 { .start = 0x1C8510, .end = 0x1C8550 }, 1054 { .start = 0x1D0030, .end = 0x1D0030 }, 1055 { .start = 0x1D0510, .end = 0x1D0550 }, 1056 { .start = 0x1D4030, .end = 0x1D4030 }, 1057 { .start = 0x1D4510, .end = 0x1D4550 }, 1058 { .start = 0x1D8030, .end = 0x1D8030 }, 1059 { .start = 0x1D8510, .end = 0x1D8550 }, 1060 1061 /* 1062 * The rest of these ranges are specific to Xe_HP and beyond, but 1063 * are reserved/unused ranges on earlier gen12 platforms, so they can 1064 * be safely added to the gen12 table. 1065 */ 1066 { .start = 0x1E0030, .end = 0x1E0030 }, 1067 { .start = 0x1E0510, .end = 0x1E0550 }, 1068 { .start = 0x1E4030, .end = 0x1E4030 }, 1069 { .start = 0x1E4510, .end = 0x1E4550 }, 1070 { .start = 0x1E8030, .end = 0x1E8030 }, 1071 { .start = 0x1E8510, .end = 0x1E8550 }, 1072 { .start = 0x1F0030, .end = 0x1F0030 }, 1073 { .start = 0x1F0510, .end = 0x1F0550 }, 1074 { .start = 0x1F4030, .end = 0x1F4030 }, 1075 { .start = 0x1F4510, .end = 0x1F4550 }, 1076 { .start = 0x1F8030, .end = 0x1F8030 }, 1077 { .start = 0x1F8510, .end = 0x1F8550 }, 1078 }; 1079 1080 static const struct i915_range dg2_shadowed_regs[] = { 1081 { .start = 0x2030, .end = 0x2030 }, 1082 { .start = 0x2510, .end = 0x2550 }, 1083 { .start = 0xA008, .end = 0xA00C }, 1084 { .start = 0xA188, .end = 0xA188 }, 1085 { .start = 0xA278, .end = 0xA278 }, 1086 { .start = 0xA540, .end = 0xA56C }, 1087 { .start = 0xC4C8, .end = 0xC4C8 }, 1088 { .start = 0xC4E0, .end = 0xC4E0 }, 1089 { .start = 0xC600, .end = 0xC600 }, 1090 { .start = 0xC658, .end = 0xC658 }, 1091 { .start = 0x22030, .end = 0x22030 }, 1092 { .start = 0x22510, .end = 0x22550 }, 1093 { .start = 0x1C0030, .end = 0x1C0030 }, 1094 { .start = 0x1C0510, .end = 0x1C0550 }, 1095 { .start = 0x1C4030, .end = 0x1C4030 }, 1096 { .start = 0x1C4510, .end = 0x1C4550 }, 1097 { .start = 0x1C8030, .end = 0x1C8030 }, 1098 { .start = 0x1C8510, .end = 0x1C8550 }, 1099 { .start = 0x1D0030, .end = 0x1D0030 }, 1100 { .start = 0x1D0510, .end = 0x1D0550 }, 1101 { .start = 0x1D4030, .end = 0x1D4030 }, 1102 { .start = 0x1D4510, .end = 0x1D4550 }, 1103 { .start = 0x1D8030, .end = 0x1D8030 }, 1104 { .start = 0x1D8510, .end = 0x1D8550 }, 1105 { .start = 0x1E0030, .end = 0x1E0030 }, 1106 { .start = 0x1E0510, .end = 0x1E0550 }, 1107 { .start = 0x1E4030, .end = 0x1E4030 }, 1108 { .start = 0x1E4510, .end = 0x1E4550 }, 1109 { .start = 0x1E8030, .end = 0x1E8030 }, 1110 { .start = 0x1E8510, .end = 0x1E8550 }, 1111 { .start = 0x1F0030, .end = 0x1F0030 }, 1112 { .start = 0x1F0510, .end = 0x1F0550 }, 1113 { .start = 0x1F4030, .end = 0x1F4030 }, 1114 { .start = 0x1F4510, .end = 0x1F4550 }, 1115 { .start = 0x1F8030, .end = 0x1F8030 }, 1116 { .start = 0x1F8510, .end = 0x1F8550 }, 1117 }; 1118 1119 static const struct i915_range mtl_shadowed_regs[] = { 1120 { .start = 0x2030, .end = 0x2030 }, 1121 { .start = 0x2510, .end = 0x2550 }, 1122 { .start = 0xA008, .end = 0xA00C }, 1123 { .start = 0xA188, .end = 0xA188 }, 1124 { .start = 0xA278, .end = 0xA278 }, 1125 { .start = 0xA540, .end = 0xA56C }, 1126 { .start = 0xC050, .end = 0xC050 }, 1127 { .start = 0xC340, .end = 0xC340 }, 1128 { .start = 0xC4C8, .end = 0xC4C8 }, 1129 { .start = 0xC4E0, .end = 0xC4E0 }, 1130 { .start = 0xC600, .end = 0xC600 }, 1131 { .start = 0xC658, .end = 0xC658 }, 1132 { .start = 0xCFD4, .end = 0xCFDC }, 1133 { .start = 0x22030, .end = 0x22030 }, 1134 { .start = 0x22510, .end = 0x22550 }, 1135 }; 1136 1137 static const struct i915_range xelpmp_shadowed_regs[] = { 1138 { .start = 0x1C0030, .end = 0x1C0030 }, 1139 { .start = 0x1C0510, .end = 0x1C0550 }, 1140 { .start = 0x1C8030, .end = 0x1C8030 }, 1141 { .start = 0x1C8510, .end = 0x1C8550 }, 1142 { .start = 0x1D0030, .end = 0x1D0030 }, 1143 { .start = 0x1D0510, .end = 0x1D0550 }, 1144 { .start = 0x38A008, .end = 0x38A00C }, 1145 { .start = 0x38A188, .end = 0x38A188 }, 1146 { .start = 0x38A278, .end = 0x38A278 }, 1147 { .start = 0x38A540, .end = 0x38A56C }, 1148 { .start = 0x38A618, .end = 0x38A618 }, 1149 { .start = 0x38C050, .end = 0x38C050 }, 1150 { .start = 0x38C340, .end = 0x38C340 }, 1151 { .start = 0x38C4C8, .end = 0x38C4C8 }, 1152 { .start = 0x38C4E0, .end = 0x38C4E4 }, 1153 { .start = 0x38C600, .end = 0x38C600 }, 1154 { .start = 0x38C658, .end = 0x38C658 }, 1155 { .start = 0x38CFD4, .end = 0x38CFDC }, 1156 }; 1157 1158 static int mmio_range_cmp(u32 key, const struct i915_range *range) 1159 { 1160 if (key < range->start) 1161 return -1; 1162 else if (key > range->end) 1163 return 1; 1164 else 1165 return 0; 1166 } 1167 1168 static bool is_shadowed(struct intel_uncore *uncore, u32 offset) 1169 { 1170 if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table)) 1171 return false; 1172 1173 if (IS_GSI_REG(offset)) 1174 offset += uncore->gsi_offset; 1175 1176 return BSEARCH(offset, 1177 uncore->shadowed_reg_table, 1178 uncore->shadowed_reg_table_entries, 1179 mmio_range_cmp); 1180 } 1181 1182 static enum forcewake_domains 1183 gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) 1184 { 1185 return FORCEWAKE_RENDER; 1186 } 1187 1188 #define __fwtable_reg_read_fw_domains(uncore, offset) \ 1189 ({ \ 1190 enum forcewake_domains __fwd = 0; \ 1191 if (NEEDS_FORCE_WAKE((offset))) \ 1192 __fwd = find_fw_domain(uncore, offset); \ 1193 __fwd; \ 1194 }) 1195 1196 #define __fwtable_reg_write_fw_domains(uncore, offset) \ 1197 ({ \ 1198 enum forcewake_domains __fwd = 0; \ 1199 const u32 __offset = (offset); \ 1200 if (NEEDS_FORCE_WAKE((__offset)) && !is_shadowed(uncore, __offset)) \ 1201 __fwd = find_fw_domain(uncore, __offset); \ 1202 __fwd; \ 1203 }) 1204 1205 #define GEN_FW_RANGE(s, e, d) \ 1206 { .start = (s), .end = (e), .domains = (d) } 1207 1208 /* 1209 * All platforms' forcewake tables below must be sorted by offset ranges. 1210 * Furthermore, new forcewake tables added should be "watertight" and have 1211 * no gaps between ranges. 1212 * 1213 * When there are multiple consecutive ranges listed in the bspec with 1214 * the same forcewake domain, it is customary to combine them into a single 1215 * row in the tables below to keep the tables small and lookups fast. 1216 * Likewise, reserved/unused ranges may be combined with the preceding and/or 1217 * following ranges since the driver will never be making MMIO accesses in 1218 * those ranges. 1219 * 1220 * For example, if the bspec were to list: 1221 * 1222 * ... 1223 * 0x1000 - 0x1fff: GT 1224 * 0x2000 - 0x2cff: GT 1225 * 0x2d00 - 0x2fff: unused/reserved 1226 * 0x3000 - 0xffff: GT 1227 * ... 1228 * 1229 * these could all be represented by a single line in the code: 1230 * 1231 * GEN_FW_RANGE(0x1000, 0xffff, FORCEWAKE_GT) 1232 * 1233 * When adding new forcewake tables here, please also add them to 1234 * intel_uncore_mock_selftests in selftests/intel_uncore.c so that they will be 1235 * scanned for obvious mistakes or typos by the selftests. 1236 */ 1237 1238 static const struct intel_forcewake_range __gen6_fw_ranges[] = { 1239 GEN_FW_RANGE(0x0, 0x3ffff, FORCEWAKE_RENDER), 1240 }; 1241 1242 static const struct intel_forcewake_range __vlv_fw_ranges[] = { 1243 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), 1244 GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER), 1245 GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER), 1246 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 1247 GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA), 1248 GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER), 1249 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 1250 }; 1251 1252 static const struct intel_forcewake_range __chv_fw_ranges[] = { 1253 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), 1254 GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1255 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 1256 GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1257 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1258 GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1259 GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA), 1260 GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1261 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1262 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 1263 GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER), 1264 GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1265 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 1266 GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA), 1267 GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA), 1268 GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA), 1269 }; 1270 1271 static const struct intel_forcewake_range __gen9_fw_ranges[] = { 1272 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_GT), 1273 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */ 1274 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1275 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT), 1276 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 1277 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), 1278 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 1279 GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_GT), 1280 GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA), 1281 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1282 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT), 1283 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1284 GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT), 1285 GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA), 1286 GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_GT), 1287 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), 1288 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_GT), 1289 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 1290 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT), 1291 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1292 GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_GT), 1293 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 1294 GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_GT), 1295 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), 1296 GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT), 1297 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA), 1298 GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_GT), 1299 GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA), 1300 GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_GT), 1301 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER), 1302 GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_GT), 1303 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 1304 }; 1305 1306 static const struct intel_forcewake_range __gen11_fw_ranges[] = { 1307 GEN_FW_RANGE(0x0, 0x1fff, 0), /* uncore range */ 1308 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1309 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT), 1310 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 1311 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), 1312 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 1313 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT), 1314 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1315 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT), 1316 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1317 GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT), 1318 GEN_FW_RANGE(0x8800, 0x8bff, 0), 1319 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), 1320 GEN_FW_RANGE(0x8d00, 0x94cf, FORCEWAKE_GT), 1321 GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER), 1322 GEN_FW_RANGE(0x9560, 0x95ff, 0), 1323 GEN_FW_RANGE(0x9600, 0xafff, FORCEWAKE_GT), 1324 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 1325 GEN_FW_RANGE(0xb480, 0xdeff, FORCEWAKE_GT), 1326 GEN_FW_RANGE(0xdf00, 0xe8ff, FORCEWAKE_RENDER), 1327 GEN_FW_RANGE(0xe900, 0x16dff, FORCEWAKE_GT), 1328 GEN_FW_RANGE(0x16e00, 0x19fff, FORCEWAKE_RENDER), 1329 GEN_FW_RANGE(0x1a000, 0x23fff, FORCEWAKE_GT), 1330 GEN_FW_RANGE(0x24000, 0x2407f, 0), 1331 GEN_FW_RANGE(0x24080, 0x2417f, FORCEWAKE_GT), 1332 GEN_FW_RANGE(0x24180, 0x242ff, FORCEWAKE_RENDER), 1333 GEN_FW_RANGE(0x24300, 0x243ff, FORCEWAKE_GT), 1334 GEN_FW_RANGE(0x24400, 0x24fff, FORCEWAKE_RENDER), 1335 GEN_FW_RANGE(0x25000, 0x3ffff, FORCEWAKE_GT), 1336 GEN_FW_RANGE(0x40000, 0x1bffff, 0), 1337 GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), 1338 GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0), 1339 GEN_FW_RANGE(0x1c8000, 0x1cffff, FORCEWAKE_MEDIA_VEBOX0), 1340 GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), 1341 GEN_FW_RANGE(0x1d4000, 0x1dbfff, 0) 1342 }; 1343 1344 static const struct intel_forcewake_range __gen12_fw_ranges[] = { 1345 GEN_FW_RANGE(0x0, 0x1fff, 0), /* 1346 0x0 - 0xaff: reserved 1347 0xb00 - 0x1fff: always on */ 1348 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1349 GEN_FW_RANGE(0x2700, 0x27ff, FORCEWAKE_GT), 1350 GEN_FW_RANGE(0x2800, 0x2aff, FORCEWAKE_RENDER), 1351 GEN_FW_RANGE(0x2b00, 0x2fff, FORCEWAKE_GT), 1352 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 1353 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /* 1354 0x4000 - 0x48ff: gt 1355 0x4900 - 0x51ff: reserved */ 1356 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /* 1357 0x5200 - 0x53ff: render 1358 0x5400 - 0x54ff: reserved 1359 0x5500 - 0x7fff: render */ 1360 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT), 1361 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1362 GEN_FW_RANGE(0x8160, 0x81ff, 0), /* 1363 0x8160 - 0x817f: reserved 1364 0x8180 - 0x81ff: always on */ 1365 GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT), 1366 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1367 GEN_FW_RANGE(0x8500, 0x94cf, FORCEWAKE_GT), /* 1368 0x8500 - 0x87ff: gt 1369 0x8800 - 0x8fff: reserved 1370 0x9000 - 0x947f: gt 1371 0x9480 - 0x94cf: reserved */ 1372 GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER), 1373 GEN_FW_RANGE(0x9560, 0x97ff, 0), /* 1374 0x9560 - 0x95ff: always on 1375 0x9600 - 0x97ff: reserved */ 1376 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT), 1377 GEN_FW_RANGE(0xb000, 0xb3ff, FORCEWAKE_RENDER), 1378 GEN_FW_RANGE(0xb400, 0xcfff, FORCEWAKE_GT), /* 1379 0xb400 - 0xbf7f: gt 1380 0xb480 - 0xbfff: reserved 1381 0xc000 - 0xcfff: gt */ 1382 GEN_FW_RANGE(0xd000, 0xd7ff, 0), 1383 GEN_FW_RANGE(0xd800, 0xd8ff, FORCEWAKE_RENDER), 1384 GEN_FW_RANGE(0xd900, 0xdbff, FORCEWAKE_GT), 1385 GEN_FW_RANGE(0xdc00, 0xefff, FORCEWAKE_RENDER), /* 1386 0xdc00 - 0xddff: render 1387 0xde00 - 0xde7f: reserved 1388 0xde80 - 0xe8ff: render 1389 0xe900 - 0xefff: reserved */ 1390 GEN_FW_RANGE(0xf000, 0x147ff, FORCEWAKE_GT), /* 1391 0xf000 - 0xffff: gt 1392 0x10000 - 0x147ff: reserved */ 1393 GEN_FW_RANGE(0x14800, 0x1ffff, FORCEWAKE_RENDER), /* 1394 0x14800 - 0x14fff: render 1395 0x15000 - 0x16dff: reserved 1396 0x16e00 - 0x1bfff: render 1397 0x1c000 - 0x1ffff: reserved */ 1398 GEN_FW_RANGE(0x20000, 0x20fff, FORCEWAKE_MEDIA_VDBOX0), 1399 GEN_FW_RANGE(0x21000, 0x21fff, FORCEWAKE_MEDIA_VDBOX2), 1400 GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT), 1401 GEN_FW_RANGE(0x24000, 0x2417f, 0), /* 1402 0x24000 - 0x2407f: always on 1403 0x24080 - 0x2417f: reserved */ 1404 GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /* 1405 0x24180 - 0x241ff: gt 1406 0x24200 - 0x249ff: reserved */ 1407 GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /* 1408 0x24a00 - 0x24a7f: render 1409 0x24a80 - 0x251ff: reserved */ 1410 GEN_FW_RANGE(0x25200, 0x255ff, FORCEWAKE_GT), /* 1411 0x25200 - 0x252ff: gt 1412 0x25300 - 0x255ff: reserved */ 1413 GEN_FW_RANGE(0x25600, 0x2567f, FORCEWAKE_MEDIA_VDBOX0), 1414 GEN_FW_RANGE(0x25680, 0x259ff, FORCEWAKE_MEDIA_VDBOX2), /* 1415 0x25680 - 0x256ff: VD2 1416 0x25700 - 0x259ff: reserved */ 1417 GEN_FW_RANGE(0x25a00, 0x25a7f, FORCEWAKE_MEDIA_VDBOX0), 1418 GEN_FW_RANGE(0x25a80, 0x2ffff, FORCEWAKE_MEDIA_VDBOX2), /* 1419 0x25a80 - 0x25aff: VD2 1420 0x25b00 - 0x2ffff: reserved */ 1421 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT), 1422 GEN_FW_RANGE(0x40000, 0x1bffff, 0), 1423 GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /* 1424 0x1c0000 - 0x1c2bff: VD0 1425 0x1c2c00 - 0x1c2cff: reserved 1426 0x1c2d00 - 0x1c2dff: VD0 1427 0x1c2e00 - 0x1c3eff: reserved 1428 0x1c3f00 - 0x1c3fff: VD0 */ 1429 GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0), 1430 GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /* 1431 0x1c8000 - 0x1ca0ff: VE0 1432 0x1ca100 - 0x1cbeff: reserved 1433 0x1cbf00 - 0x1cbfff: VE0 */ 1434 GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /* 1435 0x1cc000 - 0x1ccfff: VD0 1436 0x1cd000 - 0x1cffff: reserved */ 1437 GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /* 1438 0x1d0000 - 0x1d2bff: VD2 1439 0x1d2c00 - 0x1d2cff: reserved 1440 0x1d2d00 - 0x1d2dff: VD2 1441 0x1d2e00 - 0x1d3eff: reserved 1442 0x1d3f00 - 0x1d3fff: VD2 */ 1443 }; 1444 1445 static const struct intel_forcewake_range __dg2_fw_ranges[] = { 1446 GEN_FW_RANGE(0x0, 0x1fff, 0), /* 1447 0x0 - 0xaff: reserved 1448 0xb00 - 0x1fff: always on */ 1449 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1450 GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT), 1451 GEN_FW_RANGE(0x4b00, 0x51ff, 0), /* 1452 0x4b00 - 0x4fff: reserved 1453 0x5000 - 0x51ff: always on */ 1454 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), 1455 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT), 1456 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), 1457 GEN_FW_RANGE(0x8160, 0x81ff, 0), /* 1458 0x8160 - 0x817f: reserved 1459 0x8180 - 0x81ff: always on */ 1460 GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT), 1461 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), 1462 GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /* 1463 0x8500 - 0x87ff: gt 1464 0x8800 - 0x8c7f: reserved 1465 0x8c80 - 0x8cff: gt (DG2 only) */ 1466 GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /* 1467 0x8d00 - 0x8dff: render (DG2 only) 1468 0x8e00 - 0x8fff: reserved */ 1469 GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /* 1470 0x9000 - 0x947f: gt 1471 0x9480 - 0x94cf: reserved */ 1472 GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER), 1473 GEN_FW_RANGE(0x9560, 0x967f, 0), /* 1474 0x9560 - 0x95ff: always on 1475 0x9600 - 0x967f: reserved */ 1476 GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /* 1477 0x9680 - 0x96ff: render 1478 0x9700 - 0x97ff: reserved */ 1479 GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /* 1480 0x9800 - 0xb4ff: gt 1481 0xb500 - 0xbfff: reserved 1482 0xc000 - 0xcfff: gt */ 1483 GEN_FW_RANGE(0xd000, 0xd7ff, 0), 1484 GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER), 1485 GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT), 1486 GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER), 1487 GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /* 1488 0xdd00 - 0xddff: gt 1489 0xde00 - 0xde7f: reserved */ 1490 GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /* 1491 0xde80 - 0xdfff: render 1492 0xe000 - 0xe0ff: reserved 1493 0xe100 - 0xe8ff: render */ 1494 GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /* 1495 0xe900 - 0xe9ff: gt 1496 0xea00 - 0xefff: reserved 1497 0xf000 - 0xffff: gt */ 1498 GEN_FW_RANGE(0x10000, 0x12fff, 0), /* 1499 0x10000 - 0x11fff: reserved 1500 0x12000 - 0x127ff: always on 1501 0x12800 - 0x12fff: reserved */ 1502 GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0), 1503 GEN_FW_RANGE(0x13200, 0x147ff, FORCEWAKE_MEDIA_VDBOX2), /* 1504 0x13200 - 0x133ff: VD2 (DG2 only) 1505 0x13400 - 0x147ff: reserved */ 1506 GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER), 1507 GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /* 1508 0x15000 - 0x15fff: gt (DG2 only) 1509 0x16000 - 0x16dff: reserved */ 1510 GEN_FW_RANGE(0x16e00, 0x21fff, FORCEWAKE_RENDER), /* 1511 0x16e00 - 0x1ffff: render 1512 0x20000 - 0x21fff: reserved */ 1513 GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT), 1514 GEN_FW_RANGE(0x24000, 0x2417f, 0), /* 1515 0x24000 - 0x2407f: always on 1516 0x24080 - 0x2417f: reserved */ 1517 GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /* 1518 0x24180 - 0x241ff: gt 1519 0x24200 - 0x249ff: reserved */ 1520 GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /* 1521 0x24a00 - 0x24a7f: render 1522 0x24a80 - 0x251ff: reserved */ 1523 GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /* 1524 0x25200 - 0x252ff: gt 1525 0x25300 - 0x25fff: reserved */ 1526 GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /* 1527 0x26000 - 0x27fff: render 1528 0x28000 - 0x29fff: reserved 1529 0x2a000 - 0x2ffff: undocumented */ 1530 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT), 1531 GEN_FW_RANGE(0x40000, 0x1bffff, 0), 1532 GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /* 1533 0x1c0000 - 0x1c2bff: VD0 1534 0x1c2c00 - 0x1c2cff: reserved 1535 0x1c2d00 - 0x1c2dff: VD0 1536 0x1c2e00 - 0x1c3eff: VD0 1537 0x1c3f00 - 0x1c3fff: VD0 */ 1538 GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /* 1539 0x1c4000 - 0x1c6bff: VD1 1540 0x1c6c00 - 0x1c6cff: reserved 1541 0x1c6d00 - 0x1c6dff: VD1 1542 0x1c6e00 - 0x1c7fff: reserved */ 1543 GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /* 1544 0x1c8000 - 0x1ca0ff: VE0 1545 0x1ca100 - 0x1cbfff: reserved */ 1546 GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0), 1547 GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2), 1548 GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4), 1549 GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6), 1550 GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /* 1551 0x1d0000 - 0x1d2bff: VD2 1552 0x1d2c00 - 0x1d2cff: reserved 1553 0x1d2d00 - 0x1d2dff: VD2 1554 0x1d2e00 - 0x1d3dff: VD2 1555 0x1d3e00 - 0x1d3eff: reserved 1556 0x1d3f00 - 0x1d3fff: VD2 */ 1557 GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /* 1558 0x1d4000 - 0x1d6bff: VD3 1559 0x1d6c00 - 0x1d6cff: reserved 1560 0x1d6d00 - 0x1d6dff: VD3 1561 0x1d6e00 - 0x1d7fff: reserved */ 1562 GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /* 1563 0x1d8000 - 0x1da0ff: VE1 1564 0x1da100 - 0x1dffff: reserved */ 1565 GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /* 1566 0x1e0000 - 0x1e2bff: VD4 1567 0x1e2c00 - 0x1e2cff: reserved 1568 0x1e2d00 - 0x1e2dff: VD4 1569 0x1e2e00 - 0x1e3eff: reserved 1570 0x1e3f00 - 0x1e3fff: VD4 */ 1571 GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /* 1572 0x1e4000 - 0x1e6bff: VD5 1573 0x1e6c00 - 0x1e6cff: reserved 1574 0x1e6d00 - 0x1e6dff: VD5 1575 0x1e6e00 - 0x1e7fff: reserved */ 1576 GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /* 1577 0x1e8000 - 0x1ea0ff: VE2 1578 0x1ea100 - 0x1effff: reserved */ 1579 GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /* 1580 0x1f0000 - 0x1f2bff: VD6 1581 0x1f2c00 - 0x1f2cff: reserved 1582 0x1f2d00 - 0x1f2dff: VD6 1583 0x1f2e00 - 0x1f3eff: reserved 1584 0x1f3f00 - 0x1f3fff: VD6 */ 1585 GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /* 1586 0x1f4000 - 0x1f6bff: VD7 1587 0x1f6c00 - 0x1f6cff: reserved 1588 0x1f6d00 - 0x1f6dff: VD7 1589 0x1f6e00 - 0x1f7fff: reserved */ 1590 GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3), 1591 }; 1592 1593 static const struct intel_forcewake_range __mtl_fw_ranges[] = { 1594 GEN_FW_RANGE(0x0, 0xaff, 0), 1595 GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT), 1596 GEN_FW_RANGE(0xc00, 0xfff, 0), 1597 GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT), 1598 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), 1599 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT), 1600 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), 1601 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /* 1602 0x4000 - 0x48ff: render 1603 0x4900 - 0x51ff: reserved */ 1604 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /* 1605 0x5200 - 0x53ff: render 1606 0x5400 - 0x54ff: reserved 1607 0x5500 - 0x7fff: render */ 1608 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT), 1609 GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER), /* 1610 0x8140 - 0x815f: render 1611 0x8160 - 0x817f: reserved */ 1612 GEN_FW_RANGE(0x8180, 0x81ff, 0), 1613 GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /* 1614 0x8200 - 0x87ff: gt 1615 0x8800 - 0x8dff: reserved 1616 0x8e00 - 0x8f7f: gt 1617 0x8f80 - 0x8fff: reserved 1618 0x9000 - 0x947f: gt 1619 0x9480 - 0x94cf: reserved */ 1620 GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER), 1621 GEN_FW_RANGE(0x9560, 0x967f, 0), /* 1622 0x9560 - 0x95ff: always on 1623 0x9600 - 0x967f: reserved */ 1624 GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /* 1625 0x9680 - 0x96ff: render 1626 0x9700 - 0x97ff: reserved */ 1627 GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /* 1628 0x9800 - 0xb4ff: gt 1629 0xb500 - 0xbfff: reserved 1630 0xc000 - 0xcfff: gt */ 1631 GEN_FW_RANGE(0xd000, 0xd7ff, 0), /* 1632 0xd000 - 0xd3ff: always on 1633 0xd400 - 0xd7ff: reserved */ 1634 GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER), 1635 GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT), 1636 GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER), 1637 GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /* 1638 0xdd00 - 0xddff: gt 1639 0xde00 - 0xde7f: reserved */ 1640 GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /* 1641 0xde80 - 0xdfff: render 1642 0xe000 - 0xe0ff: reserved 1643 0xe100 - 0xe8ff: render */ 1644 GEN_FW_RANGE(0xe900, 0xe9ff, FORCEWAKE_GT), 1645 GEN_FW_RANGE(0xea00, 0x147ff, 0), /* 1646 0xea00 - 0x11fff: reserved 1647 0x12000 - 0x127ff: always on 1648 0x12800 - 0x147ff: reserved */ 1649 GEN_FW_RANGE(0x14800, 0x19fff, FORCEWAKE_GT), /* 1650 0x14800 - 0x153ff: gt 1651 0x15400 - 0x19fff: reserved */ 1652 GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /* 1653 0x1a000 - 0x1bfff: render 1654 0x1c000 - 0x21fff: reserved */ 1655 GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT), 1656 GEN_FW_RANGE(0x24000, 0x2ffff, 0), /* 1657 0x24000 - 0x2407f: always on 1658 0x24080 - 0x2ffff: reserved */ 1659 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT), 1660 GEN_FW_RANGE(0x40000, 0x1901ef, 0), 1661 GEN_FW_RANGE(0x1901f0, 0x1901f3, FORCEWAKE_GT) 1662 /* FIXME: WA to wake GT while triggering H2G */ 1663 }; 1664 1665 /* 1666 * Note that the register ranges here are the final offsets after 1667 * translation of the GSI block to the 0x380000 offset. 1668 * 1669 * NOTE: There are a couple MCR ranges near the bottom of this table 1670 * that need to power up either VD0 or VD2 depending on which replicated 1671 * instance of the register we're trying to access. Our forcewake logic 1672 * at the moment doesn't have a good way to take steering into consideration, 1673 * and the driver doesn't even access any registers in those ranges today, 1674 * so for now we just mark those ranges as FORCEWAKE_ALL. That will ensure 1675 * proper operation if we do start using the ranges in the future, and we 1676 * can determine at that time whether it's worth adding extra complexity to 1677 * the forcewake handling to take steering into consideration. 1678 */ 1679 static const struct intel_forcewake_range __xelpmp_fw_ranges[] = { 1680 GEN_FW_RANGE(0x0, 0x115fff, 0), /* render GT range */ 1681 GEN_FW_RANGE(0x116000, 0x11ffff, FORCEWAKE_GSC), /* 1682 0x116000 - 0x117fff: gsc 1683 0x118000 - 0x119fff: reserved 1684 0x11a000 - 0x11efff: gsc 1685 0x11f000 - 0x11ffff: reserved */ 1686 GEN_FW_RANGE(0x120000, 0x1bffff, 0), /* non-GT range */ 1687 GEN_FW_RANGE(0x1c0000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX0), /* 1688 0x1c0000 - 0x1c3dff: VD0 1689 0x1c3e00 - 0x1c3eff: reserved 1690 0x1c3f00 - 0x1c3fff: VD0 1691 0x1c4000 - 0x1c7fff: reserved */ 1692 GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /* 1693 0x1c8000 - 0x1ca0ff: VE0 1694 0x1ca100 - 0x1cbfff: reserved */ 1695 GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /* 1696 0x1cc000 - 0x1cdfff: VD0 1697 0x1ce000 - 0x1cffff: reserved */ 1698 GEN_FW_RANGE(0x1d0000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX2), /* 1699 0x1d0000 - 0x1d3dff: VD2 1700 0x1d3e00 - 0x1d3eff: reserved 1701 0x1d4000 - 0x1d7fff: VD2 */ 1702 GEN_FW_RANGE(0x1d8000, 0x1da0ff, FORCEWAKE_MEDIA_VEBOX1), 1703 GEN_FW_RANGE(0x1da100, 0x380aff, 0), /* 1704 0x1da100 - 0x23ffff: reserved 1705 0x240000 - 0x37ffff: non-GT range 1706 0x380000 - 0x380aff: reserved */ 1707 GEN_FW_RANGE(0x380b00, 0x380bff, FORCEWAKE_GT), 1708 GEN_FW_RANGE(0x380c00, 0x380fff, 0), 1709 GEN_FW_RANGE(0x381000, 0x38817f, FORCEWAKE_GT), /* 1710 0x381000 - 0x381fff: gt 1711 0x382000 - 0x383fff: reserved 1712 0x384000 - 0x384aff: gt 1713 0x384b00 - 0x3851ff: reserved 1714 0x385200 - 0x3871ff: gt 1715 0x387200 - 0x387fff: reserved 1716 0x388000 - 0x38813f: gt 1717 0x388140 - 0x38817f: reserved */ 1718 GEN_FW_RANGE(0x388180, 0x3882ff, 0), /* 1719 0x388180 - 0x3881ff: always on 1720 0x388200 - 0x3882ff: reserved */ 1721 GEN_FW_RANGE(0x388300, 0x38955f, FORCEWAKE_GT), /* 1722 0x388300 - 0x38887f: gt 1723 0x388880 - 0x388fff: reserved 1724 0x389000 - 0x38947f: gt 1725 0x389480 - 0x38955f: reserved */ 1726 GEN_FW_RANGE(0x389560, 0x389fff, 0), /* 1727 0x389560 - 0x3895ff: always on 1728 0x389600 - 0x389fff: reserved */ 1729 GEN_FW_RANGE(0x38a000, 0x38cfff, FORCEWAKE_GT), /* 1730 0x38a000 - 0x38afff: gt 1731 0x38b000 - 0x38bfff: reserved 1732 0x38c000 - 0x38cfff: gt */ 1733 GEN_FW_RANGE(0x38d000, 0x38d11f, 0), 1734 GEN_FW_RANGE(0x38d120, 0x391fff, FORCEWAKE_GT), /* 1735 0x38d120 - 0x38dfff: gt 1736 0x38e000 - 0x38efff: reserved 1737 0x38f000 - 0x38ffff: gt 1738 0x389000 - 0x391fff: reserved */ 1739 GEN_FW_RANGE(0x392000, 0x392fff, 0), /* 1740 0x392000 - 0x3927ff: always on 1741 0x392800 - 0x292fff: reserved */ 1742 GEN_FW_RANGE(0x393000, 0x3931ff, FORCEWAKE_GT), 1743 GEN_FW_RANGE(0x393200, 0x39323f, FORCEWAKE_ALL), /* instance-based, see note above */ 1744 GEN_FW_RANGE(0x393240, 0x3933ff, FORCEWAKE_GT), 1745 GEN_FW_RANGE(0x393400, 0x3934ff, FORCEWAKE_ALL), /* instance-based, see note above */ 1746 GEN_FW_RANGE(0x393500, 0x393c7f, 0), /* 1747 0x393500 - 0x393bff: reserved 1748 0x393c00 - 0x393c7f: always on */ 1749 GEN_FW_RANGE(0x393c80, 0x393dff, FORCEWAKE_GT), 1750 }; 1751 1752 static void 1753 ilk_dummy_write(struct intel_uncore *uncore) 1754 { 1755 /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up 1756 * the chip from rc6 before touching it for real. MI_MODE is masked, 1757 * hence harmless to write 0 into. */ 1758 __raw_uncore_write32(uncore, RING_MI_MODE(RENDER_RING_BASE), 0); 1759 } 1760 1761 static void 1762 __unclaimed_reg_debug(struct intel_uncore *uncore, 1763 const i915_reg_t reg, 1764 const bool read) 1765 { 1766 if (drm_WARN(&uncore->i915->drm, 1767 check_for_unclaimed_mmio(uncore), 1768 "Unclaimed %s register 0x%x\n", 1769 read ? "read from" : "write to", 1770 i915_mmio_reg_offset(reg))) 1771 /* Only report the first N failures */ 1772 uncore->i915->params.mmio_debug--; 1773 } 1774 1775 static void 1776 __unclaimed_previous_reg_debug(struct intel_uncore *uncore, 1777 const i915_reg_t reg, 1778 const bool read) 1779 { 1780 if (check_for_unclaimed_mmio(uncore)) 1781 drm_dbg(&uncore->i915->drm, 1782 "Unclaimed access detected before %s register 0x%x\n", 1783 read ? "read from" : "write to", 1784 i915_mmio_reg_offset(reg)); 1785 } 1786 1787 static inline bool __must_check 1788 unclaimed_reg_debug_header(struct intel_uncore *uncore, 1789 const i915_reg_t reg, const bool read) 1790 { 1791 if (likely(!uncore->i915->params.mmio_debug) || !uncore->debug) 1792 return false; 1793 1794 /* interrupts are disabled and re-enabled around uncore->lock usage */ 1795 lockdep_assert_held(&uncore->lock); 1796 1797 spin_lock(&uncore->debug->lock); 1798 __unclaimed_previous_reg_debug(uncore, reg, read); 1799 1800 return true; 1801 } 1802 1803 static inline void 1804 unclaimed_reg_debug_footer(struct intel_uncore *uncore, 1805 const i915_reg_t reg, const bool read) 1806 { 1807 /* interrupts are disabled and re-enabled around uncore->lock usage */ 1808 lockdep_assert_held(&uncore->lock); 1809 1810 __unclaimed_reg_debug(uncore, reg, read); 1811 spin_unlock(&uncore->debug->lock); 1812 } 1813 1814 #define __vgpu_read(x) \ 1815 static u##x \ 1816 vgpu_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \ 1817 u##x val = __raw_uncore_read##x(uncore, reg); \ 1818 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1819 return val; \ 1820 } 1821 __vgpu_read(8) 1822 __vgpu_read(16) 1823 __vgpu_read(32) 1824 __vgpu_read(64) 1825 1826 #define GEN2_READ_HEADER(x) \ 1827 u##x val = 0; \ 1828 assert_rpm_wakelock_held(uncore->rpm); 1829 1830 #define GEN2_READ_FOOTER \ 1831 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1832 return val 1833 1834 #define __gen2_read(x) \ 1835 static u##x \ 1836 gen2_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \ 1837 GEN2_READ_HEADER(x); \ 1838 val = __raw_uncore_read##x(uncore, reg); \ 1839 GEN2_READ_FOOTER; \ 1840 } 1841 1842 #define __gen5_read(x) \ 1843 static u##x \ 1844 gen5_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \ 1845 GEN2_READ_HEADER(x); \ 1846 ilk_dummy_write(uncore); \ 1847 val = __raw_uncore_read##x(uncore, reg); \ 1848 GEN2_READ_FOOTER; \ 1849 } 1850 1851 __gen5_read(8) 1852 __gen5_read(16) 1853 __gen5_read(32) 1854 __gen5_read(64) 1855 __gen2_read(8) 1856 __gen2_read(16) 1857 __gen2_read(32) 1858 __gen2_read(64) 1859 1860 #undef __gen5_read 1861 #undef __gen2_read 1862 1863 #undef GEN2_READ_FOOTER 1864 #undef GEN2_READ_HEADER 1865 1866 #define GEN6_READ_HEADER(x) \ 1867 u32 offset = i915_mmio_reg_offset(reg); \ 1868 unsigned long irqflags; \ 1869 bool unclaimed_reg_debug; \ 1870 u##x val = 0; \ 1871 assert_rpm_wakelock_held(uncore->rpm); \ 1872 spin_lock_irqsave(&uncore->lock, irqflags); \ 1873 unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, true) 1874 1875 #define GEN6_READ_FOOTER \ 1876 if (unclaimed_reg_debug) \ 1877 unclaimed_reg_debug_footer(uncore, reg, true); \ 1878 spin_unlock_irqrestore(&uncore->lock, irqflags); \ 1879 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 1880 return val 1881 1882 static noinline void ___force_wake_auto(struct intel_uncore *uncore, 1883 enum forcewake_domains fw_domains) 1884 { 1885 struct intel_uncore_forcewake_domain *domain; 1886 unsigned int tmp; 1887 1888 GEM_BUG_ON(fw_domains & ~uncore->fw_domains); 1889 1890 for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) 1891 fw_domain_arm_timer(domain); 1892 1893 fw_domains_get(uncore, fw_domains); 1894 } 1895 1896 static inline void __force_wake_auto(struct intel_uncore *uncore, 1897 enum forcewake_domains fw_domains) 1898 { 1899 GEM_BUG_ON(!fw_domains); 1900 1901 /* Turn on all requested but inactive supported forcewake domains. */ 1902 fw_domains &= uncore->fw_domains; 1903 fw_domains &= ~uncore->fw_domains_active; 1904 1905 if (fw_domains) 1906 ___force_wake_auto(uncore, fw_domains); 1907 } 1908 1909 #define __gen_fwtable_read(x) \ 1910 static u##x \ 1911 fwtable_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) \ 1912 { \ 1913 enum forcewake_domains fw_engine; \ 1914 GEN6_READ_HEADER(x); \ 1915 fw_engine = __fwtable_reg_read_fw_domains(uncore, offset); \ 1916 if (fw_engine) \ 1917 __force_wake_auto(uncore, fw_engine); \ 1918 val = __raw_uncore_read##x(uncore, reg); \ 1919 GEN6_READ_FOOTER; \ 1920 } 1921 1922 static enum forcewake_domains 1923 fwtable_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) { 1924 return __fwtable_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg)); 1925 } 1926 1927 __gen_fwtable_read(8) 1928 __gen_fwtable_read(16) 1929 __gen_fwtable_read(32) 1930 __gen_fwtable_read(64) 1931 1932 #undef __gen_fwtable_read 1933 #undef GEN6_READ_FOOTER 1934 #undef GEN6_READ_HEADER 1935 1936 #define GEN2_WRITE_HEADER \ 1937 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 1938 assert_rpm_wakelock_held(uncore->rpm); \ 1939 1940 #define GEN2_WRITE_FOOTER 1941 1942 #define __gen2_write(x) \ 1943 static void \ 1944 gen2_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \ 1945 GEN2_WRITE_HEADER; \ 1946 __raw_uncore_write##x(uncore, reg, val); \ 1947 GEN2_WRITE_FOOTER; \ 1948 } 1949 1950 #define __gen5_write(x) \ 1951 static void \ 1952 gen5_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \ 1953 GEN2_WRITE_HEADER; \ 1954 ilk_dummy_write(uncore); \ 1955 __raw_uncore_write##x(uncore, reg, val); \ 1956 GEN2_WRITE_FOOTER; \ 1957 } 1958 1959 __gen5_write(8) 1960 __gen5_write(16) 1961 __gen5_write(32) 1962 __gen2_write(8) 1963 __gen2_write(16) 1964 __gen2_write(32) 1965 1966 #undef __gen5_write 1967 #undef __gen2_write 1968 1969 #undef GEN2_WRITE_FOOTER 1970 #undef GEN2_WRITE_HEADER 1971 1972 #define GEN6_WRITE_HEADER \ 1973 u32 offset = i915_mmio_reg_offset(reg); \ 1974 unsigned long irqflags; \ 1975 bool unclaimed_reg_debug; \ 1976 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 1977 assert_rpm_wakelock_held(uncore->rpm); \ 1978 spin_lock_irqsave(&uncore->lock, irqflags); \ 1979 unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, false) 1980 1981 #define GEN6_WRITE_FOOTER \ 1982 if (unclaimed_reg_debug) \ 1983 unclaimed_reg_debug_footer(uncore, reg, false); \ 1984 spin_unlock_irqrestore(&uncore->lock, irqflags) 1985 1986 #define __gen6_write(x) \ 1987 static void \ 1988 gen6_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \ 1989 GEN6_WRITE_HEADER; \ 1990 if (NEEDS_FORCE_WAKE(offset)) \ 1991 __gen6_gt_wait_for_fifo(uncore); \ 1992 __raw_uncore_write##x(uncore, reg, val); \ 1993 GEN6_WRITE_FOOTER; \ 1994 } 1995 __gen6_write(8) 1996 __gen6_write(16) 1997 __gen6_write(32) 1998 1999 #define __gen_fwtable_write(x) \ 2000 static void \ 2001 fwtable_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \ 2002 enum forcewake_domains fw_engine; \ 2003 GEN6_WRITE_HEADER; \ 2004 fw_engine = __fwtable_reg_write_fw_domains(uncore, offset); \ 2005 if (fw_engine) \ 2006 __force_wake_auto(uncore, fw_engine); \ 2007 __raw_uncore_write##x(uncore, reg, val); \ 2008 GEN6_WRITE_FOOTER; \ 2009 } 2010 2011 static enum forcewake_domains 2012 fwtable_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) 2013 { 2014 return __fwtable_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg)); 2015 } 2016 2017 __gen_fwtable_write(8) 2018 __gen_fwtable_write(16) 2019 __gen_fwtable_write(32) 2020 2021 #undef __gen_fwtable_write 2022 #undef GEN6_WRITE_FOOTER 2023 #undef GEN6_WRITE_HEADER 2024 2025 #define __vgpu_write(x) \ 2026 static void \ 2027 vgpu_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \ 2028 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 2029 __raw_uncore_write##x(uncore, reg, val); \ 2030 } 2031 __vgpu_write(8) 2032 __vgpu_write(16) 2033 __vgpu_write(32) 2034 2035 #define ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, x) \ 2036 do { \ 2037 (uncore)->funcs.mmio_writeb = x##_write8; \ 2038 (uncore)->funcs.mmio_writew = x##_write16; \ 2039 (uncore)->funcs.mmio_writel = x##_write32; \ 2040 } while (0) 2041 2042 #define ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x) \ 2043 do { \ 2044 (uncore)->funcs.mmio_readb = x##_read8; \ 2045 (uncore)->funcs.mmio_readw = x##_read16; \ 2046 (uncore)->funcs.mmio_readl = x##_read32; \ 2047 (uncore)->funcs.mmio_readq = x##_read64; \ 2048 } while (0) 2049 2050 #define ASSIGN_WRITE_MMIO_VFUNCS(uncore, x) \ 2051 do { \ 2052 ASSIGN_RAW_WRITE_MMIO_VFUNCS((uncore), x); \ 2053 (uncore)->funcs.write_fw_domains = x##_reg_write_fw_domains; \ 2054 } while (0) 2055 2056 #define ASSIGN_READ_MMIO_VFUNCS(uncore, x) \ 2057 do { \ 2058 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x); \ 2059 (uncore)->funcs.read_fw_domains = x##_reg_read_fw_domains; \ 2060 } while (0) 2061 2062 static int __fw_domain_init(struct intel_uncore *uncore, 2063 enum forcewake_domain_id domain_id, 2064 i915_reg_t reg_set, 2065 i915_reg_t reg_ack) 2066 { 2067 struct intel_uncore_forcewake_domain *d; 2068 2069 GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT); 2070 GEM_BUG_ON(uncore->fw_domain[domain_id]); 2071 2072 if (i915_inject_probe_failure(uncore->i915)) 2073 return -ENOMEM; 2074 2075 d = kzalloc(sizeof(*d), GFP_KERNEL); 2076 if (!d) 2077 return -ENOMEM; 2078 2079 drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_set)); 2080 drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_ack)); 2081 2082 d->uncore = uncore; 2083 d->wake_count = 0; 2084 d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set) + uncore->gsi_offset; 2085 d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack) + uncore->gsi_offset; 2086 2087 d->id = domain_id; 2088 2089 BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER)); 2090 BUILD_BUG_ON(FORCEWAKE_GT != (1 << FW_DOMAIN_ID_GT)); 2091 BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA)); 2092 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0)); 2093 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1)); 2094 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2)); 2095 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3)); 2096 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX4 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX4)); 2097 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX5 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX5)); 2098 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX6 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX6)); 2099 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX7 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX7)); 2100 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0)); 2101 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1)); 2102 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX2)); 2103 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX3)); 2104 BUILD_BUG_ON(FORCEWAKE_GSC != (1 << FW_DOMAIN_ID_GSC)); 2105 2106 d->mask = BIT(domain_id); 2107 2108 hrtimer_setup(&d->timer, intel_uncore_fw_release_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 2109 2110 uncore->fw_domains |= BIT(domain_id); 2111 2112 fw_domain_reset(d); 2113 2114 uncore->fw_domain[domain_id] = d; 2115 2116 return 0; 2117 } 2118 2119 static void fw_domain_fini(struct intel_uncore *uncore, 2120 enum forcewake_domain_id domain_id) 2121 { 2122 struct intel_uncore_forcewake_domain *d; 2123 2124 GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT); 2125 2126 d = fetch_and_zero(&uncore->fw_domain[domain_id]); 2127 if (!d) 2128 return; 2129 2130 uncore->fw_domains &= ~BIT(domain_id); 2131 drm_WARN_ON(&uncore->i915->drm, d->wake_count); 2132 drm_WARN_ON(&uncore->i915->drm, hrtimer_cancel(&d->timer)); 2133 kfree(d); 2134 } 2135 2136 static void intel_uncore_fw_domains_fini(struct intel_uncore *uncore) 2137 { 2138 struct intel_uncore_forcewake_domain *d; 2139 int tmp; 2140 2141 for_each_fw_domain(d, uncore, tmp) 2142 fw_domain_fini(uncore, d->id); 2143 } 2144 2145 static const struct intel_uncore_fw_get uncore_get_fallback = { 2146 .force_wake_get = fw_domains_get_with_fallback 2147 }; 2148 2149 static const struct intel_uncore_fw_get uncore_get_normal = { 2150 .force_wake_get = fw_domains_get_normal, 2151 }; 2152 2153 static const struct intel_uncore_fw_get uncore_get_thread_status = { 2154 .force_wake_get = fw_domains_get_with_thread_status 2155 }; 2156 2157 static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) 2158 { 2159 struct drm_i915_private *i915 = uncore->i915; 2160 int ret = 0; 2161 2162 GEM_BUG_ON(!intel_uncore_has_forcewake(uncore)); 2163 2164 #define fw_domain_init(uncore__, id__, set__, ack__) \ 2165 (ret ?: (ret = __fw_domain_init((uncore__), (id__), (set__), (ack__)))) 2166 2167 if (GRAPHICS_VER(i915) >= 11) { 2168 intel_engine_mask_t emask; 2169 int i; 2170 2171 /* we'll prune the domains of missing engines later */ 2172 emask = uncore->gt->info.engine_mask; 2173 2174 uncore->fw_get_funcs = &uncore_get_fallback; 2175 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 2176 fw_domain_init(uncore, FW_DOMAIN_ID_GT, 2177 FORCEWAKE_GT_GEN9, 2178 FORCEWAKE_ACK_GT_MTL); 2179 else 2180 fw_domain_init(uncore, FW_DOMAIN_ID_GT, 2181 FORCEWAKE_GT_GEN9, 2182 FORCEWAKE_ACK_GT_GEN9); 2183 2184 if (RCS_MASK(uncore->gt) || CCS_MASK(uncore->gt)) 2185 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, 2186 FORCEWAKE_RENDER_GEN9, 2187 FORCEWAKE_ACK_RENDER_GEN9); 2188 2189 for (i = 0; i < I915_MAX_VCS; i++) { 2190 if (!__HAS_ENGINE(emask, _VCS(i))) 2191 continue; 2192 2193 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VDBOX0 + i, 2194 FORCEWAKE_MEDIA_VDBOX_GEN11(i), 2195 FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i)); 2196 } 2197 for (i = 0; i < I915_MAX_VECS; i++) { 2198 if (!__HAS_ENGINE(emask, _VECS(i))) 2199 continue; 2200 2201 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VEBOX0 + i, 2202 FORCEWAKE_MEDIA_VEBOX_GEN11(i), 2203 FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i)); 2204 } 2205 2206 if (uncore->gt->type == GT_MEDIA) 2207 fw_domain_init(uncore, FW_DOMAIN_ID_GSC, 2208 FORCEWAKE_REQ_GSC, FORCEWAKE_ACK_GSC); 2209 } else if (IS_GRAPHICS_VER(i915, 9, 10)) { 2210 uncore->fw_get_funcs = &uncore_get_fallback; 2211 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, 2212 FORCEWAKE_RENDER_GEN9, 2213 FORCEWAKE_ACK_RENDER_GEN9); 2214 fw_domain_init(uncore, FW_DOMAIN_ID_GT, 2215 FORCEWAKE_GT_GEN9, 2216 FORCEWAKE_ACK_GT_GEN9); 2217 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA, 2218 FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9); 2219 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2220 uncore->fw_get_funcs = &uncore_get_normal; 2221 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, 2222 FORCEWAKE_VLV, FORCEWAKE_ACK_VLV); 2223 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA, 2224 FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV); 2225 } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { 2226 uncore->fw_get_funcs = &uncore_get_thread_status; 2227 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, 2228 FORCEWAKE_MT, FORCEWAKE_ACK_HSW); 2229 } else if (IS_IVYBRIDGE(i915)) { 2230 u32 ecobus; 2231 2232 /* IVB configs may use multi-threaded forcewake */ 2233 2234 /* A small trick here - if the bios hasn't configured 2235 * MT forcewake, and if the device is in RC6, then 2236 * force_wake_mt_get will not wake the device and the 2237 * ECOBUS read will return zero. Which will be 2238 * (correctly) interpreted by the test below as MT 2239 * forcewake being disabled. 2240 */ 2241 uncore->fw_get_funcs = &uncore_get_thread_status; 2242 2243 /* We need to init first for ECOBUS access and then 2244 * determine later if we want to reinit, in case of MT access is 2245 * not working. In this stage we don't know which flavour this 2246 * ivb is, so it is better to reset also the gen6 fw registers 2247 * before the ecobus check. 2248 */ 2249 2250 __raw_uncore_write32(uncore, FORCEWAKE, 0); 2251 __raw_posting_read(uncore, ECOBUS); 2252 2253 ret = __fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, 2254 FORCEWAKE_MT, FORCEWAKE_MT_ACK); 2255 if (ret) 2256 goto out; 2257 2258 spin_lock_irq(&uncore->lock); 2259 fw_domains_get_with_thread_status(uncore, FORCEWAKE_RENDER); 2260 ecobus = __raw_uncore_read32(uncore, ECOBUS); 2261 fw_domains_put(uncore, FORCEWAKE_RENDER); 2262 spin_unlock_irq(&uncore->lock); 2263 2264 if (!(ecobus & FORCEWAKE_MT_ENABLE)) { 2265 drm_info(&i915->drm, "No MT forcewake available on Ivybridge, this can result in issues\n"); 2266 drm_info(&i915->drm, "when using vblank-synced partial screen updates.\n"); 2267 fw_domain_fini(uncore, FW_DOMAIN_ID_RENDER); 2268 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, 2269 FORCEWAKE, FORCEWAKE_ACK); 2270 } 2271 } else if (GRAPHICS_VER(i915) == 6) { 2272 uncore->fw_get_funcs = &uncore_get_thread_status; 2273 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, 2274 FORCEWAKE, FORCEWAKE_ACK); 2275 } 2276 2277 #undef fw_domain_init 2278 2279 /* All future platforms are expected to require complex power gating */ 2280 drm_WARN_ON(&i915->drm, !ret && uncore->fw_domains == 0); 2281 2282 out: 2283 if (ret) 2284 intel_uncore_fw_domains_fini(uncore); 2285 2286 return ret; 2287 } 2288 2289 #define ASSIGN_FW_DOMAINS_TABLE(uncore, d) \ 2290 { \ 2291 (uncore)->fw_domains_table = \ 2292 (struct intel_forcewake_range *)(d); \ 2293 (uncore)->fw_domains_table_entries = ARRAY_SIZE((d)); \ 2294 } 2295 2296 #define ASSIGN_SHADOW_TABLE(uncore, d) \ 2297 { \ 2298 (uncore)->shadowed_reg_table = d; \ 2299 (uncore)->shadowed_reg_table_entries = ARRAY_SIZE((d)); \ 2300 } 2301 2302 static int i915_pmic_bus_access_notifier(struct notifier_block *nb, 2303 unsigned long action, void *data) 2304 { 2305 struct intel_uncore *uncore = container_of(nb, 2306 struct intel_uncore, pmic_bus_access_nb); 2307 2308 switch (action) { 2309 case MBI_PMIC_BUS_ACCESS_BEGIN: 2310 /* 2311 * forcewake all now to make sure that we don't need to do a 2312 * forcewake later which on systems where this notifier gets 2313 * called requires the punit to access to the shared pmic i2c 2314 * bus, which will be busy after this notification, leading to: 2315 * "render: timed out waiting for forcewake ack request." 2316 * errors. 2317 * 2318 * The notifier is unregistered during intel_runtime_suspend(), 2319 * so it's ok to access the HW here without holding a RPM 2320 * wake reference -> disable wakeref asserts for the time of 2321 * the access. 2322 */ 2323 disable_rpm_wakeref_asserts(uncore->rpm); 2324 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 2325 enable_rpm_wakeref_asserts(uncore->rpm); 2326 break; 2327 case MBI_PMIC_BUS_ACCESS_END: 2328 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 2329 break; 2330 } 2331 2332 return NOTIFY_OK; 2333 } 2334 2335 static void uncore_unmap_mmio(struct drm_device *drm, void *regs) 2336 { 2337 iounmap((void __iomem *)regs); 2338 } 2339 2340 int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr) 2341 { 2342 struct drm_i915_private *i915 = uncore->i915; 2343 int mmio_size; 2344 2345 /* 2346 * Before gen4, the registers and the GTT are behind different BARs. 2347 * However, from gen4 onwards, the registers and the GTT are shared 2348 * in the same BAR, so we want to restrict this ioremap from 2349 * clobbering the GTT which we want ioremap_wc instead. Fortunately, 2350 * the register BAR remains the same size for all the earlier 2351 * generations up to Ironlake. 2352 * For dgfx chips register range is expanded to 4MB, and this larger 2353 * range is also used for integrated gpus beginning with Meteor Lake. 2354 */ 2355 if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 2356 mmio_size = 4 * 1024 * 1024; 2357 else if (GRAPHICS_VER(i915) >= 5) 2358 mmio_size = 2 * 1024 * 1024; 2359 else 2360 mmio_size = 512 * 1024; 2361 2362 uncore->regs = ioremap(phys_addr, mmio_size); 2363 if (uncore->regs == NULL) { 2364 drm_err(&i915->drm, "failed to map registers\n"); 2365 return -EIO; 2366 } 2367 2368 return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio, 2369 (void __force *)uncore->regs); 2370 } 2371 2372 void intel_uncore_init_early(struct intel_uncore *uncore, 2373 struct intel_gt *gt) 2374 { 2375 spin_lock_init(&uncore->lock); 2376 uncore->i915 = gt->i915; 2377 uncore->gt = gt; 2378 uncore->rpm = >->i915->runtime_pm; 2379 } 2380 2381 static void uncore_raw_init(struct intel_uncore *uncore) 2382 { 2383 GEM_BUG_ON(intel_uncore_has_forcewake(uncore)); 2384 2385 if (intel_vgpu_active(uncore->i915)) { 2386 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu); 2387 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu); 2388 } else if (GRAPHICS_VER(uncore->i915) == 5) { 2389 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5); 2390 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5); 2391 } else { 2392 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen2); 2393 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen2); 2394 } 2395 } 2396 2397 static int uncore_media_forcewake_init(struct intel_uncore *uncore) 2398 { 2399 struct drm_i915_private *i915 = uncore->i915; 2400 2401 if (MEDIA_VER(i915) >= 13) { 2402 ASSIGN_FW_DOMAINS_TABLE(uncore, __xelpmp_fw_ranges); 2403 ASSIGN_SHADOW_TABLE(uncore, xelpmp_shadowed_regs); 2404 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); 2405 } else { 2406 MISSING_CASE(MEDIA_VER(i915)); 2407 return -ENODEV; 2408 } 2409 2410 return 0; 2411 } 2412 2413 static int uncore_forcewake_init(struct intel_uncore *uncore) 2414 { 2415 struct drm_i915_private *i915 = uncore->i915; 2416 int ret; 2417 2418 GEM_BUG_ON(!intel_uncore_has_forcewake(uncore)); 2419 2420 ret = intel_uncore_fw_domains_init(uncore); 2421 if (ret) 2422 return ret; 2423 forcewake_early_sanitize(uncore, 0); 2424 2425 ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable); 2426 2427 if (uncore->gt->type == GT_MEDIA) 2428 return uncore_media_forcewake_init(uncore); 2429 2430 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 2431 ASSIGN_FW_DOMAINS_TABLE(uncore, __mtl_fw_ranges); 2432 ASSIGN_SHADOW_TABLE(uncore, mtl_shadowed_regs); 2433 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); 2434 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) { 2435 ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges); 2436 ASSIGN_SHADOW_TABLE(uncore, dg2_shadowed_regs); 2437 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); 2438 } else if (GRAPHICS_VER(i915) >= 12) { 2439 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges); 2440 ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs); 2441 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); 2442 } else if (GRAPHICS_VER(i915) == 11) { 2443 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges); 2444 ASSIGN_SHADOW_TABLE(uncore, gen11_shadowed_regs); 2445 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); 2446 } else if (IS_GRAPHICS_VER(i915, 9, 10)) { 2447 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges); 2448 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs); 2449 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); 2450 } else if (IS_CHERRYVIEW(i915)) { 2451 ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges); 2452 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs); 2453 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); 2454 } else if (GRAPHICS_VER(i915) == 8) { 2455 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges); 2456 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs); 2457 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable); 2458 } else if (IS_VALLEYVIEW(i915)) { 2459 ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges); 2460 ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6); 2461 } else if (IS_GRAPHICS_VER(i915, 6, 7)) { 2462 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges); 2463 ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6); 2464 } 2465 2466 uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier; 2467 iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb); 2468 2469 return 0; 2470 } 2471 2472 static int sanity_check_mmio_access(struct intel_uncore *uncore) 2473 { 2474 struct drm_i915_private *i915 = uncore->i915; 2475 2476 if (GRAPHICS_VER(i915) < 8) 2477 return 0; 2478 2479 /* 2480 * Sanitycheck that MMIO access to the device is working properly. If 2481 * the CPU is unable to communicate with a PCI device, BAR reads will 2482 * return 0xFFFFFFFF. Let's make sure the device isn't in this state 2483 * before we start trying to access registers. 2484 * 2485 * We use the primary GT's forcewake register as our guinea pig since 2486 * it's been around since HSW and it's a masked register so the upper 2487 * 16 bits can never read back as 1's if device access is operating 2488 * properly. 2489 * 2490 * If MMIO isn't working, we'll wait up to 2 seconds to see if it 2491 * recovers, then give up. 2492 */ 2493 #define COND (__raw_uncore_read32(uncore, FORCEWAKE_MT) != ~0) 2494 if (wait_for(COND, 2000) == -ETIMEDOUT) { 2495 drm_err(&i915->drm, "Device is non-operational; MMIO access returns 0xFFFFFFFF!\n"); 2496 return -EIO; 2497 } 2498 2499 return 0; 2500 } 2501 2502 int intel_uncore_init_mmio(struct intel_uncore *uncore) 2503 { 2504 struct drm_i915_private *i915 = uncore->i915; 2505 int ret; 2506 2507 ret = sanity_check_mmio_access(uncore); 2508 if (ret) 2509 return ret; 2510 2511 /* 2512 * The boot firmware initializes local memory and assesses its health. 2513 * If memory training fails, the punit will have been instructed to 2514 * keep the GT powered down; we won't be able to communicate with it 2515 * and we should not continue with driver initialization. 2516 */ 2517 if (IS_DGFX(i915) && 2518 !(__raw_uncore_read32(uncore, GU_CNTL) & LMEM_INIT)) { 2519 drm_err(&i915->drm, "LMEM not initialized by firmware\n"); 2520 return -ENODEV; 2521 } 2522 2523 if (GRAPHICS_VER(i915) > 5 && !intel_vgpu_active(i915)) 2524 uncore->flags |= UNCORE_HAS_FORCEWAKE; 2525 2526 if (!intel_uncore_has_forcewake(uncore)) { 2527 uncore_raw_init(uncore); 2528 } else { 2529 ret = uncore_forcewake_init(uncore); 2530 if (ret) 2531 return ret; 2532 } 2533 2534 /* make sure fw funcs are set if and only if we have fw*/ 2535 GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->fw_get_funcs); 2536 GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.read_fw_domains); 2537 GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.write_fw_domains); 2538 2539 if (HAS_FPGA_DBG_UNCLAIMED(i915)) 2540 uncore->flags |= UNCORE_HAS_FPGA_DBG_UNCLAIMED; 2541 2542 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 2543 uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED; 2544 2545 if (IS_GRAPHICS_VER(i915, 6, 7)) 2546 uncore->flags |= UNCORE_HAS_FIFO; 2547 2548 /* clear out unclaimed reg detection bit */ 2549 if (intel_uncore_unclaimed_mmio(uncore)) 2550 drm_dbg(&i915->drm, "unclaimed mmio detected on uncore init, clearing\n"); 2551 2552 return 0; 2553 } 2554 2555 /* 2556 * We might have detected that some engines are fused off after we initialized 2557 * the forcewake domains. Prune them, to make sure they only reference existing 2558 * engines. 2559 */ 2560 void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore, 2561 struct intel_gt *gt) 2562 { 2563 enum forcewake_domains fw_domains = uncore->fw_domains; 2564 enum forcewake_domain_id domain_id; 2565 int i; 2566 2567 if (!intel_uncore_has_forcewake(uncore) || GRAPHICS_VER(uncore->i915) < 11) 2568 return; 2569 2570 for (i = 0; i < I915_MAX_VCS; i++) { 2571 domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i; 2572 2573 if (HAS_ENGINE(gt, _VCS(i))) 2574 continue; 2575 2576 /* 2577 * Starting with XeHP, the power well for an even-numbered 2578 * VDBOX is also used for shared units within the 2579 * media slice such as SFC. So even if the engine 2580 * itself is fused off, we still need to initialize 2581 * the forcewake domain if any of the other engines 2582 * in the same media slice are present. 2583 */ 2584 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 55) && i % 2 == 0) { 2585 if ((i + 1 < I915_MAX_VCS) && HAS_ENGINE(gt, _VCS(i + 1))) 2586 continue; 2587 2588 if (HAS_ENGINE(gt, _VECS(i / 2))) 2589 continue; 2590 } 2591 2592 if (fw_domains & BIT(domain_id)) 2593 fw_domain_fini(uncore, domain_id); 2594 } 2595 2596 for (i = 0; i < I915_MAX_VECS; i++) { 2597 domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i; 2598 2599 if (HAS_ENGINE(gt, _VECS(i))) 2600 continue; 2601 2602 if (fw_domains & BIT(domain_id)) 2603 fw_domain_fini(uncore, domain_id); 2604 } 2605 2606 if ((fw_domains & BIT(FW_DOMAIN_ID_GSC)) && !HAS_ENGINE(gt, GSC0)) 2607 fw_domain_fini(uncore, FW_DOMAIN_ID_GSC); 2608 } 2609 2610 /* 2611 * The driver-initiated FLR is the highest level of reset that we can trigger 2612 * from within the driver. It is different from the PCI FLR in that it doesn't 2613 * fully reset the SGUnit and doesn't modify the PCI config space and therefore 2614 * it doesn't require a re-enumeration of the PCI BARs. However, the 2615 * driver-initiated FLR does still cause a reset of both GT and display and a 2616 * memory wipe of local and stolen memory, so recovery would require a full HW 2617 * re-init and saving/restoring (or re-populating) the wiped memory. Since we 2618 * perform the FLR as the very last action before releasing access to the HW 2619 * during the driver release flow, we don't attempt recovery at all, because 2620 * if/when a new instance of i915 is bound to the device it will do a full 2621 * re-init anyway. 2622 */ 2623 static void driver_initiated_flr(struct intel_uncore *uncore) 2624 { 2625 struct drm_i915_private *i915 = uncore->i915; 2626 unsigned int flr_timeout_ms; 2627 int ret; 2628 2629 drm_dbg(&i915->drm, "Triggering Driver-FLR\n"); 2630 2631 /* 2632 * The specification recommends a 3 seconds FLR reset timeout. To be 2633 * cautious, we will extend this to 9 seconds, three times the specified 2634 * timeout. 2635 */ 2636 flr_timeout_ms = 9000; 2637 2638 /* 2639 * Make sure any pending FLR requests have cleared by waiting for the 2640 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS 2641 * to make sure it's not still set from a prior attempt (it's a write to 2642 * clear bit). 2643 * Note that we should never be in a situation where a previous attempt 2644 * is still pending (unless the HW is totally dead), but better to be 2645 * safe in case something unexpected happens 2646 */ 2647 ret = intel_wait_for_register_fw(uncore, GU_CNTL, DRIVERFLR, 0, flr_timeout_ms, NULL); 2648 if (ret) { 2649 drm_err(&i915->drm, 2650 "Failed to wait for Driver-FLR bit to clear! %d\n", 2651 ret); 2652 return; 2653 } 2654 intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS); 2655 2656 /* Trigger the actual Driver-FLR */ 2657 intel_uncore_rmw_fw(uncore, GU_CNTL, 0, DRIVERFLR); 2658 2659 /* Wait for hardware teardown to complete */ 2660 ret = intel_wait_for_register_fw(uncore, GU_CNTL, 2661 DRIVERFLR, 0, 2662 flr_timeout_ms, NULL); 2663 if (ret) { 2664 drm_err(&i915->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret); 2665 return; 2666 } 2667 2668 /* Wait for hardware/firmware re-init to complete */ 2669 ret = intel_wait_for_register_fw(uncore, GU_DEBUG, 2670 DRIVERFLR_STATUS, DRIVERFLR_STATUS, 2671 flr_timeout_ms, NULL); 2672 if (ret) { 2673 drm_err(&i915->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret); 2674 return; 2675 } 2676 2677 /* Clear sticky completion status */ 2678 intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS); 2679 } 2680 2681 /* Called via drm-managed action */ 2682 void intel_uncore_fini_mmio(struct drm_device *dev, void *data) 2683 { 2684 struct intel_uncore *uncore = data; 2685 2686 if (intel_uncore_has_forcewake(uncore)) { 2687 iosf_mbi_punit_acquire(); 2688 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( 2689 &uncore->pmic_bus_access_nb); 2690 intel_uncore_forcewake_reset(uncore); 2691 intel_uncore_fw_domains_fini(uncore); 2692 iosf_mbi_punit_release(); 2693 } 2694 2695 if (intel_uncore_needs_flr_on_fini(uncore)) 2696 driver_initiated_flr(uncore); 2697 } 2698 2699 /** 2700 * __intel_wait_for_register_fw - wait until register matches expected state 2701 * @uncore: the struct intel_uncore 2702 * @reg: the register to read 2703 * @mask: mask to apply to register value 2704 * @value: expected value 2705 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 2706 * @slow_timeout_ms: slow timeout in millisecond 2707 * @out_value: optional placeholder to hold registry value 2708 * 2709 * This routine waits until the target register @reg contains the expected 2710 * @value after applying the @mask, i.e. it waits until :: 2711 * 2712 * (intel_uncore_read_fw(uncore, reg) & mask) == value 2713 * 2714 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. 2715 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us 2716 * must be not larger than 20,0000 microseconds. 2717 * 2718 * Note that this routine assumes the caller holds forcewake asserted, it is 2719 * not suitable for very long waits. See intel_wait_for_register() if you 2720 * wish to wait without holding forcewake for the duration (i.e. you expect 2721 * the wait to be slow). 2722 * 2723 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT. 2724 */ 2725 int __intel_wait_for_register_fw(struct intel_uncore *uncore, 2726 i915_reg_t reg, 2727 u32 mask, 2728 u32 value, 2729 unsigned int fast_timeout_us, 2730 unsigned int slow_timeout_ms, 2731 u32 *out_value) 2732 { 2733 u32 reg_value = 0; 2734 #define done (((reg_value = intel_uncore_read_fw(uncore, reg)) & mask) == value) 2735 int ret; 2736 2737 /* Catch any overuse of this function */ 2738 might_sleep_if(slow_timeout_ms); 2739 GEM_BUG_ON(fast_timeout_us > 20000); 2740 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms); 2741 2742 ret = -ETIMEDOUT; 2743 if (fast_timeout_us && fast_timeout_us <= 20000) 2744 ret = _wait_for_atomic(done, fast_timeout_us, 0); 2745 if (ret && slow_timeout_ms) 2746 ret = wait_for(done, slow_timeout_ms); 2747 2748 if (out_value) 2749 *out_value = reg_value; 2750 2751 return ret; 2752 #undef done 2753 } 2754 2755 /** 2756 * __intel_wait_for_register - wait until register matches expected state 2757 * @uncore: the struct intel_uncore 2758 * @reg: the register to read 2759 * @mask: mask to apply to register value 2760 * @value: expected value 2761 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 2762 * @slow_timeout_ms: slow timeout in millisecond 2763 * @out_value: optional placeholder to hold registry value 2764 * 2765 * This routine waits until the target register @reg contains the expected 2766 * @value after applying the @mask, i.e. it waits until :: 2767 * 2768 * (intel_uncore_read(uncore, reg) & mask) == value 2769 * 2770 * Otherwise, the wait will timeout after @timeout_ms milliseconds. 2771 * 2772 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT. 2773 */ 2774 int __intel_wait_for_register(struct intel_uncore *uncore, 2775 i915_reg_t reg, 2776 u32 mask, 2777 u32 value, 2778 unsigned int fast_timeout_us, 2779 unsigned int slow_timeout_ms, 2780 u32 *out_value) 2781 { 2782 unsigned fw = 2783 intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ); 2784 u32 reg_value; 2785 int ret; 2786 2787 might_sleep_if(slow_timeout_ms); 2788 2789 spin_lock_irq(&uncore->lock); 2790 intel_uncore_forcewake_get__locked(uncore, fw); 2791 2792 ret = __intel_wait_for_register_fw(uncore, 2793 reg, mask, value, 2794 fast_timeout_us, 0, ®_value); 2795 2796 intel_uncore_forcewake_put__locked(uncore, fw); 2797 spin_unlock_irq(&uncore->lock); 2798 2799 if (ret && slow_timeout_ms) 2800 ret = __wait_for(reg_value = intel_uncore_read_notrace(uncore, 2801 reg), 2802 (reg_value & mask) == value, 2803 slow_timeout_ms * 1000, 10, 1000); 2804 2805 /* just trace the final value */ 2806 trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); 2807 2808 if (out_value) 2809 *out_value = reg_value; 2810 2811 return ret; 2812 } 2813 2814 bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore) 2815 { 2816 bool ret; 2817 2818 if (!uncore->debug) 2819 return false; 2820 2821 spin_lock_irq(&uncore->debug->lock); 2822 ret = check_for_unclaimed_mmio(uncore); 2823 spin_unlock_irq(&uncore->debug->lock); 2824 2825 return ret; 2826 } 2827 2828 bool 2829 intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore) 2830 { 2831 bool ret = false; 2832 2833 if (drm_WARN_ON(&uncore->i915->drm, !uncore->debug)) 2834 return false; 2835 2836 spin_lock_irq(&uncore->debug->lock); 2837 2838 if (unlikely(uncore->debug->unclaimed_mmio_check <= 0)) 2839 goto out; 2840 2841 if (unlikely(check_for_unclaimed_mmio(uncore))) { 2842 if (!uncore->i915->params.mmio_debug) { 2843 drm_dbg(&uncore->i915->drm, 2844 "Unclaimed register detected, " 2845 "enabling oneshot unclaimed register reporting. " 2846 "Please use i915.mmio_debug=N for more information.\n"); 2847 uncore->i915->params.mmio_debug++; 2848 } 2849 uncore->debug->unclaimed_mmio_check--; 2850 ret = true; 2851 } 2852 2853 out: 2854 spin_unlock_irq(&uncore->debug->lock); 2855 2856 return ret; 2857 } 2858 2859 /** 2860 * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access 2861 * a register 2862 * @uncore: pointer to struct intel_uncore 2863 * @reg: register in question 2864 * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE 2865 * 2866 * Returns a set of forcewake domains required to be taken with for example 2867 * intel_uncore_forcewake_get for the specified register to be accessible in the 2868 * specified mode (read, write or read/write) with raw mmio accessors. 2869 * 2870 * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the 2871 * callers to do FIFO management on their own or risk losing writes. 2872 */ 2873 enum forcewake_domains 2874 intel_uncore_forcewake_for_reg(struct intel_uncore *uncore, 2875 i915_reg_t reg, unsigned int op) 2876 { 2877 enum forcewake_domains fw_domains = 0; 2878 2879 drm_WARN_ON(&uncore->i915->drm, !op); 2880 2881 if (!intel_uncore_has_forcewake(uncore)) 2882 return 0; 2883 2884 if (op & FW_REG_READ) 2885 fw_domains = uncore->funcs.read_fw_domains(uncore, reg); 2886 2887 if (op & FW_REG_WRITE) 2888 fw_domains |= uncore->funcs.write_fw_domains(uncore, reg); 2889 2890 drm_WARN_ON(&uncore->i915->drm, fw_domains & ~uncore->fw_domains); 2891 2892 return fw_domains; 2893 } 2894 2895 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2896 #include "selftests/mock_uncore.c" 2897 #include "selftests/intel_uncore.c" 2898 #endif 2899