1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 8 #include <drm/intel/i915_drm.h> 9 #include <drm/intel/display_parent_interface.h> 10 11 #include "display/intel_display_rps.h" 12 #include "display/vlv_clock.h" 13 14 #include "i915_drv.h" 15 #include "i915_freq.h" 16 #include "i915_irq.h" 17 #include "i915_reg.h" 18 #include "i915_wait_util.h" 19 #include "intel_breadcrumbs.h" 20 #include "intel_gt.h" 21 #include "intel_gt_clock_utils.h" 22 #include "intel_gt_irq.h" 23 #include "intel_gt_pm.h" 24 #include "intel_gt_pm_irq.h" 25 #include "intel_gt_print.h" 26 #include "intel_gt_regs.h" 27 #include "intel_mchbar_regs.h" 28 #include "intel_pcode.h" 29 #include "intel_rps.h" 30 #include "vlv_iosf_sb.h" 31 #include "../../../platform/x86/intel_ips.h" 32 33 #define BUSY_MAX_EI 20u /* ms */ 34 35 /* 36 * Lock protecting IPS related data structures 37 */ 38 static DEFINE_SPINLOCK(mchdev_lock); 39 40 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 41 { 42 return container_of(rps, struct intel_gt, rps); 43 } 44 45 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 46 { 47 return rps_to_gt(rps)->i915; 48 } 49 50 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 51 { 52 return rps_to_gt(rps)->uncore; 53 } 54 55 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 56 { 57 struct intel_gt *gt = rps_to_gt(rps); 58 59 return >_to_guc(gt)->slpc; 60 } 61 62 static bool rps_uses_slpc(struct intel_rps *rps) 63 { 64 struct intel_gt *gt = rps_to_gt(rps); 65 66 return intel_uc_uses_guc_slpc(>->uc); 67 } 68 69 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 70 { 71 return mask & ~rps->pm_intrmsk_mbz; 72 } 73 74 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 75 { 76 intel_uncore_write_fw(uncore, reg, val); 77 } 78 79 static void rps_timer(struct timer_list *t) 80 { 81 struct intel_rps *rps = timer_container_of(rps, t, timer); 82 struct intel_gt *gt = rps_to_gt(rps); 83 struct intel_engine_cs *engine; 84 ktime_t dt, last, timestamp; 85 enum intel_engine_id id; 86 s64 max_busy[3] = {}; 87 88 timestamp = 0; 89 for_each_engine(engine, gt, id) { 90 s64 busy; 91 int i; 92 93 dt = intel_engine_get_busy_time(engine, ×tamp); 94 last = engine->stats.rps; 95 engine->stats.rps = dt; 96 97 busy = ktime_to_ns(ktime_sub(dt, last)); 98 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 99 if (busy > max_busy[i]) 100 swap(busy, max_busy[i]); 101 } 102 } 103 last = rps->pm_timestamp; 104 rps->pm_timestamp = timestamp; 105 106 if (intel_rps_is_active(rps)) { 107 s64 busy; 108 int i; 109 110 dt = ktime_sub(timestamp, last); 111 112 /* 113 * Our goal is to evaluate each engine independently, so we run 114 * at the lowest clocks required to sustain the heaviest 115 * workload. However, a task may be split into sequential 116 * dependent operations across a set of engines, such that 117 * the independent contributions do not account for high load, 118 * but overall the task is GPU bound. For example, consider 119 * video decode on vcs followed by colour post-processing 120 * on vecs, followed by general post-processing on rcs. 121 * Since multi-engines being active does imply a single 122 * continuous workload across all engines, we hedge our 123 * bets by only contributing a factor of the distributed 124 * load into our busyness calculation. 125 */ 126 busy = max_busy[0]; 127 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 128 if (!max_busy[i]) 129 break; 130 131 busy += div_u64(max_busy[i], 1 << i); 132 } 133 GT_TRACE(gt, 134 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 135 busy, (int)div64_u64(100 * busy, dt), 136 max_busy[0], max_busy[1], max_busy[2], 137 rps->pm_interval); 138 139 if (100 * busy > rps->power.up_threshold * dt && 140 rps->cur_freq < rps->max_freq_softlimit) { 141 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 142 rps->pm_interval = 1; 143 queue_work(gt->i915->unordered_wq, &rps->work); 144 } else if (100 * busy < rps->power.down_threshold * dt && 145 rps->cur_freq > rps->min_freq_softlimit) { 146 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 147 rps->pm_interval = 1; 148 queue_work(gt->i915->unordered_wq, &rps->work); 149 } else { 150 rps->last_adj = 0; 151 } 152 153 mod_timer(&rps->timer, 154 jiffies + msecs_to_jiffies(rps->pm_interval)); 155 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 156 } 157 } 158 159 static void rps_start_timer(struct intel_rps *rps) 160 { 161 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 162 rps->pm_interval = 1; 163 mod_timer(&rps->timer, jiffies + 1); 164 } 165 166 static void rps_stop_timer(struct intel_rps *rps) 167 { 168 timer_delete_sync(&rps->timer); 169 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 170 cancel_work_sync(&rps->work); 171 } 172 173 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 174 { 175 u32 mask = 0; 176 177 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 178 if (val > rps->min_freq_softlimit) 179 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 180 GEN6_PM_RP_DOWN_THRESHOLD | 181 GEN6_PM_RP_DOWN_TIMEOUT); 182 183 if (val < rps->max_freq_softlimit) 184 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 185 186 mask &= rps->pm_events; 187 188 return rps_pm_sanitize_mask(rps, ~mask); 189 } 190 191 static void rps_reset_ei(struct intel_rps *rps) 192 { 193 memset(&rps->ei, 0, sizeof(rps->ei)); 194 } 195 196 static void rps_enable_interrupts(struct intel_rps *rps) 197 { 198 struct intel_gt *gt = rps_to_gt(rps); 199 200 GEM_BUG_ON(rps_uses_slpc(rps)); 201 202 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 203 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 204 205 rps_reset_ei(rps); 206 207 spin_lock_irq(gt->irq_lock); 208 gen6_gt_pm_enable_irq(gt, rps->pm_events); 209 spin_unlock_irq(gt->irq_lock); 210 211 intel_uncore_write(gt->uncore, 212 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 213 } 214 215 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 216 { 217 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 218 } 219 220 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 221 { 222 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 223 ; 224 } 225 226 static void rps_reset_interrupts(struct intel_rps *rps) 227 { 228 struct intel_gt *gt = rps_to_gt(rps); 229 230 spin_lock_irq(gt->irq_lock); 231 if (GRAPHICS_VER(gt->i915) >= 11) 232 gen11_rps_reset_interrupts(rps); 233 else 234 gen6_rps_reset_interrupts(rps); 235 236 rps->pm_iir = 0; 237 spin_unlock_irq(gt->irq_lock); 238 } 239 240 static void rps_disable_interrupts(struct intel_rps *rps) 241 { 242 struct intel_gt *gt = rps_to_gt(rps); 243 244 intel_uncore_write(gt->uncore, 245 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 246 247 spin_lock_irq(gt->irq_lock); 248 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 249 spin_unlock_irq(gt->irq_lock); 250 251 intel_synchronize_irq(gt->i915); 252 253 /* 254 * Now that we will not be generating any more work, flush any 255 * outstanding tasks. As we are called on the RPS idle path, 256 * we will reset the GPU to minimum frequencies, so the current 257 * state of the worker can be discarded. 258 */ 259 cancel_work_sync(&rps->work); 260 261 rps_reset_interrupts(rps); 262 GT_TRACE(gt, "interrupts:off\n"); 263 } 264 265 static const struct cparams { 266 u16 i; 267 u16 t; 268 u16 m; 269 u16 c; 270 } cparams[] = { 271 { 1, 1333, 301, 28664 }, 272 { 1, 1067, 294, 24460 }, 273 { 1, 800, 294, 25192 }, 274 { 0, 1333, 276, 27605 }, 275 { 0, 1067, 276, 27605 }, 276 { 0, 800, 231, 23784 }, 277 }; 278 279 static void gen5_rps_init(struct intel_rps *rps) 280 { 281 struct drm_i915_private *i915 = rps_to_i915(rps); 282 struct intel_uncore *uncore = rps_to_uncore(rps); 283 unsigned int fsb_freq, mem_freq; 284 u8 fmax, fmin, fstart; 285 u32 rgvmodectl; 286 int c_m, i; 287 288 fsb_freq = ilk_fsb_freq(i915); 289 mem_freq = ilk_mem_freq(i915); 290 291 if (fsb_freq <= 3200000) 292 c_m = 0; 293 else if (fsb_freq <= 4800000) 294 c_m = 1; 295 else 296 c_m = 2; 297 298 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 299 if (cparams[i].i == c_m && 300 cparams[i].t == DIV_ROUND_CLOSEST(mem_freq, 1000)) { 301 rps->ips.m = cparams[i].m; 302 rps->ips.c = cparams[i].c; 303 break; 304 } 305 } 306 307 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 308 309 /* Set up min, max, and cur for interrupt handling */ 310 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 311 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 312 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 313 MEMMODE_FSTART_SHIFT; 314 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 315 fmax, fmin, fstart); 316 317 rps->min_freq = fmax; 318 rps->efficient_freq = fstart; 319 rps->max_freq = fmin; 320 } 321 322 static unsigned long 323 __ips_chipset_val(struct intel_ips *ips) 324 { 325 struct intel_uncore *uncore = 326 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 327 unsigned long now = jiffies_to_msecs(jiffies), dt; 328 unsigned long result; 329 u64 total, delta; 330 331 lockdep_assert_held(&mchdev_lock); 332 333 /* 334 * Prevent division-by-zero if we are asking too fast. 335 * Also, we don't get interesting results if we are polling 336 * faster than once in 10ms, so just return the saved value 337 * in such cases. 338 */ 339 dt = now - ips->last_time1; 340 if (dt <= 10) 341 return ips->chipset_power; 342 343 /* FIXME: handle per-counter overflow */ 344 total = intel_uncore_read(uncore, DMIEC); 345 total += intel_uncore_read(uncore, DDREC); 346 total += intel_uncore_read(uncore, CSIEC); 347 348 delta = total - ips->last_count1; 349 350 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 351 352 ips->last_count1 = total; 353 ips->last_time1 = now; 354 355 ips->chipset_power = result; 356 357 return result; 358 } 359 360 static unsigned long ips_mch_val(struct intel_uncore *uncore) 361 { 362 unsigned int m, x, b; 363 u32 tsfs; 364 365 tsfs = intel_uncore_read(uncore, TSFS); 366 x = intel_uncore_read8(uncore, TR1); 367 368 b = tsfs & TSFS_INTR_MASK; 369 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 370 371 return m * x / 127 - b; 372 } 373 374 static int _pxvid_to_vd(u8 pxvid) 375 { 376 if (pxvid == 0) 377 return 0; 378 379 if (pxvid >= 8 && pxvid < 31) 380 pxvid = 31; 381 382 return (pxvid + 2) * 125; 383 } 384 385 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 386 { 387 const int vd = _pxvid_to_vd(pxvid); 388 389 if (INTEL_INFO(i915)->is_mobile) 390 return max(vd - 1125, 0); 391 392 return vd; 393 } 394 395 static void __gen5_ips_update(struct intel_ips *ips) 396 { 397 struct intel_uncore *uncore = 398 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 399 u64 now, delta, dt; 400 u32 count; 401 402 lockdep_assert_held(&mchdev_lock); 403 404 now = ktime_get_raw_ns(); 405 dt = now - ips->last_time2; 406 do_div(dt, NSEC_PER_MSEC); 407 408 /* Don't divide by 0 */ 409 if (dt <= 10) 410 return; 411 412 count = intel_uncore_read(uncore, GFXEC); 413 delta = count - ips->last_count2; 414 415 ips->last_count2 = count; 416 ips->last_time2 = now; 417 418 /* More magic constants... */ 419 ips->gfx_power = div_u64(delta * 1181, dt * 10); 420 } 421 422 static void gen5_rps_update(struct intel_rps *rps) 423 { 424 spin_lock_irq(&mchdev_lock); 425 __gen5_ips_update(&rps->ips); 426 spin_unlock_irq(&mchdev_lock); 427 } 428 429 static unsigned int gen5_invert_freq(struct intel_rps *rps, 430 unsigned int val) 431 { 432 /* Invert the frequency bin into an ips delay */ 433 val = rps->max_freq - val; 434 val = rps->min_freq + val; 435 436 return val; 437 } 438 439 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 440 { 441 struct intel_uncore *uncore = rps_to_uncore(rps); 442 u16 rgvswctl; 443 444 lockdep_assert_held(&mchdev_lock); 445 446 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 447 if (rgvswctl & MEMCTL_CMD_STS) { 448 drm_dbg(&rps_to_i915(rps)->drm, 449 "gpu busy, RCS change rejected\n"); 450 return -EBUSY; /* still busy with another command */ 451 } 452 453 /* Invert the frequency bin into an ips delay */ 454 val = gen5_invert_freq(rps, val); 455 456 rgvswctl = 457 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 458 (val << MEMCTL_FREQ_SHIFT) | 459 MEMCTL_SFCAVM; 460 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 461 intel_uncore_posting_read16(uncore, MEMSWCTL); 462 463 rgvswctl |= MEMCTL_CMD_STS; 464 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 465 466 return 0; 467 } 468 469 static int gen5_rps_set(struct intel_rps *rps, u8 val) 470 { 471 int err; 472 473 spin_lock_irq(&mchdev_lock); 474 err = __gen5_rps_set(rps, val); 475 spin_unlock_irq(&mchdev_lock); 476 477 return err; 478 } 479 480 static unsigned long intel_pxfreq(u32 vidfreq) 481 { 482 int div = (vidfreq & 0x3f0000) >> 16; 483 int post = (vidfreq & 0x3000) >> 12; 484 int pre = (vidfreq & 0x7); 485 486 if (!pre) 487 return 0; 488 489 return div * 133333 / (pre << post); 490 } 491 492 static unsigned int init_emon(struct intel_uncore *uncore) 493 { 494 u8 pxw[16]; 495 int i; 496 497 /* Disable to program */ 498 intel_uncore_write(uncore, ECR, 0); 499 intel_uncore_posting_read(uncore, ECR); 500 501 /* Program energy weights for various events */ 502 intel_uncore_write(uncore, SDEW, 0x15040d00); 503 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 504 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 505 intel_uncore_write(uncore, CSIEW2, 0x04000004); 506 507 for (i = 0; i < 5; i++) 508 intel_uncore_write(uncore, PEW(i), 0); 509 for (i = 0; i < 3; i++) 510 intel_uncore_write(uncore, DEW(i), 0); 511 512 /* Program P-state weights to account for frequency power adjustment */ 513 for (i = 0; i < 16; i++) { 514 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 515 unsigned int freq = intel_pxfreq(pxvidfreq); 516 unsigned int vid = 517 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 518 unsigned int val; 519 520 val = vid * vid * freq / 1000 * 255; 521 val /= 127 * 127 * 900; 522 523 pxw[i] = val; 524 } 525 /* Render standby states get 0 weight */ 526 pxw[14] = 0; 527 pxw[15] = 0; 528 529 for (i = 0; i < 4; i++) { 530 intel_uncore_write(uncore, PXW(i), 531 pxw[i * 4 + 0] << 24 | 532 pxw[i * 4 + 1] << 16 | 533 pxw[i * 4 + 2] << 8 | 534 pxw[i * 4 + 3] << 0); 535 } 536 537 /* Adjust magic regs to magic values (more experimental results) */ 538 intel_uncore_write(uncore, OGW0, 0); 539 intel_uncore_write(uncore, OGW1, 0); 540 intel_uncore_write(uncore, EG0, 0x00007f00); 541 intel_uncore_write(uncore, EG1, 0x0000000e); 542 intel_uncore_write(uncore, EG2, 0x000e0000); 543 intel_uncore_write(uncore, EG3, 0x68000300); 544 intel_uncore_write(uncore, EG4, 0x42000000); 545 intel_uncore_write(uncore, EG5, 0x00140031); 546 intel_uncore_write(uncore, EG6, 0); 547 intel_uncore_write(uncore, EG7, 0); 548 549 for (i = 0; i < 8; i++) 550 intel_uncore_write(uncore, PXWL(i), 0); 551 552 /* Enable PMON + select events */ 553 intel_uncore_write(uncore, ECR, 0x80000019); 554 555 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 556 } 557 558 static bool gen5_rps_enable(struct intel_rps *rps) 559 { 560 struct drm_i915_private *i915 = rps_to_i915(rps); 561 struct intel_display *display = i915->display; 562 struct intel_uncore *uncore = rps_to_uncore(rps); 563 u8 fstart, vstart; 564 u32 rgvmodectl; 565 566 spin_lock_irq(&mchdev_lock); 567 568 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 569 570 /* Enable temp reporting */ 571 intel_uncore_write16(uncore, PMMISC, 572 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 573 intel_uncore_write16(uncore, TSC1, 574 intel_uncore_read16(uncore, TSC1) | TSE); 575 576 /* 100ms RC evaluation intervals */ 577 intel_uncore_write(uncore, RCUPEI, 100000); 578 intel_uncore_write(uncore, RCDNEI, 100000); 579 580 /* Set max/min thresholds to 90ms and 80ms respectively */ 581 intel_uncore_write(uncore, RCBMAXAVG, 90000); 582 intel_uncore_write(uncore, RCBMINAVG, 80000); 583 584 intel_uncore_write(uncore, MEMIHYST, 1); 585 586 /* Set up min, max, and cur for interrupt handling */ 587 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 588 MEMMODE_FSTART_SHIFT; 589 590 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 591 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 592 593 intel_uncore_write(uncore, 594 MEMINTREN, 595 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 596 597 intel_uncore_write(uncore, VIDSTART, vstart); 598 intel_uncore_posting_read(uncore, VIDSTART); 599 600 rgvmodectl |= MEMMODE_SWMODE_EN; 601 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 602 603 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 604 MEMCTL_CMD_STS) == 0, 10)) 605 drm_err(&uncore->i915->drm, 606 "stuck trying to change perf mode\n"); 607 mdelay(1); 608 609 __gen5_rps_set(rps, rps->cur_freq); 610 611 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 612 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 613 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 614 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 615 616 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 617 rps->ips.last_time2 = ktime_get_raw_ns(); 618 619 ilk_display_rps_enable(display); 620 621 spin_unlock_irq(&mchdev_lock); 622 623 rps->ips.corr = init_emon(uncore); 624 625 return true; 626 } 627 628 static void gen5_rps_disable(struct intel_rps *rps) 629 { 630 struct drm_i915_private *i915 = rps_to_i915(rps); 631 struct intel_display *display = i915->display; 632 struct intel_uncore *uncore = rps_to_uncore(rps); 633 u16 rgvswctl; 634 635 spin_lock_irq(&mchdev_lock); 636 637 ilk_display_rps_disable(display); 638 639 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 640 641 /* Ack interrupts, disable EFC interrupt */ 642 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0); 643 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 644 645 /* Go back to the starting frequency */ 646 __gen5_rps_set(rps, rps->idle_freq); 647 mdelay(1); 648 rgvswctl |= MEMCTL_CMD_STS; 649 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 650 mdelay(1); 651 652 spin_unlock_irq(&mchdev_lock); 653 } 654 655 static u32 rps_limits(struct intel_rps *rps, u8 val) 656 { 657 u32 limits; 658 659 /* 660 * Only set the down limit when we've reached the lowest level to avoid 661 * getting more interrupts, otherwise leave this clear. This prevents a 662 * race in the hw when coming out of rc6: There's a tiny window where 663 * the hw runs at the minimal clock before selecting the desired 664 * frequency, if the down threshold expires in that window we will not 665 * receive a down interrupt. 666 */ 667 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 668 limits = rps->max_freq_softlimit << 23; 669 if (val <= rps->min_freq_softlimit) 670 limits |= rps->min_freq_softlimit << 14; 671 } else { 672 limits = rps->max_freq_softlimit << 24; 673 if (val <= rps->min_freq_softlimit) 674 limits |= rps->min_freq_softlimit << 16; 675 } 676 677 return limits; 678 } 679 680 static void rps_set_power(struct intel_rps *rps, int new_power) 681 { 682 struct intel_gt *gt = rps_to_gt(rps); 683 struct intel_uncore *uncore = gt->uncore; 684 u32 ei_up = 0, ei_down = 0; 685 686 lockdep_assert_held(&rps->power.mutex); 687 688 if (new_power == rps->power.mode) 689 return; 690 691 /* Note the units here are not exactly 1us, but 1280ns. */ 692 switch (new_power) { 693 case LOW_POWER: 694 ei_up = 16000; 695 ei_down = 32000; 696 break; 697 698 case BETWEEN: 699 ei_up = 13000; 700 ei_down = 32000; 701 break; 702 703 case HIGH_POWER: 704 ei_up = 10000; 705 ei_down = 32000; 706 break; 707 } 708 709 /* When byt can survive without system hang with dynamic 710 * sw freq adjustments, this restriction can be lifted. 711 */ 712 if (IS_VALLEYVIEW(gt->i915)) 713 goto skip_hw_write; 714 715 GT_TRACE(gt, 716 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 717 new_power, 718 rps->power.up_threshold, ei_up, 719 rps->power.down_threshold, ei_down); 720 721 set(uncore, GEN6_RP_UP_EI, 722 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 723 set(uncore, GEN6_RP_UP_THRESHOLD, 724 intel_gt_ns_to_pm_interval(gt, 725 ei_up * rps->power.up_threshold * 10)); 726 727 set(uncore, GEN6_RP_DOWN_EI, 728 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 729 set(uncore, GEN6_RP_DOWN_THRESHOLD, 730 intel_gt_ns_to_pm_interval(gt, 731 ei_down * 732 rps->power.down_threshold * 10)); 733 734 set(uncore, GEN6_RP_CONTROL, 735 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 736 GEN6_RP_MEDIA_HW_NORMAL_MODE | 737 GEN6_RP_MEDIA_IS_GFX | 738 GEN6_RP_ENABLE | 739 GEN6_RP_UP_BUSY_AVG | 740 GEN6_RP_DOWN_IDLE_AVG); 741 742 skip_hw_write: 743 rps->power.mode = new_power; 744 } 745 746 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 747 { 748 int new_power; 749 750 new_power = rps->power.mode; 751 switch (rps->power.mode) { 752 case LOW_POWER: 753 if (val > rps->efficient_freq + 1 && 754 val > rps->cur_freq) 755 new_power = BETWEEN; 756 break; 757 758 case BETWEEN: 759 if (val <= rps->efficient_freq && 760 val < rps->cur_freq) 761 new_power = LOW_POWER; 762 else if (val >= rps->rp0_freq && 763 val > rps->cur_freq) 764 new_power = HIGH_POWER; 765 break; 766 767 case HIGH_POWER: 768 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 769 val < rps->cur_freq) 770 new_power = BETWEEN; 771 break; 772 } 773 /* Max/min bins are special */ 774 if (val <= rps->min_freq_softlimit) 775 new_power = LOW_POWER; 776 if (val >= rps->max_freq_softlimit) 777 new_power = HIGH_POWER; 778 779 mutex_lock(&rps->power.mutex); 780 if (rps->power.interactive) 781 new_power = HIGH_POWER; 782 rps_set_power(rps, new_power); 783 mutex_unlock(&rps->power.mutex); 784 } 785 786 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 787 { 788 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", 789 str_yes_no(interactive)); 790 791 mutex_lock(&rps->power.mutex); 792 if (interactive) { 793 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 794 rps_set_power(rps, HIGH_POWER); 795 } else { 796 GEM_BUG_ON(!rps->power.interactive); 797 rps->power.interactive--; 798 } 799 mutex_unlock(&rps->power.mutex); 800 } 801 802 static int gen6_rps_set(struct intel_rps *rps, u8 val) 803 { 804 struct intel_uncore *uncore = rps_to_uncore(rps); 805 struct drm_i915_private *i915 = rps_to_i915(rps); 806 u32 swreq; 807 808 GEM_BUG_ON(rps_uses_slpc(rps)); 809 810 if (GRAPHICS_VER(i915) >= 9) 811 swreq = GEN9_FREQUENCY(val); 812 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 813 swreq = HSW_FREQUENCY(val); 814 else 815 swreq = (GEN6_FREQUENCY(val) | 816 GEN6_OFFSET(0) | 817 GEN6_AGGRESSIVE_TURBO); 818 set(uncore, GEN6_RPNSWREQ, swreq); 819 820 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 821 val, intel_gpu_freq(rps, val), swreq); 822 823 return 0; 824 } 825 826 static int vlv_rps_set(struct intel_rps *rps, u8 val) 827 { 828 struct drm_i915_private *i915 = rps_to_i915(rps); 829 int err; 830 831 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 832 err = vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_REQ, val); 833 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 834 835 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 836 val, intel_gpu_freq(rps, val)); 837 838 return err; 839 } 840 841 static int rps_set(struct intel_rps *rps, u8 val, bool update) 842 { 843 struct drm_i915_private *i915 = rps_to_i915(rps); 844 int err; 845 846 if (val == rps->last_freq) 847 return 0; 848 849 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 850 err = vlv_rps_set(rps, val); 851 else if (GRAPHICS_VER(i915) >= 6) 852 err = gen6_rps_set(rps, val); 853 else 854 err = gen5_rps_set(rps, val); 855 if (err) 856 return err; 857 858 if (update && GRAPHICS_VER(i915) >= 6) 859 gen6_rps_set_thresholds(rps, val); 860 rps->last_freq = val; 861 862 return 0; 863 } 864 865 void intel_rps_unpark(struct intel_rps *rps) 866 { 867 if (!intel_rps_is_enabled(rps)) 868 return; 869 870 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 871 872 /* 873 * Use the user's desired frequency as a guide, but for better 874 * performance, jump directly to RPe as our starting frequency. 875 */ 876 mutex_lock(&rps->lock); 877 878 intel_rps_set_active(rps); 879 intel_rps_set(rps, 880 clamp(rps->cur_freq, 881 rps->min_freq_softlimit, 882 rps->max_freq_softlimit)); 883 884 mutex_unlock(&rps->lock); 885 886 rps->pm_iir = 0; 887 if (intel_rps_has_interrupts(rps)) 888 rps_enable_interrupts(rps); 889 if (intel_rps_uses_timer(rps)) 890 rps_start_timer(rps); 891 892 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 893 gen5_rps_update(rps); 894 } 895 896 void intel_rps_park(struct intel_rps *rps) 897 { 898 int adj; 899 900 if (!intel_rps_is_enabled(rps)) 901 return; 902 903 if (!intel_rps_clear_active(rps)) 904 return; 905 906 if (intel_rps_uses_timer(rps)) 907 rps_stop_timer(rps); 908 if (intel_rps_has_interrupts(rps)) 909 rps_disable_interrupts(rps); 910 911 if (rps->last_freq <= rps->idle_freq) 912 return; 913 914 /* 915 * The punit delays the write of the frequency and voltage until it 916 * determines the GPU is awake. During normal usage we don't want to 917 * waste power changing the frequency if the GPU is sleeping (rc6). 918 * However, the GPU and driver is now idle and we do not want to delay 919 * switching to minimum voltage (reducing power whilst idle) as we do 920 * not expect to be woken in the near future and so must flush the 921 * change by waking the device. 922 * 923 * We choose to take the media powerwell (either would do to trick the 924 * punit into committing the voltage change) as that takes a lot less 925 * power than the render powerwell. 926 */ 927 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 928 rps_set(rps, rps->idle_freq, false); 929 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 930 931 /* 932 * Since we will try and restart from the previously requested 933 * frequency on unparking, treat this idle point as a downclock 934 * interrupt and reduce the frequency for resume. If we park/unpark 935 * more frequently than the rps worker can run, we will not respond 936 * to any EI and never see a change in frequency. 937 * 938 * (Note we accommodate Cherryview's limitation of only using an 939 * even bin by applying it to all.) 940 */ 941 adj = rps->last_adj; 942 if (adj < 0) 943 adj *= 2; 944 else /* CHV needs even encode values */ 945 adj = -2; 946 rps->last_adj = adj; 947 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 948 if (rps->cur_freq < rps->efficient_freq) { 949 rps->cur_freq = rps->efficient_freq; 950 rps->last_adj = 0; 951 } 952 953 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 954 } 955 956 u32 intel_rps_get_boost_frequency(struct intel_rps *rps) 957 { 958 struct intel_guc_slpc *slpc; 959 960 if (rps_uses_slpc(rps)) { 961 slpc = rps_to_slpc(rps); 962 963 return slpc->boost_freq; 964 } else { 965 return intel_gpu_freq(rps, rps->boost_freq); 966 } 967 } 968 969 static int rps_set_boost_freq(struct intel_rps *rps, u32 val) 970 { 971 bool boost = false; 972 973 /* Validate against (static) hardware limits */ 974 val = intel_freq_opcode(rps, val); 975 if (val < rps->min_freq || val > rps->max_freq) 976 return -EINVAL; 977 978 mutex_lock(&rps->lock); 979 if (val != rps->boost_freq) { 980 rps->boost_freq = val; 981 boost = atomic_read(&rps->num_waiters); 982 } 983 mutex_unlock(&rps->lock); 984 if (boost) 985 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 986 987 return 0; 988 } 989 990 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq) 991 { 992 struct intel_guc_slpc *slpc; 993 994 if (rps_uses_slpc(rps)) { 995 slpc = rps_to_slpc(rps); 996 997 return intel_guc_slpc_set_boost_freq(slpc, freq); 998 } else { 999 return rps_set_boost_freq(rps, freq); 1000 } 1001 } 1002 1003 void intel_rps_dec_waiters(struct intel_rps *rps) 1004 { 1005 struct intel_guc_slpc *slpc; 1006 1007 if (rps_uses_slpc(rps)) { 1008 slpc = rps_to_slpc(rps); 1009 1010 /* Don't decrement num_waiters for req where increment was skipped */ 1011 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING) 1012 return; 1013 1014 intel_guc_slpc_dec_waiters(slpc); 1015 } else { 1016 atomic_dec(&rps->num_waiters); 1017 } 1018 } 1019 1020 void intel_rps_boost(struct i915_request *rq) 1021 { 1022 struct intel_guc_slpc *slpc; 1023 1024 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 1025 return; 1026 1027 /* Waitboost is not needed for contexts marked with a Freq hint */ 1028 if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags)) 1029 return; 1030 1031 /* Serializes with i915_request_retire() */ 1032 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 1033 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 1034 1035 if (rps_uses_slpc(rps)) { 1036 slpc = rps_to_slpc(rps); 1037 1038 /* Waitboost should not be done with power saving profile */ 1039 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING) 1040 return; 1041 1042 /* Return if old value is non zero */ 1043 if (!atomic_fetch_inc(&slpc->num_waiters)) { 1044 /* 1045 * Skip queuing boost work if frequency is already boosted, 1046 * but still increment num_waiters. 1047 */ 1048 if (slpc->min_freq_softlimit >= slpc->boost_freq) 1049 return; 1050 1051 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1052 rq->fence.context, rq->fence.seqno); 1053 queue_work(rps_to_gt(rps)->i915->unordered_wq, 1054 &slpc->boost_work); 1055 } 1056 1057 return; 1058 } 1059 1060 if (atomic_fetch_inc(&rps->num_waiters)) 1061 return; 1062 1063 if (!intel_rps_is_active(rps)) 1064 return; 1065 1066 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1067 rq->fence.context, rq->fence.seqno); 1068 1069 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 1070 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 1071 1072 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 1073 } 1074 } 1075 1076 int intel_rps_set(struct intel_rps *rps, u8 val) 1077 { 1078 int err; 1079 1080 lockdep_assert_held(&rps->lock); 1081 GEM_BUG_ON(val > rps->max_freq); 1082 GEM_BUG_ON(val < rps->min_freq); 1083 1084 if (intel_rps_is_active(rps)) { 1085 err = rps_set(rps, val, true); 1086 if (err) 1087 return err; 1088 1089 /* 1090 * Make sure we continue to get interrupts 1091 * until we hit the minimum or maximum frequencies. 1092 */ 1093 if (intel_rps_has_interrupts(rps)) { 1094 struct intel_uncore *uncore = rps_to_uncore(rps); 1095 1096 set(uncore, 1097 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 1098 1099 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 1100 } 1101 } 1102 1103 rps->cur_freq = val; 1104 return 0; 1105 } 1106 1107 static u32 intel_rps_read_state_cap(struct intel_rps *rps) 1108 { 1109 struct drm_i915_private *i915 = rps_to_i915(rps); 1110 struct intel_uncore *uncore = rps_to_uncore(rps); 1111 1112 if (IS_GEN9_LP(i915)) 1113 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 1114 else 1115 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 1116 } 1117 1118 static void 1119 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1120 { 1121 struct intel_uncore *uncore = rps_to_uncore(rps); 1122 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ? 1123 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) : 1124 intel_uncore_read(uncore, MTL_RP_STATE_CAP); 1125 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ? 1126 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) : 1127 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY); 1128 1129 /* MTL values are in units of 16.67 MHz */ 1130 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap); 1131 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap); 1132 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe); 1133 } 1134 1135 static void 1136 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1137 { 1138 struct drm_i915_private *i915 = rps_to_i915(rps); 1139 u32 rp_state_cap; 1140 1141 rp_state_cap = intel_rps_read_state_cap(rps); 1142 1143 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1144 if (IS_GEN9_LP(i915)) { 1145 caps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1146 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1147 caps->min_freq = (rp_state_cap >> 0) & 0xff; 1148 } else { 1149 caps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1150 if (GRAPHICS_VER(i915) >= 10) 1151 caps->rp1_freq = REG_FIELD_GET(RPE_MASK, 1152 intel_uncore_read(to_gt(i915)->uncore, 1153 GEN10_FREQ_INFO_REC)); 1154 else 1155 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1156 caps->min_freq = (rp_state_cap >> 16) & 0xff; 1157 } 1158 1159 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1160 /* 1161 * In this case rp_state_cap register reports frequencies in 1162 * units of 50 MHz. Convert these to the actual "hw unit", i.e. 1163 * units of 16.67 MHz 1164 */ 1165 caps->rp0_freq *= GEN9_FREQ_SCALER; 1166 caps->rp1_freq *= GEN9_FREQ_SCALER; 1167 caps->min_freq *= GEN9_FREQ_SCALER; 1168 } 1169 } 1170 1171 /** 1172 * gen6_rps_get_freq_caps - Get freq caps exposed by HW 1173 * @rps: the intel_rps structure 1174 * @caps: returned freq caps 1175 * 1176 * Returned "caps" frequencies should be converted to MHz using 1177 * intel_gpu_freq() 1178 */ 1179 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1180 { 1181 struct drm_i915_private *i915 = rps_to_i915(rps); 1182 1183 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 1184 return mtl_get_freq_caps(rps, caps); 1185 else 1186 return __gen6_rps_get_freq_caps(rps, caps); 1187 } 1188 1189 static void gen6_rps_init(struct intel_rps *rps) 1190 { 1191 struct drm_i915_private *i915 = rps_to_i915(rps); 1192 struct intel_rps_freq_caps caps; 1193 1194 gen6_rps_get_freq_caps(rps, &caps); 1195 rps->rp0_freq = caps.rp0_freq; 1196 rps->rp1_freq = caps.rp1_freq; 1197 rps->min_freq = caps.min_freq; 1198 1199 /* hw_max = RP0 until we check for overclocking */ 1200 rps->max_freq = rps->rp0_freq; 1201 1202 rps->efficient_freq = rps->rp1_freq; 1203 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1204 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1205 u32 ddcc_status = 0; 1206 u32 mult = 1; 1207 1208 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) 1209 mult = GEN9_FREQ_SCALER; 1210 if (snb_pcode_read(rps_to_gt(rps)->uncore, 1211 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1212 &ddcc_status, NULL) == 0) 1213 rps->efficient_freq = 1214 clamp_t(u32, 1215 ((ddcc_status >> 8) & 0xff) * mult, 1216 rps->min_freq, 1217 rps->max_freq); 1218 } 1219 } 1220 1221 static bool rps_reset(struct intel_rps *rps) 1222 { 1223 struct drm_i915_private *i915 = rps_to_i915(rps); 1224 1225 /* force a reset */ 1226 rps->power.mode = -1; 1227 rps->last_freq = -1; 1228 1229 if (rps_set(rps, rps->min_freq, true)) { 1230 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1231 return false; 1232 } 1233 1234 rps->cur_freq = rps->min_freq; 1235 return true; 1236 } 1237 1238 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1239 static bool gen9_rps_enable(struct intel_rps *rps) 1240 { 1241 struct intel_gt *gt = rps_to_gt(rps); 1242 struct intel_uncore *uncore = gt->uncore; 1243 1244 /* Program defaults and thresholds for RPS */ 1245 if (GRAPHICS_VER(gt->i915) == 9) 1246 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1247 GEN9_FREQUENCY(rps->rp1_freq)); 1248 1249 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1250 1251 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1252 1253 return rps_reset(rps); 1254 } 1255 1256 static bool gen8_rps_enable(struct intel_rps *rps) 1257 { 1258 struct intel_uncore *uncore = rps_to_uncore(rps); 1259 1260 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1261 HSW_FREQUENCY(rps->rp1_freq)); 1262 1263 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1264 1265 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1266 1267 return rps_reset(rps); 1268 } 1269 1270 static bool gen6_rps_enable(struct intel_rps *rps) 1271 { 1272 struct intel_uncore *uncore = rps_to_uncore(rps); 1273 1274 /* Power down if completely idle for over 50ms */ 1275 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1276 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1277 1278 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1279 GEN6_PM_RP_DOWN_THRESHOLD | 1280 GEN6_PM_RP_DOWN_TIMEOUT); 1281 1282 return rps_reset(rps); 1283 } 1284 1285 static int chv_rps_max_freq(struct intel_rps *rps) 1286 { 1287 struct drm_i915_private *i915 = rps_to_i915(rps); 1288 struct intel_gt *gt = rps_to_gt(rps); 1289 u32 val; 1290 1291 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE); 1292 1293 switch (gt->info.sseu.eu_total) { 1294 case 8: 1295 /* (2 * 4) config */ 1296 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1297 break; 1298 case 12: 1299 /* (2 * 6) config */ 1300 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1301 break; 1302 case 16: 1303 /* (2 * 8) config */ 1304 default: 1305 /* Setting (2 * 8) Min RP0 for any other combination */ 1306 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1307 break; 1308 } 1309 1310 return val & FB_GFX_FREQ_FUSE_MASK; 1311 } 1312 1313 static int chv_rps_rpe_freq(struct intel_rps *rps) 1314 { 1315 struct drm_i915_private *i915 = rps_to_i915(rps); 1316 u32 val; 1317 1318 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_GPU_DUTYCYCLE_REG); 1319 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1320 1321 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1322 } 1323 1324 static int chv_rps_guar_freq(struct intel_rps *rps) 1325 { 1326 struct drm_i915_private *i915 = rps_to_i915(rps); 1327 u32 val; 1328 1329 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE); 1330 1331 return val & FB_GFX_FREQ_FUSE_MASK; 1332 } 1333 1334 static u32 chv_rps_min_freq(struct intel_rps *rps) 1335 { 1336 struct drm_i915_private *i915 = rps_to_i915(rps); 1337 u32 val; 1338 1339 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMIN_AT_VMIN_FUSE); 1340 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1341 1342 return val & FB_GFX_FREQ_FUSE_MASK; 1343 } 1344 1345 static bool chv_rps_enable(struct intel_rps *rps) 1346 { 1347 struct intel_uncore *uncore = rps_to_uncore(rps); 1348 struct drm_i915_private *i915 = rps_to_i915(rps); 1349 u32 val; 1350 1351 /* 1: Program defaults and thresholds for RPS*/ 1352 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1353 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1354 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1355 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1356 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1357 1358 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1359 1360 /* 2: Enable RPS */ 1361 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1362 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1363 GEN6_RP_MEDIA_IS_GFX | 1364 GEN6_RP_ENABLE | 1365 GEN6_RP_UP_BUSY_AVG | 1366 GEN6_RP_DOWN_IDLE_AVG); 1367 1368 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1369 GEN6_PM_RP_DOWN_THRESHOLD | 1370 GEN6_PM_RP_DOWN_TIMEOUT); 1371 1372 /* Setting Fixed Bias */ 1373 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1374 1375 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1376 vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val); 1377 1378 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 1379 1380 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1381 1382 /* RPS code assumes GPLL is used */ 1383 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1384 "GPLL not enabled\n"); 1385 1386 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1387 str_yes_no(val & GPLLENABLE)); 1388 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1389 1390 return rps_reset(rps); 1391 } 1392 1393 static int vlv_rps_guar_freq(struct intel_rps *rps) 1394 { 1395 struct drm_i915_private *i915 = rps_to_i915(rps); 1396 u32 val, rp1; 1397 1398 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE); 1399 1400 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1401 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1402 1403 return rp1; 1404 } 1405 1406 static int vlv_rps_max_freq(struct intel_rps *rps) 1407 { 1408 struct drm_i915_private *i915 = rps_to_i915(rps); 1409 u32 val, rp0; 1410 1411 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE); 1412 1413 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1414 /* Clamp to max */ 1415 rp0 = min_t(u32, rp0, 0xea); 1416 1417 return rp0; 1418 } 1419 1420 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1421 { 1422 struct drm_i915_private *i915 = rps_to_i915(rps); 1423 u32 val, rpe; 1424 1425 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1426 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1427 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1428 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1429 1430 return rpe; 1431 } 1432 1433 static int vlv_rps_min_freq(struct intel_rps *rps) 1434 { 1435 struct drm_i915_private *i915 = rps_to_i915(rps); 1436 u32 val; 1437 1438 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_LFM) & 0xff; 1439 /* 1440 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1441 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1442 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1443 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1444 * to make sure it matches what Punit accepts. 1445 */ 1446 return max_t(u32, val, 0xc0); 1447 } 1448 1449 static bool vlv_rps_enable(struct intel_rps *rps) 1450 { 1451 struct intel_uncore *uncore = rps_to_uncore(rps); 1452 struct drm_i915_private *i915 = rps_to_i915(rps); 1453 u32 val; 1454 1455 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1456 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1457 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1458 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1459 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1460 1461 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1462 1463 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1464 GEN6_RP_MEDIA_TURBO | 1465 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1466 GEN6_RP_MEDIA_IS_GFX | 1467 GEN6_RP_ENABLE | 1468 GEN6_RP_UP_BUSY_AVG | 1469 GEN6_RP_DOWN_IDLE_CONT); 1470 1471 /* WaGsvRC0ResidencyMethod:vlv */ 1472 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1473 1474 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1475 1476 /* Setting Fixed Bias */ 1477 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1478 vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val); 1479 1480 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 1481 1482 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1483 1484 /* RPS code assumes GPLL is used */ 1485 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1486 "GPLL not enabled\n"); 1487 1488 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1489 str_yes_no(val & GPLLENABLE)); 1490 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1491 1492 return rps_reset(rps); 1493 } 1494 1495 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1496 { 1497 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1498 struct intel_uncore *uncore = rps_to_uncore(rps); 1499 unsigned int t, state1, state2; 1500 u32 pxvid, ext_v; 1501 u64 corr, corr2; 1502 1503 lockdep_assert_held(&mchdev_lock); 1504 1505 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1506 pxvid = (pxvid >> 24) & 0x7f; 1507 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1508 1509 state1 = ext_v; 1510 1511 /* Revel in the empirically derived constants */ 1512 1513 /* Correction factor in 1/100000 units */ 1514 t = ips_mch_val(uncore); 1515 if (t > 80) 1516 corr = t * 2349 + 135940; 1517 else if (t >= 50) 1518 corr = t * 964 + 29317; 1519 else /* < 50 */ 1520 corr = t * 301 + 1004; 1521 1522 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1523 corr2 = div_u64(corr, 100000) * ips->corr; 1524 1525 state2 = div_u64(corr2 * state1, 10000); 1526 state2 /= 100; /* convert to mW */ 1527 1528 __gen5_ips_update(ips); 1529 1530 return ips->gfx_power + state2; 1531 } 1532 1533 static bool has_busy_stats(struct intel_rps *rps) 1534 { 1535 struct intel_engine_cs *engine; 1536 enum intel_engine_id id; 1537 1538 for_each_engine(engine, rps_to_gt(rps), id) { 1539 if (!intel_engine_supports_stats(engine)) 1540 return false; 1541 } 1542 1543 return true; 1544 } 1545 1546 void intel_rps_enable(struct intel_rps *rps) 1547 { 1548 struct drm_i915_private *i915 = rps_to_i915(rps); 1549 struct intel_uncore *uncore = rps_to_uncore(rps); 1550 bool enabled = false; 1551 1552 if (!HAS_RPS(i915)) 1553 return; 1554 1555 if (rps_uses_slpc(rps)) 1556 return; 1557 1558 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1559 1560 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1561 if (rps->max_freq <= rps->min_freq) 1562 /* leave disabled, no room for dynamic reclocking */; 1563 else if (IS_CHERRYVIEW(i915)) 1564 enabled = chv_rps_enable(rps); 1565 else if (IS_VALLEYVIEW(i915)) 1566 enabled = vlv_rps_enable(rps); 1567 else if (GRAPHICS_VER(i915) >= 9) 1568 enabled = gen9_rps_enable(rps); 1569 else if (GRAPHICS_VER(i915) >= 8) 1570 enabled = gen8_rps_enable(rps); 1571 else if (GRAPHICS_VER(i915) >= 6) 1572 enabled = gen6_rps_enable(rps); 1573 else if (IS_IRONLAKE_M(i915)) 1574 enabled = gen5_rps_enable(rps); 1575 else 1576 MISSING_CASE(GRAPHICS_VER(i915)); 1577 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1578 if (!enabled) 1579 return; 1580 1581 GT_TRACE(rps_to_gt(rps), 1582 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n", 1583 rps->min_freq, rps->max_freq, 1584 intel_gpu_freq(rps, rps->min_freq), 1585 intel_gpu_freq(rps, rps->max_freq), 1586 rps->power.up_threshold, 1587 rps->power.down_threshold); 1588 1589 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1590 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1591 1592 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1593 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1594 1595 if (has_busy_stats(rps)) 1596 intel_rps_set_timer(rps); 1597 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) 1598 intel_rps_set_interrupts(rps); 1599 else 1600 /* Ironlake currently uses intel_ips.ko */ {} 1601 1602 intel_rps_set_enabled(rps); 1603 } 1604 1605 static void gen6_rps_disable(struct intel_rps *rps) 1606 { 1607 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1608 } 1609 1610 void intel_rps_disable(struct intel_rps *rps) 1611 { 1612 struct drm_i915_private *i915 = rps_to_i915(rps); 1613 1614 if (!intel_rps_is_enabled(rps)) 1615 return; 1616 1617 intel_rps_clear_enabled(rps); 1618 intel_rps_clear_interrupts(rps); 1619 intel_rps_clear_timer(rps); 1620 1621 if (GRAPHICS_VER(i915) >= 6) 1622 gen6_rps_disable(rps); 1623 else if (IS_IRONLAKE_M(i915)) 1624 gen5_rps_disable(rps); 1625 } 1626 1627 static int byt_gpu_freq(struct intel_rps *rps, int val) 1628 { 1629 /* 1630 * N = val - 0xb7 1631 * Slow = Fast = GPLL ref * N 1632 */ 1633 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1634 } 1635 1636 static int byt_freq_opcode(struct intel_rps *rps, int val) 1637 { 1638 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1639 } 1640 1641 static int chv_gpu_freq(struct intel_rps *rps, int val) 1642 { 1643 /* 1644 * N = val / 2 1645 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1646 */ 1647 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1648 } 1649 1650 static int chv_freq_opcode(struct intel_rps *rps, int val) 1651 { 1652 /* CHV needs even values */ 1653 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1654 } 1655 1656 int intel_gpu_freq(struct intel_rps *rps, int val) 1657 { 1658 struct drm_i915_private *i915 = rps_to_i915(rps); 1659 1660 if (GRAPHICS_VER(i915) >= 9) 1661 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1662 GEN9_FREQ_SCALER); 1663 else if (IS_CHERRYVIEW(i915)) 1664 return chv_gpu_freq(rps, val); 1665 else if (IS_VALLEYVIEW(i915)) 1666 return byt_gpu_freq(rps, val); 1667 else if (GRAPHICS_VER(i915) >= 6) 1668 return val * GT_FREQUENCY_MULTIPLIER; 1669 else 1670 return val; 1671 } 1672 1673 int intel_freq_opcode(struct intel_rps *rps, int val) 1674 { 1675 struct drm_i915_private *i915 = rps_to_i915(rps); 1676 1677 if (GRAPHICS_VER(i915) >= 9) 1678 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1679 GT_FREQUENCY_MULTIPLIER); 1680 else if (IS_CHERRYVIEW(i915)) 1681 return chv_freq_opcode(rps, val); 1682 else if (IS_VALLEYVIEW(i915)) 1683 return byt_freq_opcode(rps, val); 1684 else if (GRAPHICS_VER(i915) >= 6) 1685 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1686 else 1687 return val; 1688 } 1689 1690 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1691 { 1692 struct drm_i915_private *i915 = rps_to_i915(rps); 1693 1694 rps->gpll_ref_freq = vlv_clock_get_gpll(&i915->drm); 1695 1696 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1697 rps->gpll_ref_freq); 1698 } 1699 1700 static void vlv_rps_init(struct intel_rps *rps) 1701 { 1702 struct drm_i915_private *i915 = rps_to_i915(rps); 1703 1704 vlv_init_gpll_ref_freq(rps); 1705 1706 vlv_iosf_sb_get(&i915->drm, 1707 BIT(VLV_IOSF_SB_PUNIT) | 1708 BIT(VLV_IOSF_SB_NC) | 1709 BIT(VLV_IOSF_SB_CCK)); 1710 1711 rps->max_freq = vlv_rps_max_freq(rps); 1712 rps->rp0_freq = rps->max_freq; 1713 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1714 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1715 1716 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1717 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1718 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1719 1720 rps->rp1_freq = vlv_rps_guar_freq(rps); 1721 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1722 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1723 1724 rps->min_freq = vlv_rps_min_freq(rps); 1725 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1726 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1727 1728 vlv_iosf_sb_put(&i915->drm, 1729 BIT(VLV_IOSF_SB_PUNIT) | 1730 BIT(VLV_IOSF_SB_NC) | 1731 BIT(VLV_IOSF_SB_CCK)); 1732 } 1733 1734 static void chv_rps_init(struct intel_rps *rps) 1735 { 1736 struct drm_i915_private *i915 = rps_to_i915(rps); 1737 1738 vlv_init_gpll_ref_freq(rps); 1739 1740 vlv_iosf_sb_get(&i915->drm, 1741 BIT(VLV_IOSF_SB_PUNIT) | 1742 BIT(VLV_IOSF_SB_NC) | 1743 BIT(VLV_IOSF_SB_CCK)); 1744 1745 rps->max_freq = chv_rps_max_freq(rps); 1746 rps->rp0_freq = rps->max_freq; 1747 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1748 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1749 1750 rps->efficient_freq = chv_rps_rpe_freq(rps); 1751 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1752 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1753 1754 rps->rp1_freq = chv_rps_guar_freq(rps); 1755 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1756 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1757 1758 rps->min_freq = chv_rps_min_freq(rps); 1759 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1760 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1761 1762 vlv_iosf_sb_put(&i915->drm, 1763 BIT(VLV_IOSF_SB_PUNIT) | 1764 BIT(VLV_IOSF_SB_NC) | 1765 BIT(VLV_IOSF_SB_CCK)); 1766 1767 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1768 rps->rp1_freq | rps->min_freq) & 1, 1769 "Odd GPU freq values\n"); 1770 } 1771 1772 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1773 { 1774 ei->ktime = ktime_get_raw(); 1775 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1776 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1777 } 1778 1779 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1780 { 1781 struct drm_i915_private *i915 = rps_to_i915(rps); 1782 struct intel_uncore *uncore = rps_to_uncore(rps); 1783 const struct intel_rps_ei *prev = &rps->ei; 1784 struct intel_rps_ei now; 1785 u32 events = 0; 1786 1787 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1788 return 0; 1789 1790 vlv_c0_read(uncore, &now); 1791 1792 if (prev->ktime) { 1793 u64 time, c0; 1794 u32 render, media; 1795 1796 time = ktime_us_delta(now.ktime, prev->ktime); 1797 1798 time *= vlv_clock_get_czclk(&i915->drm); 1799 1800 /* Workload can be split between render + media, 1801 * e.g. SwapBuffers being blitted in X after being rendered in 1802 * mesa. To account for this we need to combine both engines 1803 * into our activity counter. 1804 */ 1805 render = now.render_c0 - prev->render_c0; 1806 media = now.media_c0 - prev->media_c0; 1807 c0 = max(render, media); 1808 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1809 1810 if (c0 > time * rps->power.up_threshold) 1811 events = GEN6_PM_RP_UP_THRESHOLD; 1812 else if (c0 < time * rps->power.down_threshold) 1813 events = GEN6_PM_RP_DOWN_THRESHOLD; 1814 } 1815 1816 rps->ei = now; 1817 return events; 1818 } 1819 1820 static void rps_work(struct work_struct *work) 1821 { 1822 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1823 struct intel_gt *gt = rps_to_gt(rps); 1824 struct drm_i915_private *i915 = rps_to_i915(rps); 1825 bool client_boost = false; 1826 int new_freq, adj, min, max; 1827 u32 pm_iir = 0; 1828 1829 spin_lock_irq(gt->irq_lock); 1830 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1831 client_boost = atomic_read(&rps->num_waiters); 1832 spin_unlock_irq(gt->irq_lock); 1833 1834 /* Make sure we didn't queue anything we're not going to process. */ 1835 if (!pm_iir && !client_boost) 1836 goto out; 1837 1838 mutex_lock(&rps->lock); 1839 if (!intel_rps_is_active(rps)) { 1840 mutex_unlock(&rps->lock); 1841 return; 1842 } 1843 1844 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1845 1846 adj = rps->last_adj; 1847 new_freq = rps->cur_freq; 1848 min = rps->min_freq_softlimit; 1849 max = rps->max_freq_softlimit; 1850 if (client_boost) 1851 max = rps->max_freq; 1852 1853 GT_TRACE(gt, 1854 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1855 pm_iir, str_yes_no(client_boost), 1856 adj, new_freq, min, max); 1857 1858 if (client_boost && new_freq < rps->boost_freq) { 1859 new_freq = rps->boost_freq; 1860 adj = 0; 1861 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1862 if (adj > 0) 1863 adj *= 2; 1864 else /* CHV needs even encode values */ 1865 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1866 1867 if (new_freq >= rps->max_freq_softlimit) 1868 adj = 0; 1869 } else if (client_boost) { 1870 adj = 0; 1871 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1872 if (rps->cur_freq > rps->efficient_freq) 1873 new_freq = rps->efficient_freq; 1874 else if (rps->cur_freq > rps->min_freq_softlimit) 1875 new_freq = rps->min_freq_softlimit; 1876 adj = 0; 1877 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1878 if (adj < 0) 1879 adj *= 2; 1880 else /* CHV needs even encode values */ 1881 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1882 1883 if (new_freq <= rps->min_freq_softlimit) 1884 adj = 0; 1885 } else { /* unknown event */ 1886 adj = 0; 1887 } 1888 1889 /* 1890 * sysfs frequency limits may have snuck in while 1891 * servicing the interrupt 1892 */ 1893 new_freq += adj; 1894 new_freq = clamp_t(int, new_freq, min, max); 1895 1896 if (intel_rps_set(rps, new_freq)) { 1897 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1898 adj = 0; 1899 } 1900 rps->last_adj = adj; 1901 1902 mutex_unlock(&rps->lock); 1903 1904 out: 1905 spin_lock_irq(gt->irq_lock); 1906 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1907 spin_unlock_irq(gt->irq_lock); 1908 } 1909 1910 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1911 { 1912 struct intel_gt *gt = rps_to_gt(rps); 1913 const u32 events = rps->pm_events & pm_iir; 1914 1915 lockdep_assert_held(gt->irq_lock); 1916 1917 if (unlikely(!events)) 1918 return; 1919 1920 GT_TRACE(gt, "irq events:%x\n", events); 1921 1922 gen6_gt_pm_mask_irq(gt, events); 1923 1924 rps->pm_iir |= events; 1925 queue_work(gt->i915->unordered_wq, &rps->work); 1926 } 1927 1928 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1929 { 1930 struct intel_gt *gt = rps_to_gt(rps); 1931 u32 events; 1932 1933 events = pm_iir & rps->pm_events; 1934 if (events) { 1935 spin_lock(gt->irq_lock); 1936 1937 GT_TRACE(gt, "irq events:%x\n", events); 1938 1939 gen6_gt_pm_mask_irq(gt, events); 1940 rps->pm_iir |= events; 1941 1942 queue_work(gt->i915->unordered_wq, &rps->work); 1943 spin_unlock(gt->irq_lock); 1944 } 1945 1946 if (GRAPHICS_VER(gt->i915) >= 8) 1947 return; 1948 1949 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1950 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1951 1952 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1953 drm_dbg(&rps_to_i915(rps)->drm, 1954 "Command parser error, pm_iir 0x%08x\n", pm_iir); 1955 } 1956 1957 void gen5_rps_irq_handler(struct intel_rps *rps) 1958 { 1959 struct intel_uncore *uncore = rps_to_uncore(rps); 1960 u32 busy_up, busy_down, max_avg, min_avg; 1961 u8 new_freq; 1962 1963 spin_lock(&mchdev_lock); 1964 1965 intel_uncore_write16(uncore, 1966 MEMINTRSTS, 1967 intel_uncore_read(uncore, MEMINTRSTS)); 1968 1969 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1970 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1971 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1972 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1973 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1974 1975 /* Handle RCS change request from hw */ 1976 new_freq = rps->cur_freq; 1977 if (busy_up > max_avg) 1978 new_freq++; 1979 else if (busy_down < min_avg) 1980 new_freq--; 1981 new_freq = clamp(new_freq, 1982 rps->min_freq_softlimit, 1983 rps->max_freq_softlimit); 1984 1985 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1986 rps->cur_freq = new_freq; 1987 1988 spin_unlock(&mchdev_lock); 1989 } 1990 1991 void intel_rps_init_early(struct intel_rps *rps) 1992 { 1993 mutex_init(&rps->lock); 1994 mutex_init(&rps->power.mutex); 1995 1996 INIT_WORK(&rps->work, rps_work); 1997 timer_setup(&rps->timer, rps_timer, 0); 1998 1999 atomic_set(&rps->num_waiters, 0); 2000 } 2001 2002 void intel_rps_init(struct intel_rps *rps) 2003 { 2004 struct drm_i915_private *i915 = rps_to_i915(rps); 2005 2006 if (rps_uses_slpc(rps)) 2007 return; 2008 2009 if (IS_CHERRYVIEW(i915)) 2010 chv_rps_init(rps); 2011 else if (IS_VALLEYVIEW(i915)) 2012 vlv_rps_init(rps); 2013 else if (GRAPHICS_VER(i915) >= 6) 2014 gen6_rps_init(rps); 2015 else if (IS_IRONLAKE_M(i915)) 2016 gen5_rps_init(rps); 2017 2018 /* Derive initial user preferences/limits from the hardware limits */ 2019 rps->max_freq_softlimit = rps->max_freq; 2020 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit; 2021 rps->min_freq_softlimit = rps->min_freq; 2022 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit; 2023 2024 /* After setting max-softlimit, find the overclock max freq */ 2025 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 2026 u32 params = 0; 2027 2028 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL); 2029 if (params & BIT(31)) { /* OC supported */ 2030 drm_dbg(&i915->drm, 2031 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 2032 (rps->max_freq & 0xff) * 50, 2033 (params & 0xff) * 50); 2034 rps->max_freq = params & 0xff; 2035 } 2036 } 2037 2038 /* Set default thresholds in % */ 2039 rps->power.up_threshold = 95; 2040 rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold; 2041 rps->power.down_threshold = 85; 2042 rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold; 2043 2044 /* Finally allow us to boost to max by default */ 2045 rps->boost_freq = rps->max_freq; 2046 rps->idle_freq = rps->min_freq; 2047 2048 /* Start in the middle, from here we will autotune based on workload */ 2049 rps->cur_freq = rps->efficient_freq; 2050 2051 rps->pm_intrmsk_mbz = 0; 2052 2053 /* 2054 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 2055 * if GEN6_PM_UP_EI_EXPIRED is masked. 2056 * 2057 * TODO: verify if this can be reproduced on VLV,CHV. 2058 */ 2059 if (GRAPHICS_VER(i915) <= 7) 2060 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 2061 2062 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 2063 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 2064 2065 /* GuC needs ARAT expired interrupt unmasked */ 2066 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 2067 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 2068 } 2069 2070 void intel_rps_sanitize(struct intel_rps *rps) 2071 { 2072 if (rps_uses_slpc(rps)) 2073 return; 2074 2075 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 2076 rps_disable_interrupts(rps); 2077 } 2078 2079 u32 intel_rps_read_rpstat(struct intel_rps *rps) 2080 { 2081 struct drm_i915_private *i915 = rps_to_i915(rps); 2082 i915_reg_t rpstat; 2083 2084 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1; 2085 2086 return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat); 2087 } 2088 2089 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 2090 { 2091 struct drm_i915_private *i915 = rps_to_i915(rps); 2092 u32 cagf; 2093 2094 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 2095 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat); 2096 else if (GRAPHICS_VER(i915) >= 12) 2097 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat); 2098 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 2099 cagf = REG_FIELD_GET(RPE_MASK, rpstat); 2100 else if (GRAPHICS_VER(i915) >= 9) 2101 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat); 2102 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2103 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat); 2104 else if (GRAPHICS_VER(i915) >= 6) 2105 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat); 2106 else 2107 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat)); 2108 2109 return cagf; 2110 } 2111 2112 static u32 __read_cagf(struct intel_rps *rps, bool take_fw) 2113 { 2114 struct drm_i915_private *i915 = rps_to_i915(rps); 2115 struct intel_uncore *uncore = rps_to_uncore(rps); 2116 i915_reg_t r = INVALID_MMIO_REG; 2117 u32 freq; 2118 2119 /* 2120 * For Gen12+ reading freq from HW does not need a forcewake and 2121 * registers will return 0 freq when GT is in RC6 2122 */ 2123 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 2124 r = MTL_MIRROR_TARGET_WP1; 2125 } else if (GRAPHICS_VER(i915) >= 12) { 2126 r = GEN12_RPSTAT1; 2127 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2128 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 2129 freq = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 2130 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 2131 } else if (GRAPHICS_VER(i915) >= 6) { 2132 r = GEN6_RPSTAT1; 2133 } else { 2134 r = MEMSTAT_ILK; 2135 } 2136 2137 if (i915_mmio_reg_valid(r)) 2138 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r); 2139 2140 return intel_rps_get_cagf(rps, freq); 2141 } 2142 2143 static u32 read_cagf(struct intel_rps *rps) 2144 { 2145 return __read_cagf(rps, true); 2146 } 2147 2148 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 2149 { 2150 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2151 intel_wakeref_t wakeref; 2152 u32 freq = 0; 2153 2154 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2155 freq = intel_gpu_freq(rps, read_cagf(rps)); 2156 2157 return freq; 2158 } 2159 2160 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps) 2161 { 2162 return intel_gpu_freq(rps, __read_cagf(rps, false)); 2163 } 2164 2165 static u32 intel_rps_read_punit_req(struct intel_rps *rps) 2166 { 2167 struct intel_uncore *uncore = rps_to_uncore(rps); 2168 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2169 intel_wakeref_t wakeref; 2170 u32 freq = 0; 2171 2172 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2173 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2174 2175 return freq; 2176 } 2177 2178 static u32 intel_rps_get_req(u32 pureq) 2179 { 2180 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 2181 2182 return req; 2183 } 2184 2185 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 2186 { 2187 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 2188 2189 return intel_gpu_freq(rps, freq); 2190 } 2191 2192 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 2193 { 2194 if (rps_uses_slpc(rps)) 2195 return intel_rps_read_punit_req_frequency(rps); 2196 else 2197 return intel_gpu_freq(rps, rps->cur_freq); 2198 } 2199 2200 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2201 { 2202 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2203 2204 if (rps_uses_slpc(rps)) 2205 return slpc->max_freq_softlimit; 2206 else 2207 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2208 } 2209 2210 /** 2211 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format. 2212 * @rps: the intel_rps structure 2213 * 2214 * Returns the max frequency in a raw format. In newer platforms raw is in 2215 * units of 50 MHz. 2216 */ 2217 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps) 2218 { 2219 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2220 u32 freq; 2221 2222 if (rps_uses_slpc(rps)) { 2223 return DIV_ROUND_CLOSEST(slpc->rp0_freq, 2224 GT_FREQUENCY_MULTIPLIER); 2225 } else { 2226 freq = rps->max_freq; 2227 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2228 /* Convert GT frequency to 50 MHz units */ 2229 freq /= GEN9_FREQ_SCALER; 2230 } 2231 return freq; 2232 } 2233 } 2234 2235 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2236 { 2237 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2238 2239 if (rps_uses_slpc(rps)) 2240 return slpc->rp0_freq; 2241 else 2242 return intel_gpu_freq(rps, rps->rp0_freq); 2243 } 2244 2245 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2246 { 2247 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2248 2249 if (rps_uses_slpc(rps)) 2250 return slpc->rp1_freq; 2251 else 2252 return intel_gpu_freq(rps, rps->rp1_freq); 2253 } 2254 2255 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2256 { 2257 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2258 2259 if (rps_uses_slpc(rps)) 2260 return slpc->min_freq; 2261 else 2262 return intel_gpu_freq(rps, rps->min_freq); 2263 } 2264 2265 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2266 { 2267 struct intel_gt *gt = rps_to_gt(rps); 2268 struct drm_i915_private *i915 = gt->i915; 2269 struct intel_uncore *uncore = gt->uncore; 2270 struct intel_rps_freq_caps caps; 2271 u32 rp_state_limits; 2272 u32 gt_perf_status; 2273 u32 rpmodectl, rpinclimit, rpdeclimit; 2274 u32 rpstat, cagf, reqf; 2275 u32 rpcurupei, rpcurup, rpprevup; 2276 u32 rpcurdownei, rpcurdown, rpprevdown; 2277 u32 rpupei, rpupt, rpdownei, rpdownt; 2278 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; 2279 2280 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS); 2281 gen6_rps_get_freq_caps(rps, &caps); 2282 if (IS_GEN9_LP(i915)) 2283 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS); 2284 else 2285 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS); 2286 2287 /* RPSTAT1 is in the GT power well */ 2288 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 2289 2290 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2291 if (GRAPHICS_VER(i915) >= 9) { 2292 reqf >>= 23; 2293 } else { 2294 reqf &= ~GEN6_TURBO_DISABLE; 2295 if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2296 reqf >>= 24; 2297 else 2298 reqf >>= 25; 2299 } 2300 reqf = intel_gpu_freq(rps, reqf); 2301 2302 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL); 2303 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2304 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2305 2306 rpstat = intel_rps_read_rpstat(rps); 2307 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; 2308 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK; 2309 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK; 2310 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; 2311 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; 2312 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; 2313 2314 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI); 2315 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2316 2317 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI); 2318 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2319 2320 cagf = intel_rps_read_actual_frequency(rps); 2321 2322 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 2323 2324 if (GRAPHICS_VER(i915) >= 11) { 2325 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE); 2326 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK); 2327 /* 2328 * The equivalent to the PM ISR & IIR cannot be read 2329 * without affecting the current state of the system 2330 */ 2331 pm_isr = 0; 2332 pm_iir = 0; 2333 } else if (GRAPHICS_VER(i915) >= 8) { 2334 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2)); 2335 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2)); 2336 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2)); 2337 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2)); 2338 } else { 2339 pm_ier = intel_uncore_read(uncore, GEN6_PMIER); 2340 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR); 2341 pm_isr = intel_uncore_read(uncore, GEN6_PMISR); 2342 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR); 2343 } 2344 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2345 2346 drm_printf(p, "Video Turbo Mode: %s\n", 2347 str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO)); 2348 drm_printf(p, "HW control enabled: %s\n", 2349 str_yes_no(rpmodectl & GEN6_RP_ENABLE)); 2350 drm_printf(p, "SW control enabled: %s\n", 2351 str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE)); 2352 2353 drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", 2354 pm_ier, pm_imr, pm_mask); 2355 if (GRAPHICS_VER(i915) <= 10) 2356 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n", 2357 pm_isr, pm_iir); 2358 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2359 rps->pm_intrmsk_mbz); 2360 drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); 2361 drm_printf(p, "Render p-state ratio: %d\n", 2362 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8); 2363 drm_printf(p, "Render p-state VID: %d\n", 2364 gt_perf_status & 0xff); 2365 drm_printf(p, "Render p-state limit: %d\n", 2366 rp_state_limits & 0xff); 2367 drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat); 2368 drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl); 2369 drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit); 2370 drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit); 2371 drm_printf(p, "RPNSWREQ: %dMHz\n", reqf); 2372 drm_printf(p, "CAGF: %dMHz\n", cagf); 2373 drm_printf(p, "RP CUR UP EI: %d (%lldns)\n", 2374 rpcurupei, 2375 intel_gt_pm_interval_to_ns(gt, rpcurupei)); 2376 drm_printf(p, "RP CUR UP: %d (%lldns)\n", 2377 rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup)); 2378 drm_printf(p, "RP PREV UP: %d (%lldns)\n", 2379 rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup)); 2380 drm_printf(p, "Up threshold: %d%%\n", 2381 rps->power.up_threshold); 2382 drm_printf(p, "RP UP EI: %d (%lldns)\n", 2383 rpupei, intel_gt_pm_interval_to_ns(gt, rpupei)); 2384 drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n", 2385 rpupt, intel_gt_pm_interval_to_ns(gt, rpupt)); 2386 2387 drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n", 2388 rpcurdownei, 2389 intel_gt_pm_interval_to_ns(gt, rpcurdownei)); 2390 drm_printf(p, "RP CUR DOWN: %d (%lldns)\n", 2391 rpcurdown, 2392 intel_gt_pm_interval_to_ns(gt, rpcurdown)); 2393 drm_printf(p, "RP PREV DOWN: %d (%lldns)\n", 2394 rpprevdown, 2395 intel_gt_pm_interval_to_ns(gt, rpprevdown)); 2396 drm_printf(p, "Down threshold: %d%%\n", 2397 rps->power.down_threshold); 2398 drm_printf(p, "RP DOWN EI: %d (%lldns)\n", 2399 rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei)); 2400 drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n", 2401 rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt)); 2402 2403 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2404 intel_gpu_freq(rps, caps.min_freq)); 2405 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2406 intel_gpu_freq(rps, caps.rp1_freq)); 2407 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2408 intel_gpu_freq(rps, caps.rp0_freq)); 2409 drm_printf(p, "Max overclocked frequency: %dMHz\n", 2410 intel_gpu_freq(rps, rps->max_freq)); 2411 2412 drm_printf(p, "Current freq: %d MHz\n", 2413 intel_gpu_freq(rps, rps->cur_freq)); 2414 drm_printf(p, "Actual freq: %d MHz\n", cagf); 2415 drm_printf(p, "Idle freq: %d MHz\n", 2416 intel_gpu_freq(rps, rps->idle_freq)); 2417 drm_printf(p, "Min freq: %d MHz\n", 2418 intel_gpu_freq(rps, rps->min_freq)); 2419 drm_printf(p, "Boost freq: %d MHz\n", 2420 intel_gpu_freq(rps, rps->boost_freq)); 2421 drm_printf(p, "Max freq: %d MHz\n", 2422 intel_gpu_freq(rps, rps->max_freq)); 2423 drm_printf(p, 2424 "efficient (RPe) frequency: %d MHz\n", 2425 intel_gpu_freq(rps, rps->efficient_freq)); 2426 } 2427 2428 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2429 { 2430 struct intel_gt *gt = rps_to_gt(rps); 2431 struct intel_uncore *uncore = gt->uncore; 2432 struct intel_rps_freq_caps caps; 2433 u32 pm_mask; 2434 2435 gen6_rps_get_freq_caps(rps, &caps); 2436 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2437 2438 drm_printf(p, "PM MASK=0x%08x\n", pm_mask); 2439 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2440 rps->pm_intrmsk_mbz); 2441 drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps)); 2442 drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps)); 2443 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2444 intel_gpu_freq(rps, caps.min_freq)); 2445 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2446 intel_gpu_freq(rps, caps.rp1_freq)); 2447 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2448 intel_gpu_freq(rps, caps.rp0_freq)); 2449 drm_printf(p, "Current freq: %d MHz\n", 2450 intel_rps_get_requested_frequency(rps)); 2451 drm_printf(p, "Actual freq: %d MHz\n", 2452 intel_rps_read_actual_frequency(rps)); 2453 drm_printf(p, "Min freq: %d MHz\n", 2454 intel_rps_get_min_frequency(rps)); 2455 drm_printf(p, "Boost freq: %d MHz\n", 2456 intel_rps_get_boost_frequency(rps)); 2457 drm_printf(p, "Max freq: %d MHz\n", 2458 intel_rps_get_max_frequency(rps)); 2459 drm_printf(p, 2460 "efficient (RPe) frequency: %d MHz\n", 2461 intel_gpu_freq(rps, caps.rp1_freq)); 2462 } 2463 2464 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2465 { 2466 if (rps_uses_slpc(rps)) 2467 return slpc_frequency_dump(rps, p); 2468 else 2469 return rps_frequency_dump(rps, p); 2470 } 2471 2472 static int set_max_freq(struct intel_rps *rps, u32 val) 2473 { 2474 struct drm_i915_private *i915 = rps_to_i915(rps); 2475 int ret = 0; 2476 2477 mutex_lock(&rps->lock); 2478 2479 val = intel_freq_opcode(rps, val); 2480 if (val < rps->min_freq || 2481 val > rps->max_freq || 2482 val < rps->min_freq_softlimit) { 2483 ret = -EINVAL; 2484 goto unlock; 2485 } 2486 2487 if (val > rps->rp0_freq) 2488 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2489 intel_gpu_freq(rps, val)); 2490 2491 rps->max_freq_softlimit = val; 2492 2493 val = clamp_t(int, rps->cur_freq, 2494 rps->min_freq_softlimit, 2495 rps->max_freq_softlimit); 2496 2497 /* 2498 * We still need *_set_rps to process the new max_delay and 2499 * update the interrupt limits and PMINTRMSK even though 2500 * frequency request may be unchanged. 2501 */ 2502 intel_rps_set(rps, val); 2503 2504 unlock: 2505 mutex_unlock(&rps->lock); 2506 2507 return ret; 2508 } 2509 2510 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2511 { 2512 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2513 2514 if (rps_uses_slpc(rps)) 2515 return intel_guc_slpc_set_max_freq(slpc, val); 2516 else 2517 return set_max_freq(rps, val); 2518 } 2519 2520 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2521 { 2522 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2523 2524 if (rps_uses_slpc(rps)) 2525 return slpc->min_freq_softlimit; 2526 else 2527 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2528 } 2529 2530 /** 2531 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format. 2532 * @rps: the intel_rps structure 2533 * 2534 * Returns the min frequency in a raw format. In newer platforms raw is in 2535 * units of 50 MHz. 2536 */ 2537 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps) 2538 { 2539 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2540 u32 freq; 2541 2542 if (rps_uses_slpc(rps)) { 2543 return DIV_ROUND_CLOSEST(slpc->min_freq, 2544 GT_FREQUENCY_MULTIPLIER); 2545 } else { 2546 freq = rps->min_freq; 2547 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2548 /* Convert GT frequency to 50 MHz units */ 2549 freq /= GEN9_FREQ_SCALER; 2550 } 2551 return freq; 2552 } 2553 } 2554 2555 static int set_min_freq(struct intel_rps *rps, u32 val) 2556 { 2557 int ret = 0; 2558 2559 mutex_lock(&rps->lock); 2560 2561 val = intel_freq_opcode(rps, val); 2562 if (val < rps->min_freq || 2563 val > rps->max_freq || 2564 val > rps->max_freq_softlimit) { 2565 ret = -EINVAL; 2566 goto unlock; 2567 } 2568 2569 rps->min_freq_softlimit = val; 2570 2571 val = clamp_t(int, rps->cur_freq, 2572 rps->min_freq_softlimit, 2573 rps->max_freq_softlimit); 2574 2575 /* 2576 * We still need *_set_rps to process the new min_delay and 2577 * update the interrupt limits and PMINTRMSK even though 2578 * frequency request may be unchanged. 2579 */ 2580 intel_rps_set(rps, val); 2581 2582 unlock: 2583 mutex_unlock(&rps->lock); 2584 2585 return ret; 2586 } 2587 2588 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2589 { 2590 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2591 2592 if (rps_uses_slpc(rps)) 2593 return intel_guc_slpc_set_min_freq(slpc, val); 2594 else 2595 return set_min_freq(rps, val); 2596 } 2597 2598 u8 intel_rps_get_up_threshold(struct intel_rps *rps) 2599 { 2600 return rps->power.up_threshold; 2601 } 2602 2603 static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val) 2604 { 2605 int ret; 2606 2607 if (val > 100) 2608 return -EINVAL; 2609 2610 ret = mutex_lock_interruptible(&rps->lock); 2611 if (ret) 2612 return ret; 2613 2614 if (*threshold == val) 2615 goto out_unlock; 2616 2617 *threshold = val; 2618 2619 /* Force reset. */ 2620 rps->last_freq = -1; 2621 mutex_lock(&rps->power.mutex); 2622 rps->power.mode = -1; 2623 mutex_unlock(&rps->power.mutex); 2624 2625 intel_rps_set(rps, clamp(rps->cur_freq, 2626 rps->min_freq_softlimit, 2627 rps->max_freq_softlimit)); 2628 2629 out_unlock: 2630 mutex_unlock(&rps->lock); 2631 2632 return ret; 2633 } 2634 2635 int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold) 2636 { 2637 return rps_set_threshold(rps, &rps->power.up_threshold, threshold); 2638 } 2639 2640 u8 intel_rps_get_down_threshold(struct intel_rps *rps) 2641 { 2642 return rps->power.down_threshold; 2643 } 2644 2645 int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold) 2646 { 2647 return rps_set_threshold(rps, &rps->power.down_threshold, threshold); 2648 } 2649 2650 static void intel_rps_set_manual(struct intel_rps *rps, bool enable) 2651 { 2652 struct intel_uncore *uncore = rps_to_uncore(rps); 2653 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE; 2654 2655 /* Allow punit to process software requests */ 2656 intel_uncore_write(uncore, GEN6_RP_CONTROL, state); 2657 } 2658 2659 void intel_rps_raise_unslice(struct intel_rps *rps) 2660 { 2661 struct intel_uncore *uncore = rps_to_uncore(rps); 2662 2663 mutex_lock(&rps->lock); 2664 2665 if (rps_uses_slpc(rps)) { 2666 /* RP limits have not been initialized yet for SLPC path */ 2667 struct intel_rps_freq_caps caps; 2668 2669 gen6_rps_get_freq_caps(rps, &caps); 2670 2671 intel_rps_set_manual(rps, true); 2672 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2673 ((caps.rp0_freq << 2674 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2675 GEN9_IGNORE_SLICE_RATIO)); 2676 intel_rps_set_manual(rps, false); 2677 } else { 2678 intel_rps_set(rps, rps->rp0_freq); 2679 } 2680 2681 mutex_unlock(&rps->lock); 2682 } 2683 2684 void intel_rps_lower_unslice(struct intel_rps *rps) 2685 { 2686 struct intel_uncore *uncore = rps_to_uncore(rps); 2687 2688 mutex_lock(&rps->lock); 2689 2690 if (rps_uses_slpc(rps)) { 2691 /* RP limits have not been initialized yet for SLPC path */ 2692 struct intel_rps_freq_caps caps; 2693 2694 gen6_rps_get_freq_caps(rps, &caps); 2695 2696 intel_rps_set_manual(rps, true); 2697 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2698 ((caps.min_freq << 2699 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2700 GEN9_IGNORE_SLICE_RATIO)); 2701 intel_rps_set_manual(rps, false); 2702 } else { 2703 intel_rps_set(rps, rps->min_freq); 2704 } 2705 2706 mutex_unlock(&rps->lock); 2707 } 2708 2709 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32) 2710 { 2711 struct intel_gt *gt = rps_to_gt(rps); 2712 intel_wakeref_t wakeref; 2713 u32 val; 2714 2715 with_intel_runtime_pm(gt->uncore->rpm, wakeref) 2716 val = intel_uncore_read(gt->uncore, reg32); 2717 2718 return val; 2719 } 2720 2721 bool rps_read_mask_mmio(struct intel_rps *rps, 2722 i915_reg_t reg32, u32 mask) 2723 { 2724 return rps_read_mmio(rps, reg32) & mask; 2725 } 2726 2727 /* External interface for intel_ips.ko */ 2728 2729 static struct drm_i915_private __rcu *ips_mchdev; 2730 2731 /* 2732 * Tells the intel_ips driver that the i915 driver is now loaded, if 2733 * IPS got loaded first. 2734 * 2735 * This awkward dance is so that neither module has to depend on the 2736 * other in order for IPS to do the appropriate communication of 2737 * GPU turbo limits to i915. 2738 */ 2739 static void 2740 ips_ping_for_i915_load(void) 2741 { 2742 void (*link)(void); 2743 2744 link = symbol_get(ips_link_to_i915_driver); 2745 if (link) { 2746 link(); 2747 symbol_put(ips_link_to_i915_driver); 2748 } 2749 } 2750 2751 void intel_rps_driver_register(struct intel_rps *rps) 2752 { 2753 struct intel_gt *gt = rps_to_gt(rps); 2754 2755 /* 2756 * We only register the i915 ips part with intel-ips once everything is 2757 * set up, to avoid intel-ips sneaking in and reading bogus values. 2758 */ 2759 if (GRAPHICS_VER(gt->i915) == 5) { 2760 GEM_BUG_ON(ips_mchdev); 2761 rcu_assign_pointer(ips_mchdev, gt->i915); 2762 ips_ping_for_i915_load(); 2763 } 2764 } 2765 2766 void intel_rps_driver_unregister(struct intel_rps *rps) 2767 { 2768 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2769 rcu_assign_pointer(ips_mchdev, NULL); 2770 } 2771 2772 static struct drm_i915_private *mchdev_get(void) 2773 { 2774 struct drm_i915_private *i915; 2775 2776 rcu_read_lock(); 2777 i915 = rcu_dereference(ips_mchdev); 2778 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2779 i915 = NULL; 2780 rcu_read_unlock(); 2781 2782 return i915; 2783 } 2784 2785 /** 2786 * i915_read_mch_val - return value for IPS use 2787 * 2788 * Calculate and return a value for the IPS driver to use when deciding whether 2789 * we have thermal and power headroom to increase CPU or GPU power budget. 2790 */ 2791 unsigned long i915_read_mch_val(void) 2792 { 2793 struct drm_i915_private *i915; 2794 unsigned long chipset_val = 0; 2795 unsigned long graphics_val = 0; 2796 intel_wakeref_t wakeref; 2797 2798 i915 = mchdev_get(); 2799 if (!i915) 2800 return 0; 2801 2802 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2803 struct intel_ips *ips = &to_gt(i915)->rps.ips; 2804 2805 spin_lock_irq(&mchdev_lock); 2806 chipset_val = __ips_chipset_val(ips); 2807 graphics_val = __ips_gfx_val(ips); 2808 spin_unlock_irq(&mchdev_lock); 2809 } 2810 2811 drm_dev_put(&i915->drm); 2812 return chipset_val + graphics_val; 2813 } 2814 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2815 2816 /** 2817 * i915_gpu_raise - raise GPU frequency limit 2818 * 2819 * Raise the limit; IPS indicates we have thermal headroom. 2820 */ 2821 bool i915_gpu_raise(void) 2822 { 2823 struct drm_i915_private *i915; 2824 struct intel_rps *rps; 2825 2826 i915 = mchdev_get(); 2827 if (!i915) 2828 return false; 2829 2830 rps = &to_gt(i915)->rps; 2831 2832 spin_lock_irq(&mchdev_lock); 2833 if (rps->max_freq_softlimit < rps->max_freq) 2834 rps->max_freq_softlimit++; 2835 spin_unlock_irq(&mchdev_lock); 2836 2837 drm_dev_put(&i915->drm); 2838 return true; 2839 } 2840 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2841 2842 /** 2843 * i915_gpu_lower - lower GPU frequency limit 2844 * 2845 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2846 * frequency maximum. 2847 */ 2848 bool i915_gpu_lower(void) 2849 { 2850 struct drm_i915_private *i915; 2851 struct intel_rps *rps; 2852 2853 i915 = mchdev_get(); 2854 if (!i915) 2855 return false; 2856 2857 rps = &to_gt(i915)->rps; 2858 2859 spin_lock_irq(&mchdev_lock); 2860 if (rps->max_freq_softlimit > rps->min_freq) 2861 rps->max_freq_softlimit--; 2862 spin_unlock_irq(&mchdev_lock); 2863 2864 drm_dev_put(&i915->drm); 2865 return true; 2866 } 2867 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2868 2869 /** 2870 * i915_gpu_busy - indicate GPU business to IPS 2871 * 2872 * Tell the IPS driver whether or not the GPU is busy. 2873 */ 2874 bool i915_gpu_busy(void) 2875 { 2876 struct drm_i915_private *i915; 2877 bool ret; 2878 2879 i915 = mchdev_get(); 2880 if (!i915) 2881 return false; 2882 2883 ret = to_gt(i915)->awake; 2884 2885 drm_dev_put(&i915->drm); 2886 return ret; 2887 } 2888 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2889 2890 /** 2891 * i915_gpu_turbo_disable - disable graphics turbo 2892 * 2893 * Disable graphics turbo by resetting the max frequency and setting the 2894 * current frequency to the default. 2895 */ 2896 bool i915_gpu_turbo_disable(void) 2897 { 2898 struct drm_i915_private *i915; 2899 struct intel_rps *rps; 2900 bool ret; 2901 2902 i915 = mchdev_get(); 2903 if (!i915) 2904 return false; 2905 2906 rps = &to_gt(i915)->rps; 2907 2908 spin_lock_irq(&mchdev_lock); 2909 rps->max_freq_softlimit = rps->min_freq; 2910 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq); 2911 spin_unlock_irq(&mchdev_lock); 2912 2913 drm_dev_put(&i915->drm); 2914 return ret; 2915 } 2916 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2917 2918 static void boost_if_not_started(struct dma_fence *fence) 2919 { 2920 struct i915_request *rq; 2921 2922 if (!dma_fence_is_i915(fence)) 2923 return; 2924 2925 rq = to_request(fence); 2926 2927 if (!i915_request_started(rq)) 2928 intel_rps_boost(rq); 2929 } 2930 2931 static void mark_interactive(struct drm_device *drm, bool interactive) 2932 { 2933 struct drm_i915_private *i915 = to_i915(drm); 2934 2935 intel_rps_mark_interactive(&to_gt(i915)->rps, interactive); 2936 } 2937 2938 static void ilk_irq_handler(struct drm_device *drm) 2939 { 2940 struct drm_i915_private *i915 = to_i915(drm); 2941 2942 gen5_rps_irq_handler(&to_gt(i915)->rps); 2943 } 2944 2945 const struct intel_display_rps_interface i915_display_rps_interface = { 2946 .boost_if_not_started = boost_if_not_started, 2947 .mark_interactive = mark_interactive, 2948 .ilk_irq_handler = ilk_irq_handler, 2949 }; 2950 2951 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2952 #include "selftest_rps.c" 2953 #include "selftest_slpc.c" 2954 #endif 2955