1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 8 #include <drm/intel/i915_drm.h> 9 10 #include "display/intel_display.h" 11 #include "display/intel_display_rps.h" 12 #include "soc/intel_dram.h" 13 #include "i915_drv.h" 14 #include "i915_irq.h" 15 #include "i915_reg.h" 16 #include "intel_breadcrumbs.h" 17 #include "intel_gt.h" 18 #include "intel_gt_clock_utils.h" 19 #include "intel_gt_irq.h" 20 #include "intel_gt_pm.h" 21 #include "intel_gt_pm_irq.h" 22 #include "intel_gt_print.h" 23 #include "intel_gt_regs.h" 24 #include "intel_mchbar_regs.h" 25 #include "intel_pcode.h" 26 #include "intel_rps.h" 27 #include "vlv_iosf_sb.h" 28 #include "../../../platform/x86/intel_ips.h" 29 30 #define BUSY_MAX_EI 20u /* ms */ 31 32 /* 33 * Lock protecting IPS related data structures 34 */ 35 static DEFINE_SPINLOCK(mchdev_lock); 36 37 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 38 { 39 return container_of(rps, struct intel_gt, rps); 40 } 41 42 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 43 { 44 return rps_to_gt(rps)->i915; 45 } 46 47 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 48 { 49 return rps_to_gt(rps)->uncore; 50 } 51 52 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 53 { 54 struct intel_gt *gt = rps_to_gt(rps); 55 56 return >_to_guc(gt)->slpc; 57 } 58 59 static bool rps_uses_slpc(struct intel_rps *rps) 60 { 61 struct intel_gt *gt = rps_to_gt(rps); 62 63 return intel_uc_uses_guc_slpc(>->uc); 64 } 65 66 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 67 { 68 return mask & ~rps->pm_intrmsk_mbz; 69 } 70 71 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 72 { 73 intel_uncore_write_fw(uncore, reg, val); 74 } 75 76 static void rps_timer(struct timer_list *t) 77 { 78 struct intel_rps *rps = timer_container_of(rps, t, timer); 79 struct intel_gt *gt = rps_to_gt(rps); 80 struct intel_engine_cs *engine; 81 ktime_t dt, last, timestamp; 82 enum intel_engine_id id; 83 s64 max_busy[3] = {}; 84 85 timestamp = 0; 86 for_each_engine(engine, gt, id) { 87 s64 busy; 88 int i; 89 90 dt = intel_engine_get_busy_time(engine, ×tamp); 91 last = engine->stats.rps; 92 engine->stats.rps = dt; 93 94 busy = ktime_to_ns(ktime_sub(dt, last)); 95 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 96 if (busy > max_busy[i]) 97 swap(busy, max_busy[i]); 98 } 99 } 100 last = rps->pm_timestamp; 101 rps->pm_timestamp = timestamp; 102 103 if (intel_rps_is_active(rps)) { 104 s64 busy; 105 int i; 106 107 dt = ktime_sub(timestamp, last); 108 109 /* 110 * Our goal is to evaluate each engine independently, so we run 111 * at the lowest clocks required to sustain the heaviest 112 * workload. However, a task may be split into sequential 113 * dependent operations across a set of engines, such that 114 * the independent contributions do not account for high load, 115 * but overall the task is GPU bound. For example, consider 116 * video decode on vcs followed by colour post-processing 117 * on vecs, followed by general post-processing on rcs. 118 * Since multi-engines being active does imply a single 119 * continuous workload across all engines, we hedge our 120 * bets by only contributing a factor of the distributed 121 * load into our busyness calculation. 122 */ 123 busy = max_busy[0]; 124 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 125 if (!max_busy[i]) 126 break; 127 128 busy += div_u64(max_busy[i], 1 << i); 129 } 130 GT_TRACE(gt, 131 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 132 busy, (int)div64_u64(100 * busy, dt), 133 max_busy[0], max_busy[1], max_busy[2], 134 rps->pm_interval); 135 136 if (100 * busy > rps->power.up_threshold * dt && 137 rps->cur_freq < rps->max_freq_softlimit) { 138 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 139 rps->pm_interval = 1; 140 queue_work(gt->i915->unordered_wq, &rps->work); 141 } else if (100 * busy < rps->power.down_threshold * dt && 142 rps->cur_freq > rps->min_freq_softlimit) { 143 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 144 rps->pm_interval = 1; 145 queue_work(gt->i915->unordered_wq, &rps->work); 146 } else { 147 rps->last_adj = 0; 148 } 149 150 mod_timer(&rps->timer, 151 jiffies + msecs_to_jiffies(rps->pm_interval)); 152 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 153 } 154 } 155 156 static void rps_start_timer(struct intel_rps *rps) 157 { 158 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 159 rps->pm_interval = 1; 160 mod_timer(&rps->timer, jiffies + 1); 161 } 162 163 static void rps_stop_timer(struct intel_rps *rps) 164 { 165 timer_delete_sync(&rps->timer); 166 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 167 cancel_work_sync(&rps->work); 168 } 169 170 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 171 { 172 u32 mask = 0; 173 174 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 175 if (val > rps->min_freq_softlimit) 176 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 177 GEN6_PM_RP_DOWN_THRESHOLD | 178 GEN6_PM_RP_DOWN_TIMEOUT); 179 180 if (val < rps->max_freq_softlimit) 181 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 182 183 mask &= rps->pm_events; 184 185 return rps_pm_sanitize_mask(rps, ~mask); 186 } 187 188 static void rps_reset_ei(struct intel_rps *rps) 189 { 190 memset(&rps->ei, 0, sizeof(rps->ei)); 191 } 192 193 static void rps_enable_interrupts(struct intel_rps *rps) 194 { 195 struct intel_gt *gt = rps_to_gt(rps); 196 197 GEM_BUG_ON(rps_uses_slpc(rps)); 198 199 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 200 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 201 202 rps_reset_ei(rps); 203 204 spin_lock_irq(gt->irq_lock); 205 gen6_gt_pm_enable_irq(gt, rps->pm_events); 206 spin_unlock_irq(gt->irq_lock); 207 208 intel_uncore_write(gt->uncore, 209 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 210 } 211 212 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 213 { 214 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 215 } 216 217 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 218 { 219 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 220 ; 221 } 222 223 static void rps_reset_interrupts(struct intel_rps *rps) 224 { 225 struct intel_gt *gt = rps_to_gt(rps); 226 227 spin_lock_irq(gt->irq_lock); 228 if (GRAPHICS_VER(gt->i915) >= 11) 229 gen11_rps_reset_interrupts(rps); 230 else 231 gen6_rps_reset_interrupts(rps); 232 233 rps->pm_iir = 0; 234 spin_unlock_irq(gt->irq_lock); 235 } 236 237 static void rps_disable_interrupts(struct intel_rps *rps) 238 { 239 struct intel_gt *gt = rps_to_gt(rps); 240 241 intel_uncore_write(gt->uncore, 242 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 243 244 spin_lock_irq(gt->irq_lock); 245 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 246 spin_unlock_irq(gt->irq_lock); 247 248 intel_synchronize_irq(gt->i915); 249 250 /* 251 * Now that we will not be generating any more work, flush any 252 * outstanding tasks. As we are called on the RPS idle path, 253 * we will reset the GPU to minimum frequencies, so the current 254 * state of the worker can be discarded. 255 */ 256 cancel_work_sync(&rps->work); 257 258 rps_reset_interrupts(rps); 259 GT_TRACE(gt, "interrupts:off\n"); 260 } 261 262 static const struct cparams { 263 u16 i; 264 u16 t; 265 u16 m; 266 u16 c; 267 } cparams[] = { 268 { 1, 1333, 301, 28664 }, 269 { 1, 1067, 294, 24460 }, 270 { 1, 800, 294, 25192 }, 271 { 0, 1333, 276, 27605 }, 272 { 0, 1067, 276, 27605 }, 273 { 0, 800, 231, 23784 }, 274 }; 275 276 static void gen5_rps_init(struct intel_rps *rps) 277 { 278 struct drm_i915_private *i915 = rps_to_i915(rps); 279 struct intel_uncore *uncore = rps_to_uncore(rps); 280 unsigned int fsb_freq, mem_freq; 281 u8 fmax, fmin, fstart; 282 u32 rgvmodectl; 283 int c_m, i; 284 285 fsb_freq = intel_fsb_freq(i915); 286 mem_freq = intel_mem_freq(i915); 287 288 if (fsb_freq <= 3200000) 289 c_m = 0; 290 else if (fsb_freq <= 4800000) 291 c_m = 1; 292 else 293 c_m = 2; 294 295 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 296 if (cparams[i].i == c_m && 297 cparams[i].t == DIV_ROUND_CLOSEST(mem_freq, 1000)) { 298 rps->ips.m = cparams[i].m; 299 rps->ips.c = cparams[i].c; 300 break; 301 } 302 } 303 304 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 305 306 /* Set up min, max, and cur for interrupt handling */ 307 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 308 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 309 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 310 MEMMODE_FSTART_SHIFT; 311 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 312 fmax, fmin, fstart); 313 314 rps->min_freq = fmax; 315 rps->efficient_freq = fstart; 316 rps->max_freq = fmin; 317 } 318 319 static unsigned long 320 __ips_chipset_val(struct intel_ips *ips) 321 { 322 struct intel_uncore *uncore = 323 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 324 unsigned long now = jiffies_to_msecs(jiffies), dt; 325 unsigned long result; 326 u64 total, delta; 327 328 lockdep_assert_held(&mchdev_lock); 329 330 /* 331 * Prevent division-by-zero if we are asking too fast. 332 * Also, we don't get interesting results if we are polling 333 * faster than once in 10ms, so just return the saved value 334 * in such cases. 335 */ 336 dt = now - ips->last_time1; 337 if (dt <= 10) 338 return ips->chipset_power; 339 340 /* FIXME: handle per-counter overflow */ 341 total = intel_uncore_read(uncore, DMIEC); 342 total += intel_uncore_read(uncore, DDREC); 343 total += intel_uncore_read(uncore, CSIEC); 344 345 delta = total - ips->last_count1; 346 347 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 348 349 ips->last_count1 = total; 350 ips->last_time1 = now; 351 352 ips->chipset_power = result; 353 354 return result; 355 } 356 357 static unsigned long ips_mch_val(struct intel_uncore *uncore) 358 { 359 unsigned int m, x, b; 360 u32 tsfs; 361 362 tsfs = intel_uncore_read(uncore, TSFS); 363 x = intel_uncore_read8(uncore, TR1); 364 365 b = tsfs & TSFS_INTR_MASK; 366 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 367 368 return m * x / 127 - b; 369 } 370 371 static int _pxvid_to_vd(u8 pxvid) 372 { 373 if (pxvid == 0) 374 return 0; 375 376 if (pxvid >= 8 && pxvid < 31) 377 pxvid = 31; 378 379 return (pxvid + 2) * 125; 380 } 381 382 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 383 { 384 const int vd = _pxvid_to_vd(pxvid); 385 386 if (INTEL_INFO(i915)->is_mobile) 387 return max(vd - 1125, 0); 388 389 return vd; 390 } 391 392 static void __gen5_ips_update(struct intel_ips *ips) 393 { 394 struct intel_uncore *uncore = 395 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 396 u64 now, delta, dt; 397 u32 count; 398 399 lockdep_assert_held(&mchdev_lock); 400 401 now = ktime_get_raw_ns(); 402 dt = now - ips->last_time2; 403 do_div(dt, NSEC_PER_MSEC); 404 405 /* Don't divide by 0 */ 406 if (dt <= 10) 407 return; 408 409 count = intel_uncore_read(uncore, GFXEC); 410 delta = count - ips->last_count2; 411 412 ips->last_count2 = count; 413 ips->last_time2 = now; 414 415 /* More magic constants... */ 416 ips->gfx_power = div_u64(delta * 1181, dt * 10); 417 } 418 419 static void gen5_rps_update(struct intel_rps *rps) 420 { 421 spin_lock_irq(&mchdev_lock); 422 __gen5_ips_update(&rps->ips); 423 spin_unlock_irq(&mchdev_lock); 424 } 425 426 static unsigned int gen5_invert_freq(struct intel_rps *rps, 427 unsigned int val) 428 { 429 /* Invert the frequency bin into an ips delay */ 430 val = rps->max_freq - val; 431 val = rps->min_freq + val; 432 433 return val; 434 } 435 436 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 437 { 438 struct intel_uncore *uncore = rps_to_uncore(rps); 439 u16 rgvswctl; 440 441 lockdep_assert_held(&mchdev_lock); 442 443 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 444 if (rgvswctl & MEMCTL_CMD_STS) { 445 drm_dbg(&rps_to_i915(rps)->drm, 446 "gpu busy, RCS change rejected\n"); 447 return -EBUSY; /* still busy with another command */ 448 } 449 450 /* Invert the frequency bin into an ips delay */ 451 val = gen5_invert_freq(rps, val); 452 453 rgvswctl = 454 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 455 (val << MEMCTL_FREQ_SHIFT) | 456 MEMCTL_SFCAVM; 457 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 458 intel_uncore_posting_read16(uncore, MEMSWCTL); 459 460 rgvswctl |= MEMCTL_CMD_STS; 461 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 462 463 return 0; 464 } 465 466 static int gen5_rps_set(struct intel_rps *rps, u8 val) 467 { 468 int err; 469 470 spin_lock_irq(&mchdev_lock); 471 err = __gen5_rps_set(rps, val); 472 spin_unlock_irq(&mchdev_lock); 473 474 return err; 475 } 476 477 static unsigned long intel_pxfreq(u32 vidfreq) 478 { 479 int div = (vidfreq & 0x3f0000) >> 16; 480 int post = (vidfreq & 0x3000) >> 12; 481 int pre = (vidfreq & 0x7); 482 483 if (!pre) 484 return 0; 485 486 return div * 133333 / (pre << post); 487 } 488 489 static unsigned int init_emon(struct intel_uncore *uncore) 490 { 491 u8 pxw[16]; 492 int i; 493 494 /* Disable to program */ 495 intel_uncore_write(uncore, ECR, 0); 496 intel_uncore_posting_read(uncore, ECR); 497 498 /* Program energy weights for various events */ 499 intel_uncore_write(uncore, SDEW, 0x15040d00); 500 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 501 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 502 intel_uncore_write(uncore, CSIEW2, 0x04000004); 503 504 for (i = 0; i < 5; i++) 505 intel_uncore_write(uncore, PEW(i), 0); 506 for (i = 0; i < 3; i++) 507 intel_uncore_write(uncore, DEW(i), 0); 508 509 /* Program P-state weights to account for frequency power adjustment */ 510 for (i = 0; i < 16; i++) { 511 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 512 unsigned int freq = intel_pxfreq(pxvidfreq); 513 unsigned int vid = 514 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 515 unsigned int val; 516 517 val = vid * vid * freq / 1000 * 255; 518 val /= 127 * 127 * 900; 519 520 pxw[i] = val; 521 } 522 /* Render standby states get 0 weight */ 523 pxw[14] = 0; 524 pxw[15] = 0; 525 526 for (i = 0; i < 4; i++) { 527 intel_uncore_write(uncore, PXW(i), 528 pxw[i * 4 + 0] << 24 | 529 pxw[i * 4 + 1] << 16 | 530 pxw[i * 4 + 2] << 8 | 531 pxw[i * 4 + 3] << 0); 532 } 533 534 /* Adjust magic regs to magic values (more experimental results) */ 535 intel_uncore_write(uncore, OGW0, 0); 536 intel_uncore_write(uncore, OGW1, 0); 537 intel_uncore_write(uncore, EG0, 0x00007f00); 538 intel_uncore_write(uncore, EG1, 0x0000000e); 539 intel_uncore_write(uncore, EG2, 0x000e0000); 540 intel_uncore_write(uncore, EG3, 0x68000300); 541 intel_uncore_write(uncore, EG4, 0x42000000); 542 intel_uncore_write(uncore, EG5, 0x00140031); 543 intel_uncore_write(uncore, EG6, 0); 544 intel_uncore_write(uncore, EG7, 0); 545 546 for (i = 0; i < 8; i++) 547 intel_uncore_write(uncore, PXWL(i), 0); 548 549 /* Enable PMON + select events */ 550 intel_uncore_write(uncore, ECR, 0x80000019); 551 552 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 553 } 554 555 static bool gen5_rps_enable(struct intel_rps *rps) 556 { 557 struct drm_i915_private *i915 = rps_to_i915(rps); 558 struct intel_display *display = i915->display; 559 struct intel_uncore *uncore = rps_to_uncore(rps); 560 u8 fstart, vstart; 561 u32 rgvmodectl; 562 563 spin_lock_irq(&mchdev_lock); 564 565 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 566 567 /* Enable temp reporting */ 568 intel_uncore_write16(uncore, PMMISC, 569 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 570 intel_uncore_write16(uncore, TSC1, 571 intel_uncore_read16(uncore, TSC1) | TSE); 572 573 /* 100ms RC evaluation intervals */ 574 intel_uncore_write(uncore, RCUPEI, 100000); 575 intel_uncore_write(uncore, RCDNEI, 100000); 576 577 /* Set max/min thresholds to 90ms and 80ms respectively */ 578 intel_uncore_write(uncore, RCBMAXAVG, 90000); 579 intel_uncore_write(uncore, RCBMINAVG, 80000); 580 581 intel_uncore_write(uncore, MEMIHYST, 1); 582 583 /* Set up min, max, and cur for interrupt handling */ 584 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 585 MEMMODE_FSTART_SHIFT; 586 587 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 588 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 589 590 intel_uncore_write(uncore, 591 MEMINTREN, 592 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 593 594 intel_uncore_write(uncore, VIDSTART, vstart); 595 intel_uncore_posting_read(uncore, VIDSTART); 596 597 rgvmodectl |= MEMMODE_SWMODE_EN; 598 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 599 600 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 601 MEMCTL_CMD_STS) == 0, 10)) 602 drm_err(&uncore->i915->drm, 603 "stuck trying to change perf mode\n"); 604 mdelay(1); 605 606 __gen5_rps_set(rps, rps->cur_freq); 607 608 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 609 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 610 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 611 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 612 613 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 614 rps->ips.last_time2 = ktime_get_raw_ns(); 615 616 ilk_display_rps_enable(display); 617 618 spin_unlock_irq(&mchdev_lock); 619 620 rps->ips.corr = init_emon(uncore); 621 622 return true; 623 } 624 625 static void gen5_rps_disable(struct intel_rps *rps) 626 { 627 struct drm_i915_private *i915 = rps_to_i915(rps); 628 struct intel_display *display = i915->display; 629 struct intel_uncore *uncore = rps_to_uncore(rps); 630 u16 rgvswctl; 631 632 spin_lock_irq(&mchdev_lock); 633 634 ilk_display_rps_disable(display); 635 636 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 637 638 /* Ack interrupts, disable EFC interrupt */ 639 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0); 640 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 641 642 /* Go back to the starting frequency */ 643 __gen5_rps_set(rps, rps->idle_freq); 644 mdelay(1); 645 rgvswctl |= MEMCTL_CMD_STS; 646 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 647 mdelay(1); 648 649 spin_unlock_irq(&mchdev_lock); 650 } 651 652 static u32 rps_limits(struct intel_rps *rps, u8 val) 653 { 654 u32 limits; 655 656 /* 657 * Only set the down limit when we've reached the lowest level to avoid 658 * getting more interrupts, otherwise leave this clear. This prevents a 659 * race in the hw when coming out of rc6: There's a tiny window where 660 * the hw runs at the minimal clock before selecting the desired 661 * frequency, if the down threshold expires in that window we will not 662 * receive a down interrupt. 663 */ 664 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 665 limits = rps->max_freq_softlimit << 23; 666 if (val <= rps->min_freq_softlimit) 667 limits |= rps->min_freq_softlimit << 14; 668 } else { 669 limits = rps->max_freq_softlimit << 24; 670 if (val <= rps->min_freq_softlimit) 671 limits |= rps->min_freq_softlimit << 16; 672 } 673 674 return limits; 675 } 676 677 static void rps_set_power(struct intel_rps *rps, int new_power) 678 { 679 struct intel_gt *gt = rps_to_gt(rps); 680 struct intel_uncore *uncore = gt->uncore; 681 u32 ei_up = 0, ei_down = 0; 682 683 lockdep_assert_held(&rps->power.mutex); 684 685 if (new_power == rps->power.mode) 686 return; 687 688 /* Note the units here are not exactly 1us, but 1280ns. */ 689 switch (new_power) { 690 case LOW_POWER: 691 ei_up = 16000; 692 ei_down = 32000; 693 break; 694 695 case BETWEEN: 696 ei_up = 13000; 697 ei_down = 32000; 698 break; 699 700 case HIGH_POWER: 701 ei_up = 10000; 702 ei_down = 32000; 703 break; 704 } 705 706 /* When byt can survive without system hang with dynamic 707 * sw freq adjustments, this restriction can be lifted. 708 */ 709 if (IS_VALLEYVIEW(gt->i915)) 710 goto skip_hw_write; 711 712 GT_TRACE(gt, 713 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 714 new_power, 715 rps->power.up_threshold, ei_up, 716 rps->power.down_threshold, ei_down); 717 718 set(uncore, GEN6_RP_UP_EI, 719 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 720 set(uncore, GEN6_RP_UP_THRESHOLD, 721 intel_gt_ns_to_pm_interval(gt, 722 ei_up * rps->power.up_threshold * 10)); 723 724 set(uncore, GEN6_RP_DOWN_EI, 725 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 726 set(uncore, GEN6_RP_DOWN_THRESHOLD, 727 intel_gt_ns_to_pm_interval(gt, 728 ei_down * 729 rps->power.down_threshold * 10)); 730 731 set(uncore, GEN6_RP_CONTROL, 732 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 733 GEN6_RP_MEDIA_HW_NORMAL_MODE | 734 GEN6_RP_MEDIA_IS_GFX | 735 GEN6_RP_ENABLE | 736 GEN6_RP_UP_BUSY_AVG | 737 GEN6_RP_DOWN_IDLE_AVG); 738 739 skip_hw_write: 740 rps->power.mode = new_power; 741 } 742 743 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 744 { 745 int new_power; 746 747 new_power = rps->power.mode; 748 switch (rps->power.mode) { 749 case LOW_POWER: 750 if (val > rps->efficient_freq + 1 && 751 val > rps->cur_freq) 752 new_power = BETWEEN; 753 break; 754 755 case BETWEEN: 756 if (val <= rps->efficient_freq && 757 val < rps->cur_freq) 758 new_power = LOW_POWER; 759 else if (val >= rps->rp0_freq && 760 val > rps->cur_freq) 761 new_power = HIGH_POWER; 762 break; 763 764 case HIGH_POWER: 765 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 766 val < rps->cur_freq) 767 new_power = BETWEEN; 768 break; 769 } 770 /* Max/min bins are special */ 771 if (val <= rps->min_freq_softlimit) 772 new_power = LOW_POWER; 773 if (val >= rps->max_freq_softlimit) 774 new_power = HIGH_POWER; 775 776 mutex_lock(&rps->power.mutex); 777 if (rps->power.interactive) 778 new_power = HIGH_POWER; 779 rps_set_power(rps, new_power); 780 mutex_unlock(&rps->power.mutex); 781 } 782 783 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 784 { 785 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", 786 str_yes_no(interactive)); 787 788 mutex_lock(&rps->power.mutex); 789 if (interactive) { 790 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 791 rps_set_power(rps, HIGH_POWER); 792 } else { 793 GEM_BUG_ON(!rps->power.interactive); 794 rps->power.interactive--; 795 } 796 mutex_unlock(&rps->power.mutex); 797 } 798 799 static int gen6_rps_set(struct intel_rps *rps, u8 val) 800 { 801 struct intel_uncore *uncore = rps_to_uncore(rps); 802 struct drm_i915_private *i915 = rps_to_i915(rps); 803 u32 swreq; 804 805 GEM_BUG_ON(rps_uses_slpc(rps)); 806 807 if (GRAPHICS_VER(i915) >= 9) 808 swreq = GEN9_FREQUENCY(val); 809 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 810 swreq = HSW_FREQUENCY(val); 811 else 812 swreq = (GEN6_FREQUENCY(val) | 813 GEN6_OFFSET(0) | 814 GEN6_AGGRESSIVE_TURBO); 815 set(uncore, GEN6_RPNSWREQ, swreq); 816 817 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 818 val, intel_gpu_freq(rps, val), swreq); 819 820 return 0; 821 } 822 823 static int vlv_rps_set(struct intel_rps *rps, u8 val) 824 { 825 struct drm_i915_private *i915 = rps_to_i915(rps); 826 int err; 827 828 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 829 err = vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_REQ, val); 830 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 831 832 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 833 val, intel_gpu_freq(rps, val)); 834 835 return err; 836 } 837 838 static int rps_set(struct intel_rps *rps, u8 val, bool update) 839 { 840 struct drm_i915_private *i915 = rps_to_i915(rps); 841 int err; 842 843 if (val == rps->last_freq) 844 return 0; 845 846 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 847 err = vlv_rps_set(rps, val); 848 else if (GRAPHICS_VER(i915) >= 6) 849 err = gen6_rps_set(rps, val); 850 else 851 err = gen5_rps_set(rps, val); 852 if (err) 853 return err; 854 855 if (update && GRAPHICS_VER(i915) >= 6) 856 gen6_rps_set_thresholds(rps, val); 857 rps->last_freq = val; 858 859 return 0; 860 } 861 862 void intel_rps_unpark(struct intel_rps *rps) 863 { 864 if (!intel_rps_is_enabled(rps)) 865 return; 866 867 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 868 869 /* 870 * Use the user's desired frequency as a guide, but for better 871 * performance, jump directly to RPe as our starting frequency. 872 */ 873 mutex_lock(&rps->lock); 874 875 intel_rps_set_active(rps); 876 intel_rps_set(rps, 877 clamp(rps->cur_freq, 878 rps->min_freq_softlimit, 879 rps->max_freq_softlimit)); 880 881 mutex_unlock(&rps->lock); 882 883 rps->pm_iir = 0; 884 if (intel_rps_has_interrupts(rps)) 885 rps_enable_interrupts(rps); 886 if (intel_rps_uses_timer(rps)) 887 rps_start_timer(rps); 888 889 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 890 gen5_rps_update(rps); 891 } 892 893 void intel_rps_park(struct intel_rps *rps) 894 { 895 int adj; 896 897 if (!intel_rps_is_enabled(rps)) 898 return; 899 900 if (!intel_rps_clear_active(rps)) 901 return; 902 903 if (intel_rps_uses_timer(rps)) 904 rps_stop_timer(rps); 905 if (intel_rps_has_interrupts(rps)) 906 rps_disable_interrupts(rps); 907 908 if (rps->last_freq <= rps->idle_freq) 909 return; 910 911 /* 912 * The punit delays the write of the frequency and voltage until it 913 * determines the GPU is awake. During normal usage we don't want to 914 * waste power changing the frequency if the GPU is sleeping (rc6). 915 * However, the GPU and driver is now idle and we do not want to delay 916 * switching to minimum voltage (reducing power whilst idle) as we do 917 * not expect to be woken in the near future and so must flush the 918 * change by waking the device. 919 * 920 * We choose to take the media powerwell (either would do to trick the 921 * punit into committing the voltage change) as that takes a lot less 922 * power than the render powerwell. 923 */ 924 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 925 rps_set(rps, rps->idle_freq, false); 926 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 927 928 /* 929 * Since we will try and restart from the previously requested 930 * frequency on unparking, treat this idle point as a downclock 931 * interrupt and reduce the frequency for resume. If we park/unpark 932 * more frequently than the rps worker can run, we will not respond 933 * to any EI and never see a change in frequency. 934 * 935 * (Note we accommodate Cherryview's limitation of only using an 936 * even bin by applying it to all.) 937 */ 938 adj = rps->last_adj; 939 if (adj < 0) 940 adj *= 2; 941 else /* CHV needs even encode values */ 942 adj = -2; 943 rps->last_adj = adj; 944 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 945 if (rps->cur_freq < rps->efficient_freq) { 946 rps->cur_freq = rps->efficient_freq; 947 rps->last_adj = 0; 948 } 949 950 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 951 } 952 953 u32 intel_rps_get_boost_frequency(struct intel_rps *rps) 954 { 955 struct intel_guc_slpc *slpc; 956 957 if (rps_uses_slpc(rps)) { 958 slpc = rps_to_slpc(rps); 959 960 return slpc->boost_freq; 961 } else { 962 return intel_gpu_freq(rps, rps->boost_freq); 963 } 964 } 965 966 static int rps_set_boost_freq(struct intel_rps *rps, u32 val) 967 { 968 bool boost = false; 969 970 /* Validate against (static) hardware limits */ 971 val = intel_freq_opcode(rps, val); 972 if (val < rps->min_freq || val > rps->max_freq) 973 return -EINVAL; 974 975 mutex_lock(&rps->lock); 976 if (val != rps->boost_freq) { 977 rps->boost_freq = val; 978 boost = atomic_read(&rps->num_waiters); 979 } 980 mutex_unlock(&rps->lock); 981 if (boost) 982 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 983 984 return 0; 985 } 986 987 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq) 988 { 989 struct intel_guc_slpc *slpc; 990 991 if (rps_uses_slpc(rps)) { 992 slpc = rps_to_slpc(rps); 993 994 return intel_guc_slpc_set_boost_freq(slpc, freq); 995 } else { 996 return rps_set_boost_freq(rps, freq); 997 } 998 } 999 1000 void intel_rps_dec_waiters(struct intel_rps *rps) 1001 { 1002 struct intel_guc_slpc *slpc; 1003 1004 if (rps_uses_slpc(rps)) { 1005 slpc = rps_to_slpc(rps); 1006 1007 /* Don't decrement num_waiters for req where increment was skipped */ 1008 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING) 1009 return; 1010 1011 intel_guc_slpc_dec_waiters(slpc); 1012 } else { 1013 atomic_dec(&rps->num_waiters); 1014 } 1015 } 1016 1017 void intel_rps_boost(struct i915_request *rq) 1018 { 1019 struct intel_guc_slpc *slpc; 1020 1021 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 1022 return; 1023 1024 /* Waitboost is not needed for contexts marked with a Freq hint */ 1025 if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags)) 1026 return; 1027 1028 /* Serializes with i915_request_retire() */ 1029 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 1030 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 1031 1032 if (rps_uses_slpc(rps)) { 1033 slpc = rps_to_slpc(rps); 1034 1035 /* Waitboost should not be done with power saving profile */ 1036 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING) 1037 return; 1038 1039 /* Return if old value is non zero */ 1040 if (!atomic_fetch_inc(&slpc->num_waiters)) { 1041 /* 1042 * Skip queuing boost work if frequency is already boosted, 1043 * but still increment num_waiters. 1044 */ 1045 if (slpc->min_freq_softlimit >= slpc->boost_freq) 1046 return; 1047 1048 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1049 rq->fence.context, rq->fence.seqno); 1050 queue_work(rps_to_gt(rps)->i915->unordered_wq, 1051 &slpc->boost_work); 1052 } 1053 1054 return; 1055 } 1056 1057 if (atomic_fetch_inc(&rps->num_waiters)) 1058 return; 1059 1060 if (!intel_rps_is_active(rps)) 1061 return; 1062 1063 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1064 rq->fence.context, rq->fence.seqno); 1065 1066 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 1067 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 1068 1069 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 1070 } 1071 } 1072 1073 int intel_rps_set(struct intel_rps *rps, u8 val) 1074 { 1075 int err; 1076 1077 lockdep_assert_held(&rps->lock); 1078 GEM_BUG_ON(val > rps->max_freq); 1079 GEM_BUG_ON(val < rps->min_freq); 1080 1081 if (intel_rps_is_active(rps)) { 1082 err = rps_set(rps, val, true); 1083 if (err) 1084 return err; 1085 1086 /* 1087 * Make sure we continue to get interrupts 1088 * until we hit the minimum or maximum frequencies. 1089 */ 1090 if (intel_rps_has_interrupts(rps)) { 1091 struct intel_uncore *uncore = rps_to_uncore(rps); 1092 1093 set(uncore, 1094 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 1095 1096 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 1097 } 1098 } 1099 1100 rps->cur_freq = val; 1101 return 0; 1102 } 1103 1104 static u32 intel_rps_read_state_cap(struct intel_rps *rps) 1105 { 1106 struct drm_i915_private *i915 = rps_to_i915(rps); 1107 struct intel_uncore *uncore = rps_to_uncore(rps); 1108 1109 if (IS_GEN9_LP(i915)) 1110 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 1111 else 1112 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 1113 } 1114 1115 static void 1116 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1117 { 1118 struct intel_uncore *uncore = rps_to_uncore(rps); 1119 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ? 1120 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) : 1121 intel_uncore_read(uncore, MTL_RP_STATE_CAP); 1122 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ? 1123 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) : 1124 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY); 1125 1126 /* MTL values are in units of 16.67 MHz */ 1127 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap); 1128 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap); 1129 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe); 1130 } 1131 1132 static void 1133 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1134 { 1135 struct drm_i915_private *i915 = rps_to_i915(rps); 1136 u32 rp_state_cap; 1137 1138 rp_state_cap = intel_rps_read_state_cap(rps); 1139 1140 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1141 if (IS_GEN9_LP(i915)) { 1142 caps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1143 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1144 caps->min_freq = (rp_state_cap >> 0) & 0xff; 1145 } else { 1146 caps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1147 if (GRAPHICS_VER(i915) >= 10) 1148 caps->rp1_freq = REG_FIELD_GET(RPE_MASK, 1149 intel_uncore_read(to_gt(i915)->uncore, 1150 GEN10_FREQ_INFO_REC)); 1151 else 1152 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1153 caps->min_freq = (rp_state_cap >> 16) & 0xff; 1154 } 1155 1156 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1157 /* 1158 * In this case rp_state_cap register reports frequencies in 1159 * units of 50 MHz. Convert these to the actual "hw unit", i.e. 1160 * units of 16.67 MHz 1161 */ 1162 caps->rp0_freq *= GEN9_FREQ_SCALER; 1163 caps->rp1_freq *= GEN9_FREQ_SCALER; 1164 caps->min_freq *= GEN9_FREQ_SCALER; 1165 } 1166 } 1167 1168 /** 1169 * gen6_rps_get_freq_caps - Get freq caps exposed by HW 1170 * @rps: the intel_rps structure 1171 * @caps: returned freq caps 1172 * 1173 * Returned "caps" frequencies should be converted to MHz using 1174 * intel_gpu_freq() 1175 */ 1176 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1177 { 1178 struct drm_i915_private *i915 = rps_to_i915(rps); 1179 1180 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 1181 return mtl_get_freq_caps(rps, caps); 1182 else 1183 return __gen6_rps_get_freq_caps(rps, caps); 1184 } 1185 1186 static void gen6_rps_init(struct intel_rps *rps) 1187 { 1188 struct drm_i915_private *i915 = rps_to_i915(rps); 1189 struct intel_rps_freq_caps caps; 1190 1191 gen6_rps_get_freq_caps(rps, &caps); 1192 rps->rp0_freq = caps.rp0_freq; 1193 rps->rp1_freq = caps.rp1_freq; 1194 rps->min_freq = caps.min_freq; 1195 1196 /* hw_max = RP0 until we check for overclocking */ 1197 rps->max_freq = rps->rp0_freq; 1198 1199 rps->efficient_freq = rps->rp1_freq; 1200 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1201 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1202 u32 ddcc_status = 0; 1203 u32 mult = 1; 1204 1205 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) 1206 mult = GEN9_FREQ_SCALER; 1207 if (snb_pcode_read(rps_to_gt(rps)->uncore, 1208 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1209 &ddcc_status, NULL) == 0) 1210 rps->efficient_freq = 1211 clamp_t(u32, 1212 ((ddcc_status >> 8) & 0xff) * mult, 1213 rps->min_freq, 1214 rps->max_freq); 1215 } 1216 } 1217 1218 static bool rps_reset(struct intel_rps *rps) 1219 { 1220 struct drm_i915_private *i915 = rps_to_i915(rps); 1221 1222 /* force a reset */ 1223 rps->power.mode = -1; 1224 rps->last_freq = -1; 1225 1226 if (rps_set(rps, rps->min_freq, true)) { 1227 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1228 return false; 1229 } 1230 1231 rps->cur_freq = rps->min_freq; 1232 return true; 1233 } 1234 1235 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1236 static bool gen9_rps_enable(struct intel_rps *rps) 1237 { 1238 struct intel_gt *gt = rps_to_gt(rps); 1239 struct intel_uncore *uncore = gt->uncore; 1240 1241 /* Program defaults and thresholds for RPS */ 1242 if (GRAPHICS_VER(gt->i915) == 9) 1243 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1244 GEN9_FREQUENCY(rps->rp1_freq)); 1245 1246 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1247 1248 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1249 1250 return rps_reset(rps); 1251 } 1252 1253 static bool gen8_rps_enable(struct intel_rps *rps) 1254 { 1255 struct intel_uncore *uncore = rps_to_uncore(rps); 1256 1257 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1258 HSW_FREQUENCY(rps->rp1_freq)); 1259 1260 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1261 1262 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1263 1264 return rps_reset(rps); 1265 } 1266 1267 static bool gen6_rps_enable(struct intel_rps *rps) 1268 { 1269 struct intel_uncore *uncore = rps_to_uncore(rps); 1270 1271 /* Power down if completely idle for over 50ms */ 1272 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1273 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1274 1275 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1276 GEN6_PM_RP_DOWN_THRESHOLD | 1277 GEN6_PM_RP_DOWN_TIMEOUT); 1278 1279 return rps_reset(rps); 1280 } 1281 1282 static int chv_rps_max_freq(struct intel_rps *rps) 1283 { 1284 struct drm_i915_private *i915 = rps_to_i915(rps); 1285 struct intel_gt *gt = rps_to_gt(rps); 1286 u32 val; 1287 1288 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE); 1289 1290 switch (gt->info.sseu.eu_total) { 1291 case 8: 1292 /* (2 * 4) config */ 1293 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1294 break; 1295 case 12: 1296 /* (2 * 6) config */ 1297 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1298 break; 1299 case 16: 1300 /* (2 * 8) config */ 1301 default: 1302 /* Setting (2 * 8) Min RP0 for any other combination */ 1303 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1304 break; 1305 } 1306 1307 return val & FB_GFX_FREQ_FUSE_MASK; 1308 } 1309 1310 static int chv_rps_rpe_freq(struct intel_rps *rps) 1311 { 1312 struct drm_i915_private *i915 = rps_to_i915(rps); 1313 u32 val; 1314 1315 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_GPU_DUTYCYCLE_REG); 1316 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1317 1318 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1319 } 1320 1321 static int chv_rps_guar_freq(struct intel_rps *rps) 1322 { 1323 struct drm_i915_private *i915 = rps_to_i915(rps); 1324 u32 val; 1325 1326 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE); 1327 1328 return val & FB_GFX_FREQ_FUSE_MASK; 1329 } 1330 1331 static u32 chv_rps_min_freq(struct intel_rps *rps) 1332 { 1333 struct drm_i915_private *i915 = rps_to_i915(rps); 1334 u32 val; 1335 1336 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMIN_AT_VMIN_FUSE); 1337 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1338 1339 return val & FB_GFX_FREQ_FUSE_MASK; 1340 } 1341 1342 static bool chv_rps_enable(struct intel_rps *rps) 1343 { 1344 struct intel_uncore *uncore = rps_to_uncore(rps); 1345 struct drm_i915_private *i915 = rps_to_i915(rps); 1346 u32 val; 1347 1348 /* 1: Program defaults and thresholds for RPS*/ 1349 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1350 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1351 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1352 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1353 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1354 1355 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1356 1357 /* 2: Enable RPS */ 1358 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1359 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1360 GEN6_RP_MEDIA_IS_GFX | 1361 GEN6_RP_ENABLE | 1362 GEN6_RP_UP_BUSY_AVG | 1363 GEN6_RP_DOWN_IDLE_AVG); 1364 1365 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1366 GEN6_PM_RP_DOWN_THRESHOLD | 1367 GEN6_PM_RP_DOWN_TIMEOUT); 1368 1369 /* Setting Fixed Bias */ 1370 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1371 1372 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1373 vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val); 1374 1375 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 1376 1377 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1378 1379 /* RPS code assumes GPLL is used */ 1380 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1381 "GPLL not enabled\n"); 1382 1383 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1384 str_yes_no(val & GPLLENABLE)); 1385 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1386 1387 return rps_reset(rps); 1388 } 1389 1390 static int vlv_rps_guar_freq(struct intel_rps *rps) 1391 { 1392 struct drm_i915_private *i915 = rps_to_i915(rps); 1393 u32 val, rp1; 1394 1395 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE); 1396 1397 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1398 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1399 1400 return rp1; 1401 } 1402 1403 static int vlv_rps_max_freq(struct intel_rps *rps) 1404 { 1405 struct drm_i915_private *i915 = rps_to_i915(rps); 1406 u32 val, rp0; 1407 1408 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE); 1409 1410 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1411 /* Clamp to max */ 1412 rp0 = min_t(u32, rp0, 0xea); 1413 1414 return rp0; 1415 } 1416 1417 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1418 { 1419 struct drm_i915_private *i915 = rps_to_i915(rps); 1420 u32 val, rpe; 1421 1422 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1423 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1424 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1425 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1426 1427 return rpe; 1428 } 1429 1430 static int vlv_rps_min_freq(struct intel_rps *rps) 1431 { 1432 struct drm_i915_private *i915 = rps_to_i915(rps); 1433 u32 val; 1434 1435 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_LFM) & 0xff; 1436 /* 1437 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1438 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1439 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1440 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1441 * to make sure it matches what Punit accepts. 1442 */ 1443 return max_t(u32, val, 0xc0); 1444 } 1445 1446 static bool vlv_rps_enable(struct intel_rps *rps) 1447 { 1448 struct intel_uncore *uncore = rps_to_uncore(rps); 1449 struct drm_i915_private *i915 = rps_to_i915(rps); 1450 u32 val; 1451 1452 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1453 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1454 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1455 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1456 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1457 1458 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1459 1460 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1461 GEN6_RP_MEDIA_TURBO | 1462 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1463 GEN6_RP_MEDIA_IS_GFX | 1464 GEN6_RP_ENABLE | 1465 GEN6_RP_UP_BUSY_AVG | 1466 GEN6_RP_DOWN_IDLE_CONT); 1467 1468 /* WaGsvRC0ResidencyMethod:vlv */ 1469 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1470 1471 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1472 1473 /* Setting Fixed Bias */ 1474 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1475 vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val); 1476 1477 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 1478 1479 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1480 1481 /* RPS code assumes GPLL is used */ 1482 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1483 "GPLL not enabled\n"); 1484 1485 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1486 str_yes_no(val & GPLLENABLE)); 1487 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1488 1489 return rps_reset(rps); 1490 } 1491 1492 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1493 { 1494 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1495 struct intel_uncore *uncore = rps_to_uncore(rps); 1496 unsigned int t, state1, state2; 1497 u32 pxvid, ext_v; 1498 u64 corr, corr2; 1499 1500 lockdep_assert_held(&mchdev_lock); 1501 1502 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1503 pxvid = (pxvid >> 24) & 0x7f; 1504 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1505 1506 state1 = ext_v; 1507 1508 /* Revel in the empirically derived constants */ 1509 1510 /* Correction factor in 1/100000 units */ 1511 t = ips_mch_val(uncore); 1512 if (t > 80) 1513 corr = t * 2349 + 135940; 1514 else if (t >= 50) 1515 corr = t * 964 + 29317; 1516 else /* < 50 */ 1517 corr = t * 301 + 1004; 1518 1519 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1520 corr2 = div_u64(corr, 100000) * ips->corr; 1521 1522 state2 = div_u64(corr2 * state1, 10000); 1523 state2 /= 100; /* convert to mW */ 1524 1525 __gen5_ips_update(ips); 1526 1527 return ips->gfx_power + state2; 1528 } 1529 1530 static bool has_busy_stats(struct intel_rps *rps) 1531 { 1532 struct intel_engine_cs *engine; 1533 enum intel_engine_id id; 1534 1535 for_each_engine(engine, rps_to_gt(rps), id) { 1536 if (!intel_engine_supports_stats(engine)) 1537 return false; 1538 } 1539 1540 return true; 1541 } 1542 1543 void intel_rps_enable(struct intel_rps *rps) 1544 { 1545 struct drm_i915_private *i915 = rps_to_i915(rps); 1546 struct intel_uncore *uncore = rps_to_uncore(rps); 1547 bool enabled = false; 1548 1549 if (!HAS_RPS(i915)) 1550 return; 1551 1552 if (rps_uses_slpc(rps)) 1553 return; 1554 1555 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1556 1557 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1558 if (rps->max_freq <= rps->min_freq) 1559 /* leave disabled, no room for dynamic reclocking */; 1560 else if (IS_CHERRYVIEW(i915)) 1561 enabled = chv_rps_enable(rps); 1562 else if (IS_VALLEYVIEW(i915)) 1563 enabled = vlv_rps_enable(rps); 1564 else if (GRAPHICS_VER(i915) >= 9) 1565 enabled = gen9_rps_enable(rps); 1566 else if (GRAPHICS_VER(i915) >= 8) 1567 enabled = gen8_rps_enable(rps); 1568 else if (GRAPHICS_VER(i915) >= 6) 1569 enabled = gen6_rps_enable(rps); 1570 else if (IS_IRONLAKE_M(i915)) 1571 enabled = gen5_rps_enable(rps); 1572 else 1573 MISSING_CASE(GRAPHICS_VER(i915)); 1574 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1575 if (!enabled) 1576 return; 1577 1578 GT_TRACE(rps_to_gt(rps), 1579 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n", 1580 rps->min_freq, rps->max_freq, 1581 intel_gpu_freq(rps, rps->min_freq), 1582 intel_gpu_freq(rps, rps->max_freq), 1583 rps->power.up_threshold, 1584 rps->power.down_threshold); 1585 1586 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1587 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1588 1589 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1590 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1591 1592 if (has_busy_stats(rps)) 1593 intel_rps_set_timer(rps); 1594 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) 1595 intel_rps_set_interrupts(rps); 1596 else 1597 /* Ironlake currently uses intel_ips.ko */ {} 1598 1599 intel_rps_set_enabled(rps); 1600 } 1601 1602 static void gen6_rps_disable(struct intel_rps *rps) 1603 { 1604 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1605 } 1606 1607 void intel_rps_disable(struct intel_rps *rps) 1608 { 1609 struct drm_i915_private *i915 = rps_to_i915(rps); 1610 1611 if (!intel_rps_is_enabled(rps)) 1612 return; 1613 1614 intel_rps_clear_enabled(rps); 1615 intel_rps_clear_interrupts(rps); 1616 intel_rps_clear_timer(rps); 1617 1618 if (GRAPHICS_VER(i915) >= 6) 1619 gen6_rps_disable(rps); 1620 else if (IS_IRONLAKE_M(i915)) 1621 gen5_rps_disable(rps); 1622 } 1623 1624 static int byt_gpu_freq(struct intel_rps *rps, int val) 1625 { 1626 /* 1627 * N = val - 0xb7 1628 * Slow = Fast = GPLL ref * N 1629 */ 1630 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1631 } 1632 1633 static int byt_freq_opcode(struct intel_rps *rps, int val) 1634 { 1635 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1636 } 1637 1638 static int chv_gpu_freq(struct intel_rps *rps, int val) 1639 { 1640 /* 1641 * N = val / 2 1642 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1643 */ 1644 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1645 } 1646 1647 static int chv_freq_opcode(struct intel_rps *rps, int val) 1648 { 1649 /* CHV needs even values */ 1650 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1651 } 1652 1653 int intel_gpu_freq(struct intel_rps *rps, int val) 1654 { 1655 struct drm_i915_private *i915 = rps_to_i915(rps); 1656 1657 if (GRAPHICS_VER(i915) >= 9) 1658 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1659 GEN9_FREQ_SCALER); 1660 else if (IS_CHERRYVIEW(i915)) 1661 return chv_gpu_freq(rps, val); 1662 else if (IS_VALLEYVIEW(i915)) 1663 return byt_gpu_freq(rps, val); 1664 else if (GRAPHICS_VER(i915) >= 6) 1665 return val * GT_FREQUENCY_MULTIPLIER; 1666 else 1667 return val; 1668 } 1669 1670 int intel_freq_opcode(struct intel_rps *rps, int val) 1671 { 1672 struct drm_i915_private *i915 = rps_to_i915(rps); 1673 1674 if (GRAPHICS_VER(i915) >= 9) 1675 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1676 GT_FREQUENCY_MULTIPLIER); 1677 else if (IS_CHERRYVIEW(i915)) 1678 return chv_freq_opcode(rps, val); 1679 else if (IS_VALLEYVIEW(i915)) 1680 return byt_freq_opcode(rps, val); 1681 else if (GRAPHICS_VER(i915) >= 6) 1682 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1683 else 1684 return val; 1685 } 1686 1687 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1688 { 1689 struct drm_i915_private *i915 = rps_to_i915(rps); 1690 1691 rps->gpll_ref_freq = 1692 vlv_get_cck_clock(&i915->drm, "GPLL ref", 1693 CCK_GPLL_CLOCK_CONTROL, 1694 i915->czclk_freq); 1695 1696 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1697 rps->gpll_ref_freq); 1698 } 1699 1700 static void vlv_rps_init(struct intel_rps *rps) 1701 { 1702 struct drm_i915_private *i915 = rps_to_i915(rps); 1703 1704 vlv_iosf_sb_get(&i915->drm, 1705 BIT(VLV_IOSF_SB_PUNIT) | 1706 BIT(VLV_IOSF_SB_NC) | 1707 BIT(VLV_IOSF_SB_CCK)); 1708 1709 vlv_init_gpll_ref_freq(rps); 1710 1711 rps->max_freq = vlv_rps_max_freq(rps); 1712 rps->rp0_freq = rps->max_freq; 1713 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1714 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1715 1716 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1717 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1718 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1719 1720 rps->rp1_freq = vlv_rps_guar_freq(rps); 1721 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1722 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1723 1724 rps->min_freq = vlv_rps_min_freq(rps); 1725 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1726 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1727 1728 vlv_iosf_sb_put(&i915->drm, 1729 BIT(VLV_IOSF_SB_PUNIT) | 1730 BIT(VLV_IOSF_SB_NC) | 1731 BIT(VLV_IOSF_SB_CCK)); 1732 } 1733 1734 static void chv_rps_init(struct intel_rps *rps) 1735 { 1736 struct drm_i915_private *i915 = rps_to_i915(rps); 1737 1738 vlv_iosf_sb_get(&i915->drm, 1739 BIT(VLV_IOSF_SB_PUNIT) | 1740 BIT(VLV_IOSF_SB_NC) | 1741 BIT(VLV_IOSF_SB_CCK)); 1742 1743 vlv_init_gpll_ref_freq(rps); 1744 1745 rps->max_freq = chv_rps_max_freq(rps); 1746 rps->rp0_freq = rps->max_freq; 1747 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1748 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1749 1750 rps->efficient_freq = chv_rps_rpe_freq(rps); 1751 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1752 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1753 1754 rps->rp1_freq = chv_rps_guar_freq(rps); 1755 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1756 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1757 1758 rps->min_freq = chv_rps_min_freq(rps); 1759 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1760 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1761 1762 vlv_iosf_sb_put(&i915->drm, 1763 BIT(VLV_IOSF_SB_PUNIT) | 1764 BIT(VLV_IOSF_SB_NC) | 1765 BIT(VLV_IOSF_SB_CCK)); 1766 1767 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1768 rps->rp1_freq | rps->min_freq) & 1, 1769 "Odd GPU freq values\n"); 1770 } 1771 1772 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1773 { 1774 ei->ktime = ktime_get_raw(); 1775 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1776 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1777 } 1778 1779 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1780 { 1781 struct intel_uncore *uncore = rps_to_uncore(rps); 1782 const struct intel_rps_ei *prev = &rps->ei; 1783 struct intel_rps_ei now; 1784 u32 events = 0; 1785 1786 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1787 return 0; 1788 1789 vlv_c0_read(uncore, &now); 1790 1791 if (prev->ktime) { 1792 u64 time, c0; 1793 u32 render, media; 1794 1795 time = ktime_us_delta(now.ktime, prev->ktime); 1796 1797 time *= rps_to_i915(rps)->czclk_freq; 1798 1799 /* Workload can be split between render + media, 1800 * e.g. SwapBuffers being blitted in X after being rendered in 1801 * mesa. To account for this we need to combine both engines 1802 * into our activity counter. 1803 */ 1804 render = now.render_c0 - prev->render_c0; 1805 media = now.media_c0 - prev->media_c0; 1806 c0 = max(render, media); 1807 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1808 1809 if (c0 > time * rps->power.up_threshold) 1810 events = GEN6_PM_RP_UP_THRESHOLD; 1811 else if (c0 < time * rps->power.down_threshold) 1812 events = GEN6_PM_RP_DOWN_THRESHOLD; 1813 } 1814 1815 rps->ei = now; 1816 return events; 1817 } 1818 1819 static void rps_work(struct work_struct *work) 1820 { 1821 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1822 struct intel_gt *gt = rps_to_gt(rps); 1823 struct drm_i915_private *i915 = rps_to_i915(rps); 1824 bool client_boost = false; 1825 int new_freq, adj, min, max; 1826 u32 pm_iir = 0; 1827 1828 spin_lock_irq(gt->irq_lock); 1829 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1830 client_boost = atomic_read(&rps->num_waiters); 1831 spin_unlock_irq(gt->irq_lock); 1832 1833 /* Make sure we didn't queue anything we're not going to process. */ 1834 if (!pm_iir && !client_boost) 1835 goto out; 1836 1837 mutex_lock(&rps->lock); 1838 if (!intel_rps_is_active(rps)) { 1839 mutex_unlock(&rps->lock); 1840 return; 1841 } 1842 1843 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1844 1845 adj = rps->last_adj; 1846 new_freq = rps->cur_freq; 1847 min = rps->min_freq_softlimit; 1848 max = rps->max_freq_softlimit; 1849 if (client_boost) 1850 max = rps->max_freq; 1851 1852 GT_TRACE(gt, 1853 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1854 pm_iir, str_yes_no(client_boost), 1855 adj, new_freq, min, max); 1856 1857 if (client_boost && new_freq < rps->boost_freq) { 1858 new_freq = rps->boost_freq; 1859 adj = 0; 1860 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1861 if (adj > 0) 1862 adj *= 2; 1863 else /* CHV needs even encode values */ 1864 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1865 1866 if (new_freq >= rps->max_freq_softlimit) 1867 adj = 0; 1868 } else if (client_boost) { 1869 adj = 0; 1870 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1871 if (rps->cur_freq > rps->efficient_freq) 1872 new_freq = rps->efficient_freq; 1873 else if (rps->cur_freq > rps->min_freq_softlimit) 1874 new_freq = rps->min_freq_softlimit; 1875 adj = 0; 1876 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1877 if (adj < 0) 1878 adj *= 2; 1879 else /* CHV needs even encode values */ 1880 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1881 1882 if (new_freq <= rps->min_freq_softlimit) 1883 adj = 0; 1884 } else { /* unknown event */ 1885 adj = 0; 1886 } 1887 1888 /* 1889 * sysfs frequency limits may have snuck in while 1890 * servicing the interrupt 1891 */ 1892 new_freq += adj; 1893 new_freq = clamp_t(int, new_freq, min, max); 1894 1895 if (intel_rps_set(rps, new_freq)) { 1896 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1897 adj = 0; 1898 } 1899 rps->last_adj = adj; 1900 1901 mutex_unlock(&rps->lock); 1902 1903 out: 1904 spin_lock_irq(gt->irq_lock); 1905 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1906 spin_unlock_irq(gt->irq_lock); 1907 } 1908 1909 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1910 { 1911 struct intel_gt *gt = rps_to_gt(rps); 1912 const u32 events = rps->pm_events & pm_iir; 1913 1914 lockdep_assert_held(gt->irq_lock); 1915 1916 if (unlikely(!events)) 1917 return; 1918 1919 GT_TRACE(gt, "irq events:%x\n", events); 1920 1921 gen6_gt_pm_mask_irq(gt, events); 1922 1923 rps->pm_iir |= events; 1924 queue_work(gt->i915->unordered_wq, &rps->work); 1925 } 1926 1927 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1928 { 1929 struct intel_gt *gt = rps_to_gt(rps); 1930 u32 events; 1931 1932 events = pm_iir & rps->pm_events; 1933 if (events) { 1934 spin_lock(gt->irq_lock); 1935 1936 GT_TRACE(gt, "irq events:%x\n", events); 1937 1938 gen6_gt_pm_mask_irq(gt, events); 1939 rps->pm_iir |= events; 1940 1941 queue_work(gt->i915->unordered_wq, &rps->work); 1942 spin_unlock(gt->irq_lock); 1943 } 1944 1945 if (GRAPHICS_VER(gt->i915) >= 8) 1946 return; 1947 1948 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1949 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1950 1951 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1952 drm_dbg(&rps_to_i915(rps)->drm, 1953 "Command parser error, pm_iir 0x%08x\n", pm_iir); 1954 } 1955 1956 void gen5_rps_irq_handler(struct intel_rps *rps) 1957 { 1958 struct intel_uncore *uncore = rps_to_uncore(rps); 1959 u32 busy_up, busy_down, max_avg, min_avg; 1960 u8 new_freq; 1961 1962 spin_lock(&mchdev_lock); 1963 1964 intel_uncore_write16(uncore, 1965 MEMINTRSTS, 1966 intel_uncore_read(uncore, MEMINTRSTS)); 1967 1968 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1969 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1970 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1971 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1972 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1973 1974 /* Handle RCS change request from hw */ 1975 new_freq = rps->cur_freq; 1976 if (busy_up > max_avg) 1977 new_freq++; 1978 else if (busy_down < min_avg) 1979 new_freq--; 1980 new_freq = clamp(new_freq, 1981 rps->min_freq_softlimit, 1982 rps->max_freq_softlimit); 1983 1984 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1985 rps->cur_freq = new_freq; 1986 1987 spin_unlock(&mchdev_lock); 1988 } 1989 1990 void intel_rps_init_early(struct intel_rps *rps) 1991 { 1992 mutex_init(&rps->lock); 1993 mutex_init(&rps->power.mutex); 1994 1995 INIT_WORK(&rps->work, rps_work); 1996 timer_setup(&rps->timer, rps_timer, 0); 1997 1998 atomic_set(&rps->num_waiters, 0); 1999 } 2000 2001 void intel_rps_init(struct intel_rps *rps) 2002 { 2003 struct drm_i915_private *i915 = rps_to_i915(rps); 2004 2005 if (rps_uses_slpc(rps)) 2006 return; 2007 2008 if (IS_CHERRYVIEW(i915)) 2009 chv_rps_init(rps); 2010 else if (IS_VALLEYVIEW(i915)) 2011 vlv_rps_init(rps); 2012 else if (GRAPHICS_VER(i915) >= 6) 2013 gen6_rps_init(rps); 2014 else if (IS_IRONLAKE_M(i915)) 2015 gen5_rps_init(rps); 2016 2017 /* Derive initial user preferences/limits from the hardware limits */ 2018 rps->max_freq_softlimit = rps->max_freq; 2019 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit; 2020 rps->min_freq_softlimit = rps->min_freq; 2021 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit; 2022 2023 /* After setting max-softlimit, find the overclock max freq */ 2024 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 2025 u32 params = 0; 2026 2027 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL); 2028 if (params & BIT(31)) { /* OC supported */ 2029 drm_dbg(&i915->drm, 2030 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 2031 (rps->max_freq & 0xff) * 50, 2032 (params & 0xff) * 50); 2033 rps->max_freq = params & 0xff; 2034 } 2035 } 2036 2037 /* Set default thresholds in % */ 2038 rps->power.up_threshold = 95; 2039 rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold; 2040 rps->power.down_threshold = 85; 2041 rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold; 2042 2043 /* Finally allow us to boost to max by default */ 2044 rps->boost_freq = rps->max_freq; 2045 rps->idle_freq = rps->min_freq; 2046 2047 /* Start in the middle, from here we will autotune based on workload */ 2048 rps->cur_freq = rps->efficient_freq; 2049 2050 rps->pm_intrmsk_mbz = 0; 2051 2052 /* 2053 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 2054 * if GEN6_PM_UP_EI_EXPIRED is masked. 2055 * 2056 * TODO: verify if this can be reproduced on VLV,CHV. 2057 */ 2058 if (GRAPHICS_VER(i915) <= 7) 2059 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 2060 2061 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 2062 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 2063 2064 /* GuC needs ARAT expired interrupt unmasked */ 2065 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 2066 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 2067 } 2068 2069 void intel_rps_sanitize(struct intel_rps *rps) 2070 { 2071 if (rps_uses_slpc(rps)) 2072 return; 2073 2074 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 2075 rps_disable_interrupts(rps); 2076 } 2077 2078 u32 intel_rps_read_rpstat(struct intel_rps *rps) 2079 { 2080 struct drm_i915_private *i915 = rps_to_i915(rps); 2081 i915_reg_t rpstat; 2082 2083 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1; 2084 2085 return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat); 2086 } 2087 2088 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 2089 { 2090 struct drm_i915_private *i915 = rps_to_i915(rps); 2091 u32 cagf; 2092 2093 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 2094 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat); 2095 else if (GRAPHICS_VER(i915) >= 12) 2096 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat); 2097 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 2098 cagf = REG_FIELD_GET(RPE_MASK, rpstat); 2099 else if (GRAPHICS_VER(i915) >= 9) 2100 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat); 2101 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2102 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat); 2103 else if (GRAPHICS_VER(i915) >= 6) 2104 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat); 2105 else 2106 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat)); 2107 2108 return cagf; 2109 } 2110 2111 static u32 __read_cagf(struct intel_rps *rps, bool take_fw) 2112 { 2113 struct drm_i915_private *i915 = rps_to_i915(rps); 2114 struct intel_uncore *uncore = rps_to_uncore(rps); 2115 i915_reg_t r = INVALID_MMIO_REG; 2116 u32 freq; 2117 2118 /* 2119 * For Gen12+ reading freq from HW does not need a forcewake and 2120 * registers will return 0 freq when GT is in RC6 2121 */ 2122 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 2123 r = MTL_MIRROR_TARGET_WP1; 2124 } else if (GRAPHICS_VER(i915) >= 12) { 2125 r = GEN12_RPSTAT1; 2126 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2127 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 2128 freq = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 2129 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 2130 } else if (GRAPHICS_VER(i915) >= 6) { 2131 r = GEN6_RPSTAT1; 2132 } else { 2133 r = MEMSTAT_ILK; 2134 } 2135 2136 if (i915_mmio_reg_valid(r)) 2137 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r); 2138 2139 return intel_rps_get_cagf(rps, freq); 2140 } 2141 2142 static u32 read_cagf(struct intel_rps *rps) 2143 { 2144 return __read_cagf(rps, true); 2145 } 2146 2147 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 2148 { 2149 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2150 intel_wakeref_t wakeref; 2151 u32 freq = 0; 2152 2153 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2154 freq = intel_gpu_freq(rps, read_cagf(rps)); 2155 2156 return freq; 2157 } 2158 2159 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps) 2160 { 2161 return intel_gpu_freq(rps, __read_cagf(rps, false)); 2162 } 2163 2164 static u32 intel_rps_read_punit_req(struct intel_rps *rps) 2165 { 2166 struct intel_uncore *uncore = rps_to_uncore(rps); 2167 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2168 intel_wakeref_t wakeref; 2169 u32 freq = 0; 2170 2171 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2172 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2173 2174 return freq; 2175 } 2176 2177 static u32 intel_rps_get_req(u32 pureq) 2178 { 2179 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 2180 2181 return req; 2182 } 2183 2184 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 2185 { 2186 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 2187 2188 return intel_gpu_freq(rps, freq); 2189 } 2190 2191 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 2192 { 2193 if (rps_uses_slpc(rps)) 2194 return intel_rps_read_punit_req_frequency(rps); 2195 else 2196 return intel_gpu_freq(rps, rps->cur_freq); 2197 } 2198 2199 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2200 { 2201 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2202 2203 if (rps_uses_slpc(rps)) 2204 return slpc->max_freq_softlimit; 2205 else 2206 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2207 } 2208 2209 /** 2210 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format. 2211 * @rps: the intel_rps structure 2212 * 2213 * Returns the max frequency in a raw format. In newer platforms raw is in 2214 * units of 50 MHz. 2215 */ 2216 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps) 2217 { 2218 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2219 u32 freq; 2220 2221 if (rps_uses_slpc(rps)) { 2222 return DIV_ROUND_CLOSEST(slpc->rp0_freq, 2223 GT_FREQUENCY_MULTIPLIER); 2224 } else { 2225 freq = rps->max_freq; 2226 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2227 /* Convert GT frequency to 50 MHz units */ 2228 freq /= GEN9_FREQ_SCALER; 2229 } 2230 return freq; 2231 } 2232 } 2233 2234 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2235 { 2236 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2237 2238 if (rps_uses_slpc(rps)) 2239 return slpc->rp0_freq; 2240 else 2241 return intel_gpu_freq(rps, rps->rp0_freq); 2242 } 2243 2244 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2245 { 2246 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2247 2248 if (rps_uses_slpc(rps)) 2249 return slpc->rp1_freq; 2250 else 2251 return intel_gpu_freq(rps, rps->rp1_freq); 2252 } 2253 2254 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2255 { 2256 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2257 2258 if (rps_uses_slpc(rps)) 2259 return slpc->min_freq; 2260 else 2261 return intel_gpu_freq(rps, rps->min_freq); 2262 } 2263 2264 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2265 { 2266 struct intel_gt *gt = rps_to_gt(rps); 2267 struct drm_i915_private *i915 = gt->i915; 2268 struct intel_uncore *uncore = gt->uncore; 2269 struct intel_rps_freq_caps caps; 2270 u32 rp_state_limits; 2271 u32 gt_perf_status; 2272 u32 rpmodectl, rpinclimit, rpdeclimit; 2273 u32 rpstat, cagf, reqf; 2274 u32 rpcurupei, rpcurup, rpprevup; 2275 u32 rpcurdownei, rpcurdown, rpprevdown; 2276 u32 rpupei, rpupt, rpdownei, rpdownt; 2277 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; 2278 2279 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS); 2280 gen6_rps_get_freq_caps(rps, &caps); 2281 if (IS_GEN9_LP(i915)) 2282 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS); 2283 else 2284 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS); 2285 2286 /* RPSTAT1 is in the GT power well */ 2287 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 2288 2289 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2290 if (GRAPHICS_VER(i915) >= 9) { 2291 reqf >>= 23; 2292 } else { 2293 reqf &= ~GEN6_TURBO_DISABLE; 2294 if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2295 reqf >>= 24; 2296 else 2297 reqf >>= 25; 2298 } 2299 reqf = intel_gpu_freq(rps, reqf); 2300 2301 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL); 2302 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2303 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2304 2305 rpstat = intel_rps_read_rpstat(rps); 2306 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; 2307 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK; 2308 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK; 2309 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; 2310 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; 2311 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; 2312 2313 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI); 2314 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2315 2316 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI); 2317 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2318 2319 cagf = intel_rps_read_actual_frequency(rps); 2320 2321 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 2322 2323 if (GRAPHICS_VER(i915) >= 11) { 2324 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE); 2325 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK); 2326 /* 2327 * The equivalent to the PM ISR & IIR cannot be read 2328 * without affecting the current state of the system 2329 */ 2330 pm_isr = 0; 2331 pm_iir = 0; 2332 } else if (GRAPHICS_VER(i915) >= 8) { 2333 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2)); 2334 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2)); 2335 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2)); 2336 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2)); 2337 } else { 2338 pm_ier = intel_uncore_read(uncore, GEN6_PMIER); 2339 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR); 2340 pm_isr = intel_uncore_read(uncore, GEN6_PMISR); 2341 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR); 2342 } 2343 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2344 2345 drm_printf(p, "Video Turbo Mode: %s\n", 2346 str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO)); 2347 drm_printf(p, "HW control enabled: %s\n", 2348 str_yes_no(rpmodectl & GEN6_RP_ENABLE)); 2349 drm_printf(p, "SW control enabled: %s\n", 2350 str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE)); 2351 2352 drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", 2353 pm_ier, pm_imr, pm_mask); 2354 if (GRAPHICS_VER(i915) <= 10) 2355 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n", 2356 pm_isr, pm_iir); 2357 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2358 rps->pm_intrmsk_mbz); 2359 drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); 2360 drm_printf(p, "Render p-state ratio: %d\n", 2361 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8); 2362 drm_printf(p, "Render p-state VID: %d\n", 2363 gt_perf_status & 0xff); 2364 drm_printf(p, "Render p-state limit: %d\n", 2365 rp_state_limits & 0xff); 2366 drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat); 2367 drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl); 2368 drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit); 2369 drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit); 2370 drm_printf(p, "RPNSWREQ: %dMHz\n", reqf); 2371 drm_printf(p, "CAGF: %dMHz\n", cagf); 2372 drm_printf(p, "RP CUR UP EI: %d (%lldns)\n", 2373 rpcurupei, 2374 intel_gt_pm_interval_to_ns(gt, rpcurupei)); 2375 drm_printf(p, "RP CUR UP: %d (%lldns)\n", 2376 rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup)); 2377 drm_printf(p, "RP PREV UP: %d (%lldns)\n", 2378 rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup)); 2379 drm_printf(p, "Up threshold: %d%%\n", 2380 rps->power.up_threshold); 2381 drm_printf(p, "RP UP EI: %d (%lldns)\n", 2382 rpupei, intel_gt_pm_interval_to_ns(gt, rpupei)); 2383 drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n", 2384 rpupt, intel_gt_pm_interval_to_ns(gt, rpupt)); 2385 2386 drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n", 2387 rpcurdownei, 2388 intel_gt_pm_interval_to_ns(gt, rpcurdownei)); 2389 drm_printf(p, "RP CUR DOWN: %d (%lldns)\n", 2390 rpcurdown, 2391 intel_gt_pm_interval_to_ns(gt, rpcurdown)); 2392 drm_printf(p, "RP PREV DOWN: %d (%lldns)\n", 2393 rpprevdown, 2394 intel_gt_pm_interval_to_ns(gt, rpprevdown)); 2395 drm_printf(p, "Down threshold: %d%%\n", 2396 rps->power.down_threshold); 2397 drm_printf(p, "RP DOWN EI: %d (%lldns)\n", 2398 rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei)); 2399 drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n", 2400 rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt)); 2401 2402 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2403 intel_gpu_freq(rps, caps.min_freq)); 2404 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2405 intel_gpu_freq(rps, caps.rp1_freq)); 2406 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2407 intel_gpu_freq(rps, caps.rp0_freq)); 2408 drm_printf(p, "Max overclocked frequency: %dMHz\n", 2409 intel_gpu_freq(rps, rps->max_freq)); 2410 2411 drm_printf(p, "Current freq: %d MHz\n", 2412 intel_gpu_freq(rps, rps->cur_freq)); 2413 drm_printf(p, "Actual freq: %d MHz\n", cagf); 2414 drm_printf(p, "Idle freq: %d MHz\n", 2415 intel_gpu_freq(rps, rps->idle_freq)); 2416 drm_printf(p, "Min freq: %d MHz\n", 2417 intel_gpu_freq(rps, rps->min_freq)); 2418 drm_printf(p, "Boost freq: %d MHz\n", 2419 intel_gpu_freq(rps, rps->boost_freq)); 2420 drm_printf(p, "Max freq: %d MHz\n", 2421 intel_gpu_freq(rps, rps->max_freq)); 2422 drm_printf(p, 2423 "efficient (RPe) frequency: %d MHz\n", 2424 intel_gpu_freq(rps, rps->efficient_freq)); 2425 } 2426 2427 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2428 { 2429 struct intel_gt *gt = rps_to_gt(rps); 2430 struct intel_uncore *uncore = gt->uncore; 2431 struct intel_rps_freq_caps caps; 2432 u32 pm_mask; 2433 2434 gen6_rps_get_freq_caps(rps, &caps); 2435 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2436 2437 drm_printf(p, "PM MASK=0x%08x\n", pm_mask); 2438 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2439 rps->pm_intrmsk_mbz); 2440 drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps)); 2441 drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps)); 2442 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2443 intel_gpu_freq(rps, caps.min_freq)); 2444 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2445 intel_gpu_freq(rps, caps.rp1_freq)); 2446 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2447 intel_gpu_freq(rps, caps.rp0_freq)); 2448 drm_printf(p, "Current freq: %d MHz\n", 2449 intel_rps_get_requested_frequency(rps)); 2450 drm_printf(p, "Actual freq: %d MHz\n", 2451 intel_rps_read_actual_frequency(rps)); 2452 drm_printf(p, "Min freq: %d MHz\n", 2453 intel_rps_get_min_frequency(rps)); 2454 drm_printf(p, "Boost freq: %d MHz\n", 2455 intel_rps_get_boost_frequency(rps)); 2456 drm_printf(p, "Max freq: %d MHz\n", 2457 intel_rps_get_max_frequency(rps)); 2458 drm_printf(p, 2459 "efficient (RPe) frequency: %d MHz\n", 2460 intel_gpu_freq(rps, caps.rp1_freq)); 2461 } 2462 2463 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2464 { 2465 if (rps_uses_slpc(rps)) 2466 return slpc_frequency_dump(rps, p); 2467 else 2468 return rps_frequency_dump(rps, p); 2469 } 2470 2471 static int set_max_freq(struct intel_rps *rps, u32 val) 2472 { 2473 struct drm_i915_private *i915 = rps_to_i915(rps); 2474 int ret = 0; 2475 2476 mutex_lock(&rps->lock); 2477 2478 val = intel_freq_opcode(rps, val); 2479 if (val < rps->min_freq || 2480 val > rps->max_freq || 2481 val < rps->min_freq_softlimit) { 2482 ret = -EINVAL; 2483 goto unlock; 2484 } 2485 2486 if (val > rps->rp0_freq) 2487 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2488 intel_gpu_freq(rps, val)); 2489 2490 rps->max_freq_softlimit = val; 2491 2492 val = clamp_t(int, rps->cur_freq, 2493 rps->min_freq_softlimit, 2494 rps->max_freq_softlimit); 2495 2496 /* 2497 * We still need *_set_rps to process the new max_delay and 2498 * update the interrupt limits and PMINTRMSK even though 2499 * frequency request may be unchanged. 2500 */ 2501 intel_rps_set(rps, val); 2502 2503 unlock: 2504 mutex_unlock(&rps->lock); 2505 2506 return ret; 2507 } 2508 2509 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2510 { 2511 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2512 2513 if (rps_uses_slpc(rps)) 2514 return intel_guc_slpc_set_max_freq(slpc, val); 2515 else 2516 return set_max_freq(rps, val); 2517 } 2518 2519 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2520 { 2521 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2522 2523 if (rps_uses_slpc(rps)) 2524 return slpc->min_freq_softlimit; 2525 else 2526 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2527 } 2528 2529 /** 2530 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format. 2531 * @rps: the intel_rps structure 2532 * 2533 * Returns the min frequency in a raw format. In newer platforms raw is in 2534 * units of 50 MHz. 2535 */ 2536 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps) 2537 { 2538 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2539 u32 freq; 2540 2541 if (rps_uses_slpc(rps)) { 2542 return DIV_ROUND_CLOSEST(slpc->min_freq, 2543 GT_FREQUENCY_MULTIPLIER); 2544 } else { 2545 freq = rps->min_freq; 2546 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2547 /* Convert GT frequency to 50 MHz units */ 2548 freq /= GEN9_FREQ_SCALER; 2549 } 2550 return freq; 2551 } 2552 } 2553 2554 static int set_min_freq(struct intel_rps *rps, u32 val) 2555 { 2556 int ret = 0; 2557 2558 mutex_lock(&rps->lock); 2559 2560 val = intel_freq_opcode(rps, val); 2561 if (val < rps->min_freq || 2562 val > rps->max_freq || 2563 val > rps->max_freq_softlimit) { 2564 ret = -EINVAL; 2565 goto unlock; 2566 } 2567 2568 rps->min_freq_softlimit = val; 2569 2570 val = clamp_t(int, rps->cur_freq, 2571 rps->min_freq_softlimit, 2572 rps->max_freq_softlimit); 2573 2574 /* 2575 * We still need *_set_rps to process the new min_delay and 2576 * update the interrupt limits and PMINTRMSK even though 2577 * frequency request may be unchanged. 2578 */ 2579 intel_rps_set(rps, val); 2580 2581 unlock: 2582 mutex_unlock(&rps->lock); 2583 2584 return ret; 2585 } 2586 2587 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2588 { 2589 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2590 2591 if (rps_uses_slpc(rps)) 2592 return intel_guc_slpc_set_min_freq(slpc, val); 2593 else 2594 return set_min_freq(rps, val); 2595 } 2596 2597 u8 intel_rps_get_up_threshold(struct intel_rps *rps) 2598 { 2599 return rps->power.up_threshold; 2600 } 2601 2602 static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val) 2603 { 2604 int ret; 2605 2606 if (val > 100) 2607 return -EINVAL; 2608 2609 ret = mutex_lock_interruptible(&rps->lock); 2610 if (ret) 2611 return ret; 2612 2613 if (*threshold == val) 2614 goto out_unlock; 2615 2616 *threshold = val; 2617 2618 /* Force reset. */ 2619 rps->last_freq = -1; 2620 mutex_lock(&rps->power.mutex); 2621 rps->power.mode = -1; 2622 mutex_unlock(&rps->power.mutex); 2623 2624 intel_rps_set(rps, clamp(rps->cur_freq, 2625 rps->min_freq_softlimit, 2626 rps->max_freq_softlimit)); 2627 2628 out_unlock: 2629 mutex_unlock(&rps->lock); 2630 2631 return ret; 2632 } 2633 2634 int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold) 2635 { 2636 return rps_set_threshold(rps, &rps->power.up_threshold, threshold); 2637 } 2638 2639 u8 intel_rps_get_down_threshold(struct intel_rps *rps) 2640 { 2641 return rps->power.down_threshold; 2642 } 2643 2644 int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold) 2645 { 2646 return rps_set_threshold(rps, &rps->power.down_threshold, threshold); 2647 } 2648 2649 static void intel_rps_set_manual(struct intel_rps *rps, bool enable) 2650 { 2651 struct intel_uncore *uncore = rps_to_uncore(rps); 2652 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE; 2653 2654 /* Allow punit to process software requests */ 2655 intel_uncore_write(uncore, GEN6_RP_CONTROL, state); 2656 } 2657 2658 void intel_rps_raise_unslice(struct intel_rps *rps) 2659 { 2660 struct intel_uncore *uncore = rps_to_uncore(rps); 2661 2662 mutex_lock(&rps->lock); 2663 2664 if (rps_uses_slpc(rps)) { 2665 /* RP limits have not been initialized yet for SLPC path */ 2666 struct intel_rps_freq_caps caps; 2667 2668 gen6_rps_get_freq_caps(rps, &caps); 2669 2670 intel_rps_set_manual(rps, true); 2671 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2672 ((caps.rp0_freq << 2673 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2674 GEN9_IGNORE_SLICE_RATIO)); 2675 intel_rps_set_manual(rps, false); 2676 } else { 2677 intel_rps_set(rps, rps->rp0_freq); 2678 } 2679 2680 mutex_unlock(&rps->lock); 2681 } 2682 2683 void intel_rps_lower_unslice(struct intel_rps *rps) 2684 { 2685 struct intel_uncore *uncore = rps_to_uncore(rps); 2686 2687 mutex_lock(&rps->lock); 2688 2689 if (rps_uses_slpc(rps)) { 2690 /* RP limits have not been initialized yet for SLPC path */ 2691 struct intel_rps_freq_caps caps; 2692 2693 gen6_rps_get_freq_caps(rps, &caps); 2694 2695 intel_rps_set_manual(rps, true); 2696 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2697 ((caps.min_freq << 2698 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2699 GEN9_IGNORE_SLICE_RATIO)); 2700 intel_rps_set_manual(rps, false); 2701 } else { 2702 intel_rps_set(rps, rps->min_freq); 2703 } 2704 2705 mutex_unlock(&rps->lock); 2706 } 2707 2708 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32) 2709 { 2710 struct intel_gt *gt = rps_to_gt(rps); 2711 intel_wakeref_t wakeref; 2712 u32 val; 2713 2714 with_intel_runtime_pm(gt->uncore->rpm, wakeref) 2715 val = intel_uncore_read(gt->uncore, reg32); 2716 2717 return val; 2718 } 2719 2720 bool rps_read_mask_mmio(struct intel_rps *rps, 2721 i915_reg_t reg32, u32 mask) 2722 { 2723 return rps_read_mmio(rps, reg32) & mask; 2724 } 2725 2726 /* External interface for intel_ips.ko */ 2727 2728 static struct drm_i915_private __rcu *ips_mchdev; 2729 2730 /* 2731 * Tells the intel_ips driver that the i915 driver is now loaded, if 2732 * IPS got loaded first. 2733 * 2734 * This awkward dance is so that neither module has to depend on the 2735 * other in order for IPS to do the appropriate communication of 2736 * GPU turbo limits to i915. 2737 */ 2738 static void 2739 ips_ping_for_i915_load(void) 2740 { 2741 void (*link)(void); 2742 2743 link = symbol_get(ips_link_to_i915_driver); 2744 if (link) { 2745 link(); 2746 symbol_put(ips_link_to_i915_driver); 2747 } 2748 } 2749 2750 void intel_rps_driver_register(struct intel_rps *rps) 2751 { 2752 struct intel_gt *gt = rps_to_gt(rps); 2753 2754 /* 2755 * We only register the i915 ips part with intel-ips once everything is 2756 * set up, to avoid intel-ips sneaking in and reading bogus values. 2757 */ 2758 if (GRAPHICS_VER(gt->i915) == 5) { 2759 GEM_BUG_ON(ips_mchdev); 2760 rcu_assign_pointer(ips_mchdev, gt->i915); 2761 ips_ping_for_i915_load(); 2762 } 2763 } 2764 2765 void intel_rps_driver_unregister(struct intel_rps *rps) 2766 { 2767 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2768 rcu_assign_pointer(ips_mchdev, NULL); 2769 } 2770 2771 static struct drm_i915_private *mchdev_get(void) 2772 { 2773 struct drm_i915_private *i915; 2774 2775 rcu_read_lock(); 2776 i915 = rcu_dereference(ips_mchdev); 2777 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2778 i915 = NULL; 2779 rcu_read_unlock(); 2780 2781 return i915; 2782 } 2783 2784 /** 2785 * i915_read_mch_val - return value for IPS use 2786 * 2787 * Calculate and return a value for the IPS driver to use when deciding whether 2788 * we have thermal and power headroom to increase CPU or GPU power budget. 2789 */ 2790 unsigned long i915_read_mch_val(void) 2791 { 2792 struct drm_i915_private *i915; 2793 unsigned long chipset_val = 0; 2794 unsigned long graphics_val = 0; 2795 intel_wakeref_t wakeref; 2796 2797 i915 = mchdev_get(); 2798 if (!i915) 2799 return 0; 2800 2801 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2802 struct intel_ips *ips = &to_gt(i915)->rps.ips; 2803 2804 spin_lock_irq(&mchdev_lock); 2805 chipset_val = __ips_chipset_val(ips); 2806 graphics_val = __ips_gfx_val(ips); 2807 spin_unlock_irq(&mchdev_lock); 2808 } 2809 2810 drm_dev_put(&i915->drm); 2811 return chipset_val + graphics_val; 2812 } 2813 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2814 2815 /** 2816 * i915_gpu_raise - raise GPU frequency limit 2817 * 2818 * Raise the limit; IPS indicates we have thermal headroom. 2819 */ 2820 bool i915_gpu_raise(void) 2821 { 2822 struct drm_i915_private *i915; 2823 struct intel_rps *rps; 2824 2825 i915 = mchdev_get(); 2826 if (!i915) 2827 return false; 2828 2829 rps = &to_gt(i915)->rps; 2830 2831 spin_lock_irq(&mchdev_lock); 2832 if (rps->max_freq_softlimit < rps->max_freq) 2833 rps->max_freq_softlimit++; 2834 spin_unlock_irq(&mchdev_lock); 2835 2836 drm_dev_put(&i915->drm); 2837 return true; 2838 } 2839 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2840 2841 /** 2842 * i915_gpu_lower - lower GPU frequency limit 2843 * 2844 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2845 * frequency maximum. 2846 */ 2847 bool i915_gpu_lower(void) 2848 { 2849 struct drm_i915_private *i915; 2850 struct intel_rps *rps; 2851 2852 i915 = mchdev_get(); 2853 if (!i915) 2854 return false; 2855 2856 rps = &to_gt(i915)->rps; 2857 2858 spin_lock_irq(&mchdev_lock); 2859 if (rps->max_freq_softlimit > rps->min_freq) 2860 rps->max_freq_softlimit--; 2861 spin_unlock_irq(&mchdev_lock); 2862 2863 drm_dev_put(&i915->drm); 2864 return true; 2865 } 2866 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2867 2868 /** 2869 * i915_gpu_busy - indicate GPU business to IPS 2870 * 2871 * Tell the IPS driver whether or not the GPU is busy. 2872 */ 2873 bool i915_gpu_busy(void) 2874 { 2875 struct drm_i915_private *i915; 2876 bool ret; 2877 2878 i915 = mchdev_get(); 2879 if (!i915) 2880 return false; 2881 2882 ret = to_gt(i915)->awake; 2883 2884 drm_dev_put(&i915->drm); 2885 return ret; 2886 } 2887 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2888 2889 /** 2890 * i915_gpu_turbo_disable - disable graphics turbo 2891 * 2892 * Disable graphics turbo by resetting the max frequency and setting the 2893 * current frequency to the default. 2894 */ 2895 bool i915_gpu_turbo_disable(void) 2896 { 2897 struct drm_i915_private *i915; 2898 struct intel_rps *rps; 2899 bool ret; 2900 2901 i915 = mchdev_get(); 2902 if (!i915) 2903 return false; 2904 2905 rps = &to_gt(i915)->rps; 2906 2907 spin_lock_irq(&mchdev_lock); 2908 rps->max_freq_softlimit = rps->min_freq; 2909 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq); 2910 spin_unlock_irq(&mchdev_lock); 2911 2912 drm_dev_put(&i915->drm); 2913 return ret; 2914 } 2915 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2916 2917 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2918 #include "selftest_rps.c" 2919 #include "selftest_slpc.c" 2920 #endif 2921