1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 8 #include <drm/intel/i915_drm.h> 9 10 #include "display/intel_display.h" 11 #include "display/intel_display_rps.h" 12 #include "i915_drv.h" 13 #include "i915_irq.h" 14 #include "i915_reg.h" 15 #include "intel_breadcrumbs.h" 16 #include "intel_gt.h" 17 #include "intel_gt_clock_utils.h" 18 #include "intel_gt_irq.h" 19 #include "intel_gt_pm.h" 20 #include "intel_gt_pm_irq.h" 21 #include "intel_gt_print.h" 22 #include "intel_gt_regs.h" 23 #include "intel_mchbar_regs.h" 24 #include "intel_pcode.h" 25 #include "intel_rps.h" 26 #include "vlv_iosf_sb.h" 27 #include "../../../platform/x86/intel_ips.h" 28 29 #define BUSY_MAX_EI 20u /* ms */ 30 31 /* 32 * Lock protecting IPS related data structures 33 */ 34 static DEFINE_SPINLOCK(mchdev_lock); 35 36 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 37 { 38 return container_of(rps, struct intel_gt, rps); 39 } 40 41 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 42 { 43 return rps_to_gt(rps)->i915; 44 } 45 46 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 47 { 48 return rps_to_gt(rps)->uncore; 49 } 50 51 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 52 { 53 struct intel_gt *gt = rps_to_gt(rps); 54 55 return >_to_guc(gt)->slpc; 56 } 57 58 static bool rps_uses_slpc(struct intel_rps *rps) 59 { 60 struct intel_gt *gt = rps_to_gt(rps); 61 62 return intel_uc_uses_guc_slpc(>->uc); 63 } 64 65 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 66 { 67 return mask & ~rps->pm_intrmsk_mbz; 68 } 69 70 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 71 { 72 intel_uncore_write_fw(uncore, reg, val); 73 } 74 75 static void rps_timer(struct timer_list *t) 76 { 77 struct intel_rps *rps = timer_container_of(rps, t, timer); 78 struct intel_gt *gt = rps_to_gt(rps); 79 struct intel_engine_cs *engine; 80 ktime_t dt, last, timestamp; 81 enum intel_engine_id id; 82 s64 max_busy[3] = {}; 83 84 timestamp = 0; 85 for_each_engine(engine, gt, id) { 86 s64 busy; 87 int i; 88 89 dt = intel_engine_get_busy_time(engine, ×tamp); 90 last = engine->stats.rps; 91 engine->stats.rps = dt; 92 93 busy = ktime_to_ns(ktime_sub(dt, last)); 94 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 95 if (busy > max_busy[i]) 96 swap(busy, max_busy[i]); 97 } 98 } 99 last = rps->pm_timestamp; 100 rps->pm_timestamp = timestamp; 101 102 if (intel_rps_is_active(rps)) { 103 s64 busy; 104 int i; 105 106 dt = ktime_sub(timestamp, last); 107 108 /* 109 * Our goal is to evaluate each engine independently, so we run 110 * at the lowest clocks required to sustain the heaviest 111 * workload. However, a task may be split into sequential 112 * dependent operations across a set of engines, such that 113 * the independent contributions do not account for high load, 114 * but overall the task is GPU bound. For example, consider 115 * video decode on vcs followed by colour post-processing 116 * on vecs, followed by general post-processing on rcs. 117 * Since multi-engines being active does imply a single 118 * continuous workload across all engines, we hedge our 119 * bets by only contributing a factor of the distributed 120 * load into our busyness calculation. 121 */ 122 busy = max_busy[0]; 123 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 124 if (!max_busy[i]) 125 break; 126 127 busy += div_u64(max_busy[i], 1 << i); 128 } 129 GT_TRACE(gt, 130 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 131 busy, (int)div64_u64(100 * busy, dt), 132 max_busy[0], max_busy[1], max_busy[2], 133 rps->pm_interval); 134 135 if (100 * busy > rps->power.up_threshold * dt && 136 rps->cur_freq < rps->max_freq_softlimit) { 137 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 138 rps->pm_interval = 1; 139 queue_work(gt->i915->unordered_wq, &rps->work); 140 } else if (100 * busy < rps->power.down_threshold * dt && 141 rps->cur_freq > rps->min_freq_softlimit) { 142 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 143 rps->pm_interval = 1; 144 queue_work(gt->i915->unordered_wq, &rps->work); 145 } else { 146 rps->last_adj = 0; 147 } 148 149 mod_timer(&rps->timer, 150 jiffies + msecs_to_jiffies(rps->pm_interval)); 151 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 152 } 153 } 154 155 static void rps_start_timer(struct intel_rps *rps) 156 { 157 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 158 rps->pm_interval = 1; 159 mod_timer(&rps->timer, jiffies + 1); 160 } 161 162 static void rps_stop_timer(struct intel_rps *rps) 163 { 164 timer_delete_sync(&rps->timer); 165 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 166 cancel_work_sync(&rps->work); 167 } 168 169 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 170 { 171 u32 mask = 0; 172 173 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 174 if (val > rps->min_freq_softlimit) 175 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 176 GEN6_PM_RP_DOWN_THRESHOLD | 177 GEN6_PM_RP_DOWN_TIMEOUT); 178 179 if (val < rps->max_freq_softlimit) 180 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 181 182 mask &= rps->pm_events; 183 184 return rps_pm_sanitize_mask(rps, ~mask); 185 } 186 187 static void rps_reset_ei(struct intel_rps *rps) 188 { 189 memset(&rps->ei, 0, sizeof(rps->ei)); 190 } 191 192 static void rps_enable_interrupts(struct intel_rps *rps) 193 { 194 struct intel_gt *gt = rps_to_gt(rps); 195 196 GEM_BUG_ON(rps_uses_slpc(rps)); 197 198 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 199 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 200 201 rps_reset_ei(rps); 202 203 spin_lock_irq(gt->irq_lock); 204 gen6_gt_pm_enable_irq(gt, rps->pm_events); 205 spin_unlock_irq(gt->irq_lock); 206 207 intel_uncore_write(gt->uncore, 208 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 209 } 210 211 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 212 { 213 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 214 } 215 216 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 217 { 218 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 219 ; 220 } 221 222 static void rps_reset_interrupts(struct intel_rps *rps) 223 { 224 struct intel_gt *gt = rps_to_gt(rps); 225 226 spin_lock_irq(gt->irq_lock); 227 if (GRAPHICS_VER(gt->i915) >= 11) 228 gen11_rps_reset_interrupts(rps); 229 else 230 gen6_rps_reset_interrupts(rps); 231 232 rps->pm_iir = 0; 233 spin_unlock_irq(gt->irq_lock); 234 } 235 236 static void rps_disable_interrupts(struct intel_rps *rps) 237 { 238 struct intel_gt *gt = rps_to_gt(rps); 239 240 intel_uncore_write(gt->uncore, 241 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 242 243 spin_lock_irq(gt->irq_lock); 244 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 245 spin_unlock_irq(gt->irq_lock); 246 247 intel_synchronize_irq(gt->i915); 248 249 /* 250 * Now that we will not be generating any more work, flush any 251 * outstanding tasks. As we are called on the RPS idle path, 252 * we will reset the GPU to minimum frequencies, so the current 253 * state of the worker can be discarded. 254 */ 255 cancel_work_sync(&rps->work); 256 257 rps_reset_interrupts(rps); 258 GT_TRACE(gt, "interrupts:off\n"); 259 } 260 261 static const struct cparams { 262 u16 i; 263 u16 t; 264 u16 m; 265 u16 c; 266 } cparams[] = { 267 { 1, 1333, 301, 28664 }, 268 { 1, 1067, 294, 24460 }, 269 { 1, 800, 294, 25192 }, 270 { 0, 1333, 276, 27605 }, 271 { 0, 1067, 276, 27605 }, 272 { 0, 800, 231, 23784 }, 273 }; 274 275 static void gen5_rps_init(struct intel_rps *rps) 276 { 277 struct drm_i915_private *i915 = rps_to_i915(rps); 278 struct intel_uncore *uncore = rps_to_uncore(rps); 279 u8 fmax, fmin, fstart; 280 u32 rgvmodectl; 281 int c_m, i; 282 283 if (i915->fsb_freq <= 3200000) 284 c_m = 0; 285 else if (i915->fsb_freq <= 4800000) 286 c_m = 1; 287 else 288 c_m = 2; 289 290 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 291 if (cparams[i].i == c_m && 292 cparams[i].t == DIV_ROUND_CLOSEST(i915->mem_freq, 1000)) { 293 rps->ips.m = cparams[i].m; 294 rps->ips.c = cparams[i].c; 295 break; 296 } 297 } 298 299 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 300 301 /* Set up min, max, and cur for interrupt handling */ 302 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 303 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 304 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 305 MEMMODE_FSTART_SHIFT; 306 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 307 fmax, fmin, fstart); 308 309 rps->min_freq = fmax; 310 rps->efficient_freq = fstart; 311 rps->max_freq = fmin; 312 } 313 314 static unsigned long 315 __ips_chipset_val(struct intel_ips *ips) 316 { 317 struct intel_uncore *uncore = 318 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 319 unsigned long now = jiffies_to_msecs(jiffies), dt; 320 unsigned long result; 321 u64 total, delta; 322 323 lockdep_assert_held(&mchdev_lock); 324 325 /* 326 * Prevent division-by-zero if we are asking too fast. 327 * Also, we don't get interesting results if we are polling 328 * faster than once in 10ms, so just return the saved value 329 * in such cases. 330 */ 331 dt = now - ips->last_time1; 332 if (dt <= 10) 333 return ips->chipset_power; 334 335 /* FIXME: handle per-counter overflow */ 336 total = intel_uncore_read(uncore, DMIEC); 337 total += intel_uncore_read(uncore, DDREC); 338 total += intel_uncore_read(uncore, CSIEC); 339 340 delta = total - ips->last_count1; 341 342 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 343 344 ips->last_count1 = total; 345 ips->last_time1 = now; 346 347 ips->chipset_power = result; 348 349 return result; 350 } 351 352 static unsigned long ips_mch_val(struct intel_uncore *uncore) 353 { 354 unsigned int m, x, b; 355 u32 tsfs; 356 357 tsfs = intel_uncore_read(uncore, TSFS); 358 x = intel_uncore_read8(uncore, TR1); 359 360 b = tsfs & TSFS_INTR_MASK; 361 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 362 363 return m * x / 127 - b; 364 } 365 366 static int _pxvid_to_vd(u8 pxvid) 367 { 368 if (pxvid == 0) 369 return 0; 370 371 if (pxvid >= 8 && pxvid < 31) 372 pxvid = 31; 373 374 return (pxvid + 2) * 125; 375 } 376 377 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 378 { 379 const int vd = _pxvid_to_vd(pxvid); 380 381 if (INTEL_INFO(i915)->is_mobile) 382 return max(vd - 1125, 0); 383 384 return vd; 385 } 386 387 static void __gen5_ips_update(struct intel_ips *ips) 388 { 389 struct intel_uncore *uncore = 390 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 391 u64 now, delta, dt; 392 u32 count; 393 394 lockdep_assert_held(&mchdev_lock); 395 396 now = ktime_get_raw_ns(); 397 dt = now - ips->last_time2; 398 do_div(dt, NSEC_PER_MSEC); 399 400 /* Don't divide by 0 */ 401 if (dt <= 10) 402 return; 403 404 count = intel_uncore_read(uncore, GFXEC); 405 delta = count - ips->last_count2; 406 407 ips->last_count2 = count; 408 ips->last_time2 = now; 409 410 /* More magic constants... */ 411 ips->gfx_power = div_u64(delta * 1181, dt * 10); 412 } 413 414 static void gen5_rps_update(struct intel_rps *rps) 415 { 416 spin_lock_irq(&mchdev_lock); 417 __gen5_ips_update(&rps->ips); 418 spin_unlock_irq(&mchdev_lock); 419 } 420 421 static unsigned int gen5_invert_freq(struct intel_rps *rps, 422 unsigned int val) 423 { 424 /* Invert the frequency bin into an ips delay */ 425 val = rps->max_freq - val; 426 val = rps->min_freq + val; 427 428 return val; 429 } 430 431 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 432 { 433 struct intel_uncore *uncore = rps_to_uncore(rps); 434 u16 rgvswctl; 435 436 lockdep_assert_held(&mchdev_lock); 437 438 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 439 if (rgvswctl & MEMCTL_CMD_STS) { 440 drm_dbg(&rps_to_i915(rps)->drm, 441 "gpu busy, RCS change rejected\n"); 442 return -EBUSY; /* still busy with another command */ 443 } 444 445 /* Invert the frequency bin into an ips delay */ 446 val = gen5_invert_freq(rps, val); 447 448 rgvswctl = 449 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 450 (val << MEMCTL_FREQ_SHIFT) | 451 MEMCTL_SFCAVM; 452 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 453 intel_uncore_posting_read16(uncore, MEMSWCTL); 454 455 rgvswctl |= MEMCTL_CMD_STS; 456 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 457 458 return 0; 459 } 460 461 static int gen5_rps_set(struct intel_rps *rps, u8 val) 462 { 463 int err; 464 465 spin_lock_irq(&mchdev_lock); 466 err = __gen5_rps_set(rps, val); 467 spin_unlock_irq(&mchdev_lock); 468 469 return err; 470 } 471 472 static unsigned long intel_pxfreq(u32 vidfreq) 473 { 474 int div = (vidfreq & 0x3f0000) >> 16; 475 int post = (vidfreq & 0x3000) >> 12; 476 int pre = (vidfreq & 0x7); 477 478 if (!pre) 479 return 0; 480 481 return div * 133333 / (pre << post); 482 } 483 484 static unsigned int init_emon(struct intel_uncore *uncore) 485 { 486 u8 pxw[16]; 487 int i; 488 489 /* Disable to program */ 490 intel_uncore_write(uncore, ECR, 0); 491 intel_uncore_posting_read(uncore, ECR); 492 493 /* Program energy weights for various events */ 494 intel_uncore_write(uncore, SDEW, 0x15040d00); 495 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 496 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 497 intel_uncore_write(uncore, CSIEW2, 0x04000004); 498 499 for (i = 0; i < 5; i++) 500 intel_uncore_write(uncore, PEW(i), 0); 501 for (i = 0; i < 3; i++) 502 intel_uncore_write(uncore, DEW(i), 0); 503 504 /* Program P-state weights to account for frequency power adjustment */ 505 for (i = 0; i < 16; i++) { 506 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 507 unsigned int freq = intel_pxfreq(pxvidfreq); 508 unsigned int vid = 509 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 510 unsigned int val; 511 512 val = vid * vid * freq / 1000 * 255; 513 val /= 127 * 127 * 900; 514 515 pxw[i] = val; 516 } 517 /* Render standby states get 0 weight */ 518 pxw[14] = 0; 519 pxw[15] = 0; 520 521 for (i = 0; i < 4; i++) { 522 intel_uncore_write(uncore, PXW(i), 523 pxw[i * 4 + 0] << 24 | 524 pxw[i * 4 + 1] << 16 | 525 pxw[i * 4 + 2] << 8 | 526 pxw[i * 4 + 3] << 0); 527 } 528 529 /* Adjust magic regs to magic values (more experimental results) */ 530 intel_uncore_write(uncore, OGW0, 0); 531 intel_uncore_write(uncore, OGW1, 0); 532 intel_uncore_write(uncore, EG0, 0x00007f00); 533 intel_uncore_write(uncore, EG1, 0x0000000e); 534 intel_uncore_write(uncore, EG2, 0x000e0000); 535 intel_uncore_write(uncore, EG3, 0x68000300); 536 intel_uncore_write(uncore, EG4, 0x42000000); 537 intel_uncore_write(uncore, EG5, 0x00140031); 538 intel_uncore_write(uncore, EG6, 0); 539 intel_uncore_write(uncore, EG7, 0); 540 541 for (i = 0; i < 8; i++) 542 intel_uncore_write(uncore, PXWL(i), 0); 543 544 /* Enable PMON + select events */ 545 intel_uncore_write(uncore, ECR, 0x80000019); 546 547 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 548 } 549 550 static bool gen5_rps_enable(struct intel_rps *rps) 551 { 552 struct drm_i915_private *i915 = rps_to_i915(rps); 553 struct intel_display *display = i915->display; 554 struct intel_uncore *uncore = rps_to_uncore(rps); 555 u8 fstart, vstart; 556 u32 rgvmodectl; 557 558 spin_lock_irq(&mchdev_lock); 559 560 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 561 562 /* Enable temp reporting */ 563 intel_uncore_write16(uncore, PMMISC, 564 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 565 intel_uncore_write16(uncore, TSC1, 566 intel_uncore_read16(uncore, TSC1) | TSE); 567 568 /* 100ms RC evaluation intervals */ 569 intel_uncore_write(uncore, RCUPEI, 100000); 570 intel_uncore_write(uncore, RCDNEI, 100000); 571 572 /* Set max/min thresholds to 90ms and 80ms respectively */ 573 intel_uncore_write(uncore, RCBMAXAVG, 90000); 574 intel_uncore_write(uncore, RCBMINAVG, 80000); 575 576 intel_uncore_write(uncore, MEMIHYST, 1); 577 578 /* Set up min, max, and cur for interrupt handling */ 579 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 580 MEMMODE_FSTART_SHIFT; 581 582 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 583 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 584 585 intel_uncore_write(uncore, 586 MEMINTREN, 587 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 588 589 intel_uncore_write(uncore, VIDSTART, vstart); 590 intel_uncore_posting_read(uncore, VIDSTART); 591 592 rgvmodectl |= MEMMODE_SWMODE_EN; 593 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 594 595 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 596 MEMCTL_CMD_STS) == 0, 10)) 597 drm_err(&uncore->i915->drm, 598 "stuck trying to change perf mode\n"); 599 mdelay(1); 600 601 __gen5_rps_set(rps, rps->cur_freq); 602 603 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 604 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 605 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 606 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 607 608 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 609 rps->ips.last_time2 = ktime_get_raw_ns(); 610 611 ilk_display_rps_enable(display); 612 613 spin_unlock_irq(&mchdev_lock); 614 615 rps->ips.corr = init_emon(uncore); 616 617 return true; 618 } 619 620 static void gen5_rps_disable(struct intel_rps *rps) 621 { 622 struct drm_i915_private *i915 = rps_to_i915(rps); 623 struct intel_display *display = i915->display; 624 struct intel_uncore *uncore = rps_to_uncore(rps); 625 u16 rgvswctl; 626 627 spin_lock_irq(&mchdev_lock); 628 629 ilk_display_rps_disable(display); 630 631 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 632 633 /* Ack interrupts, disable EFC interrupt */ 634 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0); 635 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 636 637 /* Go back to the starting frequency */ 638 __gen5_rps_set(rps, rps->idle_freq); 639 mdelay(1); 640 rgvswctl |= MEMCTL_CMD_STS; 641 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 642 mdelay(1); 643 644 spin_unlock_irq(&mchdev_lock); 645 } 646 647 static u32 rps_limits(struct intel_rps *rps, u8 val) 648 { 649 u32 limits; 650 651 /* 652 * Only set the down limit when we've reached the lowest level to avoid 653 * getting more interrupts, otherwise leave this clear. This prevents a 654 * race in the hw when coming out of rc6: There's a tiny window where 655 * the hw runs at the minimal clock before selecting the desired 656 * frequency, if the down threshold expires in that window we will not 657 * receive a down interrupt. 658 */ 659 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 660 limits = rps->max_freq_softlimit << 23; 661 if (val <= rps->min_freq_softlimit) 662 limits |= rps->min_freq_softlimit << 14; 663 } else { 664 limits = rps->max_freq_softlimit << 24; 665 if (val <= rps->min_freq_softlimit) 666 limits |= rps->min_freq_softlimit << 16; 667 } 668 669 return limits; 670 } 671 672 static void rps_set_power(struct intel_rps *rps, int new_power) 673 { 674 struct intel_gt *gt = rps_to_gt(rps); 675 struct intel_uncore *uncore = gt->uncore; 676 u32 ei_up = 0, ei_down = 0; 677 678 lockdep_assert_held(&rps->power.mutex); 679 680 if (new_power == rps->power.mode) 681 return; 682 683 /* Note the units here are not exactly 1us, but 1280ns. */ 684 switch (new_power) { 685 case LOW_POWER: 686 ei_up = 16000; 687 ei_down = 32000; 688 break; 689 690 case BETWEEN: 691 ei_up = 13000; 692 ei_down = 32000; 693 break; 694 695 case HIGH_POWER: 696 ei_up = 10000; 697 ei_down = 32000; 698 break; 699 } 700 701 /* When byt can survive without system hang with dynamic 702 * sw freq adjustments, this restriction can be lifted. 703 */ 704 if (IS_VALLEYVIEW(gt->i915)) 705 goto skip_hw_write; 706 707 GT_TRACE(gt, 708 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 709 new_power, 710 rps->power.up_threshold, ei_up, 711 rps->power.down_threshold, ei_down); 712 713 set(uncore, GEN6_RP_UP_EI, 714 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 715 set(uncore, GEN6_RP_UP_THRESHOLD, 716 intel_gt_ns_to_pm_interval(gt, 717 ei_up * rps->power.up_threshold * 10)); 718 719 set(uncore, GEN6_RP_DOWN_EI, 720 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 721 set(uncore, GEN6_RP_DOWN_THRESHOLD, 722 intel_gt_ns_to_pm_interval(gt, 723 ei_down * 724 rps->power.down_threshold * 10)); 725 726 set(uncore, GEN6_RP_CONTROL, 727 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 728 GEN6_RP_MEDIA_HW_NORMAL_MODE | 729 GEN6_RP_MEDIA_IS_GFX | 730 GEN6_RP_ENABLE | 731 GEN6_RP_UP_BUSY_AVG | 732 GEN6_RP_DOWN_IDLE_AVG); 733 734 skip_hw_write: 735 rps->power.mode = new_power; 736 } 737 738 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 739 { 740 int new_power; 741 742 new_power = rps->power.mode; 743 switch (rps->power.mode) { 744 case LOW_POWER: 745 if (val > rps->efficient_freq + 1 && 746 val > rps->cur_freq) 747 new_power = BETWEEN; 748 break; 749 750 case BETWEEN: 751 if (val <= rps->efficient_freq && 752 val < rps->cur_freq) 753 new_power = LOW_POWER; 754 else if (val >= rps->rp0_freq && 755 val > rps->cur_freq) 756 new_power = HIGH_POWER; 757 break; 758 759 case HIGH_POWER: 760 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 761 val < rps->cur_freq) 762 new_power = BETWEEN; 763 break; 764 } 765 /* Max/min bins are special */ 766 if (val <= rps->min_freq_softlimit) 767 new_power = LOW_POWER; 768 if (val >= rps->max_freq_softlimit) 769 new_power = HIGH_POWER; 770 771 mutex_lock(&rps->power.mutex); 772 if (rps->power.interactive) 773 new_power = HIGH_POWER; 774 rps_set_power(rps, new_power); 775 mutex_unlock(&rps->power.mutex); 776 } 777 778 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 779 { 780 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", 781 str_yes_no(interactive)); 782 783 mutex_lock(&rps->power.mutex); 784 if (interactive) { 785 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 786 rps_set_power(rps, HIGH_POWER); 787 } else { 788 GEM_BUG_ON(!rps->power.interactive); 789 rps->power.interactive--; 790 } 791 mutex_unlock(&rps->power.mutex); 792 } 793 794 static int gen6_rps_set(struct intel_rps *rps, u8 val) 795 { 796 struct intel_uncore *uncore = rps_to_uncore(rps); 797 struct drm_i915_private *i915 = rps_to_i915(rps); 798 u32 swreq; 799 800 GEM_BUG_ON(rps_uses_slpc(rps)); 801 802 if (GRAPHICS_VER(i915) >= 9) 803 swreq = GEN9_FREQUENCY(val); 804 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 805 swreq = HSW_FREQUENCY(val); 806 else 807 swreq = (GEN6_FREQUENCY(val) | 808 GEN6_OFFSET(0) | 809 GEN6_AGGRESSIVE_TURBO); 810 set(uncore, GEN6_RPNSWREQ, swreq); 811 812 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 813 val, intel_gpu_freq(rps, val), swreq); 814 815 return 0; 816 } 817 818 static int vlv_rps_set(struct intel_rps *rps, u8 val) 819 { 820 struct drm_i915_private *i915 = rps_to_i915(rps); 821 int err; 822 823 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 824 err = vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_REQ, val); 825 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 826 827 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 828 val, intel_gpu_freq(rps, val)); 829 830 return err; 831 } 832 833 static int rps_set(struct intel_rps *rps, u8 val, bool update) 834 { 835 struct drm_i915_private *i915 = rps_to_i915(rps); 836 int err; 837 838 if (val == rps->last_freq) 839 return 0; 840 841 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 842 err = vlv_rps_set(rps, val); 843 else if (GRAPHICS_VER(i915) >= 6) 844 err = gen6_rps_set(rps, val); 845 else 846 err = gen5_rps_set(rps, val); 847 if (err) 848 return err; 849 850 if (update && GRAPHICS_VER(i915) >= 6) 851 gen6_rps_set_thresholds(rps, val); 852 rps->last_freq = val; 853 854 return 0; 855 } 856 857 void intel_rps_unpark(struct intel_rps *rps) 858 { 859 if (!intel_rps_is_enabled(rps)) 860 return; 861 862 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 863 864 /* 865 * Use the user's desired frequency as a guide, but for better 866 * performance, jump directly to RPe as our starting frequency. 867 */ 868 mutex_lock(&rps->lock); 869 870 intel_rps_set_active(rps); 871 intel_rps_set(rps, 872 clamp(rps->cur_freq, 873 rps->min_freq_softlimit, 874 rps->max_freq_softlimit)); 875 876 mutex_unlock(&rps->lock); 877 878 rps->pm_iir = 0; 879 if (intel_rps_has_interrupts(rps)) 880 rps_enable_interrupts(rps); 881 if (intel_rps_uses_timer(rps)) 882 rps_start_timer(rps); 883 884 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 885 gen5_rps_update(rps); 886 } 887 888 void intel_rps_park(struct intel_rps *rps) 889 { 890 int adj; 891 892 if (!intel_rps_is_enabled(rps)) 893 return; 894 895 if (!intel_rps_clear_active(rps)) 896 return; 897 898 if (intel_rps_uses_timer(rps)) 899 rps_stop_timer(rps); 900 if (intel_rps_has_interrupts(rps)) 901 rps_disable_interrupts(rps); 902 903 if (rps->last_freq <= rps->idle_freq) 904 return; 905 906 /* 907 * The punit delays the write of the frequency and voltage until it 908 * determines the GPU is awake. During normal usage we don't want to 909 * waste power changing the frequency if the GPU is sleeping (rc6). 910 * However, the GPU and driver is now idle and we do not want to delay 911 * switching to minimum voltage (reducing power whilst idle) as we do 912 * not expect to be woken in the near future and so must flush the 913 * change by waking the device. 914 * 915 * We choose to take the media powerwell (either would do to trick the 916 * punit into committing the voltage change) as that takes a lot less 917 * power than the render powerwell. 918 */ 919 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 920 rps_set(rps, rps->idle_freq, false); 921 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 922 923 /* 924 * Since we will try and restart from the previously requested 925 * frequency on unparking, treat this idle point as a downclock 926 * interrupt and reduce the frequency for resume. If we park/unpark 927 * more frequently than the rps worker can run, we will not respond 928 * to any EI and never see a change in frequency. 929 * 930 * (Note we accommodate Cherryview's limitation of only using an 931 * even bin by applying it to all.) 932 */ 933 adj = rps->last_adj; 934 if (adj < 0) 935 adj *= 2; 936 else /* CHV needs even encode values */ 937 adj = -2; 938 rps->last_adj = adj; 939 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 940 if (rps->cur_freq < rps->efficient_freq) { 941 rps->cur_freq = rps->efficient_freq; 942 rps->last_adj = 0; 943 } 944 945 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 946 } 947 948 u32 intel_rps_get_boost_frequency(struct intel_rps *rps) 949 { 950 struct intel_guc_slpc *slpc; 951 952 if (rps_uses_slpc(rps)) { 953 slpc = rps_to_slpc(rps); 954 955 return slpc->boost_freq; 956 } else { 957 return intel_gpu_freq(rps, rps->boost_freq); 958 } 959 } 960 961 static int rps_set_boost_freq(struct intel_rps *rps, u32 val) 962 { 963 bool boost = false; 964 965 /* Validate against (static) hardware limits */ 966 val = intel_freq_opcode(rps, val); 967 if (val < rps->min_freq || val > rps->max_freq) 968 return -EINVAL; 969 970 mutex_lock(&rps->lock); 971 if (val != rps->boost_freq) { 972 rps->boost_freq = val; 973 boost = atomic_read(&rps->num_waiters); 974 } 975 mutex_unlock(&rps->lock); 976 if (boost) 977 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 978 979 return 0; 980 } 981 982 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq) 983 { 984 struct intel_guc_slpc *slpc; 985 986 if (rps_uses_slpc(rps)) { 987 slpc = rps_to_slpc(rps); 988 989 return intel_guc_slpc_set_boost_freq(slpc, freq); 990 } else { 991 return rps_set_boost_freq(rps, freq); 992 } 993 } 994 995 void intel_rps_dec_waiters(struct intel_rps *rps) 996 { 997 struct intel_guc_slpc *slpc; 998 999 if (rps_uses_slpc(rps)) { 1000 slpc = rps_to_slpc(rps); 1001 1002 /* Don't decrement num_waiters for req where increment was skipped */ 1003 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING) 1004 return; 1005 1006 intel_guc_slpc_dec_waiters(slpc); 1007 } else { 1008 atomic_dec(&rps->num_waiters); 1009 } 1010 } 1011 1012 void intel_rps_boost(struct i915_request *rq) 1013 { 1014 struct intel_guc_slpc *slpc; 1015 1016 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 1017 return; 1018 1019 /* Waitboost is not needed for contexts marked with a Freq hint */ 1020 if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags)) 1021 return; 1022 1023 /* Serializes with i915_request_retire() */ 1024 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 1025 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 1026 1027 if (rps_uses_slpc(rps)) { 1028 slpc = rps_to_slpc(rps); 1029 1030 /* Waitboost should not be done with power saving profile */ 1031 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING) 1032 return; 1033 1034 /* Return if old value is non zero */ 1035 if (!atomic_fetch_inc(&slpc->num_waiters)) { 1036 /* 1037 * Skip queuing boost work if frequency is already boosted, 1038 * but still increment num_waiters. 1039 */ 1040 if (slpc->min_freq_softlimit >= slpc->boost_freq) 1041 return; 1042 1043 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1044 rq->fence.context, rq->fence.seqno); 1045 queue_work(rps_to_gt(rps)->i915->unordered_wq, 1046 &slpc->boost_work); 1047 } 1048 1049 return; 1050 } 1051 1052 if (atomic_fetch_inc(&rps->num_waiters)) 1053 return; 1054 1055 if (!intel_rps_is_active(rps)) 1056 return; 1057 1058 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1059 rq->fence.context, rq->fence.seqno); 1060 1061 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 1062 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 1063 1064 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 1065 } 1066 } 1067 1068 int intel_rps_set(struct intel_rps *rps, u8 val) 1069 { 1070 int err; 1071 1072 lockdep_assert_held(&rps->lock); 1073 GEM_BUG_ON(val > rps->max_freq); 1074 GEM_BUG_ON(val < rps->min_freq); 1075 1076 if (intel_rps_is_active(rps)) { 1077 err = rps_set(rps, val, true); 1078 if (err) 1079 return err; 1080 1081 /* 1082 * Make sure we continue to get interrupts 1083 * until we hit the minimum or maximum frequencies. 1084 */ 1085 if (intel_rps_has_interrupts(rps)) { 1086 struct intel_uncore *uncore = rps_to_uncore(rps); 1087 1088 set(uncore, 1089 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 1090 1091 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 1092 } 1093 } 1094 1095 rps->cur_freq = val; 1096 return 0; 1097 } 1098 1099 static u32 intel_rps_read_state_cap(struct intel_rps *rps) 1100 { 1101 struct drm_i915_private *i915 = rps_to_i915(rps); 1102 struct intel_uncore *uncore = rps_to_uncore(rps); 1103 1104 if (IS_GEN9_LP(i915)) 1105 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 1106 else 1107 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 1108 } 1109 1110 static void 1111 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1112 { 1113 struct intel_uncore *uncore = rps_to_uncore(rps); 1114 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ? 1115 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) : 1116 intel_uncore_read(uncore, MTL_RP_STATE_CAP); 1117 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ? 1118 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) : 1119 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY); 1120 1121 /* MTL values are in units of 16.67 MHz */ 1122 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap); 1123 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap); 1124 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe); 1125 } 1126 1127 static void 1128 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1129 { 1130 struct drm_i915_private *i915 = rps_to_i915(rps); 1131 u32 rp_state_cap; 1132 1133 rp_state_cap = intel_rps_read_state_cap(rps); 1134 1135 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1136 if (IS_GEN9_LP(i915)) { 1137 caps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1138 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1139 caps->min_freq = (rp_state_cap >> 0) & 0xff; 1140 } else { 1141 caps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1142 if (GRAPHICS_VER(i915) >= 10) 1143 caps->rp1_freq = REG_FIELD_GET(RPE_MASK, 1144 intel_uncore_read(to_gt(i915)->uncore, 1145 GEN10_FREQ_INFO_REC)); 1146 else 1147 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1148 caps->min_freq = (rp_state_cap >> 16) & 0xff; 1149 } 1150 1151 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1152 /* 1153 * In this case rp_state_cap register reports frequencies in 1154 * units of 50 MHz. Convert these to the actual "hw unit", i.e. 1155 * units of 16.67 MHz 1156 */ 1157 caps->rp0_freq *= GEN9_FREQ_SCALER; 1158 caps->rp1_freq *= GEN9_FREQ_SCALER; 1159 caps->min_freq *= GEN9_FREQ_SCALER; 1160 } 1161 } 1162 1163 /** 1164 * gen6_rps_get_freq_caps - Get freq caps exposed by HW 1165 * @rps: the intel_rps structure 1166 * @caps: returned freq caps 1167 * 1168 * Returned "caps" frequencies should be converted to MHz using 1169 * intel_gpu_freq() 1170 */ 1171 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1172 { 1173 struct drm_i915_private *i915 = rps_to_i915(rps); 1174 1175 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 1176 return mtl_get_freq_caps(rps, caps); 1177 else 1178 return __gen6_rps_get_freq_caps(rps, caps); 1179 } 1180 1181 static void gen6_rps_init(struct intel_rps *rps) 1182 { 1183 struct drm_i915_private *i915 = rps_to_i915(rps); 1184 struct intel_rps_freq_caps caps; 1185 1186 gen6_rps_get_freq_caps(rps, &caps); 1187 rps->rp0_freq = caps.rp0_freq; 1188 rps->rp1_freq = caps.rp1_freq; 1189 rps->min_freq = caps.min_freq; 1190 1191 /* hw_max = RP0 until we check for overclocking */ 1192 rps->max_freq = rps->rp0_freq; 1193 1194 rps->efficient_freq = rps->rp1_freq; 1195 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1196 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1197 u32 ddcc_status = 0; 1198 u32 mult = 1; 1199 1200 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) 1201 mult = GEN9_FREQ_SCALER; 1202 if (snb_pcode_read(rps_to_gt(rps)->uncore, 1203 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1204 &ddcc_status, NULL) == 0) 1205 rps->efficient_freq = 1206 clamp_t(u32, 1207 ((ddcc_status >> 8) & 0xff) * mult, 1208 rps->min_freq, 1209 rps->max_freq); 1210 } 1211 } 1212 1213 static bool rps_reset(struct intel_rps *rps) 1214 { 1215 struct drm_i915_private *i915 = rps_to_i915(rps); 1216 1217 /* force a reset */ 1218 rps->power.mode = -1; 1219 rps->last_freq = -1; 1220 1221 if (rps_set(rps, rps->min_freq, true)) { 1222 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1223 return false; 1224 } 1225 1226 rps->cur_freq = rps->min_freq; 1227 return true; 1228 } 1229 1230 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1231 static bool gen9_rps_enable(struct intel_rps *rps) 1232 { 1233 struct intel_gt *gt = rps_to_gt(rps); 1234 struct intel_uncore *uncore = gt->uncore; 1235 1236 /* Program defaults and thresholds for RPS */ 1237 if (GRAPHICS_VER(gt->i915) == 9) 1238 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1239 GEN9_FREQUENCY(rps->rp1_freq)); 1240 1241 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1242 1243 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1244 1245 return rps_reset(rps); 1246 } 1247 1248 static bool gen8_rps_enable(struct intel_rps *rps) 1249 { 1250 struct intel_uncore *uncore = rps_to_uncore(rps); 1251 1252 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1253 HSW_FREQUENCY(rps->rp1_freq)); 1254 1255 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1256 1257 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1258 1259 return rps_reset(rps); 1260 } 1261 1262 static bool gen6_rps_enable(struct intel_rps *rps) 1263 { 1264 struct intel_uncore *uncore = rps_to_uncore(rps); 1265 1266 /* Power down if completely idle for over 50ms */ 1267 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1268 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1269 1270 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1271 GEN6_PM_RP_DOWN_THRESHOLD | 1272 GEN6_PM_RP_DOWN_TIMEOUT); 1273 1274 return rps_reset(rps); 1275 } 1276 1277 static int chv_rps_max_freq(struct intel_rps *rps) 1278 { 1279 struct drm_i915_private *i915 = rps_to_i915(rps); 1280 struct intel_gt *gt = rps_to_gt(rps); 1281 u32 val; 1282 1283 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE); 1284 1285 switch (gt->info.sseu.eu_total) { 1286 case 8: 1287 /* (2 * 4) config */ 1288 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1289 break; 1290 case 12: 1291 /* (2 * 6) config */ 1292 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1293 break; 1294 case 16: 1295 /* (2 * 8) config */ 1296 default: 1297 /* Setting (2 * 8) Min RP0 for any other combination */ 1298 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1299 break; 1300 } 1301 1302 return val & FB_GFX_FREQ_FUSE_MASK; 1303 } 1304 1305 static int chv_rps_rpe_freq(struct intel_rps *rps) 1306 { 1307 struct drm_i915_private *i915 = rps_to_i915(rps); 1308 u32 val; 1309 1310 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_GPU_DUTYCYCLE_REG); 1311 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1312 1313 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1314 } 1315 1316 static int chv_rps_guar_freq(struct intel_rps *rps) 1317 { 1318 struct drm_i915_private *i915 = rps_to_i915(rps); 1319 u32 val; 1320 1321 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMAX_AT_VMAX_FUSE); 1322 1323 return val & FB_GFX_FREQ_FUSE_MASK; 1324 } 1325 1326 static u32 chv_rps_min_freq(struct intel_rps *rps) 1327 { 1328 struct drm_i915_private *i915 = rps_to_i915(rps); 1329 u32 val; 1330 1331 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, FB_GFX_FMIN_AT_VMIN_FUSE); 1332 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1333 1334 return val & FB_GFX_FREQ_FUSE_MASK; 1335 } 1336 1337 static bool chv_rps_enable(struct intel_rps *rps) 1338 { 1339 struct intel_uncore *uncore = rps_to_uncore(rps); 1340 struct drm_i915_private *i915 = rps_to_i915(rps); 1341 u32 val; 1342 1343 /* 1: Program defaults and thresholds for RPS*/ 1344 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1345 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1346 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1347 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1348 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1349 1350 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1351 1352 /* 2: Enable RPS */ 1353 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1354 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1355 GEN6_RP_MEDIA_IS_GFX | 1356 GEN6_RP_ENABLE | 1357 GEN6_RP_UP_BUSY_AVG | 1358 GEN6_RP_DOWN_IDLE_AVG); 1359 1360 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1361 GEN6_PM_RP_DOWN_THRESHOLD | 1362 GEN6_PM_RP_DOWN_TIMEOUT); 1363 1364 /* Setting Fixed Bias */ 1365 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1366 1367 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1368 vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val); 1369 1370 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 1371 1372 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1373 1374 /* RPS code assumes GPLL is used */ 1375 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1376 "GPLL not enabled\n"); 1377 1378 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1379 str_yes_no(val & GPLLENABLE)); 1380 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1381 1382 return rps_reset(rps); 1383 } 1384 1385 static int vlv_rps_guar_freq(struct intel_rps *rps) 1386 { 1387 struct drm_i915_private *i915 = rps_to_i915(rps); 1388 u32 val, rp1; 1389 1390 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE); 1391 1392 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1393 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1394 1395 return rp1; 1396 } 1397 1398 static int vlv_rps_max_freq(struct intel_rps *rps) 1399 { 1400 struct drm_i915_private *i915 = rps_to_i915(rps); 1401 u32 val, rp0; 1402 1403 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FREQ_FUSE); 1404 1405 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1406 /* Clamp to max */ 1407 rp0 = min_t(u32, rp0, 0xea); 1408 1409 return rp0; 1410 } 1411 1412 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1413 { 1414 struct drm_i915_private *i915 = rps_to_i915(rps); 1415 u32 val, rpe; 1416 1417 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1418 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1419 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_NC, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1420 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1421 1422 return rpe; 1423 } 1424 1425 static int vlv_rps_min_freq(struct intel_rps *rps) 1426 { 1427 struct drm_i915_private *i915 = rps_to_i915(rps); 1428 u32 val; 1429 1430 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_LFM) & 0xff; 1431 /* 1432 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1433 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1434 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1435 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1436 * to make sure it matches what Punit accepts. 1437 */ 1438 return max_t(u32, val, 0xc0); 1439 } 1440 1441 static bool vlv_rps_enable(struct intel_rps *rps) 1442 { 1443 struct intel_uncore *uncore = rps_to_uncore(rps); 1444 struct drm_i915_private *i915 = rps_to_i915(rps); 1445 u32 val; 1446 1447 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1448 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1449 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1450 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1451 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1452 1453 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1454 1455 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1456 GEN6_RP_MEDIA_TURBO | 1457 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1458 GEN6_RP_MEDIA_IS_GFX | 1459 GEN6_RP_ENABLE | 1460 GEN6_RP_UP_BUSY_AVG | 1461 GEN6_RP_DOWN_IDLE_CONT); 1462 1463 /* WaGsvRC0ResidencyMethod:vlv */ 1464 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1465 1466 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1467 1468 /* Setting Fixed Bias */ 1469 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1470 vlv_iosf_sb_write(&i915->drm, VLV_IOSF_SB_PUNIT, VLV_TURBO_SOC_OVERRIDE, val); 1471 1472 val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 1473 1474 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 1475 1476 /* RPS code assumes GPLL is used */ 1477 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1478 "GPLL not enabled\n"); 1479 1480 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1481 str_yes_no(val & GPLLENABLE)); 1482 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1483 1484 return rps_reset(rps); 1485 } 1486 1487 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1488 { 1489 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1490 struct intel_uncore *uncore = rps_to_uncore(rps); 1491 unsigned int t, state1, state2; 1492 u32 pxvid, ext_v; 1493 u64 corr, corr2; 1494 1495 lockdep_assert_held(&mchdev_lock); 1496 1497 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1498 pxvid = (pxvid >> 24) & 0x7f; 1499 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1500 1501 state1 = ext_v; 1502 1503 /* Revel in the empirically derived constants */ 1504 1505 /* Correction factor in 1/100000 units */ 1506 t = ips_mch_val(uncore); 1507 if (t > 80) 1508 corr = t * 2349 + 135940; 1509 else if (t >= 50) 1510 corr = t * 964 + 29317; 1511 else /* < 50 */ 1512 corr = t * 301 + 1004; 1513 1514 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1515 corr2 = div_u64(corr, 100000) * ips->corr; 1516 1517 state2 = div_u64(corr2 * state1, 10000); 1518 state2 /= 100; /* convert to mW */ 1519 1520 __gen5_ips_update(ips); 1521 1522 return ips->gfx_power + state2; 1523 } 1524 1525 static bool has_busy_stats(struct intel_rps *rps) 1526 { 1527 struct intel_engine_cs *engine; 1528 enum intel_engine_id id; 1529 1530 for_each_engine(engine, rps_to_gt(rps), id) { 1531 if (!intel_engine_supports_stats(engine)) 1532 return false; 1533 } 1534 1535 return true; 1536 } 1537 1538 void intel_rps_enable(struct intel_rps *rps) 1539 { 1540 struct drm_i915_private *i915 = rps_to_i915(rps); 1541 struct intel_uncore *uncore = rps_to_uncore(rps); 1542 bool enabled = false; 1543 1544 if (!HAS_RPS(i915)) 1545 return; 1546 1547 if (rps_uses_slpc(rps)) 1548 return; 1549 1550 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1551 1552 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1553 if (rps->max_freq <= rps->min_freq) 1554 /* leave disabled, no room for dynamic reclocking */; 1555 else if (IS_CHERRYVIEW(i915)) 1556 enabled = chv_rps_enable(rps); 1557 else if (IS_VALLEYVIEW(i915)) 1558 enabled = vlv_rps_enable(rps); 1559 else if (GRAPHICS_VER(i915) >= 9) 1560 enabled = gen9_rps_enable(rps); 1561 else if (GRAPHICS_VER(i915) >= 8) 1562 enabled = gen8_rps_enable(rps); 1563 else if (GRAPHICS_VER(i915) >= 6) 1564 enabled = gen6_rps_enable(rps); 1565 else if (IS_IRONLAKE_M(i915)) 1566 enabled = gen5_rps_enable(rps); 1567 else 1568 MISSING_CASE(GRAPHICS_VER(i915)); 1569 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1570 if (!enabled) 1571 return; 1572 1573 GT_TRACE(rps_to_gt(rps), 1574 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n", 1575 rps->min_freq, rps->max_freq, 1576 intel_gpu_freq(rps, rps->min_freq), 1577 intel_gpu_freq(rps, rps->max_freq), 1578 rps->power.up_threshold, 1579 rps->power.down_threshold); 1580 1581 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1582 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1583 1584 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1585 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1586 1587 if (has_busy_stats(rps)) 1588 intel_rps_set_timer(rps); 1589 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) 1590 intel_rps_set_interrupts(rps); 1591 else 1592 /* Ironlake currently uses intel_ips.ko */ {} 1593 1594 intel_rps_set_enabled(rps); 1595 } 1596 1597 static void gen6_rps_disable(struct intel_rps *rps) 1598 { 1599 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1600 } 1601 1602 void intel_rps_disable(struct intel_rps *rps) 1603 { 1604 struct drm_i915_private *i915 = rps_to_i915(rps); 1605 1606 if (!intel_rps_is_enabled(rps)) 1607 return; 1608 1609 intel_rps_clear_enabled(rps); 1610 intel_rps_clear_interrupts(rps); 1611 intel_rps_clear_timer(rps); 1612 1613 if (GRAPHICS_VER(i915) >= 6) 1614 gen6_rps_disable(rps); 1615 else if (IS_IRONLAKE_M(i915)) 1616 gen5_rps_disable(rps); 1617 } 1618 1619 static int byt_gpu_freq(struct intel_rps *rps, int val) 1620 { 1621 /* 1622 * N = val - 0xb7 1623 * Slow = Fast = GPLL ref * N 1624 */ 1625 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1626 } 1627 1628 static int byt_freq_opcode(struct intel_rps *rps, int val) 1629 { 1630 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1631 } 1632 1633 static int chv_gpu_freq(struct intel_rps *rps, int val) 1634 { 1635 /* 1636 * N = val / 2 1637 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1638 */ 1639 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1640 } 1641 1642 static int chv_freq_opcode(struct intel_rps *rps, int val) 1643 { 1644 /* CHV needs even values */ 1645 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1646 } 1647 1648 int intel_gpu_freq(struct intel_rps *rps, int val) 1649 { 1650 struct drm_i915_private *i915 = rps_to_i915(rps); 1651 1652 if (GRAPHICS_VER(i915) >= 9) 1653 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1654 GEN9_FREQ_SCALER); 1655 else if (IS_CHERRYVIEW(i915)) 1656 return chv_gpu_freq(rps, val); 1657 else if (IS_VALLEYVIEW(i915)) 1658 return byt_gpu_freq(rps, val); 1659 else if (GRAPHICS_VER(i915) >= 6) 1660 return val * GT_FREQUENCY_MULTIPLIER; 1661 else 1662 return val; 1663 } 1664 1665 int intel_freq_opcode(struct intel_rps *rps, int val) 1666 { 1667 struct drm_i915_private *i915 = rps_to_i915(rps); 1668 1669 if (GRAPHICS_VER(i915) >= 9) 1670 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1671 GT_FREQUENCY_MULTIPLIER); 1672 else if (IS_CHERRYVIEW(i915)) 1673 return chv_freq_opcode(rps, val); 1674 else if (IS_VALLEYVIEW(i915)) 1675 return byt_freq_opcode(rps, val); 1676 else if (GRAPHICS_VER(i915) >= 6) 1677 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1678 else 1679 return val; 1680 } 1681 1682 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1683 { 1684 struct drm_i915_private *i915 = rps_to_i915(rps); 1685 1686 rps->gpll_ref_freq = 1687 vlv_get_cck_clock(&i915->drm, "GPLL ref", 1688 CCK_GPLL_CLOCK_CONTROL, 1689 i915->czclk_freq); 1690 1691 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1692 rps->gpll_ref_freq); 1693 } 1694 1695 static void vlv_rps_init(struct intel_rps *rps) 1696 { 1697 struct drm_i915_private *i915 = rps_to_i915(rps); 1698 1699 vlv_iosf_sb_get(&i915->drm, 1700 BIT(VLV_IOSF_SB_PUNIT) | 1701 BIT(VLV_IOSF_SB_NC) | 1702 BIT(VLV_IOSF_SB_CCK)); 1703 1704 vlv_init_gpll_ref_freq(rps); 1705 1706 rps->max_freq = vlv_rps_max_freq(rps); 1707 rps->rp0_freq = rps->max_freq; 1708 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1709 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1710 1711 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1712 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1713 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1714 1715 rps->rp1_freq = vlv_rps_guar_freq(rps); 1716 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1717 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1718 1719 rps->min_freq = vlv_rps_min_freq(rps); 1720 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1721 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1722 1723 vlv_iosf_sb_put(&i915->drm, 1724 BIT(VLV_IOSF_SB_PUNIT) | 1725 BIT(VLV_IOSF_SB_NC) | 1726 BIT(VLV_IOSF_SB_CCK)); 1727 } 1728 1729 static void chv_rps_init(struct intel_rps *rps) 1730 { 1731 struct drm_i915_private *i915 = rps_to_i915(rps); 1732 1733 vlv_iosf_sb_get(&i915->drm, 1734 BIT(VLV_IOSF_SB_PUNIT) | 1735 BIT(VLV_IOSF_SB_NC) | 1736 BIT(VLV_IOSF_SB_CCK)); 1737 1738 vlv_init_gpll_ref_freq(rps); 1739 1740 rps->max_freq = chv_rps_max_freq(rps); 1741 rps->rp0_freq = rps->max_freq; 1742 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1743 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1744 1745 rps->efficient_freq = chv_rps_rpe_freq(rps); 1746 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1747 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1748 1749 rps->rp1_freq = chv_rps_guar_freq(rps); 1750 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1751 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1752 1753 rps->min_freq = chv_rps_min_freq(rps); 1754 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1755 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1756 1757 vlv_iosf_sb_put(&i915->drm, 1758 BIT(VLV_IOSF_SB_PUNIT) | 1759 BIT(VLV_IOSF_SB_NC) | 1760 BIT(VLV_IOSF_SB_CCK)); 1761 1762 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1763 rps->rp1_freq | rps->min_freq) & 1, 1764 "Odd GPU freq values\n"); 1765 } 1766 1767 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1768 { 1769 ei->ktime = ktime_get_raw(); 1770 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1771 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1772 } 1773 1774 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1775 { 1776 struct intel_uncore *uncore = rps_to_uncore(rps); 1777 const struct intel_rps_ei *prev = &rps->ei; 1778 struct intel_rps_ei now; 1779 u32 events = 0; 1780 1781 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1782 return 0; 1783 1784 vlv_c0_read(uncore, &now); 1785 1786 if (prev->ktime) { 1787 u64 time, c0; 1788 u32 render, media; 1789 1790 time = ktime_us_delta(now.ktime, prev->ktime); 1791 1792 time *= rps_to_i915(rps)->czclk_freq; 1793 1794 /* Workload can be split between render + media, 1795 * e.g. SwapBuffers being blitted in X after being rendered in 1796 * mesa. To account for this we need to combine both engines 1797 * into our activity counter. 1798 */ 1799 render = now.render_c0 - prev->render_c0; 1800 media = now.media_c0 - prev->media_c0; 1801 c0 = max(render, media); 1802 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1803 1804 if (c0 > time * rps->power.up_threshold) 1805 events = GEN6_PM_RP_UP_THRESHOLD; 1806 else if (c0 < time * rps->power.down_threshold) 1807 events = GEN6_PM_RP_DOWN_THRESHOLD; 1808 } 1809 1810 rps->ei = now; 1811 return events; 1812 } 1813 1814 static void rps_work(struct work_struct *work) 1815 { 1816 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1817 struct intel_gt *gt = rps_to_gt(rps); 1818 struct drm_i915_private *i915 = rps_to_i915(rps); 1819 bool client_boost = false; 1820 int new_freq, adj, min, max; 1821 u32 pm_iir = 0; 1822 1823 spin_lock_irq(gt->irq_lock); 1824 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1825 client_boost = atomic_read(&rps->num_waiters); 1826 spin_unlock_irq(gt->irq_lock); 1827 1828 /* Make sure we didn't queue anything we're not going to process. */ 1829 if (!pm_iir && !client_boost) 1830 goto out; 1831 1832 mutex_lock(&rps->lock); 1833 if (!intel_rps_is_active(rps)) { 1834 mutex_unlock(&rps->lock); 1835 return; 1836 } 1837 1838 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1839 1840 adj = rps->last_adj; 1841 new_freq = rps->cur_freq; 1842 min = rps->min_freq_softlimit; 1843 max = rps->max_freq_softlimit; 1844 if (client_boost) 1845 max = rps->max_freq; 1846 1847 GT_TRACE(gt, 1848 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1849 pm_iir, str_yes_no(client_boost), 1850 adj, new_freq, min, max); 1851 1852 if (client_boost && new_freq < rps->boost_freq) { 1853 new_freq = rps->boost_freq; 1854 adj = 0; 1855 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1856 if (adj > 0) 1857 adj *= 2; 1858 else /* CHV needs even encode values */ 1859 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1860 1861 if (new_freq >= rps->max_freq_softlimit) 1862 adj = 0; 1863 } else if (client_boost) { 1864 adj = 0; 1865 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1866 if (rps->cur_freq > rps->efficient_freq) 1867 new_freq = rps->efficient_freq; 1868 else if (rps->cur_freq > rps->min_freq_softlimit) 1869 new_freq = rps->min_freq_softlimit; 1870 adj = 0; 1871 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1872 if (adj < 0) 1873 adj *= 2; 1874 else /* CHV needs even encode values */ 1875 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1876 1877 if (new_freq <= rps->min_freq_softlimit) 1878 adj = 0; 1879 } else { /* unknown event */ 1880 adj = 0; 1881 } 1882 1883 /* 1884 * sysfs frequency limits may have snuck in while 1885 * servicing the interrupt 1886 */ 1887 new_freq += adj; 1888 new_freq = clamp_t(int, new_freq, min, max); 1889 1890 if (intel_rps_set(rps, new_freq)) { 1891 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1892 adj = 0; 1893 } 1894 rps->last_adj = adj; 1895 1896 mutex_unlock(&rps->lock); 1897 1898 out: 1899 spin_lock_irq(gt->irq_lock); 1900 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1901 spin_unlock_irq(gt->irq_lock); 1902 } 1903 1904 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1905 { 1906 struct intel_gt *gt = rps_to_gt(rps); 1907 const u32 events = rps->pm_events & pm_iir; 1908 1909 lockdep_assert_held(gt->irq_lock); 1910 1911 if (unlikely(!events)) 1912 return; 1913 1914 GT_TRACE(gt, "irq events:%x\n", events); 1915 1916 gen6_gt_pm_mask_irq(gt, events); 1917 1918 rps->pm_iir |= events; 1919 queue_work(gt->i915->unordered_wq, &rps->work); 1920 } 1921 1922 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1923 { 1924 struct intel_gt *gt = rps_to_gt(rps); 1925 u32 events; 1926 1927 events = pm_iir & rps->pm_events; 1928 if (events) { 1929 spin_lock(gt->irq_lock); 1930 1931 GT_TRACE(gt, "irq events:%x\n", events); 1932 1933 gen6_gt_pm_mask_irq(gt, events); 1934 rps->pm_iir |= events; 1935 1936 queue_work(gt->i915->unordered_wq, &rps->work); 1937 spin_unlock(gt->irq_lock); 1938 } 1939 1940 if (GRAPHICS_VER(gt->i915) >= 8) 1941 return; 1942 1943 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1944 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1945 1946 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1947 drm_dbg(&rps_to_i915(rps)->drm, 1948 "Command parser error, pm_iir 0x%08x\n", pm_iir); 1949 } 1950 1951 void gen5_rps_irq_handler(struct intel_rps *rps) 1952 { 1953 struct intel_uncore *uncore = rps_to_uncore(rps); 1954 u32 busy_up, busy_down, max_avg, min_avg; 1955 u8 new_freq; 1956 1957 spin_lock(&mchdev_lock); 1958 1959 intel_uncore_write16(uncore, 1960 MEMINTRSTS, 1961 intel_uncore_read(uncore, MEMINTRSTS)); 1962 1963 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1964 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1965 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1966 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1967 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1968 1969 /* Handle RCS change request from hw */ 1970 new_freq = rps->cur_freq; 1971 if (busy_up > max_avg) 1972 new_freq++; 1973 else if (busy_down < min_avg) 1974 new_freq--; 1975 new_freq = clamp(new_freq, 1976 rps->min_freq_softlimit, 1977 rps->max_freq_softlimit); 1978 1979 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1980 rps->cur_freq = new_freq; 1981 1982 spin_unlock(&mchdev_lock); 1983 } 1984 1985 void intel_rps_init_early(struct intel_rps *rps) 1986 { 1987 mutex_init(&rps->lock); 1988 mutex_init(&rps->power.mutex); 1989 1990 INIT_WORK(&rps->work, rps_work); 1991 timer_setup(&rps->timer, rps_timer, 0); 1992 1993 atomic_set(&rps->num_waiters, 0); 1994 } 1995 1996 void intel_rps_init(struct intel_rps *rps) 1997 { 1998 struct drm_i915_private *i915 = rps_to_i915(rps); 1999 2000 if (rps_uses_slpc(rps)) 2001 return; 2002 2003 if (IS_CHERRYVIEW(i915)) 2004 chv_rps_init(rps); 2005 else if (IS_VALLEYVIEW(i915)) 2006 vlv_rps_init(rps); 2007 else if (GRAPHICS_VER(i915) >= 6) 2008 gen6_rps_init(rps); 2009 else if (IS_IRONLAKE_M(i915)) 2010 gen5_rps_init(rps); 2011 2012 /* Derive initial user preferences/limits from the hardware limits */ 2013 rps->max_freq_softlimit = rps->max_freq; 2014 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit; 2015 rps->min_freq_softlimit = rps->min_freq; 2016 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit; 2017 2018 /* After setting max-softlimit, find the overclock max freq */ 2019 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 2020 u32 params = 0; 2021 2022 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL); 2023 if (params & BIT(31)) { /* OC supported */ 2024 drm_dbg(&i915->drm, 2025 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 2026 (rps->max_freq & 0xff) * 50, 2027 (params & 0xff) * 50); 2028 rps->max_freq = params & 0xff; 2029 } 2030 } 2031 2032 /* Set default thresholds in % */ 2033 rps->power.up_threshold = 95; 2034 rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold; 2035 rps->power.down_threshold = 85; 2036 rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold; 2037 2038 /* Finally allow us to boost to max by default */ 2039 rps->boost_freq = rps->max_freq; 2040 rps->idle_freq = rps->min_freq; 2041 2042 /* Start in the middle, from here we will autotune based on workload */ 2043 rps->cur_freq = rps->efficient_freq; 2044 2045 rps->pm_intrmsk_mbz = 0; 2046 2047 /* 2048 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 2049 * if GEN6_PM_UP_EI_EXPIRED is masked. 2050 * 2051 * TODO: verify if this can be reproduced on VLV,CHV. 2052 */ 2053 if (GRAPHICS_VER(i915) <= 7) 2054 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 2055 2056 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 2057 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 2058 2059 /* GuC needs ARAT expired interrupt unmasked */ 2060 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 2061 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 2062 } 2063 2064 void intel_rps_sanitize(struct intel_rps *rps) 2065 { 2066 if (rps_uses_slpc(rps)) 2067 return; 2068 2069 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 2070 rps_disable_interrupts(rps); 2071 } 2072 2073 u32 intel_rps_read_rpstat(struct intel_rps *rps) 2074 { 2075 struct drm_i915_private *i915 = rps_to_i915(rps); 2076 i915_reg_t rpstat; 2077 2078 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1; 2079 2080 return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat); 2081 } 2082 2083 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 2084 { 2085 struct drm_i915_private *i915 = rps_to_i915(rps); 2086 u32 cagf; 2087 2088 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 2089 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat); 2090 else if (GRAPHICS_VER(i915) >= 12) 2091 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat); 2092 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 2093 cagf = REG_FIELD_GET(RPE_MASK, rpstat); 2094 else if (GRAPHICS_VER(i915) >= 9) 2095 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat); 2096 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2097 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat); 2098 else if (GRAPHICS_VER(i915) >= 6) 2099 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat); 2100 else 2101 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat)); 2102 2103 return cagf; 2104 } 2105 2106 static u32 __read_cagf(struct intel_rps *rps, bool take_fw) 2107 { 2108 struct drm_i915_private *i915 = rps_to_i915(rps); 2109 struct intel_uncore *uncore = rps_to_uncore(rps); 2110 i915_reg_t r = INVALID_MMIO_REG; 2111 u32 freq; 2112 2113 /* 2114 * For Gen12+ reading freq from HW does not need a forcewake and 2115 * registers will return 0 freq when GT is in RC6 2116 */ 2117 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 2118 r = MTL_MIRROR_TARGET_WP1; 2119 } else if (GRAPHICS_VER(i915) >= 12) { 2120 r = GEN12_RPSTAT1; 2121 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2122 vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 2123 freq = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS); 2124 vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT)); 2125 } else if (GRAPHICS_VER(i915) >= 6) { 2126 r = GEN6_RPSTAT1; 2127 } else { 2128 r = MEMSTAT_ILK; 2129 } 2130 2131 if (i915_mmio_reg_valid(r)) 2132 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r); 2133 2134 return intel_rps_get_cagf(rps, freq); 2135 } 2136 2137 static u32 read_cagf(struct intel_rps *rps) 2138 { 2139 return __read_cagf(rps, true); 2140 } 2141 2142 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 2143 { 2144 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2145 intel_wakeref_t wakeref; 2146 u32 freq = 0; 2147 2148 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2149 freq = intel_gpu_freq(rps, read_cagf(rps)); 2150 2151 return freq; 2152 } 2153 2154 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps) 2155 { 2156 return intel_gpu_freq(rps, __read_cagf(rps, false)); 2157 } 2158 2159 static u32 intel_rps_read_punit_req(struct intel_rps *rps) 2160 { 2161 struct intel_uncore *uncore = rps_to_uncore(rps); 2162 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2163 intel_wakeref_t wakeref; 2164 u32 freq = 0; 2165 2166 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2167 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2168 2169 return freq; 2170 } 2171 2172 static u32 intel_rps_get_req(u32 pureq) 2173 { 2174 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 2175 2176 return req; 2177 } 2178 2179 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 2180 { 2181 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 2182 2183 return intel_gpu_freq(rps, freq); 2184 } 2185 2186 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 2187 { 2188 if (rps_uses_slpc(rps)) 2189 return intel_rps_read_punit_req_frequency(rps); 2190 else 2191 return intel_gpu_freq(rps, rps->cur_freq); 2192 } 2193 2194 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2195 { 2196 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2197 2198 if (rps_uses_slpc(rps)) 2199 return slpc->max_freq_softlimit; 2200 else 2201 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2202 } 2203 2204 /** 2205 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format. 2206 * @rps: the intel_rps structure 2207 * 2208 * Returns the max frequency in a raw format. In newer platforms raw is in 2209 * units of 50 MHz. 2210 */ 2211 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps) 2212 { 2213 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2214 u32 freq; 2215 2216 if (rps_uses_slpc(rps)) { 2217 return DIV_ROUND_CLOSEST(slpc->rp0_freq, 2218 GT_FREQUENCY_MULTIPLIER); 2219 } else { 2220 freq = rps->max_freq; 2221 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2222 /* Convert GT frequency to 50 MHz units */ 2223 freq /= GEN9_FREQ_SCALER; 2224 } 2225 return freq; 2226 } 2227 } 2228 2229 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2230 { 2231 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2232 2233 if (rps_uses_slpc(rps)) 2234 return slpc->rp0_freq; 2235 else 2236 return intel_gpu_freq(rps, rps->rp0_freq); 2237 } 2238 2239 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2240 { 2241 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2242 2243 if (rps_uses_slpc(rps)) 2244 return slpc->rp1_freq; 2245 else 2246 return intel_gpu_freq(rps, rps->rp1_freq); 2247 } 2248 2249 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2250 { 2251 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2252 2253 if (rps_uses_slpc(rps)) 2254 return slpc->min_freq; 2255 else 2256 return intel_gpu_freq(rps, rps->min_freq); 2257 } 2258 2259 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2260 { 2261 struct intel_gt *gt = rps_to_gt(rps); 2262 struct drm_i915_private *i915 = gt->i915; 2263 struct intel_uncore *uncore = gt->uncore; 2264 struct intel_rps_freq_caps caps; 2265 u32 rp_state_limits; 2266 u32 gt_perf_status; 2267 u32 rpmodectl, rpinclimit, rpdeclimit; 2268 u32 rpstat, cagf, reqf; 2269 u32 rpcurupei, rpcurup, rpprevup; 2270 u32 rpcurdownei, rpcurdown, rpprevdown; 2271 u32 rpupei, rpupt, rpdownei, rpdownt; 2272 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; 2273 2274 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS); 2275 gen6_rps_get_freq_caps(rps, &caps); 2276 if (IS_GEN9_LP(i915)) 2277 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS); 2278 else 2279 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS); 2280 2281 /* RPSTAT1 is in the GT power well */ 2282 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 2283 2284 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2285 if (GRAPHICS_VER(i915) >= 9) { 2286 reqf >>= 23; 2287 } else { 2288 reqf &= ~GEN6_TURBO_DISABLE; 2289 if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2290 reqf >>= 24; 2291 else 2292 reqf >>= 25; 2293 } 2294 reqf = intel_gpu_freq(rps, reqf); 2295 2296 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL); 2297 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2298 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2299 2300 rpstat = intel_rps_read_rpstat(rps); 2301 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; 2302 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK; 2303 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK; 2304 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; 2305 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; 2306 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; 2307 2308 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI); 2309 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2310 2311 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI); 2312 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2313 2314 cagf = intel_rps_read_actual_frequency(rps); 2315 2316 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 2317 2318 if (GRAPHICS_VER(i915) >= 11) { 2319 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE); 2320 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK); 2321 /* 2322 * The equivalent to the PM ISR & IIR cannot be read 2323 * without affecting the current state of the system 2324 */ 2325 pm_isr = 0; 2326 pm_iir = 0; 2327 } else if (GRAPHICS_VER(i915) >= 8) { 2328 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2)); 2329 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2)); 2330 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2)); 2331 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2)); 2332 } else { 2333 pm_ier = intel_uncore_read(uncore, GEN6_PMIER); 2334 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR); 2335 pm_isr = intel_uncore_read(uncore, GEN6_PMISR); 2336 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR); 2337 } 2338 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2339 2340 drm_printf(p, "Video Turbo Mode: %s\n", 2341 str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO)); 2342 drm_printf(p, "HW control enabled: %s\n", 2343 str_yes_no(rpmodectl & GEN6_RP_ENABLE)); 2344 drm_printf(p, "SW control enabled: %s\n", 2345 str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE)); 2346 2347 drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", 2348 pm_ier, pm_imr, pm_mask); 2349 if (GRAPHICS_VER(i915) <= 10) 2350 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n", 2351 pm_isr, pm_iir); 2352 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2353 rps->pm_intrmsk_mbz); 2354 drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); 2355 drm_printf(p, "Render p-state ratio: %d\n", 2356 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8); 2357 drm_printf(p, "Render p-state VID: %d\n", 2358 gt_perf_status & 0xff); 2359 drm_printf(p, "Render p-state limit: %d\n", 2360 rp_state_limits & 0xff); 2361 drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat); 2362 drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl); 2363 drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit); 2364 drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit); 2365 drm_printf(p, "RPNSWREQ: %dMHz\n", reqf); 2366 drm_printf(p, "CAGF: %dMHz\n", cagf); 2367 drm_printf(p, "RP CUR UP EI: %d (%lldns)\n", 2368 rpcurupei, 2369 intel_gt_pm_interval_to_ns(gt, rpcurupei)); 2370 drm_printf(p, "RP CUR UP: %d (%lldns)\n", 2371 rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup)); 2372 drm_printf(p, "RP PREV UP: %d (%lldns)\n", 2373 rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup)); 2374 drm_printf(p, "Up threshold: %d%%\n", 2375 rps->power.up_threshold); 2376 drm_printf(p, "RP UP EI: %d (%lldns)\n", 2377 rpupei, intel_gt_pm_interval_to_ns(gt, rpupei)); 2378 drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n", 2379 rpupt, intel_gt_pm_interval_to_ns(gt, rpupt)); 2380 2381 drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n", 2382 rpcurdownei, 2383 intel_gt_pm_interval_to_ns(gt, rpcurdownei)); 2384 drm_printf(p, "RP CUR DOWN: %d (%lldns)\n", 2385 rpcurdown, 2386 intel_gt_pm_interval_to_ns(gt, rpcurdown)); 2387 drm_printf(p, "RP PREV DOWN: %d (%lldns)\n", 2388 rpprevdown, 2389 intel_gt_pm_interval_to_ns(gt, rpprevdown)); 2390 drm_printf(p, "Down threshold: %d%%\n", 2391 rps->power.down_threshold); 2392 drm_printf(p, "RP DOWN EI: %d (%lldns)\n", 2393 rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei)); 2394 drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n", 2395 rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt)); 2396 2397 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2398 intel_gpu_freq(rps, caps.min_freq)); 2399 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2400 intel_gpu_freq(rps, caps.rp1_freq)); 2401 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2402 intel_gpu_freq(rps, caps.rp0_freq)); 2403 drm_printf(p, "Max overclocked frequency: %dMHz\n", 2404 intel_gpu_freq(rps, rps->max_freq)); 2405 2406 drm_printf(p, "Current freq: %d MHz\n", 2407 intel_gpu_freq(rps, rps->cur_freq)); 2408 drm_printf(p, "Actual freq: %d MHz\n", cagf); 2409 drm_printf(p, "Idle freq: %d MHz\n", 2410 intel_gpu_freq(rps, rps->idle_freq)); 2411 drm_printf(p, "Min freq: %d MHz\n", 2412 intel_gpu_freq(rps, rps->min_freq)); 2413 drm_printf(p, "Boost freq: %d MHz\n", 2414 intel_gpu_freq(rps, rps->boost_freq)); 2415 drm_printf(p, "Max freq: %d MHz\n", 2416 intel_gpu_freq(rps, rps->max_freq)); 2417 drm_printf(p, 2418 "efficient (RPe) frequency: %d MHz\n", 2419 intel_gpu_freq(rps, rps->efficient_freq)); 2420 } 2421 2422 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2423 { 2424 struct intel_gt *gt = rps_to_gt(rps); 2425 struct intel_uncore *uncore = gt->uncore; 2426 struct intel_rps_freq_caps caps; 2427 u32 pm_mask; 2428 2429 gen6_rps_get_freq_caps(rps, &caps); 2430 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2431 2432 drm_printf(p, "PM MASK=0x%08x\n", pm_mask); 2433 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2434 rps->pm_intrmsk_mbz); 2435 drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps)); 2436 drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps)); 2437 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2438 intel_gpu_freq(rps, caps.min_freq)); 2439 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2440 intel_gpu_freq(rps, caps.rp1_freq)); 2441 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2442 intel_gpu_freq(rps, caps.rp0_freq)); 2443 drm_printf(p, "Current freq: %d MHz\n", 2444 intel_rps_get_requested_frequency(rps)); 2445 drm_printf(p, "Actual freq: %d MHz\n", 2446 intel_rps_read_actual_frequency(rps)); 2447 drm_printf(p, "Min freq: %d MHz\n", 2448 intel_rps_get_min_frequency(rps)); 2449 drm_printf(p, "Boost freq: %d MHz\n", 2450 intel_rps_get_boost_frequency(rps)); 2451 drm_printf(p, "Max freq: %d MHz\n", 2452 intel_rps_get_max_frequency(rps)); 2453 drm_printf(p, 2454 "efficient (RPe) frequency: %d MHz\n", 2455 intel_gpu_freq(rps, caps.rp1_freq)); 2456 } 2457 2458 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2459 { 2460 if (rps_uses_slpc(rps)) 2461 return slpc_frequency_dump(rps, p); 2462 else 2463 return rps_frequency_dump(rps, p); 2464 } 2465 2466 static int set_max_freq(struct intel_rps *rps, u32 val) 2467 { 2468 struct drm_i915_private *i915 = rps_to_i915(rps); 2469 int ret = 0; 2470 2471 mutex_lock(&rps->lock); 2472 2473 val = intel_freq_opcode(rps, val); 2474 if (val < rps->min_freq || 2475 val > rps->max_freq || 2476 val < rps->min_freq_softlimit) { 2477 ret = -EINVAL; 2478 goto unlock; 2479 } 2480 2481 if (val > rps->rp0_freq) 2482 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2483 intel_gpu_freq(rps, val)); 2484 2485 rps->max_freq_softlimit = val; 2486 2487 val = clamp_t(int, rps->cur_freq, 2488 rps->min_freq_softlimit, 2489 rps->max_freq_softlimit); 2490 2491 /* 2492 * We still need *_set_rps to process the new max_delay and 2493 * update the interrupt limits and PMINTRMSK even though 2494 * frequency request may be unchanged. 2495 */ 2496 intel_rps_set(rps, val); 2497 2498 unlock: 2499 mutex_unlock(&rps->lock); 2500 2501 return ret; 2502 } 2503 2504 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2505 { 2506 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2507 2508 if (rps_uses_slpc(rps)) 2509 return intel_guc_slpc_set_max_freq(slpc, val); 2510 else 2511 return set_max_freq(rps, val); 2512 } 2513 2514 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2515 { 2516 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2517 2518 if (rps_uses_slpc(rps)) 2519 return slpc->min_freq_softlimit; 2520 else 2521 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2522 } 2523 2524 /** 2525 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format. 2526 * @rps: the intel_rps structure 2527 * 2528 * Returns the min frequency in a raw format. In newer platforms raw is in 2529 * units of 50 MHz. 2530 */ 2531 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps) 2532 { 2533 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2534 u32 freq; 2535 2536 if (rps_uses_slpc(rps)) { 2537 return DIV_ROUND_CLOSEST(slpc->min_freq, 2538 GT_FREQUENCY_MULTIPLIER); 2539 } else { 2540 freq = rps->min_freq; 2541 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2542 /* Convert GT frequency to 50 MHz units */ 2543 freq /= GEN9_FREQ_SCALER; 2544 } 2545 return freq; 2546 } 2547 } 2548 2549 static int set_min_freq(struct intel_rps *rps, u32 val) 2550 { 2551 int ret = 0; 2552 2553 mutex_lock(&rps->lock); 2554 2555 val = intel_freq_opcode(rps, val); 2556 if (val < rps->min_freq || 2557 val > rps->max_freq || 2558 val > rps->max_freq_softlimit) { 2559 ret = -EINVAL; 2560 goto unlock; 2561 } 2562 2563 rps->min_freq_softlimit = val; 2564 2565 val = clamp_t(int, rps->cur_freq, 2566 rps->min_freq_softlimit, 2567 rps->max_freq_softlimit); 2568 2569 /* 2570 * We still need *_set_rps to process the new min_delay and 2571 * update the interrupt limits and PMINTRMSK even though 2572 * frequency request may be unchanged. 2573 */ 2574 intel_rps_set(rps, val); 2575 2576 unlock: 2577 mutex_unlock(&rps->lock); 2578 2579 return ret; 2580 } 2581 2582 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2583 { 2584 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2585 2586 if (rps_uses_slpc(rps)) 2587 return intel_guc_slpc_set_min_freq(slpc, val); 2588 else 2589 return set_min_freq(rps, val); 2590 } 2591 2592 u8 intel_rps_get_up_threshold(struct intel_rps *rps) 2593 { 2594 return rps->power.up_threshold; 2595 } 2596 2597 static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val) 2598 { 2599 int ret; 2600 2601 if (val > 100) 2602 return -EINVAL; 2603 2604 ret = mutex_lock_interruptible(&rps->lock); 2605 if (ret) 2606 return ret; 2607 2608 if (*threshold == val) 2609 goto out_unlock; 2610 2611 *threshold = val; 2612 2613 /* Force reset. */ 2614 rps->last_freq = -1; 2615 mutex_lock(&rps->power.mutex); 2616 rps->power.mode = -1; 2617 mutex_unlock(&rps->power.mutex); 2618 2619 intel_rps_set(rps, clamp(rps->cur_freq, 2620 rps->min_freq_softlimit, 2621 rps->max_freq_softlimit)); 2622 2623 out_unlock: 2624 mutex_unlock(&rps->lock); 2625 2626 return ret; 2627 } 2628 2629 int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold) 2630 { 2631 return rps_set_threshold(rps, &rps->power.up_threshold, threshold); 2632 } 2633 2634 u8 intel_rps_get_down_threshold(struct intel_rps *rps) 2635 { 2636 return rps->power.down_threshold; 2637 } 2638 2639 int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold) 2640 { 2641 return rps_set_threshold(rps, &rps->power.down_threshold, threshold); 2642 } 2643 2644 static void intel_rps_set_manual(struct intel_rps *rps, bool enable) 2645 { 2646 struct intel_uncore *uncore = rps_to_uncore(rps); 2647 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE; 2648 2649 /* Allow punit to process software requests */ 2650 intel_uncore_write(uncore, GEN6_RP_CONTROL, state); 2651 } 2652 2653 void intel_rps_raise_unslice(struct intel_rps *rps) 2654 { 2655 struct intel_uncore *uncore = rps_to_uncore(rps); 2656 2657 mutex_lock(&rps->lock); 2658 2659 if (rps_uses_slpc(rps)) { 2660 /* RP limits have not been initialized yet for SLPC path */ 2661 struct intel_rps_freq_caps caps; 2662 2663 gen6_rps_get_freq_caps(rps, &caps); 2664 2665 intel_rps_set_manual(rps, true); 2666 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2667 ((caps.rp0_freq << 2668 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2669 GEN9_IGNORE_SLICE_RATIO)); 2670 intel_rps_set_manual(rps, false); 2671 } else { 2672 intel_rps_set(rps, rps->rp0_freq); 2673 } 2674 2675 mutex_unlock(&rps->lock); 2676 } 2677 2678 void intel_rps_lower_unslice(struct intel_rps *rps) 2679 { 2680 struct intel_uncore *uncore = rps_to_uncore(rps); 2681 2682 mutex_lock(&rps->lock); 2683 2684 if (rps_uses_slpc(rps)) { 2685 /* RP limits have not been initialized yet for SLPC path */ 2686 struct intel_rps_freq_caps caps; 2687 2688 gen6_rps_get_freq_caps(rps, &caps); 2689 2690 intel_rps_set_manual(rps, true); 2691 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2692 ((caps.min_freq << 2693 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2694 GEN9_IGNORE_SLICE_RATIO)); 2695 intel_rps_set_manual(rps, false); 2696 } else { 2697 intel_rps_set(rps, rps->min_freq); 2698 } 2699 2700 mutex_unlock(&rps->lock); 2701 } 2702 2703 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32) 2704 { 2705 struct intel_gt *gt = rps_to_gt(rps); 2706 intel_wakeref_t wakeref; 2707 u32 val; 2708 2709 with_intel_runtime_pm(gt->uncore->rpm, wakeref) 2710 val = intel_uncore_read(gt->uncore, reg32); 2711 2712 return val; 2713 } 2714 2715 bool rps_read_mask_mmio(struct intel_rps *rps, 2716 i915_reg_t reg32, u32 mask) 2717 { 2718 return rps_read_mmio(rps, reg32) & mask; 2719 } 2720 2721 /* External interface for intel_ips.ko */ 2722 2723 static struct drm_i915_private __rcu *ips_mchdev; 2724 2725 /* 2726 * Tells the intel_ips driver that the i915 driver is now loaded, if 2727 * IPS got loaded first. 2728 * 2729 * This awkward dance is so that neither module has to depend on the 2730 * other in order for IPS to do the appropriate communication of 2731 * GPU turbo limits to i915. 2732 */ 2733 static void 2734 ips_ping_for_i915_load(void) 2735 { 2736 void (*link)(void); 2737 2738 link = symbol_get(ips_link_to_i915_driver); 2739 if (link) { 2740 link(); 2741 symbol_put(ips_link_to_i915_driver); 2742 } 2743 } 2744 2745 void intel_rps_driver_register(struct intel_rps *rps) 2746 { 2747 struct intel_gt *gt = rps_to_gt(rps); 2748 2749 /* 2750 * We only register the i915 ips part with intel-ips once everything is 2751 * set up, to avoid intel-ips sneaking in and reading bogus values. 2752 */ 2753 if (GRAPHICS_VER(gt->i915) == 5) { 2754 GEM_BUG_ON(ips_mchdev); 2755 rcu_assign_pointer(ips_mchdev, gt->i915); 2756 ips_ping_for_i915_load(); 2757 } 2758 } 2759 2760 void intel_rps_driver_unregister(struct intel_rps *rps) 2761 { 2762 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2763 rcu_assign_pointer(ips_mchdev, NULL); 2764 } 2765 2766 static struct drm_i915_private *mchdev_get(void) 2767 { 2768 struct drm_i915_private *i915; 2769 2770 rcu_read_lock(); 2771 i915 = rcu_dereference(ips_mchdev); 2772 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2773 i915 = NULL; 2774 rcu_read_unlock(); 2775 2776 return i915; 2777 } 2778 2779 /** 2780 * i915_read_mch_val - return value for IPS use 2781 * 2782 * Calculate and return a value for the IPS driver to use when deciding whether 2783 * we have thermal and power headroom to increase CPU or GPU power budget. 2784 */ 2785 unsigned long i915_read_mch_val(void) 2786 { 2787 struct drm_i915_private *i915; 2788 unsigned long chipset_val = 0; 2789 unsigned long graphics_val = 0; 2790 intel_wakeref_t wakeref; 2791 2792 i915 = mchdev_get(); 2793 if (!i915) 2794 return 0; 2795 2796 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2797 struct intel_ips *ips = &to_gt(i915)->rps.ips; 2798 2799 spin_lock_irq(&mchdev_lock); 2800 chipset_val = __ips_chipset_val(ips); 2801 graphics_val = __ips_gfx_val(ips); 2802 spin_unlock_irq(&mchdev_lock); 2803 } 2804 2805 drm_dev_put(&i915->drm); 2806 return chipset_val + graphics_val; 2807 } 2808 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2809 2810 /** 2811 * i915_gpu_raise - raise GPU frequency limit 2812 * 2813 * Raise the limit; IPS indicates we have thermal headroom. 2814 */ 2815 bool i915_gpu_raise(void) 2816 { 2817 struct drm_i915_private *i915; 2818 struct intel_rps *rps; 2819 2820 i915 = mchdev_get(); 2821 if (!i915) 2822 return false; 2823 2824 rps = &to_gt(i915)->rps; 2825 2826 spin_lock_irq(&mchdev_lock); 2827 if (rps->max_freq_softlimit < rps->max_freq) 2828 rps->max_freq_softlimit++; 2829 spin_unlock_irq(&mchdev_lock); 2830 2831 drm_dev_put(&i915->drm); 2832 return true; 2833 } 2834 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2835 2836 /** 2837 * i915_gpu_lower - lower GPU frequency limit 2838 * 2839 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2840 * frequency maximum. 2841 */ 2842 bool i915_gpu_lower(void) 2843 { 2844 struct drm_i915_private *i915; 2845 struct intel_rps *rps; 2846 2847 i915 = mchdev_get(); 2848 if (!i915) 2849 return false; 2850 2851 rps = &to_gt(i915)->rps; 2852 2853 spin_lock_irq(&mchdev_lock); 2854 if (rps->max_freq_softlimit > rps->min_freq) 2855 rps->max_freq_softlimit--; 2856 spin_unlock_irq(&mchdev_lock); 2857 2858 drm_dev_put(&i915->drm); 2859 return true; 2860 } 2861 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2862 2863 /** 2864 * i915_gpu_busy - indicate GPU business to IPS 2865 * 2866 * Tell the IPS driver whether or not the GPU is busy. 2867 */ 2868 bool i915_gpu_busy(void) 2869 { 2870 struct drm_i915_private *i915; 2871 bool ret; 2872 2873 i915 = mchdev_get(); 2874 if (!i915) 2875 return false; 2876 2877 ret = to_gt(i915)->awake; 2878 2879 drm_dev_put(&i915->drm); 2880 return ret; 2881 } 2882 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2883 2884 /** 2885 * i915_gpu_turbo_disable - disable graphics turbo 2886 * 2887 * Disable graphics turbo by resetting the max frequency and setting the 2888 * current frequency to the default. 2889 */ 2890 bool i915_gpu_turbo_disable(void) 2891 { 2892 struct drm_i915_private *i915; 2893 struct intel_rps *rps; 2894 bool ret; 2895 2896 i915 = mchdev_get(); 2897 if (!i915) 2898 return false; 2899 2900 rps = &to_gt(i915)->rps; 2901 2902 spin_lock_irq(&mchdev_lock); 2903 rps->max_freq_softlimit = rps->min_freq; 2904 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq); 2905 spin_unlock_irq(&mchdev_lock); 2906 2907 drm_dev_put(&i915->drm); 2908 return ret; 2909 } 2910 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2911 2912 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2913 #include "selftest_rps.c" 2914 #include "selftest_slpc.c" 2915 #endif 2916