1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 8 #include <drm/intel/i915_drm.h> 9 10 #include "display/intel_display.h" 11 #include "display/intel_display_irq.h" 12 #include "i915_drv.h" 13 #include "i915_irq.h" 14 #include "i915_reg.h" 15 #include "intel_breadcrumbs.h" 16 #include "intel_gt.h" 17 #include "intel_gt_clock_utils.h" 18 #include "intel_gt_irq.h" 19 #include "intel_gt_pm.h" 20 #include "intel_gt_pm_irq.h" 21 #include "intel_gt_print.h" 22 #include "intel_gt_regs.h" 23 #include "intel_mchbar_regs.h" 24 #include "intel_pcode.h" 25 #include "intel_rps.h" 26 #include "vlv_sideband.h" 27 #include "../../../platform/x86/intel_ips.h" 28 29 #define BUSY_MAX_EI 20u /* ms */ 30 31 /* 32 * Lock protecting IPS related data structures 33 */ 34 static DEFINE_SPINLOCK(mchdev_lock); 35 36 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 37 { 38 return container_of(rps, struct intel_gt, rps); 39 } 40 41 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 42 { 43 return rps_to_gt(rps)->i915; 44 } 45 46 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 47 { 48 return rps_to_gt(rps)->uncore; 49 } 50 51 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 52 { 53 struct intel_gt *gt = rps_to_gt(rps); 54 55 return >_to_guc(gt)->slpc; 56 } 57 58 static bool rps_uses_slpc(struct intel_rps *rps) 59 { 60 struct intel_gt *gt = rps_to_gt(rps); 61 62 return intel_uc_uses_guc_slpc(>->uc); 63 } 64 65 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 66 { 67 return mask & ~rps->pm_intrmsk_mbz; 68 } 69 70 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 71 { 72 intel_uncore_write_fw(uncore, reg, val); 73 } 74 75 static void rps_timer(struct timer_list *t) 76 { 77 struct intel_rps *rps = from_timer(rps, t, timer); 78 struct intel_gt *gt = rps_to_gt(rps); 79 struct intel_engine_cs *engine; 80 ktime_t dt, last, timestamp; 81 enum intel_engine_id id; 82 s64 max_busy[3] = {}; 83 84 timestamp = 0; 85 for_each_engine(engine, gt, id) { 86 s64 busy; 87 int i; 88 89 dt = intel_engine_get_busy_time(engine, ×tamp); 90 last = engine->stats.rps; 91 engine->stats.rps = dt; 92 93 busy = ktime_to_ns(ktime_sub(dt, last)); 94 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 95 if (busy > max_busy[i]) 96 swap(busy, max_busy[i]); 97 } 98 } 99 last = rps->pm_timestamp; 100 rps->pm_timestamp = timestamp; 101 102 if (intel_rps_is_active(rps)) { 103 s64 busy; 104 int i; 105 106 dt = ktime_sub(timestamp, last); 107 108 /* 109 * Our goal is to evaluate each engine independently, so we run 110 * at the lowest clocks required to sustain the heaviest 111 * workload. However, a task may be split into sequential 112 * dependent operations across a set of engines, such that 113 * the independent contributions do not account for high load, 114 * but overall the task is GPU bound. For example, consider 115 * video decode on vcs followed by colour post-processing 116 * on vecs, followed by general post-processing on rcs. 117 * Since multi-engines being active does imply a single 118 * continuous workload across all engines, we hedge our 119 * bets by only contributing a factor of the distributed 120 * load into our busyness calculation. 121 */ 122 busy = max_busy[0]; 123 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 124 if (!max_busy[i]) 125 break; 126 127 busy += div_u64(max_busy[i], 1 << i); 128 } 129 GT_TRACE(gt, 130 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 131 busy, (int)div64_u64(100 * busy, dt), 132 max_busy[0], max_busy[1], max_busy[2], 133 rps->pm_interval); 134 135 if (100 * busy > rps->power.up_threshold * dt && 136 rps->cur_freq < rps->max_freq_softlimit) { 137 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 138 rps->pm_interval = 1; 139 queue_work(gt->i915->unordered_wq, &rps->work); 140 } else if (100 * busy < rps->power.down_threshold * dt && 141 rps->cur_freq > rps->min_freq_softlimit) { 142 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 143 rps->pm_interval = 1; 144 queue_work(gt->i915->unordered_wq, &rps->work); 145 } else { 146 rps->last_adj = 0; 147 } 148 149 mod_timer(&rps->timer, 150 jiffies + msecs_to_jiffies(rps->pm_interval)); 151 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 152 } 153 } 154 155 static void rps_start_timer(struct intel_rps *rps) 156 { 157 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 158 rps->pm_interval = 1; 159 mod_timer(&rps->timer, jiffies + 1); 160 } 161 162 static void rps_stop_timer(struct intel_rps *rps) 163 { 164 del_timer_sync(&rps->timer); 165 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 166 cancel_work_sync(&rps->work); 167 } 168 169 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 170 { 171 u32 mask = 0; 172 173 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 174 if (val > rps->min_freq_softlimit) 175 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 176 GEN6_PM_RP_DOWN_THRESHOLD | 177 GEN6_PM_RP_DOWN_TIMEOUT); 178 179 if (val < rps->max_freq_softlimit) 180 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 181 182 mask &= rps->pm_events; 183 184 return rps_pm_sanitize_mask(rps, ~mask); 185 } 186 187 static void rps_reset_ei(struct intel_rps *rps) 188 { 189 memset(&rps->ei, 0, sizeof(rps->ei)); 190 } 191 192 static void rps_enable_interrupts(struct intel_rps *rps) 193 { 194 struct intel_gt *gt = rps_to_gt(rps); 195 196 GEM_BUG_ON(rps_uses_slpc(rps)); 197 198 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 199 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 200 201 rps_reset_ei(rps); 202 203 spin_lock_irq(gt->irq_lock); 204 gen6_gt_pm_enable_irq(gt, rps->pm_events); 205 spin_unlock_irq(gt->irq_lock); 206 207 intel_uncore_write(gt->uncore, 208 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 209 } 210 211 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 212 { 213 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 214 } 215 216 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 217 { 218 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 219 ; 220 } 221 222 static void rps_reset_interrupts(struct intel_rps *rps) 223 { 224 struct intel_gt *gt = rps_to_gt(rps); 225 226 spin_lock_irq(gt->irq_lock); 227 if (GRAPHICS_VER(gt->i915) >= 11) 228 gen11_rps_reset_interrupts(rps); 229 else 230 gen6_rps_reset_interrupts(rps); 231 232 rps->pm_iir = 0; 233 spin_unlock_irq(gt->irq_lock); 234 } 235 236 static void rps_disable_interrupts(struct intel_rps *rps) 237 { 238 struct intel_gt *gt = rps_to_gt(rps); 239 240 intel_uncore_write(gt->uncore, 241 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 242 243 spin_lock_irq(gt->irq_lock); 244 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 245 spin_unlock_irq(gt->irq_lock); 246 247 intel_synchronize_irq(gt->i915); 248 249 /* 250 * Now that we will not be generating any more work, flush any 251 * outstanding tasks. As we are called on the RPS idle path, 252 * we will reset the GPU to minimum frequencies, so the current 253 * state of the worker can be discarded. 254 */ 255 cancel_work_sync(&rps->work); 256 257 rps_reset_interrupts(rps); 258 GT_TRACE(gt, "interrupts:off\n"); 259 } 260 261 static const struct cparams { 262 u16 i; 263 u16 t; 264 u16 m; 265 u16 c; 266 } cparams[] = { 267 { 1, 1333, 301, 28664 }, 268 { 1, 1067, 294, 24460 }, 269 { 1, 800, 294, 25192 }, 270 { 0, 1333, 276, 27605 }, 271 { 0, 1067, 276, 27605 }, 272 { 0, 800, 231, 23784 }, 273 }; 274 275 static void gen5_rps_init(struct intel_rps *rps) 276 { 277 struct drm_i915_private *i915 = rps_to_i915(rps); 278 struct intel_uncore *uncore = rps_to_uncore(rps); 279 u8 fmax, fmin, fstart; 280 u32 rgvmodectl; 281 int c_m, i; 282 283 if (i915->fsb_freq <= 3200000) 284 c_m = 0; 285 else if (i915->fsb_freq <= 4800000) 286 c_m = 1; 287 else 288 c_m = 2; 289 290 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 291 if (cparams[i].i == c_m && 292 cparams[i].t == DIV_ROUND_CLOSEST(i915->mem_freq, 1000)) { 293 rps->ips.m = cparams[i].m; 294 rps->ips.c = cparams[i].c; 295 break; 296 } 297 } 298 299 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 300 301 /* Set up min, max, and cur for interrupt handling */ 302 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 303 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 304 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 305 MEMMODE_FSTART_SHIFT; 306 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 307 fmax, fmin, fstart); 308 309 rps->min_freq = fmax; 310 rps->efficient_freq = fstart; 311 rps->max_freq = fmin; 312 } 313 314 static unsigned long 315 __ips_chipset_val(struct intel_ips *ips) 316 { 317 struct intel_uncore *uncore = 318 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 319 unsigned long now = jiffies_to_msecs(jiffies), dt; 320 unsigned long result; 321 u64 total, delta; 322 323 lockdep_assert_held(&mchdev_lock); 324 325 /* 326 * Prevent division-by-zero if we are asking too fast. 327 * Also, we don't get interesting results if we are polling 328 * faster than once in 10ms, so just return the saved value 329 * in such cases. 330 */ 331 dt = now - ips->last_time1; 332 if (dt <= 10) 333 return ips->chipset_power; 334 335 /* FIXME: handle per-counter overflow */ 336 total = intel_uncore_read(uncore, DMIEC); 337 total += intel_uncore_read(uncore, DDREC); 338 total += intel_uncore_read(uncore, CSIEC); 339 340 delta = total - ips->last_count1; 341 342 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 343 344 ips->last_count1 = total; 345 ips->last_time1 = now; 346 347 ips->chipset_power = result; 348 349 return result; 350 } 351 352 static unsigned long ips_mch_val(struct intel_uncore *uncore) 353 { 354 unsigned int m, x, b; 355 u32 tsfs; 356 357 tsfs = intel_uncore_read(uncore, TSFS); 358 x = intel_uncore_read8(uncore, TR1); 359 360 b = tsfs & TSFS_INTR_MASK; 361 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 362 363 return m * x / 127 - b; 364 } 365 366 static int _pxvid_to_vd(u8 pxvid) 367 { 368 if (pxvid == 0) 369 return 0; 370 371 if (pxvid >= 8 && pxvid < 31) 372 pxvid = 31; 373 374 return (pxvid + 2) * 125; 375 } 376 377 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 378 { 379 const int vd = _pxvid_to_vd(pxvid); 380 381 if (INTEL_INFO(i915)->is_mobile) 382 return max(vd - 1125, 0); 383 384 return vd; 385 } 386 387 static void __gen5_ips_update(struct intel_ips *ips) 388 { 389 struct intel_uncore *uncore = 390 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 391 u64 now, delta, dt; 392 u32 count; 393 394 lockdep_assert_held(&mchdev_lock); 395 396 now = ktime_get_raw_ns(); 397 dt = now - ips->last_time2; 398 do_div(dt, NSEC_PER_MSEC); 399 400 /* Don't divide by 0 */ 401 if (dt <= 10) 402 return; 403 404 count = intel_uncore_read(uncore, GFXEC); 405 delta = count - ips->last_count2; 406 407 ips->last_count2 = count; 408 ips->last_time2 = now; 409 410 /* More magic constants... */ 411 ips->gfx_power = div_u64(delta * 1181, dt * 10); 412 } 413 414 static void gen5_rps_update(struct intel_rps *rps) 415 { 416 spin_lock_irq(&mchdev_lock); 417 __gen5_ips_update(&rps->ips); 418 spin_unlock_irq(&mchdev_lock); 419 } 420 421 static unsigned int gen5_invert_freq(struct intel_rps *rps, 422 unsigned int val) 423 { 424 /* Invert the frequency bin into an ips delay */ 425 val = rps->max_freq - val; 426 val = rps->min_freq + val; 427 428 return val; 429 } 430 431 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 432 { 433 struct intel_uncore *uncore = rps_to_uncore(rps); 434 u16 rgvswctl; 435 436 lockdep_assert_held(&mchdev_lock); 437 438 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 439 if (rgvswctl & MEMCTL_CMD_STS) { 440 drm_dbg(&rps_to_i915(rps)->drm, 441 "gpu busy, RCS change rejected\n"); 442 return -EBUSY; /* still busy with another command */ 443 } 444 445 /* Invert the frequency bin into an ips delay */ 446 val = gen5_invert_freq(rps, val); 447 448 rgvswctl = 449 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 450 (val << MEMCTL_FREQ_SHIFT) | 451 MEMCTL_SFCAVM; 452 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 453 intel_uncore_posting_read16(uncore, MEMSWCTL); 454 455 rgvswctl |= MEMCTL_CMD_STS; 456 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 457 458 return 0; 459 } 460 461 static int gen5_rps_set(struct intel_rps *rps, u8 val) 462 { 463 int err; 464 465 spin_lock_irq(&mchdev_lock); 466 err = __gen5_rps_set(rps, val); 467 spin_unlock_irq(&mchdev_lock); 468 469 return err; 470 } 471 472 static unsigned long intel_pxfreq(u32 vidfreq) 473 { 474 int div = (vidfreq & 0x3f0000) >> 16; 475 int post = (vidfreq & 0x3000) >> 12; 476 int pre = (vidfreq & 0x7); 477 478 if (!pre) 479 return 0; 480 481 return div * 133333 / (pre << post); 482 } 483 484 static unsigned int init_emon(struct intel_uncore *uncore) 485 { 486 u8 pxw[16]; 487 int i; 488 489 /* Disable to program */ 490 intel_uncore_write(uncore, ECR, 0); 491 intel_uncore_posting_read(uncore, ECR); 492 493 /* Program energy weights for various events */ 494 intel_uncore_write(uncore, SDEW, 0x15040d00); 495 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 496 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 497 intel_uncore_write(uncore, CSIEW2, 0x04000004); 498 499 for (i = 0; i < 5; i++) 500 intel_uncore_write(uncore, PEW(i), 0); 501 for (i = 0; i < 3; i++) 502 intel_uncore_write(uncore, DEW(i), 0); 503 504 /* Program P-state weights to account for frequency power adjustment */ 505 for (i = 0; i < 16; i++) { 506 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 507 unsigned int freq = intel_pxfreq(pxvidfreq); 508 unsigned int vid = 509 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 510 unsigned int val; 511 512 val = vid * vid * freq / 1000 * 255; 513 val /= 127 * 127 * 900; 514 515 pxw[i] = val; 516 } 517 /* Render standby states get 0 weight */ 518 pxw[14] = 0; 519 pxw[15] = 0; 520 521 for (i = 0; i < 4; i++) { 522 intel_uncore_write(uncore, PXW(i), 523 pxw[i * 4 + 0] << 24 | 524 pxw[i * 4 + 1] << 16 | 525 pxw[i * 4 + 2] << 8 | 526 pxw[i * 4 + 3] << 0); 527 } 528 529 /* Adjust magic regs to magic values (more experimental results) */ 530 intel_uncore_write(uncore, OGW0, 0); 531 intel_uncore_write(uncore, OGW1, 0); 532 intel_uncore_write(uncore, EG0, 0x00007f00); 533 intel_uncore_write(uncore, EG1, 0x0000000e); 534 intel_uncore_write(uncore, EG2, 0x000e0000); 535 intel_uncore_write(uncore, EG3, 0x68000300); 536 intel_uncore_write(uncore, EG4, 0x42000000); 537 intel_uncore_write(uncore, EG5, 0x00140031); 538 intel_uncore_write(uncore, EG6, 0); 539 intel_uncore_write(uncore, EG7, 0); 540 541 for (i = 0; i < 8; i++) 542 intel_uncore_write(uncore, PXWL(i), 0); 543 544 /* Enable PMON + select events */ 545 intel_uncore_write(uncore, ECR, 0x80000019); 546 547 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 548 } 549 550 static bool gen5_rps_enable(struct intel_rps *rps) 551 { 552 struct drm_i915_private *i915 = rps_to_i915(rps); 553 struct intel_uncore *uncore = rps_to_uncore(rps); 554 u8 fstart, vstart; 555 u32 rgvmodectl; 556 557 spin_lock_irq(&mchdev_lock); 558 559 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 560 561 /* Enable temp reporting */ 562 intel_uncore_write16(uncore, PMMISC, 563 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 564 intel_uncore_write16(uncore, TSC1, 565 intel_uncore_read16(uncore, TSC1) | TSE); 566 567 /* 100ms RC evaluation intervals */ 568 intel_uncore_write(uncore, RCUPEI, 100000); 569 intel_uncore_write(uncore, RCDNEI, 100000); 570 571 /* Set max/min thresholds to 90ms and 80ms respectively */ 572 intel_uncore_write(uncore, RCBMAXAVG, 90000); 573 intel_uncore_write(uncore, RCBMINAVG, 80000); 574 575 intel_uncore_write(uncore, MEMIHYST, 1); 576 577 /* Set up min, max, and cur for interrupt handling */ 578 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 579 MEMMODE_FSTART_SHIFT; 580 581 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 582 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 583 584 intel_uncore_write(uncore, 585 MEMINTREN, 586 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 587 588 intel_uncore_write(uncore, VIDSTART, vstart); 589 intel_uncore_posting_read(uncore, VIDSTART); 590 591 rgvmodectl |= MEMMODE_SWMODE_EN; 592 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 593 594 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 595 MEMCTL_CMD_STS) == 0, 10)) 596 drm_err(&uncore->i915->drm, 597 "stuck trying to change perf mode\n"); 598 mdelay(1); 599 600 __gen5_rps_set(rps, rps->cur_freq); 601 602 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 603 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 604 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 605 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 606 607 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 608 rps->ips.last_time2 = ktime_get_raw_ns(); 609 610 spin_lock(&i915->irq_lock); 611 ilk_enable_display_irq(i915, DE_PCU_EVENT); 612 spin_unlock(&i915->irq_lock); 613 614 spin_unlock_irq(&mchdev_lock); 615 616 rps->ips.corr = init_emon(uncore); 617 618 return true; 619 } 620 621 static void gen5_rps_disable(struct intel_rps *rps) 622 { 623 struct drm_i915_private *i915 = rps_to_i915(rps); 624 struct intel_uncore *uncore = rps_to_uncore(rps); 625 u16 rgvswctl; 626 627 spin_lock_irq(&mchdev_lock); 628 629 spin_lock(&i915->irq_lock); 630 ilk_disable_display_irq(i915, DE_PCU_EVENT); 631 spin_unlock(&i915->irq_lock); 632 633 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 634 635 /* Ack interrupts, disable EFC interrupt */ 636 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0); 637 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 638 639 /* Go back to the starting frequency */ 640 __gen5_rps_set(rps, rps->idle_freq); 641 mdelay(1); 642 rgvswctl |= MEMCTL_CMD_STS; 643 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 644 mdelay(1); 645 646 spin_unlock_irq(&mchdev_lock); 647 } 648 649 static u32 rps_limits(struct intel_rps *rps, u8 val) 650 { 651 u32 limits; 652 653 /* 654 * Only set the down limit when we've reached the lowest level to avoid 655 * getting more interrupts, otherwise leave this clear. This prevents a 656 * race in the hw when coming out of rc6: There's a tiny window where 657 * the hw runs at the minimal clock before selecting the desired 658 * frequency, if the down threshold expires in that window we will not 659 * receive a down interrupt. 660 */ 661 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 662 limits = rps->max_freq_softlimit << 23; 663 if (val <= rps->min_freq_softlimit) 664 limits |= rps->min_freq_softlimit << 14; 665 } else { 666 limits = rps->max_freq_softlimit << 24; 667 if (val <= rps->min_freq_softlimit) 668 limits |= rps->min_freq_softlimit << 16; 669 } 670 671 return limits; 672 } 673 674 static void rps_set_power(struct intel_rps *rps, int new_power) 675 { 676 struct intel_gt *gt = rps_to_gt(rps); 677 struct intel_uncore *uncore = gt->uncore; 678 u32 ei_up = 0, ei_down = 0; 679 680 lockdep_assert_held(&rps->power.mutex); 681 682 if (new_power == rps->power.mode) 683 return; 684 685 /* Note the units here are not exactly 1us, but 1280ns. */ 686 switch (new_power) { 687 case LOW_POWER: 688 ei_up = 16000; 689 ei_down = 32000; 690 break; 691 692 case BETWEEN: 693 ei_up = 13000; 694 ei_down = 32000; 695 break; 696 697 case HIGH_POWER: 698 ei_up = 10000; 699 ei_down = 32000; 700 break; 701 } 702 703 /* When byt can survive without system hang with dynamic 704 * sw freq adjustments, this restriction can be lifted. 705 */ 706 if (IS_VALLEYVIEW(gt->i915)) 707 goto skip_hw_write; 708 709 GT_TRACE(gt, 710 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 711 new_power, 712 rps->power.up_threshold, ei_up, 713 rps->power.down_threshold, ei_down); 714 715 set(uncore, GEN6_RP_UP_EI, 716 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 717 set(uncore, GEN6_RP_UP_THRESHOLD, 718 intel_gt_ns_to_pm_interval(gt, 719 ei_up * rps->power.up_threshold * 10)); 720 721 set(uncore, GEN6_RP_DOWN_EI, 722 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 723 set(uncore, GEN6_RP_DOWN_THRESHOLD, 724 intel_gt_ns_to_pm_interval(gt, 725 ei_down * 726 rps->power.down_threshold * 10)); 727 728 set(uncore, GEN6_RP_CONTROL, 729 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 730 GEN6_RP_MEDIA_HW_NORMAL_MODE | 731 GEN6_RP_MEDIA_IS_GFX | 732 GEN6_RP_ENABLE | 733 GEN6_RP_UP_BUSY_AVG | 734 GEN6_RP_DOWN_IDLE_AVG); 735 736 skip_hw_write: 737 rps->power.mode = new_power; 738 } 739 740 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 741 { 742 int new_power; 743 744 new_power = rps->power.mode; 745 switch (rps->power.mode) { 746 case LOW_POWER: 747 if (val > rps->efficient_freq + 1 && 748 val > rps->cur_freq) 749 new_power = BETWEEN; 750 break; 751 752 case BETWEEN: 753 if (val <= rps->efficient_freq && 754 val < rps->cur_freq) 755 new_power = LOW_POWER; 756 else if (val >= rps->rp0_freq && 757 val > rps->cur_freq) 758 new_power = HIGH_POWER; 759 break; 760 761 case HIGH_POWER: 762 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 763 val < rps->cur_freq) 764 new_power = BETWEEN; 765 break; 766 } 767 /* Max/min bins are special */ 768 if (val <= rps->min_freq_softlimit) 769 new_power = LOW_POWER; 770 if (val >= rps->max_freq_softlimit) 771 new_power = HIGH_POWER; 772 773 mutex_lock(&rps->power.mutex); 774 if (rps->power.interactive) 775 new_power = HIGH_POWER; 776 rps_set_power(rps, new_power); 777 mutex_unlock(&rps->power.mutex); 778 } 779 780 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 781 { 782 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", 783 str_yes_no(interactive)); 784 785 mutex_lock(&rps->power.mutex); 786 if (interactive) { 787 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 788 rps_set_power(rps, HIGH_POWER); 789 } else { 790 GEM_BUG_ON(!rps->power.interactive); 791 rps->power.interactive--; 792 } 793 mutex_unlock(&rps->power.mutex); 794 } 795 796 static int gen6_rps_set(struct intel_rps *rps, u8 val) 797 { 798 struct intel_uncore *uncore = rps_to_uncore(rps); 799 struct drm_i915_private *i915 = rps_to_i915(rps); 800 u32 swreq; 801 802 GEM_BUG_ON(rps_uses_slpc(rps)); 803 804 if (GRAPHICS_VER(i915) >= 9) 805 swreq = GEN9_FREQUENCY(val); 806 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 807 swreq = HSW_FREQUENCY(val); 808 else 809 swreq = (GEN6_FREQUENCY(val) | 810 GEN6_OFFSET(0) | 811 GEN6_AGGRESSIVE_TURBO); 812 set(uncore, GEN6_RPNSWREQ, swreq); 813 814 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 815 val, intel_gpu_freq(rps, val), swreq); 816 817 return 0; 818 } 819 820 static int vlv_rps_set(struct intel_rps *rps, u8 val) 821 { 822 struct drm_i915_private *i915 = rps_to_i915(rps); 823 int err; 824 825 vlv_punit_get(i915); 826 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 827 vlv_punit_put(i915); 828 829 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 830 val, intel_gpu_freq(rps, val)); 831 832 return err; 833 } 834 835 static int rps_set(struct intel_rps *rps, u8 val, bool update) 836 { 837 struct drm_i915_private *i915 = rps_to_i915(rps); 838 int err; 839 840 if (val == rps->last_freq) 841 return 0; 842 843 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 844 err = vlv_rps_set(rps, val); 845 else if (GRAPHICS_VER(i915) >= 6) 846 err = gen6_rps_set(rps, val); 847 else 848 err = gen5_rps_set(rps, val); 849 if (err) 850 return err; 851 852 if (update && GRAPHICS_VER(i915) >= 6) 853 gen6_rps_set_thresholds(rps, val); 854 rps->last_freq = val; 855 856 return 0; 857 } 858 859 void intel_rps_unpark(struct intel_rps *rps) 860 { 861 if (!intel_rps_is_enabled(rps)) 862 return; 863 864 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 865 866 /* 867 * Use the user's desired frequency as a guide, but for better 868 * performance, jump directly to RPe as our starting frequency. 869 */ 870 mutex_lock(&rps->lock); 871 872 intel_rps_set_active(rps); 873 intel_rps_set(rps, 874 clamp(rps->cur_freq, 875 rps->min_freq_softlimit, 876 rps->max_freq_softlimit)); 877 878 mutex_unlock(&rps->lock); 879 880 rps->pm_iir = 0; 881 if (intel_rps_has_interrupts(rps)) 882 rps_enable_interrupts(rps); 883 if (intel_rps_uses_timer(rps)) 884 rps_start_timer(rps); 885 886 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 887 gen5_rps_update(rps); 888 } 889 890 void intel_rps_park(struct intel_rps *rps) 891 { 892 int adj; 893 894 if (!intel_rps_is_enabled(rps)) 895 return; 896 897 if (!intel_rps_clear_active(rps)) 898 return; 899 900 if (intel_rps_uses_timer(rps)) 901 rps_stop_timer(rps); 902 if (intel_rps_has_interrupts(rps)) 903 rps_disable_interrupts(rps); 904 905 if (rps->last_freq <= rps->idle_freq) 906 return; 907 908 /* 909 * The punit delays the write of the frequency and voltage until it 910 * determines the GPU is awake. During normal usage we don't want to 911 * waste power changing the frequency if the GPU is sleeping (rc6). 912 * However, the GPU and driver is now idle and we do not want to delay 913 * switching to minimum voltage (reducing power whilst idle) as we do 914 * not expect to be woken in the near future and so must flush the 915 * change by waking the device. 916 * 917 * We choose to take the media powerwell (either would do to trick the 918 * punit into committing the voltage change) as that takes a lot less 919 * power than the render powerwell. 920 */ 921 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 922 rps_set(rps, rps->idle_freq, false); 923 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 924 925 /* 926 * Since we will try and restart from the previously requested 927 * frequency on unparking, treat this idle point as a downclock 928 * interrupt and reduce the frequency for resume. If we park/unpark 929 * more frequently than the rps worker can run, we will not respond 930 * to any EI and never see a change in frequency. 931 * 932 * (Note we accommodate Cherryview's limitation of only using an 933 * even bin by applying it to all.) 934 */ 935 adj = rps->last_adj; 936 if (adj < 0) 937 adj *= 2; 938 else /* CHV needs even encode values */ 939 adj = -2; 940 rps->last_adj = adj; 941 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 942 if (rps->cur_freq < rps->efficient_freq) { 943 rps->cur_freq = rps->efficient_freq; 944 rps->last_adj = 0; 945 } 946 947 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 948 } 949 950 u32 intel_rps_get_boost_frequency(struct intel_rps *rps) 951 { 952 struct intel_guc_slpc *slpc; 953 954 if (rps_uses_slpc(rps)) { 955 slpc = rps_to_slpc(rps); 956 957 return slpc->boost_freq; 958 } else { 959 return intel_gpu_freq(rps, rps->boost_freq); 960 } 961 } 962 963 static int rps_set_boost_freq(struct intel_rps *rps, u32 val) 964 { 965 bool boost = false; 966 967 /* Validate against (static) hardware limits */ 968 val = intel_freq_opcode(rps, val); 969 if (val < rps->min_freq || val > rps->max_freq) 970 return -EINVAL; 971 972 mutex_lock(&rps->lock); 973 if (val != rps->boost_freq) { 974 rps->boost_freq = val; 975 boost = atomic_read(&rps->num_waiters); 976 } 977 mutex_unlock(&rps->lock); 978 if (boost) 979 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 980 981 return 0; 982 } 983 984 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq) 985 { 986 struct intel_guc_slpc *slpc; 987 988 if (rps_uses_slpc(rps)) { 989 slpc = rps_to_slpc(rps); 990 991 return intel_guc_slpc_set_boost_freq(slpc, freq); 992 } else { 993 return rps_set_boost_freq(rps, freq); 994 } 995 } 996 997 void intel_rps_dec_waiters(struct intel_rps *rps) 998 { 999 struct intel_guc_slpc *slpc; 1000 1001 if (rps_uses_slpc(rps)) { 1002 slpc = rps_to_slpc(rps); 1003 1004 intel_guc_slpc_dec_waiters(slpc); 1005 } else { 1006 atomic_dec(&rps->num_waiters); 1007 } 1008 } 1009 1010 void intel_rps_boost(struct i915_request *rq) 1011 { 1012 struct intel_guc_slpc *slpc; 1013 1014 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 1015 return; 1016 1017 /* Waitboost is not needed for contexts marked with a Freq hint */ 1018 if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags)) 1019 return; 1020 1021 /* Serializes with i915_request_retire() */ 1022 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 1023 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 1024 1025 if (rps_uses_slpc(rps)) { 1026 slpc = rps_to_slpc(rps); 1027 1028 /* Waitboost should not be done with power saving profile */ 1029 if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING) 1030 return; 1031 1032 if (slpc->min_freq_softlimit >= slpc->boost_freq) 1033 return; 1034 1035 /* Return if old value is non zero */ 1036 if (!atomic_fetch_inc(&slpc->num_waiters)) { 1037 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1038 rq->fence.context, rq->fence.seqno); 1039 queue_work(rps_to_gt(rps)->i915->unordered_wq, 1040 &slpc->boost_work); 1041 } 1042 1043 return; 1044 } 1045 1046 if (atomic_fetch_inc(&rps->num_waiters)) 1047 return; 1048 1049 if (!intel_rps_is_active(rps)) 1050 return; 1051 1052 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1053 rq->fence.context, rq->fence.seqno); 1054 1055 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 1056 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work); 1057 1058 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 1059 } 1060 } 1061 1062 int intel_rps_set(struct intel_rps *rps, u8 val) 1063 { 1064 int err; 1065 1066 lockdep_assert_held(&rps->lock); 1067 GEM_BUG_ON(val > rps->max_freq); 1068 GEM_BUG_ON(val < rps->min_freq); 1069 1070 if (intel_rps_is_active(rps)) { 1071 err = rps_set(rps, val, true); 1072 if (err) 1073 return err; 1074 1075 /* 1076 * Make sure we continue to get interrupts 1077 * until we hit the minimum or maximum frequencies. 1078 */ 1079 if (intel_rps_has_interrupts(rps)) { 1080 struct intel_uncore *uncore = rps_to_uncore(rps); 1081 1082 set(uncore, 1083 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 1084 1085 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 1086 } 1087 } 1088 1089 rps->cur_freq = val; 1090 return 0; 1091 } 1092 1093 static u32 intel_rps_read_state_cap(struct intel_rps *rps) 1094 { 1095 struct drm_i915_private *i915 = rps_to_i915(rps); 1096 struct intel_uncore *uncore = rps_to_uncore(rps); 1097 1098 if (IS_GEN9_LP(i915)) 1099 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 1100 else 1101 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 1102 } 1103 1104 static void 1105 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1106 { 1107 struct intel_uncore *uncore = rps_to_uncore(rps); 1108 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ? 1109 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) : 1110 intel_uncore_read(uncore, MTL_RP_STATE_CAP); 1111 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ? 1112 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) : 1113 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY); 1114 1115 /* MTL values are in units of 16.67 MHz */ 1116 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap); 1117 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap); 1118 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe); 1119 } 1120 1121 static void 1122 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1123 { 1124 struct drm_i915_private *i915 = rps_to_i915(rps); 1125 u32 rp_state_cap; 1126 1127 rp_state_cap = intel_rps_read_state_cap(rps); 1128 1129 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1130 if (IS_GEN9_LP(i915)) { 1131 caps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1132 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1133 caps->min_freq = (rp_state_cap >> 0) & 0xff; 1134 } else { 1135 caps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1136 if (GRAPHICS_VER(i915) >= 10) 1137 caps->rp1_freq = REG_FIELD_GET(RPE_MASK, 1138 intel_uncore_read(to_gt(i915)->uncore, 1139 GEN10_FREQ_INFO_REC)); 1140 else 1141 caps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1142 caps->min_freq = (rp_state_cap >> 16) & 0xff; 1143 } 1144 1145 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1146 /* 1147 * In this case rp_state_cap register reports frequencies in 1148 * units of 50 MHz. Convert these to the actual "hw unit", i.e. 1149 * units of 16.67 MHz 1150 */ 1151 caps->rp0_freq *= GEN9_FREQ_SCALER; 1152 caps->rp1_freq *= GEN9_FREQ_SCALER; 1153 caps->min_freq *= GEN9_FREQ_SCALER; 1154 } 1155 } 1156 1157 /** 1158 * gen6_rps_get_freq_caps - Get freq caps exposed by HW 1159 * @rps: the intel_rps structure 1160 * @caps: returned freq caps 1161 * 1162 * Returned "caps" frequencies should be converted to MHz using 1163 * intel_gpu_freq() 1164 */ 1165 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps) 1166 { 1167 struct drm_i915_private *i915 = rps_to_i915(rps); 1168 1169 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 1170 return mtl_get_freq_caps(rps, caps); 1171 else 1172 return __gen6_rps_get_freq_caps(rps, caps); 1173 } 1174 1175 static void gen6_rps_init(struct intel_rps *rps) 1176 { 1177 struct drm_i915_private *i915 = rps_to_i915(rps); 1178 struct intel_rps_freq_caps caps; 1179 1180 gen6_rps_get_freq_caps(rps, &caps); 1181 rps->rp0_freq = caps.rp0_freq; 1182 rps->rp1_freq = caps.rp1_freq; 1183 rps->min_freq = caps.min_freq; 1184 1185 /* hw_max = RP0 until we check for overclocking */ 1186 rps->max_freq = rps->rp0_freq; 1187 1188 rps->efficient_freq = rps->rp1_freq; 1189 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1190 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1191 u32 ddcc_status = 0; 1192 u32 mult = 1; 1193 1194 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) 1195 mult = GEN9_FREQ_SCALER; 1196 if (snb_pcode_read(rps_to_gt(rps)->uncore, 1197 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1198 &ddcc_status, NULL) == 0) 1199 rps->efficient_freq = 1200 clamp_t(u32, 1201 ((ddcc_status >> 8) & 0xff) * mult, 1202 rps->min_freq, 1203 rps->max_freq); 1204 } 1205 } 1206 1207 static bool rps_reset(struct intel_rps *rps) 1208 { 1209 struct drm_i915_private *i915 = rps_to_i915(rps); 1210 1211 /* force a reset */ 1212 rps->power.mode = -1; 1213 rps->last_freq = -1; 1214 1215 if (rps_set(rps, rps->min_freq, true)) { 1216 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1217 return false; 1218 } 1219 1220 rps->cur_freq = rps->min_freq; 1221 return true; 1222 } 1223 1224 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1225 static bool gen9_rps_enable(struct intel_rps *rps) 1226 { 1227 struct intel_gt *gt = rps_to_gt(rps); 1228 struct intel_uncore *uncore = gt->uncore; 1229 1230 /* Program defaults and thresholds for RPS */ 1231 if (GRAPHICS_VER(gt->i915) == 9) 1232 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1233 GEN9_FREQUENCY(rps->rp1_freq)); 1234 1235 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1236 1237 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1238 1239 return rps_reset(rps); 1240 } 1241 1242 static bool gen8_rps_enable(struct intel_rps *rps) 1243 { 1244 struct intel_uncore *uncore = rps_to_uncore(rps); 1245 1246 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1247 HSW_FREQUENCY(rps->rp1_freq)); 1248 1249 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1250 1251 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1252 1253 return rps_reset(rps); 1254 } 1255 1256 static bool gen6_rps_enable(struct intel_rps *rps) 1257 { 1258 struct intel_uncore *uncore = rps_to_uncore(rps); 1259 1260 /* Power down if completely idle for over 50ms */ 1261 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1262 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1263 1264 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1265 GEN6_PM_RP_DOWN_THRESHOLD | 1266 GEN6_PM_RP_DOWN_TIMEOUT); 1267 1268 return rps_reset(rps); 1269 } 1270 1271 static int chv_rps_max_freq(struct intel_rps *rps) 1272 { 1273 struct drm_i915_private *i915 = rps_to_i915(rps); 1274 struct intel_gt *gt = rps_to_gt(rps); 1275 u32 val; 1276 1277 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1278 1279 switch (gt->info.sseu.eu_total) { 1280 case 8: 1281 /* (2 * 4) config */ 1282 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1283 break; 1284 case 12: 1285 /* (2 * 6) config */ 1286 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1287 break; 1288 case 16: 1289 /* (2 * 8) config */ 1290 default: 1291 /* Setting (2 * 8) Min RP0 for any other combination */ 1292 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1293 break; 1294 } 1295 1296 return val & FB_GFX_FREQ_FUSE_MASK; 1297 } 1298 1299 static int chv_rps_rpe_freq(struct intel_rps *rps) 1300 { 1301 struct drm_i915_private *i915 = rps_to_i915(rps); 1302 u32 val; 1303 1304 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 1305 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1306 1307 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1308 } 1309 1310 static int chv_rps_guar_freq(struct intel_rps *rps) 1311 { 1312 struct drm_i915_private *i915 = rps_to_i915(rps); 1313 u32 val; 1314 1315 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1316 1317 return val & FB_GFX_FREQ_FUSE_MASK; 1318 } 1319 1320 static u32 chv_rps_min_freq(struct intel_rps *rps) 1321 { 1322 struct drm_i915_private *i915 = rps_to_i915(rps); 1323 u32 val; 1324 1325 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1326 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1327 1328 return val & FB_GFX_FREQ_FUSE_MASK; 1329 } 1330 1331 static bool chv_rps_enable(struct intel_rps *rps) 1332 { 1333 struct intel_uncore *uncore = rps_to_uncore(rps); 1334 struct drm_i915_private *i915 = rps_to_i915(rps); 1335 u32 val; 1336 1337 /* 1: Program defaults and thresholds for RPS*/ 1338 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1339 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1340 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1341 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1342 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1343 1344 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1345 1346 /* 2: Enable RPS */ 1347 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1348 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1349 GEN6_RP_MEDIA_IS_GFX | 1350 GEN6_RP_ENABLE | 1351 GEN6_RP_UP_BUSY_AVG | 1352 GEN6_RP_DOWN_IDLE_AVG); 1353 1354 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1355 GEN6_PM_RP_DOWN_THRESHOLD | 1356 GEN6_PM_RP_DOWN_TIMEOUT); 1357 1358 /* Setting Fixed Bias */ 1359 vlv_punit_get(i915); 1360 1361 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1362 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1363 1364 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1365 1366 vlv_punit_put(i915); 1367 1368 /* RPS code assumes GPLL is used */ 1369 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1370 "GPLL not enabled\n"); 1371 1372 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1373 str_yes_no(val & GPLLENABLE)); 1374 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1375 1376 return rps_reset(rps); 1377 } 1378 1379 static int vlv_rps_guar_freq(struct intel_rps *rps) 1380 { 1381 struct drm_i915_private *i915 = rps_to_i915(rps); 1382 u32 val, rp1; 1383 1384 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1385 1386 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1387 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1388 1389 return rp1; 1390 } 1391 1392 static int vlv_rps_max_freq(struct intel_rps *rps) 1393 { 1394 struct drm_i915_private *i915 = rps_to_i915(rps); 1395 u32 val, rp0; 1396 1397 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1398 1399 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1400 /* Clamp to max */ 1401 rp0 = min_t(u32, rp0, 0xea); 1402 1403 return rp0; 1404 } 1405 1406 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1407 { 1408 struct drm_i915_private *i915 = rps_to_i915(rps); 1409 u32 val, rpe; 1410 1411 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1412 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1413 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1414 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1415 1416 return rpe; 1417 } 1418 1419 static int vlv_rps_min_freq(struct intel_rps *rps) 1420 { 1421 struct drm_i915_private *i915 = rps_to_i915(rps); 1422 u32 val; 1423 1424 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1425 /* 1426 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1427 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1428 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1429 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1430 * to make sure it matches what Punit accepts. 1431 */ 1432 return max_t(u32, val, 0xc0); 1433 } 1434 1435 static bool vlv_rps_enable(struct intel_rps *rps) 1436 { 1437 struct intel_uncore *uncore = rps_to_uncore(rps); 1438 struct drm_i915_private *i915 = rps_to_i915(rps); 1439 u32 val; 1440 1441 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1442 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1443 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1444 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1445 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1446 1447 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1448 1449 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1450 GEN6_RP_MEDIA_TURBO | 1451 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1452 GEN6_RP_MEDIA_IS_GFX | 1453 GEN6_RP_ENABLE | 1454 GEN6_RP_UP_BUSY_AVG | 1455 GEN6_RP_DOWN_IDLE_CONT); 1456 1457 /* WaGsvRC0ResidencyMethod:vlv */ 1458 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1459 1460 vlv_punit_get(i915); 1461 1462 /* Setting Fixed Bias */ 1463 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1464 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1465 1466 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1467 1468 vlv_punit_put(i915); 1469 1470 /* RPS code assumes GPLL is used */ 1471 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1472 "GPLL not enabled\n"); 1473 1474 drm_dbg(&i915->drm, "GPLL enabled? %s\n", 1475 str_yes_no(val & GPLLENABLE)); 1476 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1477 1478 return rps_reset(rps); 1479 } 1480 1481 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1482 { 1483 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1484 struct intel_uncore *uncore = rps_to_uncore(rps); 1485 unsigned int t, state1, state2; 1486 u32 pxvid, ext_v; 1487 u64 corr, corr2; 1488 1489 lockdep_assert_held(&mchdev_lock); 1490 1491 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1492 pxvid = (pxvid >> 24) & 0x7f; 1493 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1494 1495 state1 = ext_v; 1496 1497 /* Revel in the empirically derived constants */ 1498 1499 /* Correction factor in 1/100000 units */ 1500 t = ips_mch_val(uncore); 1501 if (t > 80) 1502 corr = t * 2349 + 135940; 1503 else if (t >= 50) 1504 corr = t * 964 + 29317; 1505 else /* < 50 */ 1506 corr = t * 301 + 1004; 1507 1508 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1509 corr2 = div_u64(corr, 100000) * ips->corr; 1510 1511 state2 = div_u64(corr2 * state1, 10000); 1512 state2 /= 100; /* convert to mW */ 1513 1514 __gen5_ips_update(ips); 1515 1516 return ips->gfx_power + state2; 1517 } 1518 1519 static bool has_busy_stats(struct intel_rps *rps) 1520 { 1521 struct intel_engine_cs *engine; 1522 enum intel_engine_id id; 1523 1524 for_each_engine(engine, rps_to_gt(rps), id) { 1525 if (!intel_engine_supports_stats(engine)) 1526 return false; 1527 } 1528 1529 return true; 1530 } 1531 1532 void intel_rps_enable(struct intel_rps *rps) 1533 { 1534 struct drm_i915_private *i915 = rps_to_i915(rps); 1535 struct intel_uncore *uncore = rps_to_uncore(rps); 1536 bool enabled = false; 1537 1538 if (!HAS_RPS(i915)) 1539 return; 1540 1541 if (rps_uses_slpc(rps)) 1542 return; 1543 1544 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1545 1546 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1547 if (rps->max_freq <= rps->min_freq) 1548 /* leave disabled, no room for dynamic reclocking */; 1549 else if (IS_CHERRYVIEW(i915)) 1550 enabled = chv_rps_enable(rps); 1551 else if (IS_VALLEYVIEW(i915)) 1552 enabled = vlv_rps_enable(rps); 1553 else if (GRAPHICS_VER(i915) >= 9) 1554 enabled = gen9_rps_enable(rps); 1555 else if (GRAPHICS_VER(i915) >= 8) 1556 enabled = gen8_rps_enable(rps); 1557 else if (GRAPHICS_VER(i915) >= 6) 1558 enabled = gen6_rps_enable(rps); 1559 else if (IS_IRONLAKE_M(i915)) 1560 enabled = gen5_rps_enable(rps); 1561 else 1562 MISSING_CASE(GRAPHICS_VER(i915)); 1563 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1564 if (!enabled) 1565 return; 1566 1567 GT_TRACE(rps_to_gt(rps), 1568 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n", 1569 rps->min_freq, rps->max_freq, 1570 intel_gpu_freq(rps, rps->min_freq), 1571 intel_gpu_freq(rps, rps->max_freq), 1572 rps->power.up_threshold, 1573 rps->power.down_threshold); 1574 1575 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1576 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1577 1578 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1579 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1580 1581 if (has_busy_stats(rps)) 1582 intel_rps_set_timer(rps); 1583 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) 1584 intel_rps_set_interrupts(rps); 1585 else 1586 /* Ironlake currently uses intel_ips.ko */ {} 1587 1588 intel_rps_set_enabled(rps); 1589 } 1590 1591 static void gen6_rps_disable(struct intel_rps *rps) 1592 { 1593 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1594 } 1595 1596 void intel_rps_disable(struct intel_rps *rps) 1597 { 1598 struct drm_i915_private *i915 = rps_to_i915(rps); 1599 1600 if (!intel_rps_is_enabled(rps)) 1601 return; 1602 1603 intel_rps_clear_enabled(rps); 1604 intel_rps_clear_interrupts(rps); 1605 intel_rps_clear_timer(rps); 1606 1607 if (GRAPHICS_VER(i915) >= 6) 1608 gen6_rps_disable(rps); 1609 else if (IS_IRONLAKE_M(i915)) 1610 gen5_rps_disable(rps); 1611 } 1612 1613 static int byt_gpu_freq(struct intel_rps *rps, int val) 1614 { 1615 /* 1616 * N = val - 0xb7 1617 * Slow = Fast = GPLL ref * N 1618 */ 1619 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1620 } 1621 1622 static int byt_freq_opcode(struct intel_rps *rps, int val) 1623 { 1624 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1625 } 1626 1627 static int chv_gpu_freq(struct intel_rps *rps, int val) 1628 { 1629 /* 1630 * N = val / 2 1631 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1632 */ 1633 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1634 } 1635 1636 static int chv_freq_opcode(struct intel_rps *rps, int val) 1637 { 1638 /* CHV needs even values */ 1639 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1640 } 1641 1642 int intel_gpu_freq(struct intel_rps *rps, int val) 1643 { 1644 struct drm_i915_private *i915 = rps_to_i915(rps); 1645 1646 if (GRAPHICS_VER(i915) >= 9) 1647 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1648 GEN9_FREQ_SCALER); 1649 else if (IS_CHERRYVIEW(i915)) 1650 return chv_gpu_freq(rps, val); 1651 else if (IS_VALLEYVIEW(i915)) 1652 return byt_gpu_freq(rps, val); 1653 else if (GRAPHICS_VER(i915) >= 6) 1654 return val * GT_FREQUENCY_MULTIPLIER; 1655 else 1656 return val; 1657 } 1658 1659 int intel_freq_opcode(struct intel_rps *rps, int val) 1660 { 1661 struct drm_i915_private *i915 = rps_to_i915(rps); 1662 1663 if (GRAPHICS_VER(i915) >= 9) 1664 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1665 GT_FREQUENCY_MULTIPLIER); 1666 else if (IS_CHERRYVIEW(i915)) 1667 return chv_freq_opcode(rps, val); 1668 else if (IS_VALLEYVIEW(i915)) 1669 return byt_freq_opcode(rps, val); 1670 else if (GRAPHICS_VER(i915) >= 6) 1671 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1672 else 1673 return val; 1674 } 1675 1676 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1677 { 1678 struct drm_i915_private *i915 = rps_to_i915(rps); 1679 1680 rps->gpll_ref_freq = 1681 vlv_get_cck_clock(i915, "GPLL ref", 1682 CCK_GPLL_CLOCK_CONTROL, 1683 i915->czclk_freq); 1684 1685 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1686 rps->gpll_ref_freq); 1687 } 1688 1689 static void vlv_rps_init(struct intel_rps *rps) 1690 { 1691 struct drm_i915_private *i915 = rps_to_i915(rps); 1692 1693 vlv_iosf_sb_get(i915, 1694 BIT(VLV_IOSF_SB_PUNIT) | 1695 BIT(VLV_IOSF_SB_NC) | 1696 BIT(VLV_IOSF_SB_CCK)); 1697 1698 vlv_init_gpll_ref_freq(rps); 1699 1700 rps->max_freq = vlv_rps_max_freq(rps); 1701 rps->rp0_freq = rps->max_freq; 1702 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1703 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1704 1705 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1706 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1707 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1708 1709 rps->rp1_freq = vlv_rps_guar_freq(rps); 1710 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1711 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1712 1713 rps->min_freq = vlv_rps_min_freq(rps); 1714 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1715 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1716 1717 vlv_iosf_sb_put(i915, 1718 BIT(VLV_IOSF_SB_PUNIT) | 1719 BIT(VLV_IOSF_SB_NC) | 1720 BIT(VLV_IOSF_SB_CCK)); 1721 } 1722 1723 static void chv_rps_init(struct intel_rps *rps) 1724 { 1725 struct drm_i915_private *i915 = rps_to_i915(rps); 1726 1727 vlv_iosf_sb_get(i915, 1728 BIT(VLV_IOSF_SB_PUNIT) | 1729 BIT(VLV_IOSF_SB_NC) | 1730 BIT(VLV_IOSF_SB_CCK)); 1731 1732 vlv_init_gpll_ref_freq(rps); 1733 1734 rps->max_freq = chv_rps_max_freq(rps); 1735 rps->rp0_freq = rps->max_freq; 1736 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1737 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1738 1739 rps->efficient_freq = chv_rps_rpe_freq(rps); 1740 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1741 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1742 1743 rps->rp1_freq = chv_rps_guar_freq(rps); 1744 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1745 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1746 1747 rps->min_freq = chv_rps_min_freq(rps); 1748 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1749 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1750 1751 vlv_iosf_sb_put(i915, 1752 BIT(VLV_IOSF_SB_PUNIT) | 1753 BIT(VLV_IOSF_SB_NC) | 1754 BIT(VLV_IOSF_SB_CCK)); 1755 1756 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1757 rps->rp1_freq | rps->min_freq) & 1, 1758 "Odd GPU freq values\n"); 1759 } 1760 1761 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1762 { 1763 ei->ktime = ktime_get_raw(); 1764 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1765 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1766 } 1767 1768 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1769 { 1770 struct intel_uncore *uncore = rps_to_uncore(rps); 1771 const struct intel_rps_ei *prev = &rps->ei; 1772 struct intel_rps_ei now; 1773 u32 events = 0; 1774 1775 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1776 return 0; 1777 1778 vlv_c0_read(uncore, &now); 1779 1780 if (prev->ktime) { 1781 u64 time, c0; 1782 u32 render, media; 1783 1784 time = ktime_us_delta(now.ktime, prev->ktime); 1785 1786 time *= rps_to_i915(rps)->czclk_freq; 1787 1788 /* Workload can be split between render + media, 1789 * e.g. SwapBuffers being blitted in X after being rendered in 1790 * mesa. To account for this we need to combine both engines 1791 * into our activity counter. 1792 */ 1793 render = now.render_c0 - prev->render_c0; 1794 media = now.media_c0 - prev->media_c0; 1795 c0 = max(render, media); 1796 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1797 1798 if (c0 > time * rps->power.up_threshold) 1799 events = GEN6_PM_RP_UP_THRESHOLD; 1800 else if (c0 < time * rps->power.down_threshold) 1801 events = GEN6_PM_RP_DOWN_THRESHOLD; 1802 } 1803 1804 rps->ei = now; 1805 return events; 1806 } 1807 1808 static void rps_work(struct work_struct *work) 1809 { 1810 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1811 struct intel_gt *gt = rps_to_gt(rps); 1812 struct drm_i915_private *i915 = rps_to_i915(rps); 1813 bool client_boost = false; 1814 int new_freq, adj, min, max; 1815 u32 pm_iir = 0; 1816 1817 spin_lock_irq(gt->irq_lock); 1818 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1819 client_boost = atomic_read(&rps->num_waiters); 1820 spin_unlock_irq(gt->irq_lock); 1821 1822 /* Make sure we didn't queue anything we're not going to process. */ 1823 if (!pm_iir && !client_boost) 1824 goto out; 1825 1826 mutex_lock(&rps->lock); 1827 if (!intel_rps_is_active(rps)) { 1828 mutex_unlock(&rps->lock); 1829 return; 1830 } 1831 1832 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1833 1834 adj = rps->last_adj; 1835 new_freq = rps->cur_freq; 1836 min = rps->min_freq_softlimit; 1837 max = rps->max_freq_softlimit; 1838 if (client_boost) 1839 max = rps->max_freq; 1840 1841 GT_TRACE(gt, 1842 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1843 pm_iir, str_yes_no(client_boost), 1844 adj, new_freq, min, max); 1845 1846 if (client_boost && new_freq < rps->boost_freq) { 1847 new_freq = rps->boost_freq; 1848 adj = 0; 1849 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1850 if (adj > 0) 1851 adj *= 2; 1852 else /* CHV needs even encode values */ 1853 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1854 1855 if (new_freq >= rps->max_freq_softlimit) 1856 adj = 0; 1857 } else if (client_boost) { 1858 adj = 0; 1859 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1860 if (rps->cur_freq > rps->efficient_freq) 1861 new_freq = rps->efficient_freq; 1862 else if (rps->cur_freq > rps->min_freq_softlimit) 1863 new_freq = rps->min_freq_softlimit; 1864 adj = 0; 1865 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1866 if (adj < 0) 1867 adj *= 2; 1868 else /* CHV needs even encode values */ 1869 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1870 1871 if (new_freq <= rps->min_freq_softlimit) 1872 adj = 0; 1873 } else { /* unknown event */ 1874 adj = 0; 1875 } 1876 1877 /* 1878 * sysfs frequency limits may have snuck in while 1879 * servicing the interrupt 1880 */ 1881 new_freq += adj; 1882 new_freq = clamp_t(int, new_freq, min, max); 1883 1884 if (intel_rps_set(rps, new_freq)) { 1885 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1886 adj = 0; 1887 } 1888 rps->last_adj = adj; 1889 1890 mutex_unlock(&rps->lock); 1891 1892 out: 1893 spin_lock_irq(gt->irq_lock); 1894 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1895 spin_unlock_irq(gt->irq_lock); 1896 } 1897 1898 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1899 { 1900 struct intel_gt *gt = rps_to_gt(rps); 1901 const u32 events = rps->pm_events & pm_iir; 1902 1903 lockdep_assert_held(gt->irq_lock); 1904 1905 if (unlikely(!events)) 1906 return; 1907 1908 GT_TRACE(gt, "irq events:%x\n", events); 1909 1910 gen6_gt_pm_mask_irq(gt, events); 1911 1912 rps->pm_iir |= events; 1913 queue_work(gt->i915->unordered_wq, &rps->work); 1914 } 1915 1916 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1917 { 1918 struct intel_gt *gt = rps_to_gt(rps); 1919 u32 events; 1920 1921 events = pm_iir & rps->pm_events; 1922 if (events) { 1923 spin_lock(gt->irq_lock); 1924 1925 GT_TRACE(gt, "irq events:%x\n", events); 1926 1927 gen6_gt_pm_mask_irq(gt, events); 1928 rps->pm_iir |= events; 1929 1930 queue_work(gt->i915->unordered_wq, &rps->work); 1931 spin_unlock(gt->irq_lock); 1932 } 1933 1934 if (GRAPHICS_VER(gt->i915) >= 8) 1935 return; 1936 1937 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1938 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1939 1940 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1941 drm_dbg(&rps_to_i915(rps)->drm, 1942 "Command parser error, pm_iir 0x%08x\n", pm_iir); 1943 } 1944 1945 void gen5_rps_irq_handler(struct intel_rps *rps) 1946 { 1947 struct intel_uncore *uncore = rps_to_uncore(rps); 1948 u32 busy_up, busy_down, max_avg, min_avg; 1949 u8 new_freq; 1950 1951 spin_lock(&mchdev_lock); 1952 1953 intel_uncore_write16(uncore, 1954 MEMINTRSTS, 1955 intel_uncore_read(uncore, MEMINTRSTS)); 1956 1957 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1958 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1959 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1960 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1961 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1962 1963 /* Handle RCS change request from hw */ 1964 new_freq = rps->cur_freq; 1965 if (busy_up > max_avg) 1966 new_freq++; 1967 else if (busy_down < min_avg) 1968 new_freq--; 1969 new_freq = clamp(new_freq, 1970 rps->min_freq_softlimit, 1971 rps->max_freq_softlimit); 1972 1973 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1974 rps->cur_freq = new_freq; 1975 1976 spin_unlock(&mchdev_lock); 1977 } 1978 1979 void intel_rps_init_early(struct intel_rps *rps) 1980 { 1981 mutex_init(&rps->lock); 1982 mutex_init(&rps->power.mutex); 1983 1984 INIT_WORK(&rps->work, rps_work); 1985 timer_setup(&rps->timer, rps_timer, 0); 1986 1987 atomic_set(&rps->num_waiters, 0); 1988 } 1989 1990 void intel_rps_init(struct intel_rps *rps) 1991 { 1992 struct drm_i915_private *i915 = rps_to_i915(rps); 1993 1994 if (rps_uses_slpc(rps)) 1995 return; 1996 1997 if (IS_CHERRYVIEW(i915)) 1998 chv_rps_init(rps); 1999 else if (IS_VALLEYVIEW(i915)) 2000 vlv_rps_init(rps); 2001 else if (GRAPHICS_VER(i915) >= 6) 2002 gen6_rps_init(rps); 2003 else if (IS_IRONLAKE_M(i915)) 2004 gen5_rps_init(rps); 2005 2006 /* Derive initial user preferences/limits from the hardware limits */ 2007 rps->max_freq_softlimit = rps->max_freq; 2008 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit; 2009 rps->min_freq_softlimit = rps->min_freq; 2010 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit; 2011 2012 /* After setting max-softlimit, find the overclock max freq */ 2013 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 2014 u32 params = 0; 2015 2016 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL); 2017 if (params & BIT(31)) { /* OC supported */ 2018 drm_dbg(&i915->drm, 2019 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 2020 (rps->max_freq & 0xff) * 50, 2021 (params & 0xff) * 50); 2022 rps->max_freq = params & 0xff; 2023 } 2024 } 2025 2026 /* Set default thresholds in % */ 2027 rps->power.up_threshold = 95; 2028 rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold; 2029 rps->power.down_threshold = 85; 2030 rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold; 2031 2032 /* Finally allow us to boost to max by default */ 2033 rps->boost_freq = rps->max_freq; 2034 rps->idle_freq = rps->min_freq; 2035 2036 /* Start in the middle, from here we will autotune based on workload */ 2037 rps->cur_freq = rps->efficient_freq; 2038 2039 rps->pm_intrmsk_mbz = 0; 2040 2041 /* 2042 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 2043 * if GEN6_PM_UP_EI_EXPIRED is masked. 2044 * 2045 * TODO: verify if this can be reproduced on VLV,CHV. 2046 */ 2047 if (GRAPHICS_VER(i915) <= 7) 2048 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 2049 2050 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 2051 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 2052 2053 /* GuC needs ARAT expired interrupt unmasked */ 2054 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 2055 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 2056 } 2057 2058 void intel_rps_sanitize(struct intel_rps *rps) 2059 { 2060 if (rps_uses_slpc(rps)) 2061 return; 2062 2063 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 2064 rps_disable_interrupts(rps); 2065 } 2066 2067 u32 intel_rps_read_rpstat(struct intel_rps *rps) 2068 { 2069 struct drm_i915_private *i915 = rps_to_i915(rps); 2070 i915_reg_t rpstat; 2071 2072 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1; 2073 2074 return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat); 2075 } 2076 2077 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 2078 { 2079 struct drm_i915_private *i915 = rps_to_i915(rps); 2080 u32 cagf; 2081 2082 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 2083 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat); 2084 else if (GRAPHICS_VER(i915) >= 12) 2085 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat); 2086 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 2087 cagf = REG_FIELD_GET(RPE_MASK, rpstat); 2088 else if (GRAPHICS_VER(i915) >= 9) 2089 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat); 2090 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2091 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat); 2092 else if (GRAPHICS_VER(i915) >= 6) 2093 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat); 2094 else 2095 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat)); 2096 2097 return cagf; 2098 } 2099 2100 static u32 __read_cagf(struct intel_rps *rps, bool take_fw) 2101 { 2102 struct drm_i915_private *i915 = rps_to_i915(rps); 2103 struct intel_uncore *uncore = rps_to_uncore(rps); 2104 i915_reg_t r = INVALID_MMIO_REG; 2105 u32 freq; 2106 2107 /* 2108 * For Gen12+ reading freq from HW does not need a forcewake and 2109 * registers will return 0 freq when GT is in RC6 2110 */ 2111 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 2112 r = MTL_MIRROR_TARGET_WP1; 2113 } else if (GRAPHICS_VER(i915) >= 12) { 2114 r = GEN12_RPSTAT1; 2115 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2116 vlv_punit_get(i915); 2117 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 2118 vlv_punit_put(i915); 2119 } else if (GRAPHICS_VER(i915) >= 6) { 2120 r = GEN6_RPSTAT1; 2121 } else { 2122 r = MEMSTAT_ILK; 2123 } 2124 2125 if (i915_mmio_reg_valid(r)) 2126 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r); 2127 2128 return intel_rps_get_cagf(rps, freq); 2129 } 2130 2131 static u32 read_cagf(struct intel_rps *rps) 2132 { 2133 return __read_cagf(rps, true); 2134 } 2135 2136 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 2137 { 2138 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2139 intel_wakeref_t wakeref; 2140 u32 freq = 0; 2141 2142 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2143 freq = intel_gpu_freq(rps, read_cagf(rps)); 2144 2145 return freq; 2146 } 2147 2148 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps) 2149 { 2150 return intel_gpu_freq(rps, __read_cagf(rps, false)); 2151 } 2152 2153 static u32 intel_rps_read_punit_req(struct intel_rps *rps) 2154 { 2155 struct intel_uncore *uncore = rps_to_uncore(rps); 2156 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2157 intel_wakeref_t wakeref; 2158 u32 freq = 0; 2159 2160 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2161 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2162 2163 return freq; 2164 } 2165 2166 static u32 intel_rps_get_req(u32 pureq) 2167 { 2168 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 2169 2170 return req; 2171 } 2172 2173 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 2174 { 2175 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 2176 2177 return intel_gpu_freq(rps, freq); 2178 } 2179 2180 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 2181 { 2182 if (rps_uses_slpc(rps)) 2183 return intel_rps_read_punit_req_frequency(rps); 2184 else 2185 return intel_gpu_freq(rps, rps->cur_freq); 2186 } 2187 2188 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2189 { 2190 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2191 2192 if (rps_uses_slpc(rps)) 2193 return slpc->max_freq_softlimit; 2194 else 2195 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2196 } 2197 2198 /** 2199 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format. 2200 * @rps: the intel_rps structure 2201 * 2202 * Returns the max frequency in a raw format. In newer platforms raw is in 2203 * units of 50 MHz. 2204 */ 2205 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps) 2206 { 2207 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2208 u32 freq; 2209 2210 if (rps_uses_slpc(rps)) { 2211 return DIV_ROUND_CLOSEST(slpc->rp0_freq, 2212 GT_FREQUENCY_MULTIPLIER); 2213 } else { 2214 freq = rps->max_freq; 2215 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2216 /* Convert GT frequency to 50 MHz units */ 2217 freq /= GEN9_FREQ_SCALER; 2218 } 2219 return freq; 2220 } 2221 } 2222 2223 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2224 { 2225 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2226 2227 if (rps_uses_slpc(rps)) 2228 return slpc->rp0_freq; 2229 else 2230 return intel_gpu_freq(rps, rps->rp0_freq); 2231 } 2232 2233 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2234 { 2235 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2236 2237 if (rps_uses_slpc(rps)) 2238 return slpc->rp1_freq; 2239 else 2240 return intel_gpu_freq(rps, rps->rp1_freq); 2241 } 2242 2243 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2244 { 2245 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2246 2247 if (rps_uses_slpc(rps)) 2248 return slpc->min_freq; 2249 else 2250 return intel_gpu_freq(rps, rps->min_freq); 2251 } 2252 2253 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2254 { 2255 struct intel_gt *gt = rps_to_gt(rps); 2256 struct drm_i915_private *i915 = gt->i915; 2257 struct intel_uncore *uncore = gt->uncore; 2258 struct intel_rps_freq_caps caps; 2259 u32 rp_state_limits; 2260 u32 gt_perf_status; 2261 u32 rpmodectl, rpinclimit, rpdeclimit; 2262 u32 rpstat, cagf, reqf; 2263 u32 rpcurupei, rpcurup, rpprevup; 2264 u32 rpcurdownei, rpcurdown, rpprevdown; 2265 u32 rpupei, rpupt, rpdownei, rpdownt; 2266 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; 2267 2268 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS); 2269 gen6_rps_get_freq_caps(rps, &caps); 2270 if (IS_GEN9_LP(i915)) 2271 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS); 2272 else 2273 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS); 2274 2275 /* RPSTAT1 is in the GT power well */ 2276 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 2277 2278 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2279 if (GRAPHICS_VER(i915) >= 9) { 2280 reqf >>= 23; 2281 } else { 2282 reqf &= ~GEN6_TURBO_DISABLE; 2283 if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 2284 reqf >>= 24; 2285 else 2286 reqf >>= 25; 2287 } 2288 reqf = intel_gpu_freq(rps, reqf); 2289 2290 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL); 2291 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2292 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2293 2294 rpstat = intel_rps_read_rpstat(rps); 2295 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; 2296 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK; 2297 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK; 2298 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; 2299 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; 2300 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; 2301 2302 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI); 2303 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); 2304 2305 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI); 2306 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); 2307 2308 cagf = intel_rps_read_actual_frequency(rps); 2309 2310 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 2311 2312 if (GRAPHICS_VER(i915) >= 11) { 2313 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE); 2314 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK); 2315 /* 2316 * The equivalent to the PM ISR & IIR cannot be read 2317 * without affecting the current state of the system 2318 */ 2319 pm_isr = 0; 2320 pm_iir = 0; 2321 } else if (GRAPHICS_VER(i915) >= 8) { 2322 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2)); 2323 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2)); 2324 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2)); 2325 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2)); 2326 } else { 2327 pm_ier = intel_uncore_read(uncore, GEN6_PMIER); 2328 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR); 2329 pm_isr = intel_uncore_read(uncore, GEN6_PMISR); 2330 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR); 2331 } 2332 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2333 2334 drm_printf(p, "Video Turbo Mode: %s\n", 2335 str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO)); 2336 drm_printf(p, "HW control enabled: %s\n", 2337 str_yes_no(rpmodectl & GEN6_RP_ENABLE)); 2338 drm_printf(p, "SW control enabled: %s\n", 2339 str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE)); 2340 2341 drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", 2342 pm_ier, pm_imr, pm_mask); 2343 if (GRAPHICS_VER(i915) <= 10) 2344 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n", 2345 pm_isr, pm_iir); 2346 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2347 rps->pm_intrmsk_mbz); 2348 drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); 2349 drm_printf(p, "Render p-state ratio: %d\n", 2350 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8); 2351 drm_printf(p, "Render p-state VID: %d\n", 2352 gt_perf_status & 0xff); 2353 drm_printf(p, "Render p-state limit: %d\n", 2354 rp_state_limits & 0xff); 2355 drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat); 2356 drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl); 2357 drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit); 2358 drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit); 2359 drm_printf(p, "RPNSWREQ: %dMHz\n", reqf); 2360 drm_printf(p, "CAGF: %dMHz\n", cagf); 2361 drm_printf(p, "RP CUR UP EI: %d (%lldns)\n", 2362 rpcurupei, 2363 intel_gt_pm_interval_to_ns(gt, rpcurupei)); 2364 drm_printf(p, "RP CUR UP: %d (%lldns)\n", 2365 rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup)); 2366 drm_printf(p, "RP PREV UP: %d (%lldns)\n", 2367 rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup)); 2368 drm_printf(p, "Up threshold: %d%%\n", 2369 rps->power.up_threshold); 2370 drm_printf(p, "RP UP EI: %d (%lldns)\n", 2371 rpupei, intel_gt_pm_interval_to_ns(gt, rpupei)); 2372 drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n", 2373 rpupt, intel_gt_pm_interval_to_ns(gt, rpupt)); 2374 2375 drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n", 2376 rpcurdownei, 2377 intel_gt_pm_interval_to_ns(gt, rpcurdownei)); 2378 drm_printf(p, "RP CUR DOWN: %d (%lldns)\n", 2379 rpcurdown, 2380 intel_gt_pm_interval_to_ns(gt, rpcurdown)); 2381 drm_printf(p, "RP PREV DOWN: %d (%lldns)\n", 2382 rpprevdown, 2383 intel_gt_pm_interval_to_ns(gt, rpprevdown)); 2384 drm_printf(p, "Down threshold: %d%%\n", 2385 rps->power.down_threshold); 2386 drm_printf(p, "RP DOWN EI: %d (%lldns)\n", 2387 rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei)); 2388 drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n", 2389 rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt)); 2390 2391 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2392 intel_gpu_freq(rps, caps.min_freq)); 2393 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2394 intel_gpu_freq(rps, caps.rp1_freq)); 2395 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2396 intel_gpu_freq(rps, caps.rp0_freq)); 2397 drm_printf(p, "Max overclocked frequency: %dMHz\n", 2398 intel_gpu_freq(rps, rps->max_freq)); 2399 2400 drm_printf(p, "Current freq: %d MHz\n", 2401 intel_gpu_freq(rps, rps->cur_freq)); 2402 drm_printf(p, "Actual freq: %d MHz\n", cagf); 2403 drm_printf(p, "Idle freq: %d MHz\n", 2404 intel_gpu_freq(rps, rps->idle_freq)); 2405 drm_printf(p, "Min freq: %d MHz\n", 2406 intel_gpu_freq(rps, rps->min_freq)); 2407 drm_printf(p, "Boost freq: %d MHz\n", 2408 intel_gpu_freq(rps, rps->boost_freq)); 2409 drm_printf(p, "Max freq: %d MHz\n", 2410 intel_gpu_freq(rps, rps->max_freq)); 2411 drm_printf(p, 2412 "efficient (RPe) frequency: %d MHz\n", 2413 intel_gpu_freq(rps, rps->efficient_freq)); 2414 } 2415 2416 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2417 { 2418 struct intel_gt *gt = rps_to_gt(rps); 2419 struct intel_uncore *uncore = gt->uncore; 2420 struct intel_rps_freq_caps caps; 2421 u32 pm_mask; 2422 2423 gen6_rps_get_freq_caps(rps, &caps); 2424 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK); 2425 2426 drm_printf(p, "PM MASK=0x%08x\n", pm_mask); 2427 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n", 2428 rps->pm_intrmsk_mbz); 2429 drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps)); 2430 drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps)); 2431 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n", 2432 intel_gpu_freq(rps, caps.min_freq)); 2433 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n", 2434 intel_gpu_freq(rps, caps.rp1_freq)); 2435 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n", 2436 intel_gpu_freq(rps, caps.rp0_freq)); 2437 drm_printf(p, "Current freq: %d MHz\n", 2438 intel_rps_get_requested_frequency(rps)); 2439 drm_printf(p, "Actual freq: %d MHz\n", 2440 intel_rps_read_actual_frequency(rps)); 2441 drm_printf(p, "Min freq: %d MHz\n", 2442 intel_rps_get_min_frequency(rps)); 2443 drm_printf(p, "Boost freq: %d MHz\n", 2444 intel_rps_get_boost_frequency(rps)); 2445 drm_printf(p, "Max freq: %d MHz\n", 2446 intel_rps_get_max_frequency(rps)); 2447 drm_printf(p, 2448 "efficient (RPe) frequency: %d MHz\n", 2449 intel_gpu_freq(rps, caps.rp1_freq)); 2450 } 2451 2452 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p) 2453 { 2454 if (rps_uses_slpc(rps)) 2455 return slpc_frequency_dump(rps, p); 2456 else 2457 return rps_frequency_dump(rps, p); 2458 } 2459 2460 static int set_max_freq(struct intel_rps *rps, u32 val) 2461 { 2462 struct drm_i915_private *i915 = rps_to_i915(rps); 2463 int ret = 0; 2464 2465 mutex_lock(&rps->lock); 2466 2467 val = intel_freq_opcode(rps, val); 2468 if (val < rps->min_freq || 2469 val > rps->max_freq || 2470 val < rps->min_freq_softlimit) { 2471 ret = -EINVAL; 2472 goto unlock; 2473 } 2474 2475 if (val > rps->rp0_freq) 2476 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2477 intel_gpu_freq(rps, val)); 2478 2479 rps->max_freq_softlimit = val; 2480 2481 val = clamp_t(int, rps->cur_freq, 2482 rps->min_freq_softlimit, 2483 rps->max_freq_softlimit); 2484 2485 /* 2486 * We still need *_set_rps to process the new max_delay and 2487 * update the interrupt limits and PMINTRMSK even though 2488 * frequency request may be unchanged. 2489 */ 2490 intel_rps_set(rps, val); 2491 2492 unlock: 2493 mutex_unlock(&rps->lock); 2494 2495 return ret; 2496 } 2497 2498 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2499 { 2500 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2501 2502 if (rps_uses_slpc(rps)) 2503 return intel_guc_slpc_set_max_freq(slpc, val); 2504 else 2505 return set_max_freq(rps, val); 2506 } 2507 2508 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2509 { 2510 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2511 2512 if (rps_uses_slpc(rps)) 2513 return slpc->min_freq_softlimit; 2514 else 2515 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2516 } 2517 2518 /** 2519 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format. 2520 * @rps: the intel_rps structure 2521 * 2522 * Returns the min frequency in a raw format. In newer platforms raw is in 2523 * units of 50 MHz. 2524 */ 2525 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps) 2526 { 2527 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2528 u32 freq; 2529 2530 if (rps_uses_slpc(rps)) { 2531 return DIV_ROUND_CLOSEST(slpc->min_freq, 2532 GT_FREQUENCY_MULTIPLIER); 2533 } else { 2534 freq = rps->min_freq; 2535 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 2536 /* Convert GT frequency to 50 MHz units */ 2537 freq /= GEN9_FREQ_SCALER; 2538 } 2539 return freq; 2540 } 2541 } 2542 2543 static int set_min_freq(struct intel_rps *rps, u32 val) 2544 { 2545 int ret = 0; 2546 2547 mutex_lock(&rps->lock); 2548 2549 val = intel_freq_opcode(rps, val); 2550 if (val < rps->min_freq || 2551 val > rps->max_freq || 2552 val > rps->max_freq_softlimit) { 2553 ret = -EINVAL; 2554 goto unlock; 2555 } 2556 2557 rps->min_freq_softlimit = val; 2558 2559 val = clamp_t(int, rps->cur_freq, 2560 rps->min_freq_softlimit, 2561 rps->max_freq_softlimit); 2562 2563 /* 2564 * We still need *_set_rps to process the new min_delay and 2565 * update the interrupt limits and PMINTRMSK even though 2566 * frequency request may be unchanged. 2567 */ 2568 intel_rps_set(rps, val); 2569 2570 unlock: 2571 mutex_unlock(&rps->lock); 2572 2573 return ret; 2574 } 2575 2576 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2577 { 2578 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2579 2580 if (rps_uses_slpc(rps)) 2581 return intel_guc_slpc_set_min_freq(slpc, val); 2582 else 2583 return set_min_freq(rps, val); 2584 } 2585 2586 u8 intel_rps_get_up_threshold(struct intel_rps *rps) 2587 { 2588 return rps->power.up_threshold; 2589 } 2590 2591 static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val) 2592 { 2593 int ret; 2594 2595 if (val > 100) 2596 return -EINVAL; 2597 2598 ret = mutex_lock_interruptible(&rps->lock); 2599 if (ret) 2600 return ret; 2601 2602 if (*threshold == val) 2603 goto out_unlock; 2604 2605 *threshold = val; 2606 2607 /* Force reset. */ 2608 rps->last_freq = -1; 2609 mutex_lock(&rps->power.mutex); 2610 rps->power.mode = -1; 2611 mutex_unlock(&rps->power.mutex); 2612 2613 intel_rps_set(rps, clamp(rps->cur_freq, 2614 rps->min_freq_softlimit, 2615 rps->max_freq_softlimit)); 2616 2617 out_unlock: 2618 mutex_unlock(&rps->lock); 2619 2620 return ret; 2621 } 2622 2623 int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold) 2624 { 2625 return rps_set_threshold(rps, &rps->power.up_threshold, threshold); 2626 } 2627 2628 u8 intel_rps_get_down_threshold(struct intel_rps *rps) 2629 { 2630 return rps->power.down_threshold; 2631 } 2632 2633 int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold) 2634 { 2635 return rps_set_threshold(rps, &rps->power.down_threshold, threshold); 2636 } 2637 2638 static void intel_rps_set_manual(struct intel_rps *rps, bool enable) 2639 { 2640 struct intel_uncore *uncore = rps_to_uncore(rps); 2641 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE; 2642 2643 /* Allow punit to process software requests */ 2644 intel_uncore_write(uncore, GEN6_RP_CONTROL, state); 2645 } 2646 2647 void intel_rps_raise_unslice(struct intel_rps *rps) 2648 { 2649 struct intel_uncore *uncore = rps_to_uncore(rps); 2650 2651 mutex_lock(&rps->lock); 2652 2653 if (rps_uses_slpc(rps)) { 2654 /* RP limits have not been initialized yet for SLPC path */ 2655 struct intel_rps_freq_caps caps; 2656 2657 gen6_rps_get_freq_caps(rps, &caps); 2658 2659 intel_rps_set_manual(rps, true); 2660 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2661 ((caps.rp0_freq << 2662 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2663 GEN9_IGNORE_SLICE_RATIO)); 2664 intel_rps_set_manual(rps, false); 2665 } else { 2666 intel_rps_set(rps, rps->rp0_freq); 2667 } 2668 2669 mutex_unlock(&rps->lock); 2670 } 2671 2672 void intel_rps_lower_unslice(struct intel_rps *rps) 2673 { 2674 struct intel_uncore *uncore = rps_to_uncore(rps); 2675 2676 mutex_lock(&rps->lock); 2677 2678 if (rps_uses_slpc(rps)) { 2679 /* RP limits have not been initialized yet for SLPC path */ 2680 struct intel_rps_freq_caps caps; 2681 2682 gen6_rps_get_freq_caps(rps, &caps); 2683 2684 intel_rps_set_manual(rps, true); 2685 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2686 ((caps.min_freq << 2687 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2688 GEN9_IGNORE_SLICE_RATIO)); 2689 intel_rps_set_manual(rps, false); 2690 } else { 2691 intel_rps_set(rps, rps->min_freq); 2692 } 2693 2694 mutex_unlock(&rps->lock); 2695 } 2696 2697 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32) 2698 { 2699 struct intel_gt *gt = rps_to_gt(rps); 2700 intel_wakeref_t wakeref; 2701 u32 val; 2702 2703 with_intel_runtime_pm(gt->uncore->rpm, wakeref) 2704 val = intel_uncore_read(gt->uncore, reg32); 2705 2706 return val; 2707 } 2708 2709 bool rps_read_mask_mmio(struct intel_rps *rps, 2710 i915_reg_t reg32, u32 mask) 2711 { 2712 return rps_read_mmio(rps, reg32) & mask; 2713 } 2714 2715 /* External interface for intel_ips.ko */ 2716 2717 static struct drm_i915_private __rcu *ips_mchdev; 2718 2719 /* 2720 * Tells the intel_ips driver that the i915 driver is now loaded, if 2721 * IPS got loaded first. 2722 * 2723 * This awkward dance is so that neither module has to depend on the 2724 * other in order for IPS to do the appropriate communication of 2725 * GPU turbo limits to i915. 2726 */ 2727 static void 2728 ips_ping_for_i915_load(void) 2729 { 2730 void (*link)(void); 2731 2732 link = symbol_get(ips_link_to_i915_driver); 2733 if (link) { 2734 link(); 2735 symbol_put(ips_link_to_i915_driver); 2736 } 2737 } 2738 2739 void intel_rps_driver_register(struct intel_rps *rps) 2740 { 2741 struct intel_gt *gt = rps_to_gt(rps); 2742 2743 /* 2744 * We only register the i915 ips part with intel-ips once everything is 2745 * set up, to avoid intel-ips sneaking in and reading bogus values. 2746 */ 2747 if (GRAPHICS_VER(gt->i915) == 5) { 2748 GEM_BUG_ON(ips_mchdev); 2749 rcu_assign_pointer(ips_mchdev, gt->i915); 2750 ips_ping_for_i915_load(); 2751 } 2752 } 2753 2754 void intel_rps_driver_unregister(struct intel_rps *rps) 2755 { 2756 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2757 rcu_assign_pointer(ips_mchdev, NULL); 2758 } 2759 2760 static struct drm_i915_private *mchdev_get(void) 2761 { 2762 struct drm_i915_private *i915; 2763 2764 rcu_read_lock(); 2765 i915 = rcu_dereference(ips_mchdev); 2766 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2767 i915 = NULL; 2768 rcu_read_unlock(); 2769 2770 return i915; 2771 } 2772 2773 /** 2774 * i915_read_mch_val - return value for IPS use 2775 * 2776 * Calculate and return a value for the IPS driver to use when deciding whether 2777 * we have thermal and power headroom to increase CPU or GPU power budget. 2778 */ 2779 unsigned long i915_read_mch_val(void) 2780 { 2781 struct drm_i915_private *i915; 2782 unsigned long chipset_val = 0; 2783 unsigned long graphics_val = 0; 2784 intel_wakeref_t wakeref; 2785 2786 i915 = mchdev_get(); 2787 if (!i915) 2788 return 0; 2789 2790 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2791 struct intel_ips *ips = &to_gt(i915)->rps.ips; 2792 2793 spin_lock_irq(&mchdev_lock); 2794 chipset_val = __ips_chipset_val(ips); 2795 graphics_val = __ips_gfx_val(ips); 2796 spin_unlock_irq(&mchdev_lock); 2797 } 2798 2799 drm_dev_put(&i915->drm); 2800 return chipset_val + graphics_val; 2801 } 2802 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2803 2804 /** 2805 * i915_gpu_raise - raise GPU frequency limit 2806 * 2807 * Raise the limit; IPS indicates we have thermal headroom. 2808 */ 2809 bool i915_gpu_raise(void) 2810 { 2811 struct drm_i915_private *i915; 2812 struct intel_rps *rps; 2813 2814 i915 = mchdev_get(); 2815 if (!i915) 2816 return false; 2817 2818 rps = &to_gt(i915)->rps; 2819 2820 spin_lock_irq(&mchdev_lock); 2821 if (rps->max_freq_softlimit < rps->max_freq) 2822 rps->max_freq_softlimit++; 2823 spin_unlock_irq(&mchdev_lock); 2824 2825 drm_dev_put(&i915->drm); 2826 return true; 2827 } 2828 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2829 2830 /** 2831 * i915_gpu_lower - lower GPU frequency limit 2832 * 2833 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2834 * frequency maximum. 2835 */ 2836 bool i915_gpu_lower(void) 2837 { 2838 struct drm_i915_private *i915; 2839 struct intel_rps *rps; 2840 2841 i915 = mchdev_get(); 2842 if (!i915) 2843 return false; 2844 2845 rps = &to_gt(i915)->rps; 2846 2847 spin_lock_irq(&mchdev_lock); 2848 if (rps->max_freq_softlimit > rps->min_freq) 2849 rps->max_freq_softlimit--; 2850 spin_unlock_irq(&mchdev_lock); 2851 2852 drm_dev_put(&i915->drm); 2853 return true; 2854 } 2855 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2856 2857 /** 2858 * i915_gpu_busy - indicate GPU business to IPS 2859 * 2860 * Tell the IPS driver whether or not the GPU is busy. 2861 */ 2862 bool i915_gpu_busy(void) 2863 { 2864 struct drm_i915_private *i915; 2865 bool ret; 2866 2867 i915 = mchdev_get(); 2868 if (!i915) 2869 return false; 2870 2871 ret = to_gt(i915)->awake; 2872 2873 drm_dev_put(&i915->drm); 2874 return ret; 2875 } 2876 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2877 2878 /** 2879 * i915_gpu_turbo_disable - disable graphics turbo 2880 * 2881 * Disable graphics turbo by resetting the max frequency and setting the 2882 * current frequency to the default. 2883 */ 2884 bool i915_gpu_turbo_disable(void) 2885 { 2886 struct drm_i915_private *i915; 2887 struct intel_rps *rps; 2888 bool ret; 2889 2890 i915 = mchdev_get(); 2891 if (!i915) 2892 return false; 2893 2894 rps = &to_gt(i915)->rps; 2895 2896 spin_lock_irq(&mchdev_lock); 2897 rps->max_freq_softlimit = rps->min_freq; 2898 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq); 2899 spin_unlock_irq(&mchdev_lock); 2900 2901 drm_dev_put(&i915->drm); 2902 return ret; 2903 } 2904 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2905 2906 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2907 #include "selftest_rps.c" 2908 #include "selftest_slpc.c" 2909 #endif 2910