1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <drm/i915_drm.h> 7 8 #include "i915_drv.h" 9 #include "i915_irq.h" 10 #include "intel_breadcrumbs.h" 11 #include "intel_gt.h" 12 #include "intel_gt_clock_utils.h" 13 #include "intel_gt_irq.h" 14 #include "intel_gt_pm_irq.h" 15 #include "intel_gt_regs.h" 16 #include "intel_pcode.h" 17 #include "intel_rps.h" 18 #include "vlv_sideband.h" 19 #include "../../../platform/x86/intel_ips.h" 20 21 #define BUSY_MAX_EI 20u /* ms */ 22 23 /* 24 * Lock protecting IPS related data structures 25 */ 26 static DEFINE_SPINLOCK(mchdev_lock); 27 28 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 29 { 30 return container_of(rps, struct intel_gt, rps); 31 } 32 33 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 34 { 35 return rps_to_gt(rps)->i915; 36 } 37 38 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 39 { 40 return rps_to_gt(rps)->uncore; 41 } 42 43 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps) 44 { 45 struct intel_gt *gt = rps_to_gt(rps); 46 47 return >->uc.guc.slpc; 48 } 49 50 static bool rps_uses_slpc(struct intel_rps *rps) 51 { 52 struct intel_gt *gt = rps_to_gt(rps); 53 54 return intel_uc_uses_guc_slpc(>->uc); 55 } 56 57 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 58 { 59 return mask & ~rps->pm_intrmsk_mbz; 60 } 61 62 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 63 { 64 intel_uncore_write_fw(uncore, reg, val); 65 } 66 67 static void rps_timer(struct timer_list *t) 68 { 69 struct intel_rps *rps = from_timer(rps, t, timer); 70 struct intel_engine_cs *engine; 71 ktime_t dt, last, timestamp; 72 enum intel_engine_id id; 73 s64 max_busy[3] = {}; 74 75 timestamp = 0; 76 for_each_engine(engine, rps_to_gt(rps), id) { 77 s64 busy; 78 int i; 79 80 dt = intel_engine_get_busy_time(engine, ×tamp); 81 last = engine->stats.rps; 82 engine->stats.rps = dt; 83 84 busy = ktime_to_ns(ktime_sub(dt, last)); 85 for (i = 0; i < ARRAY_SIZE(max_busy); i++) { 86 if (busy > max_busy[i]) 87 swap(busy, max_busy[i]); 88 } 89 } 90 last = rps->pm_timestamp; 91 rps->pm_timestamp = timestamp; 92 93 if (intel_rps_is_active(rps)) { 94 s64 busy; 95 int i; 96 97 dt = ktime_sub(timestamp, last); 98 99 /* 100 * Our goal is to evaluate each engine independently, so we run 101 * at the lowest clocks required to sustain the heaviest 102 * workload. However, a task may be split into sequential 103 * dependent operations across a set of engines, such that 104 * the independent contributions do not account for high load, 105 * but overall the task is GPU bound. For example, consider 106 * video decode on vcs followed by colour post-processing 107 * on vecs, followed by general post-processing on rcs. 108 * Since multi-engines being active does imply a single 109 * continuous workload across all engines, we hedge our 110 * bets by only contributing a factor of the distributed 111 * load into our busyness calculation. 112 */ 113 busy = max_busy[0]; 114 for (i = 1; i < ARRAY_SIZE(max_busy); i++) { 115 if (!max_busy[i]) 116 break; 117 118 busy += div_u64(max_busy[i], 1 << i); 119 } 120 GT_TRACE(rps_to_gt(rps), 121 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", 122 busy, (int)div64_u64(100 * busy, dt), 123 max_busy[0], max_busy[1], max_busy[2], 124 rps->pm_interval); 125 126 if (100 * busy > rps->power.up_threshold * dt && 127 rps->cur_freq < rps->max_freq_softlimit) { 128 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; 129 rps->pm_interval = 1; 130 schedule_work(&rps->work); 131 } else if (100 * busy < rps->power.down_threshold * dt && 132 rps->cur_freq > rps->min_freq_softlimit) { 133 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; 134 rps->pm_interval = 1; 135 schedule_work(&rps->work); 136 } else { 137 rps->last_adj = 0; 138 } 139 140 mod_timer(&rps->timer, 141 jiffies + msecs_to_jiffies(rps->pm_interval)); 142 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); 143 } 144 } 145 146 static void rps_start_timer(struct intel_rps *rps) 147 { 148 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 149 rps->pm_interval = 1; 150 mod_timer(&rps->timer, jiffies + 1); 151 } 152 153 static void rps_stop_timer(struct intel_rps *rps) 154 { 155 del_timer_sync(&rps->timer); 156 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); 157 cancel_work_sync(&rps->work); 158 } 159 160 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 161 { 162 u32 mask = 0; 163 164 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 165 if (val > rps->min_freq_softlimit) 166 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 167 GEN6_PM_RP_DOWN_THRESHOLD | 168 GEN6_PM_RP_DOWN_TIMEOUT); 169 170 if (val < rps->max_freq_softlimit) 171 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 172 173 mask &= rps->pm_events; 174 175 return rps_pm_sanitize_mask(rps, ~mask); 176 } 177 178 static void rps_reset_ei(struct intel_rps *rps) 179 { 180 memset(&rps->ei, 0, sizeof(rps->ei)); 181 } 182 183 static void rps_enable_interrupts(struct intel_rps *rps) 184 { 185 struct intel_gt *gt = rps_to_gt(rps); 186 187 GEM_BUG_ON(rps_uses_slpc(rps)); 188 189 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", 190 rps->pm_events, rps_pm_mask(rps, rps->last_freq)); 191 192 rps_reset_ei(rps); 193 194 spin_lock_irq(>->irq_lock); 195 gen6_gt_pm_enable_irq(gt, rps->pm_events); 196 spin_unlock_irq(>->irq_lock); 197 198 intel_uncore_write(gt->uncore, 199 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 200 } 201 202 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 203 { 204 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 205 } 206 207 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 208 { 209 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 210 ; 211 } 212 213 static void rps_reset_interrupts(struct intel_rps *rps) 214 { 215 struct intel_gt *gt = rps_to_gt(rps); 216 217 spin_lock_irq(>->irq_lock); 218 if (GRAPHICS_VER(gt->i915) >= 11) 219 gen11_rps_reset_interrupts(rps); 220 else 221 gen6_rps_reset_interrupts(rps); 222 223 rps->pm_iir = 0; 224 spin_unlock_irq(>->irq_lock); 225 } 226 227 static void rps_disable_interrupts(struct intel_rps *rps) 228 { 229 struct intel_gt *gt = rps_to_gt(rps); 230 231 intel_uncore_write(gt->uncore, 232 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 233 234 spin_lock_irq(>->irq_lock); 235 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 236 spin_unlock_irq(>->irq_lock); 237 238 intel_synchronize_irq(gt->i915); 239 240 /* 241 * Now that we will not be generating any more work, flush any 242 * outstanding tasks. As we are called on the RPS idle path, 243 * we will reset the GPU to minimum frequencies, so the current 244 * state of the worker can be discarded. 245 */ 246 cancel_work_sync(&rps->work); 247 248 rps_reset_interrupts(rps); 249 GT_TRACE(gt, "interrupts:off\n"); 250 } 251 252 static const struct cparams { 253 u16 i; 254 u16 t; 255 u16 m; 256 u16 c; 257 } cparams[] = { 258 { 1, 1333, 301, 28664 }, 259 { 1, 1066, 294, 24460 }, 260 { 1, 800, 294, 25192 }, 261 { 0, 1333, 276, 27605 }, 262 { 0, 1066, 276, 27605 }, 263 { 0, 800, 231, 23784 }, 264 }; 265 266 static void gen5_rps_init(struct intel_rps *rps) 267 { 268 struct drm_i915_private *i915 = rps_to_i915(rps); 269 struct intel_uncore *uncore = rps_to_uncore(rps); 270 u8 fmax, fmin, fstart; 271 u32 rgvmodectl; 272 int c_m, i; 273 274 if (i915->fsb_freq <= 3200) 275 c_m = 0; 276 else if (i915->fsb_freq <= 4800) 277 c_m = 1; 278 else 279 c_m = 2; 280 281 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 282 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { 283 rps->ips.m = cparams[i].m; 284 rps->ips.c = cparams[i].c; 285 break; 286 } 287 } 288 289 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 290 291 /* Set up min, max, and cur for interrupt handling */ 292 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 293 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 294 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 295 MEMMODE_FSTART_SHIFT; 296 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", 297 fmax, fmin, fstart); 298 299 rps->min_freq = fmax; 300 rps->efficient_freq = fstart; 301 rps->max_freq = fmin; 302 } 303 304 static unsigned long 305 __ips_chipset_val(struct intel_ips *ips) 306 { 307 struct intel_uncore *uncore = 308 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 309 unsigned long now = jiffies_to_msecs(jiffies), dt; 310 unsigned long result; 311 u64 total, delta; 312 313 lockdep_assert_held(&mchdev_lock); 314 315 /* 316 * Prevent division-by-zero if we are asking too fast. 317 * Also, we don't get interesting results if we are polling 318 * faster than once in 10ms, so just return the saved value 319 * in such cases. 320 */ 321 dt = now - ips->last_time1; 322 if (dt <= 10) 323 return ips->chipset_power; 324 325 /* FIXME: handle per-counter overflow */ 326 total = intel_uncore_read(uncore, DMIEC); 327 total += intel_uncore_read(uncore, DDREC); 328 total += intel_uncore_read(uncore, CSIEC); 329 330 delta = total - ips->last_count1; 331 332 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 333 334 ips->last_count1 = total; 335 ips->last_time1 = now; 336 337 ips->chipset_power = result; 338 339 return result; 340 } 341 342 static unsigned long ips_mch_val(struct intel_uncore *uncore) 343 { 344 unsigned int m, x, b; 345 u32 tsfs; 346 347 tsfs = intel_uncore_read(uncore, TSFS); 348 x = intel_uncore_read8(uncore, TR1); 349 350 b = tsfs & TSFS_INTR_MASK; 351 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 352 353 return m * x / 127 - b; 354 } 355 356 static int _pxvid_to_vd(u8 pxvid) 357 { 358 if (pxvid == 0) 359 return 0; 360 361 if (pxvid >= 8 && pxvid < 31) 362 pxvid = 31; 363 364 return (pxvid + 2) * 125; 365 } 366 367 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 368 { 369 const int vd = _pxvid_to_vd(pxvid); 370 371 if (INTEL_INFO(i915)->is_mobile) 372 return max(vd - 1125, 0); 373 374 return vd; 375 } 376 377 static void __gen5_ips_update(struct intel_ips *ips) 378 { 379 struct intel_uncore *uncore = 380 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 381 u64 now, delta, dt; 382 u32 count; 383 384 lockdep_assert_held(&mchdev_lock); 385 386 now = ktime_get_raw_ns(); 387 dt = now - ips->last_time2; 388 do_div(dt, NSEC_PER_MSEC); 389 390 /* Don't divide by 0 */ 391 if (dt <= 10) 392 return; 393 394 count = intel_uncore_read(uncore, GFXEC); 395 delta = count - ips->last_count2; 396 397 ips->last_count2 = count; 398 ips->last_time2 = now; 399 400 /* More magic constants... */ 401 ips->gfx_power = div_u64(delta * 1181, dt * 10); 402 } 403 404 static void gen5_rps_update(struct intel_rps *rps) 405 { 406 spin_lock_irq(&mchdev_lock); 407 __gen5_ips_update(&rps->ips); 408 spin_unlock_irq(&mchdev_lock); 409 } 410 411 static unsigned int gen5_invert_freq(struct intel_rps *rps, 412 unsigned int val) 413 { 414 /* Invert the frequency bin into an ips delay */ 415 val = rps->max_freq - val; 416 val = rps->min_freq + val; 417 418 return val; 419 } 420 421 static int __gen5_rps_set(struct intel_rps *rps, u8 val) 422 { 423 struct intel_uncore *uncore = rps_to_uncore(rps); 424 u16 rgvswctl; 425 426 lockdep_assert_held(&mchdev_lock); 427 428 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 429 if (rgvswctl & MEMCTL_CMD_STS) { 430 DRM_DEBUG("gpu busy, RCS change rejected\n"); 431 return -EBUSY; /* still busy with another command */ 432 } 433 434 /* Invert the frequency bin into an ips delay */ 435 val = gen5_invert_freq(rps, val); 436 437 rgvswctl = 438 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 439 (val << MEMCTL_FREQ_SHIFT) | 440 MEMCTL_SFCAVM; 441 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 442 intel_uncore_posting_read16(uncore, MEMSWCTL); 443 444 rgvswctl |= MEMCTL_CMD_STS; 445 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 446 447 return 0; 448 } 449 450 static int gen5_rps_set(struct intel_rps *rps, u8 val) 451 { 452 int err; 453 454 spin_lock_irq(&mchdev_lock); 455 err = __gen5_rps_set(rps, val); 456 spin_unlock_irq(&mchdev_lock); 457 458 return err; 459 } 460 461 static unsigned long intel_pxfreq(u32 vidfreq) 462 { 463 int div = (vidfreq & 0x3f0000) >> 16; 464 int post = (vidfreq & 0x3000) >> 12; 465 int pre = (vidfreq & 0x7); 466 467 if (!pre) 468 return 0; 469 470 return div * 133333 / (pre << post); 471 } 472 473 static unsigned int init_emon(struct intel_uncore *uncore) 474 { 475 u8 pxw[16]; 476 int i; 477 478 /* Disable to program */ 479 intel_uncore_write(uncore, ECR, 0); 480 intel_uncore_posting_read(uncore, ECR); 481 482 /* Program energy weights for various events */ 483 intel_uncore_write(uncore, SDEW, 0x15040d00); 484 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 485 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 486 intel_uncore_write(uncore, CSIEW2, 0x04000004); 487 488 for (i = 0; i < 5; i++) 489 intel_uncore_write(uncore, PEW(i), 0); 490 for (i = 0; i < 3; i++) 491 intel_uncore_write(uncore, DEW(i), 0); 492 493 /* Program P-state weights to account for frequency power adjustment */ 494 for (i = 0; i < 16; i++) { 495 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 496 unsigned int freq = intel_pxfreq(pxvidfreq); 497 unsigned int vid = 498 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 499 unsigned int val; 500 501 val = vid * vid * freq / 1000 * 255; 502 val /= 127 * 127 * 900; 503 504 pxw[i] = val; 505 } 506 /* Render standby states get 0 weight */ 507 pxw[14] = 0; 508 pxw[15] = 0; 509 510 for (i = 0; i < 4; i++) { 511 intel_uncore_write(uncore, PXW(i), 512 pxw[i * 4 + 0] << 24 | 513 pxw[i * 4 + 1] << 16 | 514 pxw[i * 4 + 2] << 8 | 515 pxw[i * 4 + 3] << 0); 516 } 517 518 /* Adjust magic regs to magic values (more experimental results) */ 519 intel_uncore_write(uncore, OGW0, 0); 520 intel_uncore_write(uncore, OGW1, 0); 521 intel_uncore_write(uncore, EG0, 0x00007f00); 522 intel_uncore_write(uncore, EG1, 0x0000000e); 523 intel_uncore_write(uncore, EG2, 0x000e0000); 524 intel_uncore_write(uncore, EG3, 0x68000300); 525 intel_uncore_write(uncore, EG4, 0x42000000); 526 intel_uncore_write(uncore, EG5, 0x00140031); 527 intel_uncore_write(uncore, EG6, 0); 528 intel_uncore_write(uncore, EG7, 0); 529 530 for (i = 0; i < 8; i++) 531 intel_uncore_write(uncore, PXWL(i), 0); 532 533 /* Enable PMON + select events */ 534 intel_uncore_write(uncore, ECR, 0x80000019); 535 536 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 537 } 538 539 static bool gen5_rps_enable(struct intel_rps *rps) 540 { 541 struct drm_i915_private *i915 = rps_to_i915(rps); 542 struct intel_uncore *uncore = rps_to_uncore(rps); 543 u8 fstart, vstart; 544 u32 rgvmodectl; 545 546 spin_lock_irq(&mchdev_lock); 547 548 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 549 550 /* Enable temp reporting */ 551 intel_uncore_write16(uncore, PMMISC, 552 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 553 intel_uncore_write16(uncore, TSC1, 554 intel_uncore_read16(uncore, TSC1) | TSE); 555 556 /* 100ms RC evaluation intervals */ 557 intel_uncore_write(uncore, RCUPEI, 100000); 558 intel_uncore_write(uncore, RCDNEI, 100000); 559 560 /* Set max/min thresholds to 90ms and 80ms respectively */ 561 intel_uncore_write(uncore, RCBMAXAVG, 90000); 562 intel_uncore_write(uncore, RCBMINAVG, 80000); 563 564 intel_uncore_write(uncore, MEMIHYST, 1); 565 566 /* Set up min, max, and cur for interrupt handling */ 567 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 568 MEMMODE_FSTART_SHIFT; 569 570 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 571 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 572 573 intel_uncore_write(uncore, 574 MEMINTREN, 575 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 576 577 intel_uncore_write(uncore, VIDSTART, vstart); 578 intel_uncore_posting_read(uncore, VIDSTART); 579 580 rgvmodectl |= MEMMODE_SWMODE_EN; 581 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 582 583 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 584 MEMCTL_CMD_STS) == 0, 10)) 585 drm_err(&uncore->i915->drm, 586 "stuck trying to change perf mode\n"); 587 mdelay(1); 588 589 __gen5_rps_set(rps, rps->cur_freq); 590 591 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 592 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 593 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 594 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 595 596 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 597 rps->ips.last_time2 = ktime_get_raw_ns(); 598 599 spin_lock(&i915->irq_lock); 600 ilk_enable_display_irq(i915, DE_PCU_EVENT); 601 spin_unlock(&i915->irq_lock); 602 603 spin_unlock_irq(&mchdev_lock); 604 605 rps->ips.corr = init_emon(uncore); 606 607 return true; 608 } 609 610 static void gen5_rps_disable(struct intel_rps *rps) 611 { 612 struct drm_i915_private *i915 = rps_to_i915(rps); 613 struct intel_uncore *uncore = rps_to_uncore(rps); 614 u16 rgvswctl; 615 616 spin_lock_irq(&mchdev_lock); 617 618 spin_lock(&i915->irq_lock); 619 ilk_disable_display_irq(i915, DE_PCU_EVENT); 620 spin_unlock(&i915->irq_lock); 621 622 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 623 624 /* Ack interrupts, disable EFC interrupt */ 625 intel_uncore_write(uncore, MEMINTREN, 626 intel_uncore_read(uncore, MEMINTREN) & 627 ~MEMINT_EVAL_CHG_EN); 628 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 629 630 /* Go back to the starting frequency */ 631 __gen5_rps_set(rps, rps->idle_freq); 632 mdelay(1); 633 rgvswctl |= MEMCTL_CMD_STS; 634 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 635 mdelay(1); 636 637 spin_unlock_irq(&mchdev_lock); 638 } 639 640 static u32 rps_limits(struct intel_rps *rps, u8 val) 641 { 642 u32 limits; 643 644 /* 645 * Only set the down limit when we've reached the lowest level to avoid 646 * getting more interrupts, otherwise leave this clear. This prevents a 647 * race in the hw when coming out of rc6: There's a tiny window where 648 * the hw runs at the minimal clock before selecting the desired 649 * frequency, if the down threshold expires in that window we will not 650 * receive a down interrupt. 651 */ 652 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) { 653 limits = rps->max_freq_softlimit << 23; 654 if (val <= rps->min_freq_softlimit) 655 limits |= rps->min_freq_softlimit << 14; 656 } else { 657 limits = rps->max_freq_softlimit << 24; 658 if (val <= rps->min_freq_softlimit) 659 limits |= rps->min_freq_softlimit << 16; 660 } 661 662 return limits; 663 } 664 665 static void rps_set_power(struct intel_rps *rps, int new_power) 666 { 667 struct intel_gt *gt = rps_to_gt(rps); 668 struct intel_uncore *uncore = gt->uncore; 669 u32 threshold_up = 0, threshold_down = 0; /* in % */ 670 u32 ei_up = 0, ei_down = 0; 671 672 lockdep_assert_held(&rps->power.mutex); 673 674 if (new_power == rps->power.mode) 675 return; 676 677 threshold_up = 95; 678 threshold_down = 85; 679 680 /* Note the units here are not exactly 1us, but 1280ns. */ 681 switch (new_power) { 682 case LOW_POWER: 683 ei_up = 16000; 684 ei_down = 32000; 685 break; 686 687 case BETWEEN: 688 ei_up = 13000; 689 ei_down = 32000; 690 break; 691 692 case HIGH_POWER: 693 ei_up = 10000; 694 ei_down = 32000; 695 break; 696 } 697 698 /* When byt can survive without system hang with dynamic 699 * sw freq adjustments, this restriction can be lifted. 700 */ 701 if (IS_VALLEYVIEW(gt->i915)) 702 goto skip_hw_write; 703 704 GT_TRACE(gt, 705 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", 706 new_power, threshold_up, ei_up, threshold_down, ei_down); 707 708 set(uncore, GEN6_RP_UP_EI, 709 intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); 710 set(uncore, GEN6_RP_UP_THRESHOLD, 711 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10)); 712 713 set(uncore, GEN6_RP_DOWN_EI, 714 intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); 715 set(uncore, GEN6_RP_DOWN_THRESHOLD, 716 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10)); 717 718 set(uncore, GEN6_RP_CONTROL, 719 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 720 GEN6_RP_MEDIA_HW_NORMAL_MODE | 721 GEN6_RP_MEDIA_IS_GFX | 722 GEN6_RP_ENABLE | 723 GEN6_RP_UP_BUSY_AVG | 724 GEN6_RP_DOWN_IDLE_AVG); 725 726 skip_hw_write: 727 rps->power.mode = new_power; 728 rps->power.up_threshold = threshold_up; 729 rps->power.down_threshold = threshold_down; 730 } 731 732 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 733 { 734 int new_power; 735 736 new_power = rps->power.mode; 737 switch (rps->power.mode) { 738 case LOW_POWER: 739 if (val > rps->efficient_freq + 1 && 740 val > rps->cur_freq) 741 new_power = BETWEEN; 742 break; 743 744 case BETWEEN: 745 if (val <= rps->efficient_freq && 746 val < rps->cur_freq) 747 new_power = LOW_POWER; 748 else if (val >= rps->rp0_freq && 749 val > rps->cur_freq) 750 new_power = HIGH_POWER; 751 break; 752 753 case HIGH_POWER: 754 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 755 val < rps->cur_freq) 756 new_power = BETWEEN; 757 break; 758 } 759 /* Max/min bins are special */ 760 if (val <= rps->min_freq_softlimit) 761 new_power = LOW_POWER; 762 if (val >= rps->max_freq_softlimit) 763 new_power = HIGH_POWER; 764 765 mutex_lock(&rps->power.mutex); 766 if (rps->power.interactive) 767 new_power = HIGH_POWER; 768 rps_set_power(rps, new_power); 769 mutex_unlock(&rps->power.mutex); 770 } 771 772 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 773 { 774 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive)); 775 776 mutex_lock(&rps->power.mutex); 777 if (interactive) { 778 if (!rps->power.interactive++ && intel_rps_is_active(rps)) 779 rps_set_power(rps, HIGH_POWER); 780 } else { 781 GEM_BUG_ON(!rps->power.interactive); 782 rps->power.interactive--; 783 } 784 mutex_unlock(&rps->power.mutex); 785 } 786 787 static int gen6_rps_set(struct intel_rps *rps, u8 val) 788 { 789 struct intel_uncore *uncore = rps_to_uncore(rps); 790 struct drm_i915_private *i915 = rps_to_i915(rps); 791 u32 swreq; 792 793 GEM_BUG_ON(rps_uses_slpc(rps)); 794 795 if (GRAPHICS_VER(i915) >= 9) 796 swreq = GEN9_FREQUENCY(val); 797 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 798 swreq = HSW_FREQUENCY(val); 799 else 800 swreq = (GEN6_FREQUENCY(val) | 801 GEN6_OFFSET(0) | 802 GEN6_AGGRESSIVE_TURBO); 803 set(uncore, GEN6_RPNSWREQ, swreq); 804 805 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", 806 val, intel_gpu_freq(rps, val), swreq); 807 808 return 0; 809 } 810 811 static int vlv_rps_set(struct intel_rps *rps, u8 val) 812 { 813 struct drm_i915_private *i915 = rps_to_i915(rps); 814 int err; 815 816 vlv_punit_get(i915); 817 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 818 vlv_punit_put(i915); 819 820 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", 821 val, intel_gpu_freq(rps, val)); 822 823 return err; 824 } 825 826 static int rps_set(struct intel_rps *rps, u8 val, bool update) 827 { 828 struct drm_i915_private *i915 = rps_to_i915(rps); 829 int err; 830 831 if (val == rps->last_freq) 832 return 0; 833 834 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 835 err = vlv_rps_set(rps, val); 836 else if (GRAPHICS_VER(i915) >= 6) 837 err = gen6_rps_set(rps, val); 838 else 839 err = gen5_rps_set(rps, val); 840 if (err) 841 return err; 842 843 if (update && GRAPHICS_VER(i915) >= 6) 844 gen6_rps_set_thresholds(rps, val); 845 rps->last_freq = val; 846 847 return 0; 848 } 849 850 void intel_rps_unpark(struct intel_rps *rps) 851 { 852 if (!intel_rps_is_enabled(rps)) 853 return; 854 855 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); 856 857 /* 858 * Use the user's desired frequency as a guide, but for better 859 * performance, jump directly to RPe as our starting frequency. 860 */ 861 mutex_lock(&rps->lock); 862 863 intel_rps_set_active(rps); 864 intel_rps_set(rps, 865 clamp(rps->cur_freq, 866 rps->min_freq_softlimit, 867 rps->max_freq_softlimit)); 868 869 mutex_unlock(&rps->lock); 870 871 rps->pm_iir = 0; 872 if (intel_rps_has_interrupts(rps)) 873 rps_enable_interrupts(rps); 874 if (intel_rps_uses_timer(rps)) 875 rps_start_timer(rps); 876 877 if (GRAPHICS_VER(rps_to_i915(rps)) == 5) 878 gen5_rps_update(rps); 879 } 880 881 void intel_rps_park(struct intel_rps *rps) 882 { 883 int adj; 884 885 if (!intel_rps_is_enabled(rps)) 886 return; 887 888 if (!intel_rps_clear_active(rps)) 889 return; 890 891 if (intel_rps_uses_timer(rps)) 892 rps_stop_timer(rps); 893 if (intel_rps_has_interrupts(rps)) 894 rps_disable_interrupts(rps); 895 896 if (rps->last_freq <= rps->idle_freq) 897 return; 898 899 /* 900 * The punit delays the write of the frequency and voltage until it 901 * determines the GPU is awake. During normal usage we don't want to 902 * waste power changing the frequency if the GPU is sleeping (rc6). 903 * However, the GPU and driver is now idle and we do not want to delay 904 * switching to minimum voltage (reducing power whilst idle) as we do 905 * not expect to be woken in the near future and so must flush the 906 * change by waking the device. 907 * 908 * We choose to take the media powerwell (either would do to trick the 909 * punit into committing the voltage change) as that takes a lot less 910 * power than the render powerwell. 911 */ 912 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 913 rps_set(rps, rps->idle_freq, false); 914 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 915 916 /* 917 * Since we will try and restart from the previously requested 918 * frequency on unparking, treat this idle point as a downclock 919 * interrupt and reduce the frequency for resume. If we park/unpark 920 * more frequently than the rps worker can run, we will not respond 921 * to any EI and never see a change in frequency. 922 * 923 * (Note we accommodate Cherryview's limitation of only using an 924 * even bin by applying it to all.) 925 */ 926 adj = rps->last_adj; 927 if (adj < 0) 928 adj *= 2; 929 else /* CHV needs even encode values */ 930 adj = -2; 931 rps->last_adj = adj; 932 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); 933 if (rps->cur_freq < rps->efficient_freq) { 934 rps->cur_freq = rps->efficient_freq; 935 rps->last_adj = 0; 936 } 937 938 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); 939 } 940 941 u32 intel_rps_get_boost_frequency(struct intel_rps *rps) 942 { 943 struct intel_guc_slpc *slpc; 944 945 if (rps_uses_slpc(rps)) { 946 slpc = rps_to_slpc(rps); 947 948 return slpc->boost_freq; 949 } else { 950 return intel_gpu_freq(rps, rps->boost_freq); 951 } 952 } 953 954 static int rps_set_boost_freq(struct intel_rps *rps, u32 val) 955 { 956 bool boost = false; 957 958 /* Validate against (static) hardware limits */ 959 val = intel_freq_opcode(rps, val); 960 if (val < rps->min_freq || val > rps->max_freq) 961 return -EINVAL; 962 963 mutex_lock(&rps->lock); 964 if (val != rps->boost_freq) { 965 rps->boost_freq = val; 966 boost = atomic_read(&rps->num_waiters); 967 } 968 mutex_unlock(&rps->lock); 969 if (boost) 970 schedule_work(&rps->work); 971 972 return 0; 973 } 974 975 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq) 976 { 977 struct intel_guc_slpc *slpc; 978 979 if (rps_uses_slpc(rps)) { 980 slpc = rps_to_slpc(rps); 981 982 return intel_guc_slpc_set_boost_freq(slpc, freq); 983 } else { 984 return rps_set_boost_freq(rps, freq); 985 } 986 } 987 988 void intel_rps_dec_waiters(struct intel_rps *rps) 989 { 990 struct intel_guc_slpc *slpc; 991 992 if (rps_uses_slpc(rps)) { 993 slpc = rps_to_slpc(rps); 994 995 intel_guc_slpc_dec_waiters(slpc); 996 } else { 997 atomic_dec(&rps->num_waiters); 998 } 999 } 1000 1001 void intel_rps_boost(struct i915_request *rq) 1002 { 1003 struct intel_guc_slpc *slpc; 1004 1005 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq)) 1006 return; 1007 1008 /* Serializes with i915_request_retire() */ 1009 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) { 1010 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 1011 1012 if (rps_uses_slpc(rps)) { 1013 slpc = rps_to_slpc(rps); 1014 1015 /* Return if old value is non zero */ 1016 if (!atomic_fetch_inc(&slpc->num_waiters)) 1017 schedule_work(&slpc->boost_work); 1018 1019 return; 1020 } 1021 1022 if (atomic_fetch_inc(&rps->num_waiters)) 1023 return; 1024 1025 if (!intel_rps_is_active(rps)) 1026 return; 1027 1028 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", 1029 rq->fence.context, rq->fence.seqno); 1030 1031 if (READ_ONCE(rps->cur_freq) < rps->boost_freq) 1032 schedule_work(&rps->work); 1033 1034 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */ 1035 } 1036 } 1037 1038 int intel_rps_set(struct intel_rps *rps, u8 val) 1039 { 1040 int err; 1041 1042 lockdep_assert_held(&rps->lock); 1043 GEM_BUG_ON(val > rps->max_freq); 1044 GEM_BUG_ON(val < rps->min_freq); 1045 1046 if (intel_rps_is_active(rps)) { 1047 err = rps_set(rps, val, true); 1048 if (err) 1049 return err; 1050 1051 /* 1052 * Make sure we continue to get interrupts 1053 * until we hit the minimum or maximum frequencies. 1054 */ 1055 if (intel_rps_has_interrupts(rps)) { 1056 struct intel_uncore *uncore = rps_to_uncore(rps); 1057 1058 set(uncore, 1059 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 1060 1061 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 1062 } 1063 } 1064 1065 rps->cur_freq = val; 1066 return 0; 1067 } 1068 1069 static void gen6_rps_init(struct intel_rps *rps) 1070 { 1071 struct drm_i915_private *i915 = rps_to_i915(rps); 1072 u32 rp_state_cap = intel_rps_read_state_cap(rps); 1073 1074 /* All of these values are in units of 50MHz */ 1075 1076 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 1077 if (IS_GEN9_LP(i915)) { 1078 rps->rp0_freq = (rp_state_cap >> 16) & 0xff; 1079 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1080 rps->min_freq = (rp_state_cap >> 0) & 0xff; 1081 } else { 1082 rps->rp0_freq = (rp_state_cap >> 0) & 0xff; 1083 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 1084 rps->min_freq = (rp_state_cap >> 16) & 0xff; 1085 } 1086 1087 /* hw_max = RP0 until we check for overclocking */ 1088 rps->max_freq = rps->rp0_freq; 1089 1090 rps->efficient_freq = rps->rp1_freq; 1091 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 1092 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1093 u32 ddcc_status = 0; 1094 1095 if (snb_pcode_read(i915, HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 1096 &ddcc_status, NULL) == 0) 1097 rps->efficient_freq = 1098 clamp_t(u8, 1099 (ddcc_status >> 8) & 0xff, 1100 rps->min_freq, 1101 rps->max_freq); 1102 } 1103 1104 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { 1105 /* Store the frequency values in 16.66 MHZ units, which is 1106 * the natural hardware unit for SKL 1107 */ 1108 rps->rp0_freq *= GEN9_FREQ_SCALER; 1109 rps->rp1_freq *= GEN9_FREQ_SCALER; 1110 rps->min_freq *= GEN9_FREQ_SCALER; 1111 rps->max_freq *= GEN9_FREQ_SCALER; 1112 rps->efficient_freq *= GEN9_FREQ_SCALER; 1113 } 1114 } 1115 1116 static bool rps_reset(struct intel_rps *rps) 1117 { 1118 struct drm_i915_private *i915 = rps_to_i915(rps); 1119 1120 /* force a reset */ 1121 rps->power.mode = -1; 1122 rps->last_freq = -1; 1123 1124 if (rps_set(rps, rps->min_freq, true)) { 1125 drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); 1126 return false; 1127 } 1128 1129 rps->cur_freq = rps->min_freq; 1130 return true; 1131 } 1132 1133 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 1134 static bool gen9_rps_enable(struct intel_rps *rps) 1135 { 1136 struct intel_gt *gt = rps_to_gt(rps); 1137 struct intel_uncore *uncore = gt->uncore; 1138 1139 /* Program defaults and thresholds for RPS */ 1140 if (GRAPHICS_VER(gt->i915) == 9) 1141 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1142 GEN9_FREQUENCY(rps->rp1_freq)); 1143 1144 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 1145 1146 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1147 1148 return rps_reset(rps); 1149 } 1150 1151 static bool gen8_rps_enable(struct intel_rps *rps) 1152 { 1153 struct intel_uncore *uncore = rps_to_uncore(rps); 1154 1155 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 1156 HSW_FREQUENCY(rps->rp1_freq)); 1157 1158 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1159 1160 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; 1161 1162 return rps_reset(rps); 1163 } 1164 1165 static bool gen6_rps_enable(struct intel_rps *rps) 1166 { 1167 struct intel_uncore *uncore = rps_to_uncore(rps); 1168 1169 /* Power down if completely idle for over 50ms */ 1170 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 1171 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1172 1173 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1174 GEN6_PM_RP_DOWN_THRESHOLD | 1175 GEN6_PM_RP_DOWN_TIMEOUT); 1176 1177 return rps_reset(rps); 1178 } 1179 1180 static int chv_rps_max_freq(struct intel_rps *rps) 1181 { 1182 struct drm_i915_private *i915 = rps_to_i915(rps); 1183 struct intel_gt *gt = rps_to_gt(rps); 1184 u32 val; 1185 1186 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1187 1188 switch (gt->info.sseu.eu_total) { 1189 case 8: 1190 /* (2 * 4) config */ 1191 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 1192 break; 1193 case 12: 1194 /* (2 * 6) config */ 1195 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 1196 break; 1197 case 16: 1198 /* (2 * 8) config */ 1199 default: 1200 /* Setting (2 * 8) Min RP0 for any other combination */ 1201 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 1202 break; 1203 } 1204 1205 return val & FB_GFX_FREQ_FUSE_MASK; 1206 } 1207 1208 static int chv_rps_rpe_freq(struct intel_rps *rps) 1209 { 1210 struct drm_i915_private *i915 = rps_to_i915(rps); 1211 u32 val; 1212 1213 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 1214 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 1215 1216 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 1217 } 1218 1219 static int chv_rps_guar_freq(struct intel_rps *rps) 1220 { 1221 struct drm_i915_private *i915 = rps_to_i915(rps); 1222 u32 val; 1223 1224 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1225 1226 return val & FB_GFX_FREQ_FUSE_MASK; 1227 } 1228 1229 static u32 chv_rps_min_freq(struct intel_rps *rps) 1230 { 1231 struct drm_i915_private *i915 = rps_to_i915(rps); 1232 u32 val; 1233 1234 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1235 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1236 1237 return val & FB_GFX_FREQ_FUSE_MASK; 1238 } 1239 1240 static bool chv_rps_enable(struct intel_rps *rps) 1241 { 1242 struct intel_uncore *uncore = rps_to_uncore(rps); 1243 struct drm_i915_private *i915 = rps_to_i915(rps); 1244 u32 val; 1245 1246 /* 1: Program defaults and thresholds for RPS*/ 1247 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1248 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1249 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1250 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1251 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1252 1253 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1254 1255 /* 2: Enable RPS */ 1256 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1257 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1258 GEN6_RP_MEDIA_IS_GFX | 1259 GEN6_RP_ENABLE | 1260 GEN6_RP_UP_BUSY_AVG | 1261 GEN6_RP_DOWN_IDLE_AVG); 1262 1263 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | 1264 GEN6_PM_RP_DOWN_THRESHOLD | 1265 GEN6_PM_RP_DOWN_TIMEOUT); 1266 1267 /* Setting Fixed Bias */ 1268 vlv_punit_get(i915); 1269 1270 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1271 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1272 1273 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1274 1275 vlv_punit_put(i915); 1276 1277 /* RPS code assumes GPLL is used */ 1278 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1279 "GPLL not enabled\n"); 1280 1281 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1282 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1283 1284 return rps_reset(rps); 1285 } 1286 1287 static int vlv_rps_guar_freq(struct intel_rps *rps) 1288 { 1289 struct drm_i915_private *i915 = rps_to_i915(rps); 1290 u32 val, rp1; 1291 1292 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1293 1294 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1295 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1296 1297 return rp1; 1298 } 1299 1300 static int vlv_rps_max_freq(struct intel_rps *rps) 1301 { 1302 struct drm_i915_private *i915 = rps_to_i915(rps); 1303 u32 val, rp0; 1304 1305 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1306 1307 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1308 /* Clamp to max */ 1309 rp0 = min_t(u32, rp0, 0xea); 1310 1311 return rp0; 1312 } 1313 1314 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1315 { 1316 struct drm_i915_private *i915 = rps_to_i915(rps); 1317 u32 val, rpe; 1318 1319 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1320 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1321 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1322 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1323 1324 return rpe; 1325 } 1326 1327 static int vlv_rps_min_freq(struct intel_rps *rps) 1328 { 1329 struct drm_i915_private *i915 = rps_to_i915(rps); 1330 u32 val; 1331 1332 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1333 /* 1334 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1335 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1336 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1337 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1338 * to make sure it matches what Punit accepts. 1339 */ 1340 return max_t(u32, val, 0xc0); 1341 } 1342 1343 static bool vlv_rps_enable(struct intel_rps *rps) 1344 { 1345 struct intel_uncore *uncore = rps_to_uncore(rps); 1346 struct drm_i915_private *i915 = rps_to_i915(rps); 1347 u32 val; 1348 1349 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1350 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1351 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1352 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1353 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1354 1355 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1356 1357 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1358 GEN6_RP_MEDIA_TURBO | 1359 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1360 GEN6_RP_MEDIA_IS_GFX | 1361 GEN6_RP_ENABLE | 1362 GEN6_RP_UP_BUSY_AVG | 1363 GEN6_RP_DOWN_IDLE_CONT); 1364 1365 /* WaGsvRC0ResidencyMethod:vlv */ 1366 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; 1367 1368 vlv_punit_get(i915); 1369 1370 /* Setting Fixed Bias */ 1371 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1372 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1373 1374 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1375 1376 vlv_punit_put(i915); 1377 1378 /* RPS code assumes GPLL is used */ 1379 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1380 "GPLL not enabled\n"); 1381 1382 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1383 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); 1384 1385 return rps_reset(rps); 1386 } 1387 1388 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1389 { 1390 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1391 struct intel_uncore *uncore = rps_to_uncore(rps); 1392 unsigned int t, state1, state2; 1393 u32 pxvid, ext_v; 1394 u64 corr, corr2; 1395 1396 lockdep_assert_held(&mchdev_lock); 1397 1398 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1399 pxvid = (pxvid >> 24) & 0x7f; 1400 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1401 1402 state1 = ext_v; 1403 1404 /* Revel in the empirically derived constants */ 1405 1406 /* Correction factor in 1/100000 units */ 1407 t = ips_mch_val(uncore); 1408 if (t > 80) 1409 corr = t * 2349 + 135940; 1410 else if (t >= 50) 1411 corr = t * 964 + 29317; 1412 else /* < 50 */ 1413 corr = t * 301 + 1004; 1414 1415 corr = div_u64(corr * 150142 * state1, 10000) - 78642; 1416 corr2 = div_u64(corr, 100000) * ips->corr; 1417 1418 state2 = div_u64(corr2 * state1, 10000); 1419 state2 /= 100; /* convert to mW */ 1420 1421 __gen5_ips_update(ips); 1422 1423 return ips->gfx_power + state2; 1424 } 1425 1426 static bool has_busy_stats(struct intel_rps *rps) 1427 { 1428 struct intel_engine_cs *engine; 1429 enum intel_engine_id id; 1430 1431 for_each_engine(engine, rps_to_gt(rps), id) { 1432 if (!intel_engine_supports_stats(engine)) 1433 return false; 1434 } 1435 1436 return true; 1437 } 1438 1439 void intel_rps_enable(struct intel_rps *rps) 1440 { 1441 struct drm_i915_private *i915 = rps_to_i915(rps); 1442 struct intel_uncore *uncore = rps_to_uncore(rps); 1443 bool enabled = false; 1444 1445 if (!HAS_RPS(i915)) 1446 return; 1447 1448 if (rps_uses_slpc(rps)) 1449 return; 1450 1451 intel_gt_check_clock_frequency(rps_to_gt(rps)); 1452 1453 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1454 if (rps->max_freq <= rps->min_freq) 1455 /* leave disabled, no room for dynamic reclocking */; 1456 else if (IS_CHERRYVIEW(i915)) 1457 enabled = chv_rps_enable(rps); 1458 else if (IS_VALLEYVIEW(i915)) 1459 enabled = vlv_rps_enable(rps); 1460 else if (GRAPHICS_VER(i915) >= 9) 1461 enabled = gen9_rps_enable(rps); 1462 else if (GRAPHICS_VER(i915) >= 8) 1463 enabled = gen8_rps_enable(rps); 1464 else if (GRAPHICS_VER(i915) >= 6) 1465 enabled = gen6_rps_enable(rps); 1466 else if (IS_IRONLAKE_M(i915)) 1467 enabled = gen5_rps_enable(rps); 1468 else 1469 MISSING_CASE(GRAPHICS_VER(i915)); 1470 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1471 if (!enabled) 1472 return; 1473 1474 GT_TRACE(rps_to_gt(rps), 1475 "min:%x, max:%x, freq:[%d, %d]\n", 1476 rps->min_freq, rps->max_freq, 1477 intel_gpu_freq(rps, rps->min_freq), 1478 intel_gpu_freq(rps, rps->max_freq)); 1479 1480 GEM_BUG_ON(rps->max_freq < rps->min_freq); 1481 GEM_BUG_ON(rps->idle_freq > rps->max_freq); 1482 1483 GEM_BUG_ON(rps->efficient_freq < rps->min_freq); 1484 GEM_BUG_ON(rps->efficient_freq > rps->max_freq); 1485 1486 if (has_busy_stats(rps)) 1487 intel_rps_set_timer(rps); 1488 else if (GRAPHICS_VER(i915) >= 6) 1489 intel_rps_set_interrupts(rps); 1490 else 1491 /* Ironlake currently uses intel_ips.ko */ {} 1492 1493 intel_rps_set_enabled(rps); 1494 } 1495 1496 static void gen6_rps_disable(struct intel_rps *rps) 1497 { 1498 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1499 } 1500 1501 void intel_rps_disable(struct intel_rps *rps) 1502 { 1503 struct drm_i915_private *i915 = rps_to_i915(rps); 1504 1505 intel_rps_clear_enabled(rps); 1506 intel_rps_clear_interrupts(rps); 1507 intel_rps_clear_timer(rps); 1508 1509 if (GRAPHICS_VER(i915) >= 6) 1510 gen6_rps_disable(rps); 1511 else if (IS_IRONLAKE_M(i915)) 1512 gen5_rps_disable(rps); 1513 } 1514 1515 static int byt_gpu_freq(struct intel_rps *rps, int val) 1516 { 1517 /* 1518 * N = val - 0xb7 1519 * Slow = Fast = GPLL ref * N 1520 */ 1521 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1522 } 1523 1524 static int byt_freq_opcode(struct intel_rps *rps, int val) 1525 { 1526 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1527 } 1528 1529 static int chv_gpu_freq(struct intel_rps *rps, int val) 1530 { 1531 /* 1532 * N = val / 2 1533 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1534 */ 1535 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1536 } 1537 1538 static int chv_freq_opcode(struct intel_rps *rps, int val) 1539 { 1540 /* CHV needs even values */ 1541 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1542 } 1543 1544 int intel_gpu_freq(struct intel_rps *rps, int val) 1545 { 1546 struct drm_i915_private *i915 = rps_to_i915(rps); 1547 1548 if (GRAPHICS_VER(i915) >= 9) 1549 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1550 GEN9_FREQ_SCALER); 1551 else if (IS_CHERRYVIEW(i915)) 1552 return chv_gpu_freq(rps, val); 1553 else if (IS_VALLEYVIEW(i915)) 1554 return byt_gpu_freq(rps, val); 1555 else if (GRAPHICS_VER(i915) >= 6) 1556 return val * GT_FREQUENCY_MULTIPLIER; 1557 else 1558 return val; 1559 } 1560 1561 int intel_freq_opcode(struct intel_rps *rps, int val) 1562 { 1563 struct drm_i915_private *i915 = rps_to_i915(rps); 1564 1565 if (GRAPHICS_VER(i915) >= 9) 1566 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1567 GT_FREQUENCY_MULTIPLIER); 1568 else if (IS_CHERRYVIEW(i915)) 1569 return chv_freq_opcode(rps, val); 1570 else if (IS_VALLEYVIEW(i915)) 1571 return byt_freq_opcode(rps, val); 1572 else if (GRAPHICS_VER(i915) >= 6) 1573 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1574 else 1575 return val; 1576 } 1577 1578 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1579 { 1580 struct drm_i915_private *i915 = rps_to_i915(rps); 1581 1582 rps->gpll_ref_freq = 1583 vlv_get_cck_clock(i915, "GPLL ref", 1584 CCK_GPLL_CLOCK_CONTROL, 1585 i915->czclk_freq); 1586 1587 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", 1588 rps->gpll_ref_freq); 1589 } 1590 1591 static void vlv_rps_init(struct intel_rps *rps) 1592 { 1593 struct drm_i915_private *i915 = rps_to_i915(rps); 1594 u32 val; 1595 1596 vlv_iosf_sb_get(i915, 1597 BIT(VLV_IOSF_SB_PUNIT) | 1598 BIT(VLV_IOSF_SB_NC) | 1599 BIT(VLV_IOSF_SB_CCK)); 1600 1601 vlv_init_gpll_ref_freq(rps); 1602 1603 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1604 switch ((val >> 6) & 3) { 1605 case 0: 1606 case 1: 1607 i915->mem_freq = 800; 1608 break; 1609 case 2: 1610 i915->mem_freq = 1066; 1611 break; 1612 case 3: 1613 i915->mem_freq = 1333; 1614 break; 1615 } 1616 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1617 1618 rps->max_freq = vlv_rps_max_freq(rps); 1619 rps->rp0_freq = rps->max_freq; 1620 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1621 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1622 1623 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1624 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1625 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1626 1627 rps->rp1_freq = vlv_rps_guar_freq(rps); 1628 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1629 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1630 1631 rps->min_freq = vlv_rps_min_freq(rps); 1632 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1633 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1634 1635 vlv_iosf_sb_put(i915, 1636 BIT(VLV_IOSF_SB_PUNIT) | 1637 BIT(VLV_IOSF_SB_NC) | 1638 BIT(VLV_IOSF_SB_CCK)); 1639 } 1640 1641 static void chv_rps_init(struct intel_rps *rps) 1642 { 1643 struct drm_i915_private *i915 = rps_to_i915(rps); 1644 u32 val; 1645 1646 vlv_iosf_sb_get(i915, 1647 BIT(VLV_IOSF_SB_PUNIT) | 1648 BIT(VLV_IOSF_SB_NC) | 1649 BIT(VLV_IOSF_SB_CCK)); 1650 1651 vlv_init_gpll_ref_freq(rps); 1652 1653 val = vlv_cck_read(i915, CCK_FUSE_REG); 1654 1655 switch ((val >> 2) & 0x7) { 1656 case 3: 1657 i915->mem_freq = 2000; 1658 break; 1659 default: 1660 i915->mem_freq = 1600; 1661 break; 1662 } 1663 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); 1664 1665 rps->max_freq = chv_rps_max_freq(rps); 1666 rps->rp0_freq = rps->max_freq; 1667 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", 1668 intel_gpu_freq(rps, rps->max_freq), rps->max_freq); 1669 1670 rps->efficient_freq = chv_rps_rpe_freq(rps); 1671 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", 1672 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); 1673 1674 rps->rp1_freq = chv_rps_guar_freq(rps); 1675 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", 1676 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); 1677 1678 rps->min_freq = chv_rps_min_freq(rps); 1679 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", 1680 intel_gpu_freq(rps, rps->min_freq), rps->min_freq); 1681 1682 vlv_iosf_sb_put(i915, 1683 BIT(VLV_IOSF_SB_PUNIT) | 1684 BIT(VLV_IOSF_SB_NC) | 1685 BIT(VLV_IOSF_SB_CCK)); 1686 1687 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1688 rps->rp1_freq | rps->min_freq) & 1, 1689 "Odd GPU freq values\n"); 1690 } 1691 1692 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1693 { 1694 ei->ktime = ktime_get_raw(); 1695 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1696 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1697 } 1698 1699 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1700 { 1701 struct intel_uncore *uncore = rps_to_uncore(rps); 1702 const struct intel_rps_ei *prev = &rps->ei; 1703 struct intel_rps_ei now; 1704 u32 events = 0; 1705 1706 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1707 return 0; 1708 1709 vlv_c0_read(uncore, &now); 1710 1711 if (prev->ktime) { 1712 u64 time, c0; 1713 u32 render, media; 1714 1715 time = ktime_us_delta(now.ktime, prev->ktime); 1716 1717 time *= rps_to_i915(rps)->czclk_freq; 1718 1719 /* Workload can be split between render + media, 1720 * e.g. SwapBuffers being blitted in X after being rendered in 1721 * mesa. To account for this we need to combine both engines 1722 * into our activity counter. 1723 */ 1724 render = now.render_c0 - prev->render_c0; 1725 media = now.media_c0 - prev->media_c0; 1726 c0 = max(render, media); 1727 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1728 1729 if (c0 > time * rps->power.up_threshold) 1730 events = GEN6_PM_RP_UP_THRESHOLD; 1731 else if (c0 < time * rps->power.down_threshold) 1732 events = GEN6_PM_RP_DOWN_THRESHOLD; 1733 } 1734 1735 rps->ei = now; 1736 return events; 1737 } 1738 1739 static void rps_work(struct work_struct *work) 1740 { 1741 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1742 struct intel_gt *gt = rps_to_gt(rps); 1743 struct drm_i915_private *i915 = rps_to_i915(rps); 1744 bool client_boost = false; 1745 int new_freq, adj, min, max; 1746 u32 pm_iir = 0; 1747 1748 spin_lock_irq(>->irq_lock); 1749 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; 1750 client_boost = atomic_read(&rps->num_waiters); 1751 spin_unlock_irq(>->irq_lock); 1752 1753 /* Make sure we didn't queue anything we're not going to process. */ 1754 if (!pm_iir && !client_boost) 1755 goto out; 1756 1757 mutex_lock(&rps->lock); 1758 if (!intel_rps_is_active(rps)) { 1759 mutex_unlock(&rps->lock); 1760 return; 1761 } 1762 1763 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1764 1765 adj = rps->last_adj; 1766 new_freq = rps->cur_freq; 1767 min = rps->min_freq_softlimit; 1768 max = rps->max_freq_softlimit; 1769 if (client_boost) 1770 max = rps->max_freq; 1771 1772 GT_TRACE(gt, 1773 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", 1774 pm_iir, yesno(client_boost), 1775 adj, new_freq, min, max); 1776 1777 if (client_boost && new_freq < rps->boost_freq) { 1778 new_freq = rps->boost_freq; 1779 adj = 0; 1780 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1781 if (adj > 0) 1782 adj *= 2; 1783 else /* CHV needs even encode values */ 1784 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1785 1786 if (new_freq >= rps->max_freq_softlimit) 1787 adj = 0; 1788 } else if (client_boost) { 1789 adj = 0; 1790 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1791 if (rps->cur_freq > rps->efficient_freq) 1792 new_freq = rps->efficient_freq; 1793 else if (rps->cur_freq > rps->min_freq_softlimit) 1794 new_freq = rps->min_freq_softlimit; 1795 adj = 0; 1796 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1797 if (adj < 0) 1798 adj *= 2; 1799 else /* CHV needs even encode values */ 1800 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1801 1802 if (new_freq <= rps->min_freq_softlimit) 1803 adj = 0; 1804 } else { /* unknown event */ 1805 adj = 0; 1806 } 1807 1808 /* 1809 * sysfs frequency limits may have snuck in while 1810 * servicing the interrupt 1811 */ 1812 new_freq += adj; 1813 new_freq = clamp_t(int, new_freq, min, max); 1814 1815 if (intel_rps_set(rps, new_freq)) { 1816 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); 1817 adj = 0; 1818 } 1819 rps->last_adj = adj; 1820 1821 mutex_unlock(&rps->lock); 1822 1823 out: 1824 spin_lock_irq(>->irq_lock); 1825 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1826 spin_unlock_irq(>->irq_lock); 1827 } 1828 1829 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1830 { 1831 struct intel_gt *gt = rps_to_gt(rps); 1832 const u32 events = rps->pm_events & pm_iir; 1833 1834 lockdep_assert_held(>->irq_lock); 1835 1836 if (unlikely(!events)) 1837 return; 1838 1839 GT_TRACE(gt, "irq events:%x\n", events); 1840 1841 gen6_gt_pm_mask_irq(gt, events); 1842 1843 rps->pm_iir |= events; 1844 schedule_work(&rps->work); 1845 } 1846 1847 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1848 { 1849 struct intel_gt *gt = rps_to_gt(rps); 1850 u32 events; 1851 1852 events = pm_iir & rps->pm_events; 1853 if (events) { 1854 spin_lock(>->irq_lock); 1855 1856 GT_TRACE(gt, "irq events:%x\n", events); 1857 1858 gen6_gt_pm_mask_irq(gt, events); 1859 rps->pm_iir |= events; 1860 1861 schedule_work(&rps->work); 1862 spin_unlock(>->irq_lock); 1863 } 1864 1865 if (GRAPHICS_VER(gt->i915) >= 8) 1866 return; 1867 1868 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1869 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); 1870 1871 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1872 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1873 } 1874 1875 void gen5_rps_irq_handler(struct intel_rps *rps) 1876 { 1877 struct intel_uncore *uncore = rps_to_uncore(rps); 1878 u32 busy_up, busy_down, max_avg, min_avg; 1879 u8 new_freq; 1880 1881 spin_lock(&mchdev_lock); 1882 1883 intel_uncore_write16(uncore, 1884 MEMINTRSTS, 1885 intel_uncore_read(uncore, MEMINTRSTS)); 1886 1887 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1888 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1889 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1890 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1891 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1892 1893 /* Handle RCS change request from hw */ 1894 new_freq = rps->cur_freq; 1895 if (busy_up > max_avg) 1896 new_freq++; 1897 else if (busy_down < min_avg) 1898 new_freq--; 1899 new_freq = clamp(new_freq, 1900 rps->min_freq_softlimit, 1901 rps->max_freq_softlimit); 1902 1903 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq)) 1904 rps->cur_freq = new_freq; 1905 1906 spin_unlock(&mchdev_lock); 1907 } 1908 1909 void intel_rps_init_early(struct intel_rps *rps) 1910 { 1911 mutex_init(&rps->lock); 1912 mutex_init(&rps->power.mutex); 1913 1914 INIT_WORK(&rps->work, rps_work); 1915 timer_setup(&rps->timer, rps_timer, 0); 1916 1917 atomic_set(&rps->num_waiters, 0); 1918 } 1919 1920 void intel_rps_init(struct intel_rps *rps) 1921 { 1922 struct drm_i915_private *i915 = rps_to_i915(rps); 1923 1924 if (rps_uses_slpc(rps)) 1925 return; 1926 1927 if (IS_CHERRYVIEW(i915)) 1928 chv_rps_init(rps); 1929 else if (IS_VALLEYVIEW(i915)) 1930 vlv_rps_init(rps); 1931 else if (GRAPHICS_VER(i915) >= 6) 1932 gen6_rps_init(rps); 1933 else if (IS_IRONLAKE_M(i915)) 1934 gen5_rps_init(rps); 1935 1936 /* Derive initial user preferences/limits from the hardware limits */ 1937 rps->max_freq_softlimit = rps->max_freq; 1938 rps->min_freq_softlimit = rps->min_freq; 1939 1940 /* After setting max-softlimit, find the overclock max freq */ 1941 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 1942 u32 params = 0; 1943 1944 snb_pcode_read(i915, GEN6_READ_OC_PARAMS, ¶ms, NULL); 1945 if (params & BIT(31)) { /* OC supported */ 1946 drm_dbg(&i915->drm, 1947 "Overclocking supported, max: %dMHz, overclock: %dMHz\n", 1948 (rps->max_freq & 0xff) * 50, 1949 (params & 0xff) * 50); 1950 rps->max_freq = params & 0xff; 1951 } 1952 } 1953 1954 /* Finally allow us to boost to max by default */ 1955 rps->boost_freq = rps->max_freq; 1956 rps->idle_freq = rps->min_freq; 1957 1958 /* Start in the middle, from here we will autotune based on workload */ 1959 rps->cur_freq = rps->efficient_freq; 1960 1961 rps->pm_intrmsk_mbz = 0; 1962 1963 /* 1964 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 1965 * if GEN6_PM_UP_EI_EXPIRED is masked. 1966 * 1967 * TODO: verify if this can be reproduced on VLV,CHV. 1968 */ 1969 if (GRAPHICS_VER(i915) <= 7) 1970 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 1971 1972 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11) 1973 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 1974 1975 /* GuC needs ARAT expired interrupt unmasked */ 1976 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc)) 1977 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK; 1978 } 1979 1980 void intel_rps_sanitize(struct intel_rps *rps) 1981 { 1982 if (rps_uses_slpc(rps)) 1983 return; 1984 1985 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6) 1986 rps_disable_interrupts(rps); 1987 } 1988 1989 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 1990 { 1991 struct drm_i915_private *i915 = rps_to_i915(rps); 1992 u32 cagf; 1993 1994 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 1995 cagf = (rpstat >> 8) & 0xff; 1996 else if (GRAPHICS_VER(i915) >= 9) 1997 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; 1998 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 1999 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; 2000 else if (GRAPHICS_VER(i915) >= 6) 2001 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; 2002 else 2003 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >> 2004 MEMSTAT_PSTATE_SHIFT); 2005 2006 return cagf; 2007 } 2008 2009 static u32 read_cagf(struct intel_rps *rps) 2010 { 2011 struct drm_i915_private *i915 = rps_to_i915(rps); 2012 struct intel_uncore *uncore = rps_to_uncore(rps); 2013 u32 freq; 2014 2015 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 2016 vlv_punit_get(i915); 2017 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 2018 vlv_punit_put(i915); 2019 } else if (GRAPHICS_VER(i915) >= 6) { 2020 freq = intel_uncore_read(uncore, GEN6_RPSTAT1); 2021 } else { 2022 freq = intel_uncore_read(uncore, MEMSTAT_ILK); 2023 } 2024 2025 return intel_rps_get_cagf(rps, freq); 2026 } 2027 2028 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 2029 { 2030 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2031 intel_wakeref_t wakeref; 2032 u32 freq = 0; 2033 2034 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2035 freq = intel_gpu_freq(rps, read_cagf(rps)); 2036 2037 return freq; 2038 } 2039 2040 u32 intel_rps_read_punit_req(struct intel_rps *rps) 2041 { 2042 struct intel_uncore *uncore = rps_to_uncore(rps); 2043 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; 2044 intel_wakeref_t wakeref; 2045 u32 freq = 0; 2046 2047 with_intel_runtime_pm_if_in_use(rpm, wakeref) 2048 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ); 2049 2050 return freq; 2051 } 2052 2053 static u32 intel_rps_get_req(u32 pureq) 2054 { 2055 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT; 2056 2057 return req; 2058 } 2059 2060 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps) 2061 { 2062 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps)); 2063 2064 return intel_gpu_freq(rps, freq); 2065 } 2066 2067 u32 intel_rps_get_requested_frequency(struct intel_rps *rps) 2068 { 2069 if (rps_uses_slpc(rps)) 2070 return intel_rps_read_punit_req_frequency(rps); 2071 else 2072 return intel_gpu_freq(rps, rps->cur_freq); 2073 } 2074 2075 u32 intel_rps_get_max_frequency(struct intel_rps *rps) 2076 { 2077 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2078 2079 if (rps_uses_slpc(rps)) 2080 return slpc->max_freq_softlimit; 2081 else 2082 return intel_gpu_freq(rps, rps->max_freq_softlimit); 2083 } 2084 2085 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps) 2086 { 2087 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2088 2089 if (rps_uses_slpc(rps)) 2090 return slpc->rp0_freq; 2091 else 2092 return intel_gpu_freq(rps, rps->rp0_freq); 2093 } 2094 2095 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps) 2096 { 2097 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2098 2099 if (rps_uses_slpc(rps)) 2100 return slpc->rp1_freq; 2101 else 2102 return intel_gpu_freq(rps, rps->rp1_freq); 2103 } 2104 2105 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps) 2106 { 2107 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2108 2109 if (rps_uses_slpc(rps)) 2110 return slpc->min_freq; 2111 else 2112 return intel_gpu_freq(rps, rps->min_freq); 2113 } 2114 2115 static int set_max_freq(struct intel_rps *rps, u32 val) 2116 { 2117 struct drm_i915_private *i915 = rps_to_i915(rps); 2118 int ret = 0; 2119 2120 mutex_lock(&rps->lock); 2121 2122 val = intel_freq_opcode(rps, val); 2123 if (val < rps->min_freq || 2124 val > rps->max_freq || 2125 val < rps->min_freq_softlimit) { 2126 ret = -EINVAL; 2127 goto unlock; 2128 } 2129 2130 if (val > rps->rp0_freq) 2131 drm_dbg(&i915->drm, "User requested overclocking to %d\n", 2132 intel_gpu_freq(rps, val)); 2133 2134 rps->max_freq_softlimit = val; 2135 2136 val = clamp_t(int, rps->cur_freq, 2137 rps->min_freq_softlimit, 2138 rps->max_freq_softlimit); 2139 2140 /* 2141 * We still need *_set_rps to process the new max_delay and 2142 * update the interrupt limits and PMINTRMSK even though 2143 * frequency request may be unchanged. 2144 */ 2145 intel_rps_set(rps, val); 2146 2147 unlock: 2148 mutex_unlock(&rps->lock); 2149 2150 return ret; 2151 } 2152 2153 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val) 2154 { 2155 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2156 2157 if (rps_uses_slpc(rps)) 2158 return intel_guc_slpc_set_max_freq(slpc, val); 2159 else 2160 return set_max_freq(rps, val); 2161 } 2162 2163 u32 intel_rps_get_min_frequency(struct intel_rps *rps) 2164 { 2165 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2166 2167 if (rps_uses_slpc(rps)) 2168 return slpc->min_freq_softlimit; 2169 else 2170 return intel_gpu_freq(rps, rps->min_freq_softlimit); 2171 } 2172 2173 static int set_min_freq(struct intel_rps *rps, u32 val) 2174 { 2175 int ret = 0; 2176 2177 mutex_lock(&rps->lock); 2178 2179 val = intel_freq_opcode(rps, val); 2180 if (val < rps->min_freq || 2181 val > rps->max_freq || 2182 val > rps->max_freq_softlimit) { 2183 ret = -EINVAL; 2184 goto unlock; 2185 } 2186 2187 rps->min_freq_softlimit = val; 2188 2189 val = clamp_t(int, rps->cur_freq, 2190 rps->min_freq_softlimit, 2191 rps->max_freq_softlimit); 2192 2193 /* 2194 * We still need *_set_rps to process the new min_delay and 2195 * update the interrupt limits and PMINTRMSK even though 2196 * frequency request may be unchanged. 2197 */ 2198 intel_rps_set(rps, val); 2199 2200 unlock: 2201 mutex_unlock(&rps->lock); 2202 2203 return ret; 2204 } 2205 2206 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val) 2207 { 2208 struct intel_guc_slpc *slpc = rps_to_slpc(rps); 2209 2210 if (rps_uses_slpc(rps)) 2211 return intel_guc_slpc_set_min_freq(slpc, val); 2212 else 2213 return set_min_freq(rps, val); 2214 } 2215 2216 u32 intel_rps_read_state_cap(struct intel_rps *rps) 2217 { 2218 struct drm_i915_private *i915 = rps_to_i915(rps); 2219 struct intel_uncore *uncore = rps_to_uncore(rps); 2220 2221 if (IS_XEHPSDV(i915)) 2222 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP); 2223 else if (IS_GEN9_LP(i915)) 2224 return intel_uncore_read(uncore, BXT_RP_STATE_CAP); 2225 else 2226 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 2227 } 2228 2229 static void intel_rps_set_manual(struct intel_rps *rps, bool enable) 2230 { 2231 struct intel_uncore *uncore = rps_to_uncore(rps); 2232 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE; 2233 2234 /* Allow punit to process software requests */ 2235 intel_uncore_write(uncore, GEN6_RP_CONTROL, state); 2236 } 2237 2238 void intel_rps_raise_unslice(struct intel_rps *rps) 2239 { 2240 struct intel_uncore *uncore = rps_to_uncore(rps); 2241 u32 rp0_unslice_req; 2242 2243 mutex_lock(&rps->lock); 2244 2245 if (rps_uses_slpc(rps)) { 2246 /* RP limits have not been initialized yet for SLPC path */ 2247 rp0_unslice_req = ((intel_rps_read_state_cap(rps) >> 0) 2248 & 0xff) * GEN9_FREQ_SCALER; 2249 2250 intel_rps_set_manual(rps, true); 2251 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2252 ((rp0_unslice_req << 2253 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2254 GEN9_IGNORE_SLICE_RATIO)); 2255 intel_rps_set_manual(rps, false); 2256 } else { 2257 intel_rps_set(rps, rps->rp0_freq); 2258 } 2259 2260 mutex_unlock(&rps->lock); 2261 } 2262 2263 void intel_rps_lower_unslice(struct intel_rps *rps) 2264 { 2265 struct intel_uncore *uncore = rps_to_uncore(rps); 2266 u32 rpn_unslice_req; 2267 2268 mutex_lock(&rps->lock); 2269 2270 if (rps_uses_slpc(rps)) { 2271 /* RP limits have not been initialized yet for SLPC path */ 2272 rpn_unslice_req = ((intel_rps_read_state_cap(rps) >> 16) 2273 & 0xff) * GEN9_FREQ_SCALER; 2274 2275 intel_rps_set_manual(rps, true); 2276 intel_uncore_write(uncore, GEN6_RPNSWREQ, 2277 ((rpn_unslice_req << 2278 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) | 2279 GEN9_IGNORE_SLICE_RATIO)); 2280 intel_rps_set_manual(rps, false); 2281 } else { 2282 intel_rps_set(rps, rps->min_freq); 2283 } 2284 2285 mutex_unlock(&rps->lock); 2286 } 2287 2288 /* External interface for intel_ips.ko */ 2289 2290 static struct drm_i915_private __rcu *ips_mchdev; 2291 2292 /** 2293 * Tells the intel_ips driver that the i915 driver is now loaded, if 2294 * IPS got loaded first. 2295 * 2296 * This awkward dance is so that neither module has to depend on the 2297 * other in order for IPS to do the appropriate communication of 2298 * GPU turbo limits to i915. 2299 */ 2300 static void 2301 ips_ping_for_i915_load(void) 2302 { 2303 void (*link)(void); 2304 2305 link = symbol_get(ips_link_to_i915_driver); 2306 if (link) { 2307 link(); 2308 symbol_put(ips_link_to_i915_driver); 2309 } 2310 } 2311 2312 void intel_rps_driver_register(struct intel_rps *rps) 2313 { 2314 struct intel_gt *gt = rps_to_gt(rps); 2315 2316 /* 2317 * We only register the i915 ips part with intel-ips once everything is 2318 * set up, to avoid intel-ips sneaking in and reading bogus values. 2319 */ 2320 if (GRAPHICS_VER(gt->i915) == 5) { 2321 GEM_BUG_ON(ips_mchdev); 2322 rcu_assign_pointer(ips_mchdev, gt->i915); 2323 ips_ping_for_i915_load(); 2324 } 2325 } 2326 2327 void intel_rps_driver_unregister(struct intel_rps *rps) 2328 { 2329 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 2330 rcu_assign_pointer(ips_mchdev, NULL); 2331 } 2332 2333 static struct drm_i915_private *mchdev_get(void) 2334 { 2335 struct drm_i915_private *i915; 2336 2337 rcu_read_lock(); 2338 i915 = rcu_dereference(ips_mchdev); 2339 if (i915 && !kref_get_unless_zero(&i915->drm.ref)) 2340 i915 = NULL; 2341 rcu_read_unlock(); 2342 2343 return i915; 2344 } 2345 2346 /** 2347 * i915_read_mch_val - return value for IPS use 2348 * 2349 * Calculate and return a value for the IPS driver to use when deciding whether 2350 * we have thermal and power headroom to increase CPU or GPU power budget. 2351 */ 2352 unsigned long i915_read_mch_val(void) 2353 { 2354 struct drm_i915_private *i915; 2355 unsigned long chipset_val = 0; 2356 unsigned long graphics_val = 0; 2357 intel_wakeref_t wakeref; 2358 2359 i915 = mchdev_get(); 2360 if (!i915) 2361 return 0; 2362 2363 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 2364 struct intel_ips *ips = &to_gt(i915)->rps.ips; 2365 2366 spin_lock_irq(&mchdev_lock); 2367 chipset_val = __ips_chipset_val(ips); 2368 graphics_val = __ips_gfx_val(ips); 2369 spin_unlock_irq(&mchdev_lock); 2370 } 2371 2372 drm_dev_put(&i915->drm); 2373 return chipset_val + graphics_val; 2374 } 2375 EXPORT_SYMBOL_GPL(i915_read_mch_val); 2376 2377 /** 2378 * i915_gpu_raise - raise GPU frequency limit 2379 * 2380 * Raise the limit; IPS indicates we have thermal headroom. 2381 */ 2382 bool i915_gpu_raise(void) 2383 { 2384 struct drm_i915_private *i915; 2385 struct intel_rps *rps; 2386 2387 i915 = mchdev_get(); 2388 if (!i915) 2389 return false; 2390 2391 rps = &to_gt(i915)->rps; 2392 2393 spin_lock_irq(&mchdev_lock); 2394 if (rps->max_freq_softlimit < rps->max_freq) 2395 rps->max_freq_softlimit++; 2396 spin_unlock_irq(&mchdev_lock); 2397 2398 drm_dev_put(&i915->drm); 2399 return true; 2400 } 2401 EXPORT_SYMBOL_GPL(i915_gpu_raise); 2402 2403 /** 2404 * i915_gpu_lower - lower GPU frequency limit 2405 * 2406 * IPS indicates we're close to a thermal limit, so throttle back the GPU 2407 * frequency maximum. 2408 */ 2409 bool i915_gpu_lower(void) 2410 { 2411 struct drm_i915_private *i915; 2412 struct intel_rps *rps; 2413 2414 i915 = mchdev_get(); 2415 if (!i915) 2416 return false; 2417 2418 rps = &to_gt(i915)->rps; 2419 2420 spin_lock_irq(&mchdev_lock); 2421 if (rps->max_freq_softlimit > rps->min_freq) 2422 rps->max_freq_softlimit--; 2423 spin_unlock_irq(&mchdev_lock); 2424 2425 drm_dev_put(&i915->drm); 2426 return true; 2427 } 2428 EXPORT_SYMBOL_GPL(i915_gpu_lower); 2429 2430 /** 2431 * i915_gpu_busy - indicate GPU business to IPS 2432 * 2433 * Tell the IPS driver whether or not the GPU is busy. 2434 */ 2435 bool i915_gpu_busy(void) 2436 { 2437 struct drm_i915_private *i915; 2438 bool ret; 2439 2440 i915 = mchdev_get(); 2441 if (!i915) 2442 return false; 2443 2444 ret = to_gt(i915)->awake; 2445 2446 drm_dev_put(&i915->drm); 2447 return ret; 2448 } 2449 EXPORT_SYMBOL_GPL(i915_gpu_busy); 2450 2451 /** 2452 * i915_gpu_turbo_disable - disable graphics turbo 2453 * 2454 * Disable graphics turbo by resetting the max frequency and setting the 2455 * current frequency to the default. 2456 */ 2457 bool i915_gpu_turbo_disable(void) 2458 { 2459 struct drm_i915_private *i915; 2460 struct intel_rps *rps; 2461 bool ret; 2462 2463 i915 = mchdev_get(); 2464 if (!i915) 2465 return false; 2466 2467 rps = &to_gt(i915)->rps; 2468 2469 spin_lock_irq(&mchdev_lock); 2470 rps->max_freq_softlimit = rps->min_freq; 2471 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq); 2472 spin_unlock_irq(&mchdev_lock); 2473 2474 drm_dev_put(&i915->drm); 2475 return ret; 2476 } 2477 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 2478 2479 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2480 #include "selftest_rps.c" 2481 #include "selftest_slpc.c" 2482 #endif 2483