xref: /linux/drivers/gpu/drm/i915/gt/intel_rps.c (revision c771600c6af14749609b49565ffb4cac2959710d)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 
8 #include <drm/intel/i915_drm.h>
9 
10 #include "display/intel_display.h"
11 #include "display/intel_display_irq.h"
12 #include "i915_drv.h"
13 #include "i915_irq.h"
14 #include "i915_reg.h"
15 #include "intel_breadcrumbs.h"
16 #include "intel_gt.h"
17 #include "intel_gt_clock_utils.h"
18 #include "intel_gt_irq.h"
19 #include "intel_gt_pm.h"
20 #include "intel_gt_pm_irq.h"
21 #include "intel_gt_print.h"
22 #include "intel_gt_regs.h"
23 #include "intel_mchbar_regs.h"
24 #include "intel_pcode.h"
25 #include "intel_rps.h"
26 #include "vlv_sideband.h"
27 #include "../../../platform/x86/intel_ips.h"
28 
29 #define BUSY_MAX_EI	20u /* ms */
30 
31 /*
32  * Lock protecting IPS related data structures
33  */
34 static DEFINE_SPINLOCK(mchdev_lock);
35 
36 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
37 {
38 	return container_of(rps, struct intel_gt, rps);
39 }
40 
41 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
42 {
43 	return rps_to_gt(rps)->i915;
44 }
45 
46 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
47 {
48 	return rps_to_gt(rps)->uncore;
49 }
50 
51 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
52 {
53 	struct intel_gt *gt = rps_to_gt(rps);
54 
55 	return &gt_to_guc(gt)->slpc;
56 }
57 
58 static bool rps_uses_slpc(struct intel_rps *rps)
59 {
60 	struct intel_gt *gt = rps_to_gt(rps);
61 
62 	return intel_uc_uses_guc_slpc(&gt->uc);
63 }
64 
65 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
66 {
67 	return mask & ~rps->pm_intrmsk_mbz;
68 }
69 
70 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
71 {
72 	intel_uncore_write_fw(uncore, reg, val);
73 }
74 
75 static void rps_timer(struct timer_list *t)
76 {
77 	struct intel_rps *rps = from_timer(rps, t, timer);
78 	struct intel_gt *gt = rps_to_gt(rps);
79 	struct intel_engine_cs *engine;
80 	ktime_t dt, last, timestamp;
81 	enum intel_engine_id id;
82 	s64 max_busy[3] = {};
83 
84 	timestamp = 0;
85 	for_each_engine(engine, gt, id) {
86 		s64 busy;
87 		int i;
88 
89 		dt = intel_engine_get_busy_time(engine, &timestamp);
90 		last = engine->stats.rps;
91 		engine->stats.rps = dt;
92 
93 		busy = ktime_to_ns(ktime_sub(dt, last));
94 		for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
95 			if (busy > max_busy[i])
96 				swap(busy, max_busy[i]);
97 		}
98 	}
99 	last = rps->pm_timestamp;
100 	rps->pm_timestamp = timestamp;
101 
102 	if (intel_rps_is_active(rps)) {
103 		s64 busy;
104 		int i;
105 
106 		dt = ktime_sub(timestamp, last);
107 
108 		/*
109 		 * Our goal is to evaluate each engine independently, so we run
110 		 * at the lowest clocks required to sustain the heaviest
111 		 * workload. However, a task may be split into sequential
112 		 * dependent operations across a set of engines, such that
113 		 * the independent contributions do not account for high load,
114 		 * but overall the task is GPU bound. For example, consider
115 		 * video decode on vcs followed by colour post-processing
116 		 * on vecs, followed by general post-processing on rcs.
117 		 * Since multi-engines being active does imply a single
118 		 * continuous workload across all engines, we hedge our
119 		 * bets by only contributing a factor of the distributed
120 		 * load into our busyness calculation.
121 		 */
122 		busy = max_busy[0];
123 		for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
124 			if (!max_busy[i])
125 				break;
126 
127 			busy += div_u64(max_busy[i], 1 << i);
128 		}
129 		GT_TRACE(gt,
130 			 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
131 			 busy, (int)div64_u64(100 * busy, dt),
132 			 max_busy[0], max_busy[1], max_busy[2],
133 			 rps->pm_interval);
134 
135 		if (100 * busy > rps->power.up_threshold * dt &&
136 		    rps->cur_freq < rps->max_freq_softlimit) {
137 			rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
138 			rps->pm_interval = 1;
139 			queue_work(gt->i915->unordered_wq, &rps->work);
140 		} else if (100 * busy < rps->power.down_threshold * dt &&
141 			   rps->cur_freq > rps->min_freq_softlimit) {
142 			rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
143 			rps->pm_interval = 1;
144 			queue_work(gt->i915->unordered_wq, &rps->work);
145 		} else {
146 			rps->last_adj = 0;
147 		}
148 
149 		mod_timer(&rps->timer,
150 			  jiffies + msecs_to_jiffies(rps->pm_interval));
151 		rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
152 	}
153 }
154 
155 static void rps_start_timer(struct intel_rps *rps)
156 {
157 	rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
158 	rps->pm_interval = 1;
159 	mod_timer(&rps->timer, jiffies + 1);
160 }
161 
162 static void rps_stop_timer(struct intel_rps *rps)
163 {
164 	del_timer_sync(&rps->timer);
165 	rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
166 	cancel_work_sync(&rps->work);
167 }
168 
169 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
170 {
171 	u32 mask = 0;
172 
173 	/* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
174 	if (val > rps->min_freq_softlimit)
175 		mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
176 			 GEN6_PM_RP_DOWN_THRESHOLD |
177 			 GEN6_PM_RP_DOWN_TIMEOUT);
178 
179 	if (val < rps->max_freq_softlimit)
180 		mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
181 
182 	mask &= rps->pm_events;
183 
184 	return rps_pm_sanitize_mask(rps, ~mask);
185 }
186 
187 static void rps_reset_ei(struct intel_rps *rps)
188 {
189 	memset(&rps->ei, 0, sizeof(rps->ei));
190 }
191 
192 static void rps_enable_interrupts(struct intel_rps *rps)
193 {
194 	struct intel_gt *gt = rps_to_gt(rps);
195 
196 	GEM_BUG_ON(rps_uses_slpc(rps));
197 
198 	GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
199 		 rps->pm_events, rps_pm_mask(rps, rps->last_freq));
200 
201 	rps_reset_ei(rps);
202 
203 	spin_lock_irq(gt->irq_lock);
204 	gen6_gt_pm_enable_irq(gt, rps->pm_events);
205 	spin_unlock_irq(gt->irq_lock);
206 
207 	intel_uncore_write(gt->uncore,
208 			   GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
209 }
210 
211 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
212 {
213 	gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
214 }
215 
216 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
217 {
218 	while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
219 		;
220 }
221 
222 static void rps_reset_interrupts(struct intel_rps *rps)
223 {
224 	struct intel_gt *gt = rps_to_gt(rps);
225 
226 	spin_lock_irq(gt->irq_lock);
227 	if (GRAPHICS_VER(gt->i915) >= 11)
228 		gen11_rps_reset_interrupts(rps);
229 	else
230 		gen6_rps_reset_interrupts(rps);
231 
232 	rps->pm_iir = 0;
233 	spin_unlock_irq(gt->irq_lock);
234 }
235 
236 static void rps_disable_interrupts(struct intel_rps *rps)
237 {
238 	struct intel_gt *gt = rps_to_gt(rps);
239 
240 	intel_uncore_write(gt->uncore,
241 			   GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
242 
243 	spin_lock_irq(gt->irq_lock);
244 	gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
245 	spin_unlock_irq(gt->irq_lock);
246 
247 	intel_synchronize_irq(gt->i915);
248 
249 	/*
250 	 * Now that we will not be generating any more work, flush any
251 	 * outstanding tasks. As we are called on the RPS idle path,
252 	 * we will reset the GPU to minimum frequencies, so the current
253 	 * state of the worker can be discarded.
254 	 */
255 	cancel_work_sync(&rps->work);
256 
257 	rps_reset_interrupts(rps);
258 	GT_TRACE(gt, "interrupts:off\n");
259 }
260 
261 static const struct cparams {
262 	u16 i;
263 	u16 t;
264 	u16 m;
265 	u16 c;
266 } cparams[] = {
267 	{ 1, 1333, 301, 28664 },
268 	{ 1, 1067, 294, 24460 },
269 	{ 1, 800, 294, 25192 },
270 	{ 0, 1333, 276, 27605 },
271 	{ 0, 1067, 276, 27605 },
272 	{ 0, 800, 231, 23784 },
273 };
274 
275 static void gen5_rps_init(struct intel_rps *rps)
276 {
277 	struct drm_i915_private *i915 = rps_to_i915(rps);
278 	struct intel_uncore *uncore = rps_to_uncore(rps);
279 	u8 fmax, fmin, fstart;
280 	u32 rgvmodectl;
281 	int c_m, i;
282 
283 	if (i915->fsb_freq <= 3200000)
284 		c_m = 0;
285 	else if (i915->fsb_freq <= 4800000)
286 		c_m = 1;
287 	else
288 		c_m = 2;
289 
290 	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
291 		if (cparams[i].i == c_m &&
292 		    cparams[i].t == DIV_ROUND_CLOSEST(i915->mem_freq, 1000)) {
293 			rps->ips.m = cparams[i].m;
294 			rps->ips.c = cparams[i].c;
295 			break;
296 		}
297 	}
298 
299 	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
300 
301 	/* Set up min, max, and cur for interrupt handling */
302 	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
303 	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
304 	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
305 		MEMMODE_FSTART_SHIFT;
306 	drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
307 		fmax, fmin, fstart);
308 
309 	rps->min_freq = fmax;
310 	rps->efficient_freq = fstart;
311 	rps->max_freq = fmin;
312 }
313 
314 static unsigned long
315 __ips_chipset_val(struct intel_ips *ips)
316 {
317 	struct intel_uncore *uncore =
318 		rps_to_uncore(container_of(ips, struct intel_rps, ips));
319 	unsigned long now = jiffies_to_msecs(jiffies), dt;
320 	unsigned long result;
321 	u64 total, delta;
322 
323 	lockdep_assert_held(&mchdev_lock);
324 
325 	/*
326 	 * Prevent division-by-zero if we are asking too fast.
327 	 * Also, we don't get interesting results if we are polling
328 	 * faster than once in 10ms, so just return the saved value
329 	 * in such cases.
330 	 */
331 	dt = now - ips->last_time1;
332 	if (dt <= 10)
333 		return ips->chipset_power;
334 
335 	/* FIXME: handle per-counter overflow */
336 	total = intel_uncore_read(uncore, DMIEC);
337 	total += intel_uncore_read(uncore, DDREC);
338 	total += intel_uncore_read(uncore, CSIEC);
339 
340 	delta = total - ips->last_count1;
341 
342 	result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
343 
344 	ips->last_count1 = total;
345 	ips->last_time1 = now;
346 
347 	ips->chipset_power = result;
348 
349 	return result;
350 }
351 
352 static unsigned long ips_mch_val(struct intel_uncore *uncore)
353 {
354 	unsigned int m, x, b;
355 	u32 tsfs;
356 
357 	tsfs = intel_uncore_read(uncore, TSFS);
358 	x = intel_uncore_read8(uncore, TR1);
359 
360 	b = tsfs & TSFS_INTR_MASK;
361 	m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
362 
363 	return m * x / 127 - b;
364 }
365 
366 static int _pxvid_to_vd(u8 pxvid)
367 {
368 	if (pxvid == 0)
369 		return 0;
370 
371 	if (pxvid >= 8 && pxvid < 31)
372 		pxvid = 31;
373 
374 	return (pxvid + 2) * 125;
375 }
376 
377 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
378 {
379 	const int vd = _pxvid_to_vd(pxvid);
380 
381 	if (INTEL_INFO(i915)->is_mobile)
382 		return max(vd - 1125, 0);
383 
384 	return vd;
385 }
386 
387 static void __gen5_ips_update(struct intel_ips *ips)
388 {
389 	struct intel_uncore *uncore =
390 		rps_to_uncore(container_of(ips, struct intel_rps, ips));
391 	u64 now, delta, dt;
392 	u32 count;
393 
394 	lockdep_assert_held(&mchdev_lock);
395 
396 	now = ktime_get_raw_ns();
397 	dt = now - ips->last_time2;
398 	do_div(dt, NSEC_PER_MSEC);
399 
400 	/* Don't divide by 0 */
401 	if (dt <= 10)
402 		return;
403 
404 	count = intel_uncore_read(uncore, GFXEC);
405 	delta = count - ips->last_count2;
406 
407 	ips->last_count2 = count;
408 	ips->last_time2 = now;
409 
410 	/* More magic constants... */
411 	ips->gfx_power = div_u64(delta * 1181, dt * 10);
412 }
413 
414 static void gen5_rps_update(struct intel_rps *rps)
415 {
416 	spin_lock_irq(&mchdev_lock);
417 	__gen5_ips_update(&rps->ips);
418 	spin_unlock_irq(&mchdev_lock);
419 }
420 
421 static unsigned int gen5_invert_freq(struct intel_rps *rps,
422 				     unsigned int val)
423 {
424 	/* Invert the frequency bin into an ips delay */
425 	val = rps->max_freq - val;
426 	val = rps->min_freq + val;
427 
428 	return val;
429 }
430 
431 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
432 {
433 	struct intel_uncore *uncore = rps_to_uncore(rps);
434 	u16 rgvswctl;
435 
436 	lockdep_assert_held(&mchdev_lock);
437 
438 	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
439 	if (rgvswctl & MEMCTL_CMD_STS) {
440 		drm_dbg(&rps_to_i915(rps)->drm,
441 			"gpu busy, RCS change rejected\n");
442 		return -EBUSY; /* still busy with another command */
443 	}
444 
445 	/* Invert the frequency bin into an ips delay */
446 	val = gen5_invert_freq(rps, val);
447 
448 	rgvswctl =
449 		(MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
450 		(val << MEMCTL_FREQ_SHIFT) |
451 		MEMCTL_SFCAVM;
452 	intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
453 	intel_uncore_posting_read16(uncore, MEMSWCTL);
454 
455 	rgvswctl |= MEMCTL_CMD_STS;
456 	intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
457 
458 	return 0;
459 }
460 
461 static int gen5_rps_set(struct intel_rps *rps, u8 val)
462 {
463 	int err;
464 
465 	spin_lock_irq(&mchdev_lock);
466 	err = __gen5_rps_set(rps, val);
467 	spin_unlock_irq(&mchdev_lock);
468 
469 	return err;
470 }
471 
472 static unsigned long intel_pxfreq(u32 vidfreq)
473 {
474 	int div = (vidfreq & 0x3f0000) >> 16;
475 	int post = (vidfreq & 0x3000) >> 12;
476 	int pre = (vidfreq & 0x7);
477 
478 	if (!pre)
479 		return 0;
480 
481 	return div * 133333 / (pre << post);
482 }
483 
484 static unsigned int init_emon(struct intel_uncore *uncore)
485 {
486 	u8 pxw[16];
487 	int i;
488 
489 	/* Disable to program */
490 	intel_uncore_write(uncore, ECR, 0);
491 	intel_uncore_posting_read(uncore, ECR);
492 
493 	/* Program energy weights for various events */
494 	intel_uncore_write(uncore, SDEW, 0x15040d00);
495 	intel_uncore_write(uncore, CSIEW0, 0x007f0000);
496 	intel_uncore_write(uncore, CSIEW1, 0x1e220004);
497 	intel_uncore_write(uncore, CSIEW2, 0x04000004);
498 
499 	for (i = 0; i < 5; i++)
500 		intel_uncore_write(uncore, PEW(i), 0);
501 	for (i = 0; i < 3; i++)
502 		intel_uncore_write(uncore, DEW(i), 0);
503 
504 	/* Program P-state weights to account for frequency power adjustment */
505 	for (i = 0; i < 16; i++) {
506 		u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
507 		unsigned int freq = intel_pxfreq(pxvidfreq);
508 		unsigned int vid =
509 			(pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
510 		unsigned int val;
511 
512 		val = vid * vid * freq / 1000 * 255;
513 		val /= 127 * 127 * 900;
514 
515 		pxw[i] = val;
516 	}
517 	/* Render standby states get 0 weight */
518 	pxw[14] = 0;
519 	pxw[15] = 0;
520 
521 	for (i = 0; i < 4; i++) {
522 		intel_uncore_write(uncore, PXW(i),
523 				   pxw[i * 4 + 0] << 24 |
524 				   pxw[i * 4 + 1] << 16 |
525 				   pxw[i * 4 + 2] <<  8 |
526 				   pxw[i * 4 + 3] <<  0);
527 	}
528 
529 	/* Adjust magic regs to magic values (more experimental results) */
530 	intel_uncore_write(uncore, OGW0, 0);
531 	intel_uncore_write(uncore, OGW1, 0);
532 	intel_uncore_write(uncore, EG0, 0x00007f00);
533 	intel_uncore_write(uncore, EG1, 0x0000000e);
534 	intel_uncore_write(uncore, EG2, 0x000e0000);
535 	intel_uncore_write(uncore, EG3, 0x68000300);
536 	intel_uncore_write(uncore, EG4, 0x42000000);
537 	intel_uncore_write(uncore, EG5, 0x00140031);
538 	intel_uncore_write(uncore, EG6, 0);
539 	intel_uncore_write(uncore, EG7, 0);
540 
541 	for (i = 0; i < 8; i++)
542 		intel_uncore_write(uncore, PXWL(i), 0);
543 
544 	/* Enable PMON + select events */
545 	intel_uncore_write(uncore, ECR, 0x80000019);
546 
547 	return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
548 }
549 
550 static bool gen5_rps_enable(struct intel_rps *rps)
551 {
552 	struct drm_i915_private *i915 = rps_to_i915(rps);
553 	struct intel_uncore *uncore = rps_to_uncore(rps);
554 	u8 fstart, vstart;
555 	u32 rgvmodectl;
556 
557 	spin_lock_irq(&mchdev_lock);
558 
559 	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
560 
561 	/* Enable temp reporting */
562 	intel_uncore_write16(uncore, PMMISC,
563 			     intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
564 	intel_uncore_write16(uncore, TSC1,
565 			     intel_uncore_read16(uncore, TSC1) | TSE);
566 
567 	/* 100ms RC evaluation intervals */
568 	intel_uncore_write(uncore, RCUPEI, 100000);
569 	intel_uncore_write(uncore, RCDNEI, 100000);
570 
571 	/* Set max/min thresholds to 90ms and 80ms respectively */
572 	intel_uncore_write(uncore, RCBMAXAVG, 90000);
573 	intel_uncore_write(uncore, RCBMINAVG, 80000);
574 
575 	intel_uncore_write(uncore, MEMIHYST, 1);
576 
577 	/* Set up min, max, and cur for interrupt handling */
578 	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
579 		MEMMODE_FSTART_SHIFT;
580 
581 	vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
582 		  PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
583 
584 	intel_uncore_write(uncore,
585 			   MEMINTREN,
586 			   MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
587 
588 	intel_uncore_write(uncore, VIDSTART, vstart);
589 	intel_uncore_posting_read(uncore, VIDSTART);
590 
591 	rgvmodectl |= MEMMODE_SWMODE_EN;
592 	intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
593 
594 	if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
595 			     MEMCTL_CMD_STS) == 0, 10))
596 		drm_err(&uncore->i915->drm,
597 			"stuck trying to change perf mode\n");
598 	mdelay(1);
599 
600 	__gen5_rps_set(rps, rps->cur_freq);
601 
602 	rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
603 	rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
604 	rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
605 	rps->ips.last_time1 = jiffies_to_msecs(jiffies);
606 
607 	rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
608 	rps->ips.last_time2 = ktime_get_raw_ns();
609 
610 	spin_lock(&i915->irq_lock);
611 	ilk_enable_display_irq(i915, DE_PCU_EVENT);
612 	spin_unlock(&i915->irq_lock);
613 
614 	spin_unlock_irq(&mchdev_lock);
615 
616 	rps->ips.corr = init_emon(uncore);
617 
618 	return true;
619 }
620 
621 static void gen5_rps_disable(struct intel_rps *rps)
622 {
623 	struct drm_i915_private *i915 = rps_to_i915(rps);
624 	struct intel_uncore *uncore = rps_to_uncore(rps);
625 	u16 rgvswctl;
626 
627 	spin_lock_irq(&mchdev_lock);
628 
629 	spin_lock(&i915->irq_lock);
630 	ilk_disable_display_irq(i915, DE_PCU_EVENT);
631 	spin_unlock(&i915->irq_lock);
632 
633 	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
634 
635 	/* Ack interrupts, disable EFC interrupt */
636 	intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0);
637 	intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
638 
639 	/* Go back to the starting frequency */
640 	__gen5_rps_set(rps, rps->idle_freq);
641 	mdelay(1);
642 	rgvswctl |= MEMCTL_CMD_STS;
643 	intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
644 	mdelay(1);
645 
646 	spin_unlock_irq(&mchdev_lock);
647 }
648 
649 static u32 rps_limits(struct intel_rps *rps, u8 val)
650 {
651 	u32 limits;
652 
653 	/*
654 	 * Only set the down limit when we've reached the lowest level to avoid
655 	 * getting more interrupts, otherwise leave this clear. This prevents a
656 	 * race in the hw when coming out of rc6: There's a tiny window where
657 	 * the hw runs at the minimal clock before selecting the desired
658 	 * frequency, if the down threshold expires in that window we will not
659 	 * receive a down interrupt.
660 	 */
661 	if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
662 		limits = rps->max_freq_softlimit << 23;
663 		if (val <= rps->min_freq_softlimit)
664 			limits |= rps->min_freq_softlimit << 14;
665 	} else {
666 		limits = rps->max_freq_softlimit << 24;
667 		if (val <= rps->min_freq_softlimit)
668 			limits |= rps->min_freq_softlimit << 16;
669 	}
670 
671 	return limits;
672 }
673 
674 static void rps_set_power(struct intel_rps *rps, int new_power)
675 {
676 	struct intel_gt *gt = rps_to_gt(rps);
677 	struct intel_uncore *uncore = gt->uncore;
678 	u32 ei_up = 0, ei_down = 0;
679 
680 	lockdep_assert_held(&rps->power.mutex);
681 
682 	if (new_power == rps->power.mode)
683 		return;
684 
685 	/* Note the units here are not exactly 1us, but 1280ns. */
686 	switch (new_power) {
687 	case LOW_POWER:
688 		ei_up = 16000;
689 		ei_down = 32000;
690 		break;
691 
692 	case BETWEEN:
693 		ei_up = 13000;
694 		ei_down = 32000;
695 		break;
696 
697 	case HIGH_POWER:
698 		ei_up = 10000;
699 		ei_down = 32000;
700 		break;
701 	}
702 
703 	/* When byt can survive without system hang with dynamic
704 	 * sw freq adjustments, this restriction can be lifted.
705 	 */
706 	if (IS_VALLEYVIEW(gt->i915))
707 		goto skip_hw_write;
708 
709 	GT_TRACE(gt,
710 		 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
711 		 new_power,
712 		 rps->power.up_threshold, ei_up,
713 		 rps->power.down_threshold, ei_down);
714 
715 	set(uncore, GEN6_RP_UP_EI,
716 	    intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
717 	set(uncore, GEN6_RP_UP_THRESHOLD,
718 	    intel_gt_ns_to_pm_interval(gt,
719 				       ei_up * rps->power.up_threshold * 10));
720 
721 	set(uncore, GEN6_RP_DOWN_EI,
722 	    intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
723 	set(uncore, GEN6_RP_DOWN_THRESHOLD,
724 	    intel_gt_ns_to_pm_interval(gt,
725 				       ei_down *
726 				       rps->power.down_threshold * 10));
727 
728 	set(uncore, GEN6_RP_CONTROL,
729 	    (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
730 	    GEN6_RP_MEDIA_HW_NORMAL_MODE |
731 	    GEN6_RP_MEDIA_IS_GFX |
732 	    GEN6_RP_ENABLE |
733 	    GEN6_RP_UP_BUSY_AVG |
734 	    GEN6_RP_DOWN_IDLE_AVG);
735 
736 skip_hw_write:
737 	rps->power.mode = new_power;
738 }
739 
740 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
741 {
742 	int new_power;
743 
744 	new_power = rps->power.mode;
745 	switch (rps->power.mode) {
746 	case LOW_POWER:
747 		if (val > rps->efficient_freq + 1 &&
748 		    val > rps->cur_freq)
749 			new_power = BETWEEN;
750 		break;
751 
752 	case BETWEEN:
753 		if (val <= rps->efficient_freq &&
754 		    val < rps->cur_freq)
755 			new_power = LOW_POWER;
756 		else if (val >= rps->rp0_freq &&
757 			 val > rps->cur_freq)
758 			new_power = HIGH_POWER;
759 		break;
760 
761 	case HIGH_POWER:
762 		if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
763 		    val < rps->cur_freq)
764 			new_power = BETWEEN;
765 		break;
766 	}
767 	/* Max/min bins are special */
768 	if (val <= rps->min_freq_softlimit)
769 		new_power = LOW_POWER;
770 	if (val >= rps->max_freq_softlimit)
771 		new_power = HIGH_POWER;
772 
773 	mutex_lock(&rps->power.mutex);
774 	if (rps->power.interactive)
775 		new_power = HIGH_POWER;
776 	rps_set_power(rps, new_power);
777 	mutex_unlock(&rps->power.mutex);
778 }
779 
780 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
781 {
782 	GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
783 		 str_yes_no(interactive));
784 
785 	mutex_lock(&rps->power.mutex);
786 	if (interactive) {
787 		if (!rps->power.interactive++ && intel_rps_is_active(rps))
788 			rps_set_power(rps, HIGH_POWER);
789 	} else {
790 		GEM_BUG_ON(!rps->power.interactive);
791 		rps->power.interactive--;
792 	}
793 	mutex_unlock(&rps->power.mutex);
794 }
795 
796 static int gen6_rps_set(struct intel_rps *rps, u8 val)
797 {
798 	struct intel_uncore *uncore = rps_to_uncore(rps);
799 	struct drm_i915_private *i915 = rps_to_i915(rps);
800 	u32 swreq;
801 
802 	GEM_BUG_ON(rps_uses_slpc(rps));
803 
804 	if (GRAPHICS_VER(i915) >= 9)
805 		swreq = GEN9_FREQUENCY(val);
806 	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
807 		swreq = HSW_FREQUENCY(val);
808 	else
809 		swreq = (GEN6_FREQUENCY(val) |
810 			 GEN6_OFFSET(0) |
811 			 GEN6_AGGRESSIVE_TURBO);
812 	set(uncore, GEN6_RPNSWREQ, swreq);
813 
814 	GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
815 		 val, intel_gpu_freq(rps, val), swreq);
816 
817 	return 0;
818 }
819 
820 static int vlv_rps_set(struct intel_rps *rps, u8 val)
821 {
822 	struct drm_i915_private *i915 = rps_to_i915(rps);
823 	int err;
824 
825 	vlv_punit_get(i915);
826 	err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
827 	vlv_punit_put(i915);
828 
829 	GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
830 		 val, intel_gpu_freq(rps, val));
831 
832 	return err;
833 }
834 
835 static int rps_set(struct intel_rps *rps, u8 val, bool update)
836 {
837 	struct drm_i915_private *i915 = rps_to_i915(rps);
838 	int err;
839 
840 	if (val == rps->last_freq)
841 		return 0;
842 
843 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
844 		err = vlv_rps_set(rps, val);
845 	else if (GRAPHICS_VER(i915) >= 6)
846 		err = gen6_rps_set(rps, val);
847 	else
848 		err = gen5_rps_set(rps, val);
849 	if (err)
850 		return err;
851 
852 	if (update && GRAPHICS_VER(i915) >= 6)
853 		gen6_rps_set_thresholds(rps, val);
854 	rps->last_freq = val;
855 
856 	return 0;
857 }
858 
859 void intel_rps_unpark(struct intel_rps *rps)
860 {
861 	if (!intel_rps_is_enabled(rps))
862 		return;
863 
864 	GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
865 
866 	/*
867 	 * Use the user's desired frequency as a guide, but for better
868 	 * performance, jump directly to RPe as our starting frequency.
869 	 */
870 	mutex_lock(&rps->lock);
871 
872 	intel_rps_set_active(rps);
873 	intel_rps_set(rps,
874 		      clamp(rps->cur_freq,
875 			    rps->min_freq_softlimit,
876 			    rps->max_freq_softlimit));
877 
878 	mutex_unlock(&rps->lock);
879 
880 	rps->pm_iir = 0;
881 	if (intel_rps_has_interrupts(rps))
882 		rps_enable_interrupts(rps);
883 	if (intel_rps_uses_timer(rps))
884 		rps_start_timer(rps);
885 
886 	if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
887 		gen5_rps_update(rps);
888 }
889 
890 void intel_rps_park(struct intel_rps *rps)
891 {
892 	int adj;
893 
894 	if (!intel_rps_is_enabled(rps))
895 		return;
896 
897 	if (!intel_rps_clear_active(rps))
898 		return;
899 
900 	if (intel_rps_uses_timer(rps))
901 		rps_stop_timer(rps);
902 	if (intel_rps_has_interrupts(rps))
903 		rps_disable_interrupts(rps);
904 
905 	if (rps->last_freq <= rps->idle_freq)
906 		return;
907 
908 	/*
909 	 * The punit delays the write of the frequency and voltage until it
910 	 * determines the GPU is awake. During normal usage we don't want to
911 	 * waste power changing the frequency if the GPU is sleeping (rc6).
912 	 * However, the GPU and driver is now idle and we do not want to delay
913 	 * switching to minimum voltage (reducing power whilst idle) as we do
914 	 * not expect to be woken in the near future and so must flush the
915 	 * change by waking the device.
916 	 *
917 	 * We choose to take the media powerwell (either would do to trick the
918 	 * punit into committing the voltage change) as that takes a lot less
919 	 * power than the render powerwell.
920 	 */
921 	intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
922 	rps_set(rps, rps->idle_freq, false);
923 	intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
924 
925 	/*
926 	 * Since we will try and restart from the previously requested
927 	 * frequency on unparking, treat this idle point as a downclock
928 	 * interrupt and reduce the frequency for resume. If we park/unpark
929 	 * more frequently than the rps worker can run, we will not respond
930 	 * to any EI and never see a change in frequency.
931 	 *
932 	 * (Note we accommodate Cherryview's limitation of only using an
933 	 * even bin by applying it to all.)
934 	 */
935 	adj = rps->last_adj;
936 	if (adj < 0)
937 		adj *= 2;
938 	else /* CHV needs even encode values */
939 		adj = -2;
940 	rps->last_adj = adj;
941 	rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
942 	if (rps->cur_freq < rps->efficient_freq) {
943 		rps->cur_freq = rps->efficient_freq;
944 		rps->last_adj = 0;
945 	}
946 
947 	GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
948 }
949 
950 u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
951 {
952 	struct intel_guc_slpc *slpc;
953 
954 	if (rps_uses_slpc(rps)) {
955 		slpc = rps_to_slpc(rps);
956 
957 		return slpc->boost_freq;
958 	} else {
959 		return intel_gpu_freq(rps, rps->boost_freq);
960 	}
961 }
962 
963 static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
964 {
965 	bool boost = false;
966 
967 	/* Validate against (static) hardware limits */
968 	val = intel_freq_opcode(rps, val);
969 	if (val < rps->min_freq || val > rps->max_freq)
970 		return -EINVAL;
971 
972 	mutex_lock(&rps->lock);
973 	if (val != rps->boost_freq) {
974 		rps->boost_freq = val;
975 		boost = atomic_read(&rps->num_waiters);
976 	}
977 	mutex_unlock(&rps->lock);
978 	if (boost)
979 		queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);
980 
981 	return 0;
982 }
983 
984 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
985 {
986 	struct intel_guc_slpc *slpc;
987 
988 	if (rps_uses_slpc(rps)) {
989 		slpc = rps_to_slpc(rps);
990 
991 		return intel_guc_slpc_set_boost_freq(slpc, freq);
992 	} else {
993 		return rps_set_boost_freq(rps, freq);
994 	}
995 }
996 
997 void intel_rps_dec_waiters(struct intel_rps *rps)
998 {
999 	struct intel_guc_slpc *slpc;
1000 
1001 	if (rps_uses_slpc(rps)) {
1002 		slpc = rps_to_slpc(rps);
1003 
1004 		intel_guc_slpc_dec_waiters(slpc);
1005 	} else {
1006 		atomic_dec(&rps->num_waiters);
1007 	}
1008 }
1009 
1010 void intel_rps_boost(struct i915_request *rq)
1011 {
1012 	struct intel_guc_slpc *slpc;
1013 
1014 	if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1015 		return;
1016 
1017 	/* Waitboost is not needed for contexts marked with a Freq hint */
1018 	if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags))
1019 		return;
1020 
1021 	/* Serializes with i915_request_retire() */
1022 	if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1023 		struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1024 
1025 		if (rps_uses_slpc(rps)) {
1026 			slpc = rps_to_slpc(rps);
1027 
1028 			/* Waitboost should not be done with power saving profile */
1029 			if (slpc->power_profile == SLPC_POWER_PROFILES_POWER_SAVING)
1030 				return;
1031 
1032 			if (slpc->min_freq_softlimit >= slpc->boost_freq)
1033 				return;
1034 
1035 			/* Return if old value is non zero */
1036 			if (!atomic_fetch_inc(&slpc->num_waiters)) {
1037 				GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1038 					 rq->fence.context, rq->fence.seqno);
1039 				queue_work(rps_to_gt(rps)->i915->unordered_wq,
1040 					   &slpc->boost_work);
1041 			}
1042 
1043 			return;
1044 		}
1045 
1046 		if (atomic_fetch_inc(&rps->num_waiters))
1047 			return;
1048 
1049 		if (!intel_rps_is_active(rps))
1050 			return;
1051 
1052 		GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1053 			 rq->fence.context, rq->fence.seqno);
1054 
1055 		if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1056 			queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);
1057 
1058 		WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1059 	}
1060 }
1061 
1062 int intel_rps_set(struct intel_rps *rps, u8 val)
1063 {
1064 	int err;
1065 
1066 	lockdep_assert_held(&rps->lock);
1067 	GEM_BUG_ON(val > rps->max_freq);
1068 	GEM_BUG_ON(val < rps->min_freq);
1069 
1070 	if (intel_rps_is_active(rps)) {
1071 		err = rps_set(rps, val, true);
1072 		if (err)
1073 			return err;
1074 
1075 		/*
1076 		 * Make sure we continue to get interrupts
1077 		 * until we hit the minimum or maximum frequencies.
1078 		 */
1079 		if (intel_rps_has_interrupts(rps)) {
1080 			struct intel_uncore *uncore = rps_to_uncore(rps);
1081 
1082 			set(uncore,
1083 			    GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1084 
1085 			set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1086 		}
1087 	}
1088 
1089 	rps->cur_freq = val;
1090 	return 0;
1091 }
1092 
1093 static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1094 {
1095 	struct drm_i915_private *i915 = rps_to_i915(rps);
1096 	struct intel_uncore *uncore = rps_to_uncore(rps);
1097 
1098 	if (IS_GEN9_LP(i915))
1099 		return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1100 	else
1101 		return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1102 }
1103 
1104 static void
1105 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1106 {
1107 	struct intel_uncore *uncore = rps_to_uncore(rps);
1108 	u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?
1109 				intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :
1110 				intel_uncore_read(uncore, MTL_RP_STATE_CAP);
1111 	u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?
1112 			intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :
1113 			intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);
1114 
1115 	/* MTL values are in units of 16.67 MHz */
1116 	caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);
1117 	caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);
1118 	caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);
1119 }
1120 
1121 static void
1122 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1123 {
1124 	struct drm_i915_private *i915 = rps_to_i915(rps);
1125 	u32 rp_state_cap;
1126 
1127 	rp_state_cap = intel_rps_read_state_cap(rps);
1128 
1129 	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
1130 	if (IS_GEN9_LP(i915)) {
1131 		caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1132 		caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1133 		caps->min_freq = (rp_state_cap >>  0) & 0xff;
1134 	} else {
1135 		caps->rp0_freq = (rp_state_cap >>  0) & 0xff;
1136 		if (GRAPHICS_VER(i915) >= 10)
1137 			caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1138 						       intel_uncore_read(to_gt(i915)->uncore,
1139 						       GEN10_FREQ_INFO_REC));
1140 		else
1141 			caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1142 		caps->min_freq = (rp_state_cap >> 16) & 0xff;
1143 	}
1144 
1145 	if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1146 		/*
1147 		 * In this case rp_state_cap register reports frequencies in
1148 		 * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1149 		 * units of 16.67 MHz
1150 		 */
1151 		caps->rp0_freq *= GEN9_FREQ_SCALER;
1152 		caps->rp1_freq *= GEN9_FREQ_SCALER;
1153 		caps->min_freq *= GEN9_FREQ_SCALER;
1154 	}
1155 }
1156 
1157 /**
1158  * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1159  * @rps: the intel_rps structure
1160  * @caps: returned freq caps
1161  *
1162  * Returned "caps" frequencies should be converted to MHz using
1163  * intel_gpu_freq()
1164  */
1165 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1166 {
1167 	struct drm_i915_private *i915 = rps_to_i915(rps);
1168 
1169 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1170 		return mtl_get_freq_caps(rps, caps);
1171 	else
1172 		return __gen6_rps_get_freq_caps(rps, caps);
1173 }
1174 
1175 static void gen6_rps_init(struct intel_rps *rps)
1176 {
1177 	struct drm_i915_private *i915 = rps_to_i915(rps);
1178 	struct intel_rps_freq_caps caps;
1179 
1180 	gen6_rps_get_freq_caps(rps, &caps);
1181 	rps->rp0_freq = caps.rp0_freq;
1182 	rps->rp1_freq = caps.rp1_freq;
1183 	rps->min_freq = caps.min_freq;
1184 
1185 	/* hw_max = RP0 until we check for overclocking */
1186 	rps->max_freq = rps->rp0_freq;
1187 
1188 	rps->efficient_freq = rps->rp1_freq;
1189 	if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1190 	    IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1191 		u32 ddcc_status = 0;
1192 		u32 mult = 1;
1193 
1194 		if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1195 			mult = GEN9_FREQ_SCALER;
1196 		if (snb_pcode_read(rps_to_gt(rps)->uncore,
1197 				   HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1198 				   &ddcc_status, NULL) == 0)
1199 			rps->efficient_freq =
1200 				clamp_t(u32,
1201 					((ddcc_status >> 8) & 0xff) * mult,
1202 					rps->min_freq,
1203 					rps->max_freq);
1204 	}
1205 }
1206 
1207 static bool rps_reset(struct intel_rps *rps)
1208 {
1209 	struct drm_i915_private *i915 = rps_to_i915(rps);
1210 
1211 	/* force a reset */
1212 	rps->power.mode = -1;
1213 	rps->last_freq = -1;
1214 
1215 	if (rps_set(rps, rps->min_freq, true)) {
1216 		drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1217 		return false;
1218 	}
1219 
1220 	rps->cur_freq = rps->min_freq;
1221 	return true;
1222 }
1223 
1224 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
1225 static bool gen9_rps_enable(struct intel_rps *rps)
1226 {
1227 	struct intel_gt *gt = rps_to_gt(rps);
1228 	struct intel_uncore *uncore = gt->uncore;
1229 
1230 	/* Program defaults and thresholds for RPS */
1231 	if (GRAPHICS_VER(gt->i915) == 9)
1232 		intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1233 				      GEN9_FREQUENCY(rps->rp1_freq));
1234 
1235 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1236 
1237 	rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1238 
1239 	return rps_reset(rps);
1240 }
1241 
1242 static bool gen8_rps_enable(struct intel_rps *rps)
1243 {
1244 	struct intel_uncore *uncore = rps_to_uncore(rps);
1245 
1246 	intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1247 			      HSW_FREQUENCY(rps->rp1_freq));
1248 
1249 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1250 
1251 	rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1252 
1253 	return rps_reset(rps);
1254 }
1255 
1256 static bool gen6_rps_enable(struct intel_rps *rps)
1257 {
1258 	struct intel_uncore *uncore = rps_to_uncore(rps);
1259 
1260 	/* Power down if completely idle for over 50ms */
1261 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1262 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1263 
1264 	rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1265 			  GEN6_PM_RP_DOWN_THRESHOLD |
1266 			  GEN6_PM_RP_DOWN_TIMEOUT);
1267 
1268 	return rps_reset(rps);
1269 }
1270 
1271 static int chv_rps_max_freq(struct intel_rps *rps)
1272 {
1273 	struct drm_i915_private *i915 = rps_to_i915(rps);
1274 	struct intel_gt *gt = rps_to_gt(rps);
1275 	u32 val;
1276 
1277 	val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1278 
1279 	switch (gt->info.sseu.eu_total) {
1280 	case 8:
1281 		/* (2 * 4) config */
1282 		val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1283 		break;
1284 	case 12:
1285 		/* (2 * 6) config */
1286 		val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1287 		break;
1288 	case 16:
1289 		/* (2 * 8) config */
1290 	default:
1291 		/* Setting (2 * 8) Min RP0 for any other combination */
1292 		val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1293 		break;
1294 	}
1295 
1296 	return val & FB_GFX_FREQ_FUSE_MASK;
1297 }
1298 
1299 static int chv_rps_rpe_freq(struct intel_rps *rps)
1300 {
1301 	struct drm_i915_private *i915 = rps_to_i915(rps);
1302 	u32 val;
1303 
1304 	val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1305 	val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1306 
1307 	return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1308 }
1309 
1310 static int chv_rps_guar_freq(struct intel_rps *rps)
1311 {
1312 	struct drm_i915_private *i915 = rps_to_i915(rps);
1313 	u32 val;
1314 
1315 	val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1316 
1317 	return val & FB_GFX_FREQ_FUSE_MASK;
1318 }
1319 
1320 static u32 chv_rps_min_freq(struct intel_rps *rps)
1321 {
1322 	struct drm_i915_private *i915 = rps_to_i915(rps);
1323 	u32 val;
1324 
1325 	val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1326 	val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1327 
1328 	return val & FB_GFX_FREQ_FUSE_MASK;
1329 }
1330 
1331 static bool chv_rps_enable(struct intel_rps *rps)
1332 {
1333 	struct intel_uncore *uncore = rps_to_uncore(rps);
1334 	struct drm_i915_private *i915 = rps_to_i915(rps);
1335 	u32 val;
1336 
1337 	/* 1: Program defaults and thresholds for RPS*/
1338 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1339 	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1340 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1341 	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1342 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1343 
1344 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1345 
1346 	/* 2: Enable RPS */
1347 	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1348 			      GEN6_RP_MEDIA_HW_NORMAL_MODE |
1349 			      GEN6_RP_MEDIA_IS_GFX |
1350 			      GEN6_RP_ENABLE |
1351 			      GEN6_RP_UP_BUSY_AVG |
1352 			      GEN6_RP_DOWN_IDLE_AVG);
1353 
1354 	rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1355 			  GEN6_PM_RP_DOWN_THRESHOLD |
1356 			  GEN6_PM_RP_DOWN_TIMEOUT);
1357 
1358 	/* Setting Fixed Bias */
1359 	vlv_punit_get(i915);
1360 
1361 	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1362 	vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1363 
1364 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1365 
1366 	vlv_punit_put(i915);
1367 
1368 	/* RPS code assumes GPLL is used */
1369 	drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1370 		      "GPLL not enabled\n");
1371 
1372 	drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1373 		str_yes_no(val & GPLLENABLE));
1374 	drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1375 
1376 	return rps_reset(rps);
1377 }
1378 
1379 static int vlv_rps_guar_freq(struct intel_rps *rps)
1380 {
1381 	struct drm_i915_private *i915 = rps_to_i915(rps);
1382 	u32 val, rp1;
1383 
1384 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1385 
1386 	rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1387 	rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1388 
1389 	return rp1;
1390 }
1391 
1392 static int vlv_rps_max_freq(struct intel_rps *rps)
1393 {
1394 	struct drm_i915_private *i915 = rps_to_i915(rps);
1395 	u32 val, rp0;
1396 
1397 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1398 
1399 	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1400 	/* Clamp to max */
1401 	rp0 = min_t(u32, rp0, 0xea);
1402 
1403 	return rp0;
1404 }
1405 
1406 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1407 {
1408 	struct drm_i915_private *i915 = rps_to_i915(rps);
1409 	u32 val, rpe;
1410 
1411 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1412 	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1413 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1414 	rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1415 
1416 	return rpe;
1417 }
1418 
1419 static int vlv_rps_min_freq(struct intel_rps *rps)
1420 {
1421 	struct drm_i915_private *i915 = rps_to_i915(rps);
1422 	u32 val;
1423 
1424 	val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1425 	/*
1426 	 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1427 	 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1428 	 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1429 	 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1430 	 * to make sure it matches what Punit accepts.
1431 	 */
1432 	return max_t(u32, val, 0xc0);
1433 }
1434 
1435 static bool vlv_rps_enable(struct intel_rps *rps)
1436 {
1437 	struct intel_uncore *uncore = rps_to_uncore(rps);
1438 	struct drm_i915_private *i915 = rps_to_i915(rps);
1439 	u32 val;
1440 
1441 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1442 	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1443 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1444 	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1445 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1446 
1447 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1448 
1449 	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1450 			      GEN6_RP_MEDIA_TURBO |
1451 			      GEN6_RP_MEDIA_HW_NORMAL_MODE |
1452 			      GEN6_RP_MEDIA_IS_GFX |
1453 			      GEN6_RP_ENABLE |
1454 			      GEN6_RP_UP_BUSY_AVG |
1455 			      GEN6_RP_DOWN_IDLE_CONT);
1456 
1457 	/* WaGsvRC0ResidencyMethod:vlv */
1458 	rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1459 
1460 	vlv_punit_get(i915);
1461 
1462 	/* Setting Fixed Bias */
1463 	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1464 	vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1465 
1466 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1467 
1468 	vlv_punit_put(i915);
1469 
1470 	/* RPS code assumes GPLL is used */
1471 	drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1472 		      "GPLL not enabled\n");
1473 
1474 	drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1475 		str_yes_no(val & GPLLENABLE));
1476 	drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1477 
1478 	return rps_reset(rps);
1479 }
1480 
1481 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1482 {
1483 	struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1484 	struct intel_uncore *uncore = rps_to_uncore(rps);
1485 	unsigned int t, state1, state2;
1486 	u32 pxvid, ext_v;
1487 	u64 corr, corr2;
1488 
1489 	lockdep_assert_held(&mchdev_lock);
1490 
1491 	pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1492 	pxvid = (pxvid >> 24) & 0x7f;
1493 	ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1494 
1495 	state1 = ext_v;
1496 
1497 	/* Revel in the empirically derived constants */
1498 
1499 	/* Correction factor in 1/100000 units */
1500 	t = ips_mch_val(uncore);
1501 	if (t > 80)
1502 		corr = t * 2349 + 135940;
1503 	else if (t >= 50)
1504 		corr = t * 964 + 29317;
1505 	else /* < 50 */
1506 		corr = t * 301 + 1004;
1507 
1508 	corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1509 	corr2 = div_u64(corr, 100000) * ips->corr;
1510 
1511 	state2 = div_u64(corr2 * state1, 10000);
1512 	state2 /= 100; /* convert to mW */
1513 
1514 	__gen5_ips_update(ips);
1515 
1516 	return ips->gfx_power + state2;
1517 }
1518 
1519 static bool has_busy_stats(struct intel_rps *rps)
1520 {
1521 	struct intel_engine_cs *engine;
1522 	enum intel_engine_id id;
1523 
1524 	for_each_engine(engine, rps_to_gt(rps), id) {
1525 		if (!intel_engine_supports_stats(engine))
1526 			return false;
1527 	}
1528 
1529 	return true;
1530 }
1531 
1532 void intel_rps_enable(struct intel_rps *rps)
1533 {
1534 	struct drm_i915_private *i915 = rps_to_i915(rps);
1535 	struct intel_uncore *uncore = rps_to_uncore(rps);
1536 	bool enabled = false;
1537 
1538 	if (!HAS_RPS(i915))
1539 		return;
1540 
1541 	if (rps_uses_slpc(rps))
1542 		return;
1543 
1544 	intel_gt_check_clock_frequency(rps_to_gt(rps));
1545 
1546 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1547 	if (rps->max_freq <= rps->min_freq)
1548 		/* leave disabled, no room for dynamic reclocking */;
1549 	else if (IS_CHERRYVIEW(i915))
1550 		enabled = chv_rps_enable(rps);
1551 	else if (IS_VALLEYVIEW(i915))
1552 		enabled = vlv_rps_enable(rps);
1553 	else if (GRAPHICS_VER(i915) >= 9)
1554 		enabled = gen9_rps_enable(rps);
1555 	else if (GRAPHICS_VER(i915) >= 8)
1556 		enabled = gen8_rps_enable(rps);
1557 	else if (GRAPHICS_VER(i915) >= 6)
1558 		enabled = gen6_rps_enable(rps);
1559 	else if (IS_IRONLAKE_M(i915))
1560 		enabled = gen5_rps_enable(rps);
1561 	else
1562 		MISSING_CASE(GRAPHICS_VER(i915));
1563 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1564 	if (!enabled)
1565 		return;
1566 
1567 	GT_TRACE(rps_to_gt(rps),
1568 		 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n",
1569 		 rps->min_freq, rps->max_freq,
1570 		 intel_gpu_freq(rps, rps->min_freq),
1571 		 intel_gpu_freq(rps, rps->max_freq),
1572 		 rps->power.up_threshold,
1573 		 rps->power.down_threshold);
1574 
1575 	GEM_BUG_ON(rps->max_freq < rps->min_freq);
1576 	GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1577 
1578 	GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1579 	GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1580 
1581 	if (has_busy_stats(rps))
1582 		intel_rps_set_timer(rps);
1583 	else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1584 		intel_rps_set_interrupts(rps);
1585 	else
1586 		/* Ironlake currently uses intel_ips.ko */ {}
1587 
1588 	intel_rps_set_enabled(rps);
1589 }
1590 
1591 static void gen6_rps_disable(struct intel_rps *rps)
1592 {
1593 	set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1594 }
1595 
1596 void intel_rps_disable(struct intel_rps *rps)
1597 {
1598 	struct drm_i915_private *i915 = rps_to_i915(rps);
1599 
1600 	if (!intel_rps_is_enabled(rps))
1601 		return;
1602 
1603 	intel_rps_clear_enabled(rps);
1604 	intel_rps_clear_interrupts(rps);
1605 	intel_rps_clear_timer(rps);
1606 
1607 	if (GRAPHICS_VER(i915) >= 6)
1608 		gen6_rps_disable(rps);
1609 	else if (IS_IRONLAKE_M(i915))
1610 		gen5_rps_disable(rps);
1611 }
1612 
1613 static int byt_gpu_freq(struct intel_rps *rps, int val)
1614 {
1615 	/*
1616 	 * N = val - 0xb7
1617 	 * Slow = Fast = GPLL ref * N
1618 	 */
1619 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1620 }
1621 
1622 static int byt_freq_opcode(struct intel_rps *rps, int val)
1623 {
1624 	return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1625 }
1626 
1627 static int chv_gpu_freq(struct intel_rps *rps, int val)
1628 {
1629 	/*
1630 	 * N = val / 2
1631 	 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1632 	 */
1633 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1634 }
1635 
1636 static int chv_freq_opcode(struct intel_rps *rps, int val)
1637 {
1638 	/* CHV needs even values */
1639 	return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1640 }
1641 
1642 int intel_gpu_freq(struct intel_rps *rps, int val)
1643 {
1644 	struct drm_i915_private *i915 = rps_to_i915(rps);
1645 
1646 	if (GRAPHICS_VER(i915) >= 9)
1647 		return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1648 					 GEN9_FREQ_SCALER);
1649 	else if (IS_CHERRYVIEW(i915))
1650 		return chv_gpu_freq(rps, val);
1651 	else if (IS_VALLEYVIEW(i915))
1652 		return byt_gpu_freq(rps, val);
1653 	else if (GRAPHICS_VER(i915) >= 6)
1654 		return val * GT_FREQUENCY_MULTIPLIER;
1655 	else
1656 		return val;
1657 }
1658 
1659 int intel_freq_opcode(struct intel_rps *rps, int val)
1660 {
1661 	struct drm_i915_private *i915 = rps_to_i915(rps);
1662 
1663 	if (GRAPHICS_VER(i915) >= 9)
1664 		return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1665 					 GT_FREQUENCY_MULTIPLIER);
1666 	else if (IS_CHERRYVIEW(i915))
1667 		return chv_freq_opcode(rps, val);
1668 	else if (IS_VALLEYVIEW(i915))
1669 		return byt_freq_opcode(rps, val);
1670 	else if (GRAPHICS_VER(i915) >= 6)
1671 		return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1672 	else
1673 		return val;
1674 }
1675 
1676 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1677 {
1678 	struct drm_i915_private *i915 = rps_to_i915(rps);
1679 
1680 	rps->gpll_ref_freq =
1681 		vlv_get_cck_clock(i915, "GPLL ref",
1682 				  CCK_GPLL_CLOCK_CONTROL,
1683 				  i915->czclk_freq);
1684 
1685 	drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1686 		rps->gpll_ref_freq);
1687 }
1688 
1689 static void vlv_rps_init(struct intel_rps *rps)
1690 {
1691 	struct drm_i915_private *i915 = rps_to_i915(rps);
1692 
1693 	vlv_iosf_sb_get(i915,
1694 			BIT(VLV_IOSF_SB_PUNIT) |
1695 			BIT(VLV_IOSF_SB_NC) |
1696 			BIT(VLV_IOSF_SB_CCK));
1697 
1698 	vlv_init_gpll_ref_freq(rps);
1699 
1700 	rps->max_freq = vlv_rps_max_freq(rps);
1701 	rps->rp0_freq = rps->max_freq;
1702 	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1703 		intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1704 
1705 	rps->efficient_freq = vlv_rps_rpe_freq(rps);
1706 	drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1707 		intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1708 
1709 	rps->rp1_freq = vlv_rps_guar_freq(rps);
1710 	drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1711 		intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1712 
1713 	rps->min_freq = vlv_rps_min_freq(rps);
1714 	drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1715 		intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1716 
1717 	vlv_iosf_sb_put(i915,
1718 			BIT(VLV_IOSF_SB_PUNIT) |
1719 			BIT(VLV_IOSF_SB_NC) |
1720 			BIT(VLV_IOSF_SB_CCK));
1721 }
1722 
1723 static void chv_rps_init(struct intel_rps *rps)
1724 {
1725 	struct drm_i915_private *i915 = rps_to_i915(rps);
1726 
1727 	vlv_iosf_sb_get(i915,
1728 			BIT(VLV_IOSF_SB_PUNIT) |
1729 			BIT(VLV_IOSF_SB_NC) |
1730 			BIT(VLV_IOSF_SB_CCK));
1731 
1732 	vlv_init_gpll_ref_freq(rps);
1733 
1734 	rps->max_freq = chv_rps_max_freq(rps);
1735 	rps->rp0_freq = rps->max_freq;
1736 	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1737 		intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1738 
1739 	rps->efficient_freq = chv_rps_rpe_freq(rps);
1740 	drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1741 		intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1742 
1743 	rps->rp1_freq = chv_rps_guar_freq(rps);
1744 	drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1745 		intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1746 
1747 	rps->min_freq = chv_rps_min_freq(rps);
1748 	drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1749 		intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1750 
1751 	vlv_iosf_sb_put(i915,
1752 			BIT(VLV_IOSF_SB_PUNIT) |
1753 			BIT(VLV_IOSF_SB_NC) |
1754 			BIT(VLV_IOSF_SB_CCK));
1755 
1756 	drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1757 				   rps->rp1_freq | rps->min_freq) & 1,
1758 		      "Odd GPU freq values\n");
1759 }
1760 
1761 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1762 {
1763 	ei->ktime = ktime_get_raw();
1764 	ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1765 	ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1766 }
1767 
1768 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1769 {
1770 	struct intel_uncore *uncore = rps_to_uncore(rps);
1771 	const struct intel_rps_ei *prev = &rps->ei;
1772 	struct intel_rps_ei now;
1773 	u32 events = 0;
1774 
1775 	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1776 		return 0;
1777 
1778 	vlv_c0_read(uncore, &now);
1779 
1780 	if (prev->ktime) {
1781 		u64 time, c0;
1782 		u32 render, media;
1783 
1784 		time = ktime_us_delta(now.ktime, prev->ktime);
1785 
1786 		time *= rps_to_i915(rps)->czclk_freq;
1787 
1788 		/* Workload can be split between render + media,
1789 		 * e.g. SwapBuffers being blitted in X after being rendered in
1790 		 * mesa. To account for this we need to combine both engines
1791 		 * into our activity counter.
1792 		 */
1793 		render = now.render_c0 - prev->render_c0;
1794 		media = now.media_c0 - prev->media_c0;
1795 		c0 = max(render, media);
1796 		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1797 
1798 		if (c0 > time * rps->power.up_threshold)
1799 			events = GEN6_PM_RP_UP_THRESHOLD;
1800 		else if (c0 < time * rps->power.down_threshold)
1801 			events = GEN6_PM_RP_DOWN_THRESHOLD;
1802 	}
1803 
1804 	rps->ei = now;
1805 	return events;
1806 }
1807 
1808 static void rps_work(struct work_struct *work)
1809 {
1810 	struct intel_rps *rps = container_of(work, typeof(*rps), work);
1811 	struct intel_gt *gt = rps_to_gt(rps);
1812 	struct drm_i915_private *i915 = rps_to_i915(rps);
1813 	bool client_boost = false;
1814 	int new_freq, adj, min, max;
1815 	u32 pm_iir = 0;
1816 
1817 	spin_lock_irq(gt->irq_lock);
1818 	pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1819 	client_boost = atomic_read(&rps->num_waiters);
1820 	spin_unlock_irq(gt->irq_lock);
1821 
1822 	/* Make sure we didn't queue anything we're not going to process. */
1823 	if (!pm_iir && !client_boost)
1824 		goto out;
1825 
1826 	mutex_lock(&rps->lock);
1827 	if (!intel_rps_is_active(rps)) {
1828 		mutex_unlock(&rps->lock);
1829 		return;
1830 	}
1831 
1832 	pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1833 
1834 	adj = rps->last_adj;
1835 	new_freq = rps->cur_freq;
1836 	min = rps->min_freq_softlimit;
1837 	max = rps->max_freq_softlimit;
1838 	if (client_boost)
1839 		max = rps->max_freq;
1840 
1841 	GT_TRACE(gt,
1842 		 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1843 		 pm_iir, str_yes_no(client_boost),
1844 		 adj, new_freq, min, max);
1845 
1846 	if (client_boost && new_freq < rps->boost_freq) {
1847 		new_freq = rps->boost_freq;
1848 		adj = 0;
1849 	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1850 		if (adj > 0)
1851 			adj *= 2;
1852 		else /* CHV needs even encode values */
1853 			adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1854 
1855 		if (new_freq >= rps->max_freq_softlimit)
1856 			adj = 0;
1857 	} else if (client_boost) {
1858 		adj = 0;
1859 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1860 		if (rps->cur_freq > rps->efficient_freq)
1861 			new_freq = rps->efficient_freq;
1862 		else if (rps->cur_freq > rps->min_freq_softlimit)
1863 			new_freq = rps->min_freq_softlimit;
1864 		adj = 0;
1865 	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1866 		if (adj < 0)
1867 			adj *= 2;
1868 		else /* CHV needs even encode values */
1869 			adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1870 
1871 		if (new_freq <= rps->min_freq_softlimit)
1872 			adj = 0;
1873 	} else { /* unknown event */
1874 		adj = 0;
1875 	}
1876 
1877 	/*
1878 	 * sysfs frequency limits may have snuck in while
1879 	 * servicing the interrupt
1880 	 */
1881 	new_freq += adj;
1882 	new_freq = clamp_t(int, new_freq, min, max);
1883 
1884 	if (intel_rps_set(rps, new_freq)) {
1885 		drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1886 		adj = 0;
1887 	}
1888 	rps->last_adj = adj;
1889 
1890 	mutex_unlock(&rps->lock);
1891 
1892 out:
1893 	spin_lock_irq(gt->irq_lock);
1894 	gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1895 	spin_unlock_irq(gt->irq_lock);
1896 }
1897 
1898 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1899 {
1900 	struct intel_gt *gt = rps_to_gt(rps);
1901 	const u32 events = rps->pm_events & pm_iir;
1902 
1903 	lockdep_assert_held(gt->irq_lock);
1904 
1905 	if (unlikely(!events))
1906 		return;
1907 
1908 	GT_TRACE(gt, "irq events:%x\n", events);
1909 
1910 	gen6_gt_pm_mask_irq(gt, events);
1911 
1912 	rps->pm_iir |= events;
1913 	queue_work(gt->i915->unordered_wq, &rps->work);
1914 }
1915 
1916 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1917 {
1918 	struct intel_gt *gt = rps_to_gt(rps);
1919 	u32 events;
1920 
1921 	events = pm_iir & rps->pm_events;
1922 	if (events) {
1923 		spin_lock(gt->irq_lock);
1924 
1925 		GT_TRACE(gt, "irq events:%x\n", events);
1926 
1927 		gen6_gt_pm_mask_irq(gt, events);
1928 		rps->pm_iir |= events;
1929 
1930 		queue_work(gt->i915->unordered_wq, &rps->work);
1931 		spin_unlock(gt->irq_lock);
1932 	}
1933 
1934 	if (GRAPHICS_VER(gt->i915) >= 8)
1935 		return;
1936 
1937 	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1938 		intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1939 
1940 	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1941 		drm_dbg(&rps_to_i915(rps)->drm,
1942 			"Command parser error, pm_iir 0x%08x\n", pm_iir);
1943 }
1944 
1945 void gen5_rps_irq_handler(struct intel_rps *rps)
1946 {
1947 	struct intel_uncore *uncore = rps_to_uncore(rps);
1948 	u32 busy_up, busy_down, max_avg, min_avg;
1949 	u8 new_freq;
1950 
1951 	spin_lock(&mchdev_lock);
1952 
1953 	intel_uncore_write16(uncore,
1954 			     MEMINTRSTS,
1955 			     intel_uncore_read(uncore, MEMINTRSTS));
1956 
1957 	intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1958 	busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1959 	busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1960 	max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1961 	min_avg = intel_uncore_read(uncore, RCBMINAVG);
1962 
1963 	/* Handle RCS change request from hw */
1964 	new_freq = rps->cur_freq;
1965 	if (busy_up > max_avg)
1966 		new_freq++;
1967 	else if (busy_down < min_avg)
1968 		new_freq--;
1969 	new_freq = clamp(new_freq,
1970 			 rps->min_freq_softlimit,
1971 			 rps->max_freq_softlimit);
1972 
1973 	if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1974 		rps->cur_freq = new_freq;
1975 
1976 	spin_unlock(&mchdev_lock);
1977 }
1978 
1979 void intel_rps_init_early(struct intel_rps *rps)
1980 {
1981 	mutex_init(&rps->lock);
1982 	mutex_init(&rps->power.mutex);
1983 
1984 	INIT_WORK(&rps->work, rps_work);
1985 	timer_setup(&rps->timer, rps_timer, 0);
1986 
1987 	atomic_set(&rps->num_waiters, 0);
1988 }
1989 
1990 void intel_rps_init(struct intel_rps *rps)
1991 {
1992 	struct drm_i915_private *i915 = rps_to_i915(rps);
1993 
1994 	if (rps_uses_slpc(rps))
1995 		return;
1996 
1997 	if (IS_CHERRYVIEW(i915))
1998 		chv_rps_init(rps);
1999 	else if (IS_VALLEYVIEW(i915))
2000 		vlv_rps_init(rps);
2001 	else if (GRAPHICS_VER(i915) >= 6)
2002 		gen6_rps_init(rps);
2003 	else if (IS_IRONLAKE_M(i915))
2004 		gen5_rps_init(rps);
2005 
2006 	/* Derive initial user preferences/limits from the hardware limits */
2007 	rps->max_freq_softlimit = rps->max_freq;
2008 	rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
2009 	rps->min_freq_softlimit = rps->min_freq;
2010 	rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
2011 
2012 	/* After setting max-softlimit, find the overclock max freq */
2013 	if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
2014 		u32 params = 0;
2015 
2016 		snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, &params, NULL);
2017 		if (params & BIT(31)) { /* OC supported */
2018 			drm_dbg(&i915->drm,
2019 				"Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2020 				(rps->max_freq & 0xff) * 50,
2021 				(params & 0xff) * 50);
2022 			rps->max_freq = params & 0xff;
2023 		}
2024 	}
2025 
2026 	/* Set default thresholds in % */
2027 	rps->power.up_threshold = 95;
2028 	rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold;
2029 	rps->power.down_threshold = 85;
2030 	rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold;
2031 
2032 	/* Finally allow us to boost to max by default */
2033 	rps->boost_freq = rps->max_freq;
2034 	rps->idle_freq = rps->min_freq;
2035 
2036 	/* Start in the middle, from here we will autotune based on workload */
2037 	rps->cur_freq = rps->efficient_freq;
2038 
2039 	rps->pm_intrmsk_mbz = 0;
2040 
2041 	/*
2042 	 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2043 	 * if GEN6_PM_UP_EI_EXPIRED is masked.
2044 	 *
2045 	 * TODO: verify if this can be reproduced on VLV,CHV.
2046 	 */
2047 	if (GRAPHICS_VER(i915) <= 7)
2048 		rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2049 
2050 	if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2051 		rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2052 
2053 	/* GuC needs ARAT expired interrupt unmasked */
2054 	if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2055 		rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2056 }
2057 
2058 void intel_rps_sanitize(struct intel_rps *rps)
2059 {
2060 	if (rps_uses_slpc(rps))
2061 		return;
2062 
2063 	if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2064 		rps_disable_interrupts(rps);
2065 }
2066 
2067 u32 intel_rps_read_rpstat(struct intel_rps *rps)
2068 {
2069 	struct drm_i915_private *i915 = rps_to_i915(rps);
2070 	i915_reg_t rpstat;
2071 
2072 	rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;
2073 
2074 	return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat);
2075 }
2076 
2077 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2078 {
2079 	struct drm_i915_private *i915 = rps_to_i915(rps);
2080 	u32 cagf;
2081 
2082 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2083 		cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);
2084 	else if (GRAPHICS_VER(i915) >= 12)
2085 		cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);
2086 	else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2087 		cagf = REG_FIELD_GET(RPE_MASK, rpstat);
2088 	else if (GRAPHICS_VER(i915) >= 9)
2089 		cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);
2090 	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2091 		cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);
2092 	else if (GRAPHICS_VER(i915) >= 6)
2093 		cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);
2094 	else
2095 		cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));
2096 
2097 	return cagf;
2098 }
2099 
2100 static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
2101 {
2102 	struct drm_i915_private *i915 = rps_to_i915(rps);
2103 	struct intel_uncore *uncore = rps_to_uncore(rps);
2104 	i915_reg_t r = INVALID_MMIO_REG;
2105 	u32 freq;
2106 
2107 	/*
2108 	 * For Gen12+ reading freq from HW does not need a forcewake and
2109 	 * registers will return 0 freq when GT is in RC6
2110 	 */
2111 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2112 		r = MTL_MIRROR_TARGET_WP1;
2113 	} else if (GRAPHICS_VER(i915) >= 12) {
2114 		r = GEN12_RPSTAT1;
2115 	} else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2116 		vlv_punit_get(i915);
2117 		freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2118 		vlv_punit_put(i915);
2119 	} else if (GRAPHICS_VER(i915) >= 6) {
2120 		r = GEN6_RPSTAT1;
2121 	} else {
2122 		r = MEMSTAT_ILK;
2123 	}
2124 
2125 	if (i915_mmio_reg_valid(r))
2126 		freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r);
2127 
2128 	return intel_rps_get_cagf(rps, freq);
2129 }
2130 
2131 static u32 read_cagf(struct intel_rps *rps)
2132 {
2133 	return __read_cagf(rps, true);
2134 }
2135 
2136 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2137 {
2138 	struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2139 	intel_wakeref_t wakeref;
2140 	u32 freq = 0;
2141 
2142 	with_intel_runtime_pm_if_in_use(rpm, wakeref)
2143 		freq = intel_gpu_freq(rps, read_cagf(rps));
2144 
2145 	return freq;
2146 }
2147 
2148 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
2149 {
2150 	return intel_gpu_freq(rps, __read_cagf(rps, false));
2151 }
2152 
2153 static u32 intel_rps_read_punit_req(struct intel_rps *rps)
2154 {
2155 	struct intel_uncore *uncore = rps_to_uncore(rps);
2156 	struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2157 	intel_wakeref_t wakeref;
2158 	u32 freq = 0;
2159 
2160 	with_intel_runtime_pm_if_in_use(rpm, wakeref)
2161 		freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2162 
2163 	return freq;
2164 }
2165 
2166 static u32 intel_rps_get_req(u32 pureq)
2167 {
2168 	u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2169 
2170 	return req;
2171 }
2172 
2173 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2174 {
2175 	u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2176 
2177 	return intel_gpu_freq(rps, freq);
2178 }
2179 
2180 u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2181 {
2182 	if (rps_uses_slpc(rps))
2183 		return intel_rps_read_punit_req_frequency(rps);
2184 	else
2185 		return intel_gpu_freq(rps, rps->cur_freq);
2186 }
2187 
2188 u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2189 {
2190 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2191 
2192 	if (rps_uses_slpc(rps))
2193 		return slpc->max_freq_softlimit;
2194 	else
2195 		return intel_gpu_freq(rps, rps->max_freq_softlimit);
2196 }
2197 
2198 /**
2199  * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2200  * @rps: the intel_rps structure
2201  *
2202  * Returns the max frequency in a raw format. In newer platforms raw is in
2203  * units of 50 MHz.
2204  */
2205 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2206 {
2207 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2208 	u32 freq;
2209 
2210 	if (rps_uses_slpc(rps)) {
2211 		return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2212 					 GT_FREQUENCY_MULTIPLIER);
2213 	} else {
2214 		freq = rps->max_freq;
2215 		if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2216 			/* Convert GT frequency to 50 MHz units */
2217 			freq /= GEN9_FREQ_SCALER;
2218 		}
2219 		return freq;
2220 	}
2221 }
2222 
2223 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2224 {
2225 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2226 
2227 	if (rps_uses_slpc(rps))
2228 		return slpc->rp0_freq;
2229 	else
2230 		return intel_gpu_freq(rps, rps->rp0_freq);
2231 }
2232 
2233 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2234 {
2235 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2236 
2237 	if (rps_uses_slpc(rps))
2238 		return slpc->rp1_freq;
2239 	else
2240 		return intel_gpu_freq(rps, rps->rp1_freq);
2241 }
2242 
2243 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2244 {
2245 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2246 
2247 	if (rps_uses_slpc(rps))
2248 		return slpc->min_freq;
2249 	else
2250 		return intel_gpu_freq(rps, rps->min_freq);
2251 }
2252 
2253 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2254 {
2255 	struct intel_gt *gt = rps_to_gt(rps);
2256 	struct drm_i915_private *i915 = gt->i915;
2257 	struct intel_uncore *uncore = gt->uncore;
2258 	struct intel_rps_freq_caps caps;
2259 	u32 rp_state_limits;
2260 	u32 gt_perf_status;
2261 	u32 rpmodectl, rpinclimit, rpdeclimit;
2262 	u32 rpstat, cagf, reqf;
2263 	u32 rpcurupei, rpcurup, rpprevup;
2264 	u32 rpcurdownei, rpcurdown, rpprevdown;
2265 	u32 rpupei, rpupt, rpdownei, rpdownt;
2266 	u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
2267 
2268 	rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);
2269 	gen6_rps_get_freq_caps(rps, &caps);
2270 	if (IS_GEN9_LP(i915))
2271 		gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);
2272 	else
2273 		gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);
2274 
2275 	/* RPSTAT1 is in the GT power well */
2276 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2277 
2278 	reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2279 	if (GRAPHICS_VER(i915) >= 9) {
2280 		reqf >>= 23;
2281 	} else {
2282 		reqf &= ~GEN6_TURBO_DISABLE;
2283 		if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2284 			reqf >>= 24;
2285 		else
2286 			reqf >>= 25;
2287 	}
2288 	reqf = intel_gpu_freq(rps, reqf);
2289 
2290 	rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);
2291 	rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2292 	rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2293 
2294 	rpstat = intel_rps_read_rpstat(rps);
2295 	rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;
2296 	rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;
2297 	rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;
2298 	rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
2299 	rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
2300 	rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
2301 
2302 	rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);
2303 	rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2304 
2305 	rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);
2306 	rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2307 
2308 	cagf = intel_rps_read_actual_frequency(rps);
2309 
2310 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2311 
2312 	if (GRAPHICS_VER(i915) >= 11) {
2313 		pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);
2314 		pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);
2315 		/*
2316 		 * The equivalent to the PM ISR & IIR cannot be read
2317 		 * without affecting the current state of the system
2318 		 */
2319 		pm_isr = 0;
2320 		pm_iir = 0;
2321 	} else if (GRAPHICS_VER(i915) >= 8) {
2322 		pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));
2323 		pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));
2324 		pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));
2325 		pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));
2326 	} else {
2327 		pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
2328 		pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
2329 		pm_isr = intel_uncore_read(uncore, GEN6_PMISR);
2330 		pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);
2331 	}
2332 	pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2333 
2334 	drm_printf(p, "Video Turbo Mode: %s\n",
2335 		   str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO));
2336 	drm_printf(p, "HW control enabled: %s\n",
2337 		   str_yes_no(rpmodectl & GEN6_RP_ENABLE));
2338 	drm_printf(p, "SW control enabled: %s\n",
2339 		   str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));
2340 
2341 	drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",
2342 		   pm_ier, pm_imr, pm_mask);
2343 	if (GRAPHICS_VER(i915) <= 10)
2344 		drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n",
2345 			   pm_isr, pm_iir);
2346 	drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2347 		   rps->pm_intrmsk_mbz);
2348 	drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
2349 	drm_printf(p, "Render p-state ratio: %d\n",
2350 		   (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
2351 	drm_printf(p, "Render p-state VID: %d\n",
2352 		   gt_perf_status & 0xff);
2353 	drm_printf(p, "Render p-state limit: %d\n",
2354 		   rp_state_limits & 0xff);
2355 	drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat);
2356 	drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl);
2357 	drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit);
2358 	drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit);
2359 	drm_printf(p, "RPNSWREQ: %dMHz\n", reqf);
2360 	drm_printf(p, "CAGF: %dMHz\n", cagf);
2361 	drm_printf(p, "RP CUR UP EI: %d (%lldns)\n",
2362 		   rpcurupei,
2363 		   intel_gt_pm_interval_to_ns(gt, rpcurupei));
2364 	drm_printf(p, "RP CUR UP: %d (%lldns)\n",
2365 		   rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup));
2366 	drm_printf(p, "RP PREV UP: %d (%lldns)\n",
2367 		   rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup));
2368 	drm_printf(p, "Up threshold: %d%%\n",
2369 		   rps->power.up_threshold);
2370 	drm_printf(p, "RP UP EI: %d (%lldns)\n",
2371 		   rpupei, intel_gt_pm_interval_to_ns(gt, rpupei));
2372 	drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n",
2373 		   rpupt, intel_gt_pm_interval_to_ns(gt, rpupt));
2374 
2375 	drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n",
2376 		   rpcurdownei,
2377 		   intel_gt_pm_interval_to_ns(gt, rpcurdownei));
2378 	drm_printf(p, "RP CUR DOWN: %d (%lldns)\n",
2379 		   rpcurdown,
2380 		   intel_gt_pm_interval_to_ns(gt, rpcurdown));
2381 	drm_printf(p, "RP PREV DOWN: %d (%lldns)\n",
2382 		   rpprevdown,
2383 		   intel_gt_pm_interval_to_ns(gt, rpprevdown));
2384 	drm_printf(p, "Down threshold: %d%%\n",
2385 		   rps->power.down_threshold);
2386 	drm_printf(p, "RP DOWN EI: %d (%lldns)\n",
2387 		   rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei));
2388 	drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n",
2389 		   rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt));
2390 
2391 	drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2392 		   intel_gpu_freq(rps, caps.min_freq));
2393 	drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2394 		   intel_gpu_freq(rps, caps.rp1_freq));
2395 	drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2396 		   intel_gpu_freq(rps, caps.rp0_freq));
2397 	drm_printf(p, "Max overclocked frequency: %dMHz\n",
2398 		   intel_gpu_freq(rps, rps->max_freq));
2399 
2400 	drm_printf(p, "Current freq: %d MHz\n",
2401 		   intel_gpu_freq(rps, rps->cur_freq));
2402 	drm_printf(p, "Actual freq: %d MHz\n", cagf);
2403 	drm_printf(p, "Idle freq: %d MHz\n",
2404 		   intel_gpu_freq(rps, rps->idle_freq));
2405 	drm_printf(p, "Min freq: %d MHz\n",
2406 		   intel_gpu_freq(rps, rps->min_freq));
2407 	drm_printf(p, "Boost freq: %d MHz\n",
2408 		   intel_gpu_freq(rps, rps->boost_freq));
2409 	drm_printf(p, "Max freq: %d MHz\n",
2410 		   intel_gpu_freq(rps, rps->max_freq));
2411 	drm_printf(p,
2412 		   "efficient (RPe) frequency: %d MHz\n",
2413 		   intel_gpu_freq(rps, rps->efficient_freq));
2414 }
2415 
2416 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2417 {
2418 	struct intel_gt *gt = rps_to_gt(rps);
2419 	struct intel_uncore *uncore = gt->uncore;
2420 	struct intel_rps_freq_caps caps;
2421 	u32 pm_mask;
2422 
2423 	gen6_rps_get_freq_caps(rps, &caps);
2424 	pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2425 
2426 	drm_printf(p, "PM MASK=0x%08x\n", pm_mask);
2427 	drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2428 		   rps->pm_intrmsk_mbz);
2429 	drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));
2430 	drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));
2431 	drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2432 		   intel_gpu_freq(rps, caps.min_freq));
2433 	drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2434 		   intel_gpu_freq(rps, caps.rp1_freq));
2435 	drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2436 		   intel_gpu_freq(rps, caps.rp0_freq));
2437 	drm_printf(p, "Current freq: %d MHz\n",
2438 		   intel_rps_get_requested_frequency(rps));
2439 	drm_printf(p, "Actual freq: %d MHz\n",
2440 		   intel_rps_read_actual_frequency(rps));
2441 	drm_printf(p, "Min freq: %d MHz\n",
2442 		   intel_rps_get_min_frequency(rps));
2443 	drm_printf(p, "Boost freq: %d MHz\n",
2444 		   intel_rps_get_boost_frequency(rps));
2445 	drm_printf(p, "Max freq: %d MHz\n",
2446 		   intel_rps_get_max_frequency(rps));
2447 	drm_printf(p,
2448 		   "efficient (RPe) frequency: %d MHz\n",
2449 		   intel_gpu_freq(rps, caps.rp1_freq));
2450 }
2451 
2452 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2453 {
2454 	if (rps_uses_slpc(rps))
2455 		return slpc_frequency_dump(rps, p);
2456 	else
2457 		return rps_frequency_dump(rps, p);
2458 }
2459 
2460 static int set_max_freq(struct intel_rps *rps, u32 val)
2461 {
2462 	struct drm_i915_private *i915 = rps_to_i915(rps);
2463 	int ret = 0;
2464 
2465 	mutex_lock(&rps->lock);
2466 
2467 	val = intel_freq_opcode(rps, val);
2468 	if (val < rps->min_freq ||
2469 	    val > rps->max_freq ||
2470 	    val < rps->min_freq_softlimit) {
2471 		ret = -EINVAL;
2472 		goto unlock;
2473 	}
2474 
2475 	if (val > rps->rp0_freq)
2476 		drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2477 			intel_gpu_freq(rps, val));
2478 
2479 	rps->max_freq_softlimit = val;
2480 
2481 	val = clamp_t(int, rps->cur_freq,
2482 		      rps->min_freq_softlimit,
2483 		      rps->max_freq_softlimit);
2484 
2485 	/*
2486 	 * We still need *_set_rps to process the new max_delay and
2487 	 * update the interrupt limits and PMINTRMSK even though
2488 	 * frequency request may be unchanged.
2489 	 */
2490 	intel_rps_set(rps, val);
2491 
2492 unlock:
2493 	mutex_unlock(&rps->lock);
2494 
2495 	return ret;
2496 }
2497 
2498 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2499 {
2500 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2501 
2502 	if (rps_uses_slpc(rps))
2503 		return intel_guc_slpc_set_max_freq(slpc, val);
2504 	else
2505 		return set_max_freq(rps, val);
2506 }
2507 
2508 u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2509 {
2510 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2511 
2512 	if (rps_uses_slpc(rps))
2513 		return slpc->min_freq_softlimit;
2514 	else
2515 		return intel_gpu_freq(rps, rps->min_freq_softlimit);
2516 }
2517 
2518 /**
2519  * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2520  * @rps: the intel_rps structure
2521  *
2522  * Returns the min frequency in a raw format. In newer platforms raw is in
2523  * units of 50 MHz.
2524  */
2525 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2526 {
2527 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2528 	u32 freq;
2529 
2530 	if (rps_uses_slpc(rps)) {
2531 		return DIV_ROUND_CLOSEST(slpc->min_freq,
2532 					 GT_FREQUENCY_MULTIPLIER);
2533 	} else {
2534 		freq = rps->min_freq;
2535 		if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2536 			/* Convert GT frequency to 50 MHz units */
2537 			freq /= GEN9_FREQ_SCALER;
2538 		}
2539 		return freq;
2540 	}
2541 }
2542 
2543 static int set_min_freq(struct intel_rps *rps, u32 val)
2544 {
2545 	int ret = 0;
2546 
2547 	mutex_lock(&rps->lock);
2548 
2549 	val = intel_freq_opcode(rps, val);
2550 	if (val < rps->min_freq ||
2551 	    val > rps->max_freq ||
2552 	    val > rps->max_freq_softlimit) {
2553 		ret = -EINVAL;
2554 		goto unlock;
2555 	}
2556 
2557 	rps->min_freq_softlimit = val;
2558 
2559 	val = clamp_t(int, rps->cur_freq,
2560 		      rps->min_freq_softlimit,
2561 		      rps->max_freq_softlimit);
2562 
2563 	/*
2564 	 * We still need *_set_rps to process the new min_delay and
2565 	 * update the interrupt limits and PMINTRMSK even though
2566 	 * frequency request may be unchanged.
2567 	 */
2568 	intel_rps_set(rps, val);
2569 
2570 unlock:
2571 	mutex_unlock(&rps->lock);
2572 
2573 	return ret;
2574 }
2575 
2576 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2577 {
2578 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2579 
2580 	if (rps_uses_slpc(rps))
2581 		return intel_guc_slpc_set_min_freq(slpc, val);
2582 	else
2583 		return set_min_freq(rps, val);
2584 }
2585 
2586 u8 intel_rps_get_up_threshold(struct intel_rps *rps)
2587 {
2588 	return rps->power.up_threshold;
2589 }
2590 
2591 static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val)
2592 {
2593 	int ret;
2594 
2595 	if (val > 100)
2596 		return -EINVAL;
2597 
2598 	ret = mutex_lock_interruptible(&rps->lock);
2599 	if (ret)
2600 		return ret;
2601 
2602 	if (*threshold == val)
2603 		goto out_unlock;
2604 
2605 	*threshold = val;
2606 
2607 	/* Force reset. */
2608 	rps->last_freq = -1;
2609 	mutex_lock(&rps->power.mutex);
2610 	rps->power.mode = -1;
2611 	mutex_unlock(&rps->power.mutex);
2612 
2613 	intel_rps_set(rps, clamp(rps->cur_freq,
2614 				 rps->min_freq_softlimit,
2615 				 rps->max_freq_softlimit));
2616 
2617 out_unlock:
2618 	mutex_unlock(&rps->lock);
2619 
2620 	return ret;
2621 }
2622 
2623 int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold)
2624 {
2625 	return rps_set_threshold(rps, &rps->power.up_threshold, threshold);
2626 }
2627 
2628 u8 intel_rps_get_down_threshold(struct intel_rps *rps)
2629 {
2630 	return rps->power.down_threshold;
2631 }
2632 
2633 int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold)
2634 {
2635 	return rps_set_threshold(rps, &rps->power.down_threshold, threshold);
2636 }
2637 
2638 static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2639 {
2640 	struct intel_uncore *uncore = rps_to_uncore(rps);
2641 	u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2642 
2643 	/* Allow punit to process software requests */
2644 	intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2645 }
2646 
2647 void intel_rps_raise_unslice(struct intel_rps *rps)
2648 {
2649 	struct intel_uncore *uncore = rps_to_uncore(rps);
2650 
2651 	mutex_lock(&rps->lock);
2652 
2653 	if (rps_uses_slpc(rps)) {
2654 		/* RP limits have not been initialized yet for SLPC path */
2655 		struct intel_rps_freq_caps caps;
2656 
2657 		gen6_rps_get_freq_caps(rps, &caps);
2658 
2659 		intel_rps_set_manual(rps, true);
2660 		intel_uncore_write(uncore, GEN6_RPNSWREQ,
2661 				   ((caps.rp0_freq <<
2662 				   GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2663 				   GEN9_IGNORE_SLICE_RATIO));
2664 		intel_rps_set_manual(rps, false);
2665 	} else {
2666 		intel_rps_set(rps, rps->rp0_freq);
2667 	}
2668 
2669 	mutex_unlock(&rps->lock);
2670 }
2671 
2672 void intel_rps_lower_unslice(struct intel_rps *rps)
2673 {
2674 	struct intel_uncore *uncore = rps_to_uncore(rps);
2675 
2676 	mutex_lock(&rps->lock);
2677 
2678 	if (rps_uses_slpc(rps)) {
2679 		/* RP limits have not been initialized yet for SLPC path */
2680 		struct intel_rps_freq_caps caps;
2681 
2682 		gen6_rps_get_freq_caps(rps, &caps);
2683 
2684 		intel_rps_set_manual(rps, true);
2685 		intel_uncore_write(uncore, GEN6_RPNSWREQ,
2686 				   ((caps.min_freq <<
2687 				   GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2688 				   GEN9_IGNORE_SLICE_RATIO));
2689 		intel_rps_set_manual(rps, false);
2690 	} else {
2691 		intel_rps_set(rps, rps->min_freq);
2692 	}
2693 
2694 	mutex_unlock(&rps->lock);
2695 }
2696 
2697 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2698 {
2699 	struct intel_gt *gt = rps_to_gt(rps);
2700 	intel_wakeref_t wakeref;
2701 	u32 val;
2702 
2703 	with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2704 		val = intel_uncore_read(gt->uncore, reg32);
2705 
2706 	return val;
2707 }
2708 
2709 bool rps_read_mask_mmio(struct intel_rps *rps,
2710 			i915_reg_t reg32, u32 mask)
2711 {
2712 	return rps_read_mmio(rps, reg32) & mask;
2713 }
2714 
2715 /* External interface for intel_ips.ko */
2716 
2717 static struct drm_i915_private __rcu *ips_mchdev;
2718 
2719 /*
2720  * Tells the intel_ips driver that the i915 driver is now loaded, if
2721  * IPS got loaded first.
2722  *
2723  * This awkward dance is so that neither module has to depend on the
2724  * other in order for IPS to do the appropriate communication of
2725  * GPU turbo limits to i915.
2726  */
2727 static void
2728 ips_ping_for_i915_load(void)
2729 {
2730 	void (*link)(void);
2731 
2732 	link = symbol_get(ips_link_to_i915_driver);
2733 	if (link) {
2734 		link();
2735 		symbol_put(ips_link_to_i915_driver);
2736 	}
2737 }
2738 
2739 void intel_rps_driver_register(struct intel_rps *rps)
2740 {
2741 	struct intel_gt *gt = rps_to_gt(rps);
2742 
2743 	/*
2744 	 * We only register the i915 ips part with intel-ips once everything is
2745 	 * set up, to avoid intel-ips sneaking in and reading bogus values.
2746 	 */
2747 	if (GRAPHICS_VER(gt->i915) == 5) {
2748 		GEM_BUG_ON(ips_mchdev);
2749 		rcu_assign_pointer(ips_mchdev, gt->i915);
2750 		ips_ping_for_i915_load();
2751 	}
2752 }
2753 
2754 void intel_rps_driver_unregister(struct intel_rps *rps)
2755 {
2756 	if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2757 		rcu_assign_pointer(ips_mchdev, NULL);
2758 }
2759 
2760 static struct drm_i915_private *mchdev_get(void)
2761 {
2762 	struct drm_i915_private *i915;
2763 
2764 	rcu_read_lock();
2765 	i915 = rcu_dereference(ips_mchdev);
2766 	if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2767 		i915 = NULL;
2768 	rcu_read_unlock();
2769 
2770 	return i915;
2771 }
2772 
2773 /**
2774  * i915_read_mch_val - return value for IPS use
2775  *
2776  * Calculate and return a value for the IPS driver to use when deciding whether
2777  * we have thermal and power headroom to increase CPU or GPU power budget.
2778  */
2779 unsigned long i915_read_mch_val(void)
2780 {
2781 	struct drm_i915_private *i915;
2782 	unsigned long chipset_val = 0;
2783 	unsigned long graphics_val = 0;
2784 	intel_wakeref_t wakeref;
2785 
2786 	i915 = mchdev_get();
2787 	if (!i915)
2788 		return 0;
2789 
2790 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2791 		struct intel_ips *ips = &to_gt(i915)->rps.ips;
2792 
2793 		spin_lock_irq(&mchdev_lock);
2794 		chipset_val = __ips_chipset_val(ips);
2795 		graphics_val = __ips_gfx_val(ips);
2796 		spin_unlock_irq(&mchdev_lock);
2797 	}
2798 
2799 	drm_dev_put(&i915->drm);
2800 	return chipset_val + graphics_val;
2801 }
2802 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2803 
2804 /**
2805  * i915_gpu_raise - raise GPU frequency limit
2806  *
2807  * Raise the limit; IPS indicates we have thermal headroom.
2808  */
2809 bool i915_gpu_raise(void)
2810 {
2811 	struct drm_i915_private *i915;
2812 	struct intel_rps *rps;
2813 
2814 	i915 = mchdev_get();
2815 	if (!i915)
2816 		return false;
2817 
2818 	rps = &to_gt(i915)->rps;
2819 
2820 	spin_lock_irq(&mchdev_lock);
2821 	if (rps->max_freq_softlimit < rps->max_freq)
2822 		rps->max_freq_softlimit++;
2823 	spin_unlock_irq(&mchdev_lock);
2824 
2825 	drm_dev_put(&i915->drm);
2826 	return true;
2827 }
2828 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2829 
2830 /**
2831  * i915_gpu_lower - lower GPU frequency limit
2832  *
2833  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2834  * frequency maximum.
2835  */
2836 bool i915_gpu_lower(void)
2837 {
2838 	struct drm_i915_private *i915;
2839 	struct intel_rps *rps;
2840 
2841 	i915 = mchdev_get();
2842 	if (!i915)
2843 		return false;
2844 
2845 	rps = &to_gt(i915)->rps;
2846 
2847 	spin_lock_irq(&mchdev_lock);
2848 	if (rps->max_freq_softlimit > rps->min_freq)
2849 		rps->max_freq_softlimit--;
2850 	spin_unlock_irq(&mchdev_lock);
2851 
2852 	drm_dev_put(&i915->drm);
2853 	return true;
2854 }
2855 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2856 
2857 /**
2858  * i915_gpu_busy - indicate GPU business to IPS
2859  *
2860  * Tell the IPS driver whether or not the GPU is busy.
2861  */
2862 bool i915_gpu_busy(void)
2863 {
2864 	struct drm_i915_private *i915;
2865 	bool ret;
2866 
2867 	i915 = mchdev_get();
2868 	if (!i915)
2869 		return false;
2870 
2871 	ret = to_gt(i915)->awake;
2872 
2873 	drm_dev_put(&i915->drm);
2874 	return ret;
2875 }
2876 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2877 
2878 /**
2879  * i915_gpu_turbo_disable - disable graphics turbo
2880  *
2881  * Disable graphics turbo by resetting the max frequency and setting the
2882  * current frequency to the default.
2883  */
2884 bool i915_gpu_turbo_disable(void)
2885 {
2886 	struct drm_i915_private *i915;
2887 	struct intel_rps *rps;
2888 	bool ret;
2889 
2890 	i915 = mchdev_get();
2891 	if (!i915)
2892 		return false;
2893 
2894 	rps = &to_gt(i915)->rps;
2895 
2896 	spin_lock_irq(&mchdev_lock);
2897 	rps->max_freq_softlimit = rps->min_freq;
2898 	ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2899 	spin_unlock_irq(&mchdev_lock);
2900 
2901 	drm_dev_put(&i915->drm);
2902 	return ret;
2903 }
2904 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2905 
2906 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2907 #include "selftest_rps.c"
2908 #include "selftest_slpc.c"
2909 #endif
2910