1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include <drm/drm_atomic_state_helper.h>
7
8 #include "i915_drv.h"
9 #include "i915_reg.h"
10 #include "i915_utils.h"
11 #include "intel_atomic.h"
12 #include "intel_bw.h"
13 #include "intel_cdclk.h"
14 #include "intel_display_core.h"
15 #include "intel_display_types.h"
16 #include "skl_watermark.h"
17 #include "intel_mchbar_regs.h"
18 #include "intel_pcode.h"
19
20 /* Parameters for Qclk Geyserville (QGV) */
21 struct intel_qgv_point {
22 u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
23 };
24
25 #define DEPROGBWPCLIMIT 60
26
27 struct intel_psf_gv_point {
28 u8 clk; /* clock in multiples of 16.6666 MHz */
29 };
30
31 struct intel_qgv_info {
32 struct intel_qgv_point points[I915_NUM_QGV_POINTS];
33 struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
34 u8 num_points;
35 u8 num_psf_points;
36 u8 t_bl;
37 u8 max_numchannels;
38 u8 channel_width;
39 u8 deinterleave;
40 };
41
dg1_mchbar_read_qgv_point_info(struct intel_display * display,struct intel_qgv_point * sp,int point)42 static int dg1_mchbar_read_qgv_point_info(struct intel_display *display,
43 struct intel_qgv_point *sp,
44 int point)
45 {
46 struct drm_i915_private *i915 = to_i915(display->drm);
47 u32 dclk_ratio, dclk_reference;
48 u32 val;
49
50 val = intel_uncore_read(&i915->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
51 dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
52 if (val & DG1_QCLK_REFERENCE)
53 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
54 else
55 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
56 sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
57
58 val = intel_uncore_read(&i915->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
59 if (val & DG1_GEAR_TYPE)
60 sp->dclk *= 2;
61
62 if (sp->dclk == 0)
63 return -EINVAL;
64
65 val = intel_uncore_read(&i915->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
66 sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
67 sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
68
69 val = intel_uncore_read(&i915->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
70 sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
71 sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
72
73 sp->t_rc = sp->t_rp + sp->t_ras;
74
75 return 0;
76 }
77
icl_pcode_read_qgv_point_info(struct intel_display * display,struct intel_qgv_point * sp,int point)78 static int icl_pcode_read_qgv_point_info(struct intel_display *display,
79 struct intel_qgv_point *sp,
80 int point)
81 {
82 struct drm_i915_private *i915 = to_i915(display->drm);
83 u32 val = 0, val2 = 0;
84 u16 dclk;
85 int ret;
86
87 ret = snb_pcode_read(&i915->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
88 ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
89 &val, &val2);
90 if (ret)
91 return ret;
92
93 dclk = val & 0xffff;
94 sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(display) >= 12 ? 500 : 0),
95 1000);
96 sp->t_rp = (val & 0xff0000) >> 16;
97 sp->t_rcd = (val & 0xff000000) >> 24;
98
99 sp->t_rdpre = val2 & 0xff;
100 sp->t_ras = (val2 & 0xff00) >> 8;
101
102 sp->t_rc = sp->t_rp + sp->t_ras;
103
104 return 0;
105 }
106
adls_pcode_read_psf_gv_point_info(struct intel_display * display,struct intel_psf_gv_point * points)107 static int adls_pcode_read_psf_gv_point_info(struct intel_display *display,
108 struct intel_psf_gv_point *points)
109 {
110 struct drm_i915_private *i915 = to_i915(display->drm);
111 u32 val = 0;
112 int ret;
113 int i;
114
115 ret = snb_pcode_read(&i915->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
116 ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
117 if (ret)
118 return ret;
119
120 for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
121 points[i].clk = val & 0xff;
122 val >>= 8;
123 }
124
125 return 0;
126 }
127
icl_qgv_points_mask(struct intel_display * display)128 static u16 icl_qgv_points_mask(struct intel_display *display)
129 {
130 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
131 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
132 u16 qgv_points = 0, psf_points = 0;
133
134 /*
135 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
136 * it with failure if we try masking any unadvertised points.
137 * So need to operate only with those returned from PCode.
138 */
139 if (num_qgv_points > 0)
140 qgv_points = GENMASK(num_qgv_points - 1, 0);
141
142 if (num_psf_gv_points > 0)
143 psf_points = GENMASK(num_psf_gv_points - 1, 0);
144
145 return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
146 }
147
is_sagv_enabled(struct intel_display * display,u16 points_mask)148 static bool is_sagv_enabled(struct intel_display *display, u16 points_mask)
149 {
150 return !is_power_of_2(~points_mask & icl_qgv_points_mask(display) &
151 ICL_PCODE_REQ_QGV_PT_MASK);
152 }
153
icl_pcode_restrict_qgv_points(struct intel_display * display,u32 points_mask)154 int icl_pcode_restrict_qgv_points(struct intel_display *display,
155 u32 points_mask)
156 {
157 struct drm_i915_private *i915 = to_i915(display->drm);
158 int ret;
159
160 if (DISPLAY_VER(display) >= 14)
161 return 0;
162
163 /* bspec says to keep retrying for at least 1 ms */
164 ret = skl_pcode_request(&i915->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
165 points_mask,
166 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
167 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
168 1);
169
170 if (ret < 0) {
171 drm_err(display->drm,
172 "Failed to disable qgv points (0x%x) points: 0x%x\n",
173 ret, points_mask);
174 return ret;
175 }
176
177 display->sagv.status = is_sagv_enabled(display, points_mask) ?
178 I915_SAGV_ENABLED : I915_SAGV_DISABLED;
179
180 return 0;
181 }
182
mtl_read_qgv_point_info(struct intel_display * display,struct intel_qgv_point * sp,int point)183 static int mtl_read_qgv_point_info(struct intel_display *display,
184 struct intel_qgv_point *sp, int point)
185 {
186 struct drm_i915_private *i915 = to_i915(display->drm);
187 u32 val, val2;
188 u16 dclk;
189
190 val = intel_uncore_read(&i915->uncore,
191 MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
192 val2 = intel_uncore_read(&i915->uncore,
193 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
194 dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
195 sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
196 sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
197 sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
198
199 sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
200 sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
201
202 sp->t_rc = sp->t_rp + sp->t_ras;
203
204 return 0;
205 }
206
207 static int
intel_read_qgv_point_info(struct intel_display * display,struct intel_qgv_point * sp,int point)208 intel_read_qgv_point_info(struct intel_display *display,
209 struct intel_qgv_point *sp,
210 int point)
211 {
212 if (DISPLAY_VER(display) >= 14)
213 return mtl_read_qgv_point_info(display, sp, point);
214 else if (display->platform.dg1)
215 return dg1_mchbar_read_qgv_point_info(display, sp, point);
216 else
217 return icl_pcode_read_qgv_point_info(display, sp, point);
218 }
219
icl_get_qgv_points(struct intel_display * display,struct intel_qgv_info * qi,bool is_y_tile)220 static int icl_get_qgv_points(struct intel_display *display,
221 struct intel_qgv_info *qi,
222 bool is_y_tile)
223 {
224 struct drm_i915_private *i915 = to_i915(display->drm);
225 const struct dram_info *dram_info = &i915->dram_info;
226 int i, ret;
227
228 qi->num_points = dram_info->num_qgv_points;
229 qi->num_psf_points = dram_info->num_psf_gv_points;
230
231 if (DISPLAY_VER(display) >= 14) {
232 switch (dram_info->type) {
233 case INTEL_DRAM_DDR4:
234 qi->t_bl = 4;
235 qi->max_numchannels = 2;
236 qi->channel_width = 64;
237 qi->deinterleave = 2;
238 break;
239 case INTEL_DRAM_DDR5:
240 qi->t_bl = 8;
241 qi->max_numchannels = 4;
242 qi->channel_width = 32;
243 qi->deinterleave = 2;
244 break;
245 case INTEL_DRAM_LPDDR4:
246 case INTEL_DRAM_LPDDR5:
247 qi->t_bl = 16;
248 qi->max_numchannels = 8;
249 qi->channel_width = 16;
250 qi->deinterleave = 4;
251 break;
252 case INTEL_DRAM_GDDR:
253 case INTEL_DRAM_GDDR_ECC:
254 qi->channel_width = 32;
255 break;
256 default:
257 MISSING_CASE(dram_info->type);
258 return -EINVAL;
259 }
260 } else if (DISPLAY_VER(display) >= 12) {
261 switch (dram_info->type) {
262 case INTEL_DRAM_DDR4:
263 qi->t_bl = is_y_tile ? 8 : 4;
264 qi->max_numchannels = 2;
265 qi->channel_width = 64;
266 qi->deinterleave = is_y_tile ? 1 : 2;
267 break;
268 case INTEL_DRAM_DDR5:
269 qi->t_bl = is_y_tile ? 16 : 8;
270 qi->max_numchannels = 4;
271 qi->channel_width = 32;
272 qi->deinterleave = is_y_tile ? 1 : 2;
273 break;
274 case INTEL_DRAM_LPDDR4:
275 if (display->platform.rocketlake) {
276 qi->t_bl = 8;
277 qi->max_numchannels = 4;
278 qi->channel_width = 32;
279 qi->deinterleave = 2;
280 break;
281 }
282 fallthrough;
283 case INTEL_DRAM_LPDDR5:
284 qi->t_bl = 16;
285 qi->max_numchannels = 8;
286 qi->channel_width = 16;
287 qi->deinterleave = is_y_tile ? 2 : 4;
288 break;
289 default:
290 qi->t_bl = 16;
291 qi->max_numchannels = 1;
292 break;
293 }
294 } else if (DISPLAY_VER(display) == 11) {
295 qi->t_bl = dram_info->type == INTEL_DRAM_DDR4 ? 4 : 8;
296 qi->max_numchannels = 1;
297 }
298
299 if (drm_WARN_ON(display->drm,
300 qi->num_points > ARRAY_SIZE(qi->points)))
301 qi->num_points = ARRAY_SIZE(qi->points);
302
303 for (i = 0; i < qi->num_points; i++) {
304 struct intel_qgv_point *sp = &qi->points[i];
305
306 ret = intel_read_qgv_point_info(display, sp, i);
307 if (ret) {
308 drm_dbg_kms(display->drm, "Could not read QGV %d info\n", i);
309 return ret;
310 }
311
312 drm_dbg_kms(display->drm,
313 "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
314 i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
315 sp->t_rcd, sp->t_rc);
316 }
317
318 if (qi->num_psf_points > 0) {
319 ret = adls_pcode_read_psf_gv_point_info(display, qi->psf_points);
320 if (ret) {
321 drm_err(display->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
322 qi->num_psf_points = 0;
323 }
324
325 for (i = 0; i < qi->num_psf_points; i++)
326 drm_dbg_kms(display->drm,
327 "PSF GV %d: CLK=%d \n",
328 i, qi->psf_points[i].clk);
329 }
330
331 return 0;
332 }
333
adl_calc_psf_bw(int clk)334 static int adl_calc_psf_bw(int clk)
335 {
336 /*
337 * clk is multiples of 16.666MHz (100/6)
338 * According to BSpec PSF GV bandwidth is
339 * calculated as BW = 64 * clk * 16.666Mhz
340 */
341 return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
342 }
343
icl_sagv_max_dclk(const struct intel_qgv_info * qi)344 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
345 {
346 u16 dclk = 0;
347 int i;
348
349 for (i = 0; i < qi->num_points; i++)
350 dclk = max(dclk, qi->points[i].dclk);
351
352 return dclk;
353 }
354
355 struct intel_sa_info {
356 u16 displayrtids;
357 u8 deburst, deprogbwlimit, derating;
358 };
359
360 static const struct intel_sa_info icl_sa_info = {
361 .deburst = 8,
362 .deprogbwlimit = 25, /* GB/s */
363 .displayrtids = 128,
364 .derating = 10,
365 };
366
367 static const struct intel_sa_info tgl_sa_info = {
368 .deburst = 16,
369 .deprogbwlimit = 34, /* GB/s */
370 .displayrtids = 256,
371 .derating = 10,
372 };
373
374 static const struct intel_sa_info rkl_sa_info = {
375 .deburst = 8,
376 .deprogbwlimit = 20, /* GB/s */
377 .displayrtids = 128,
378 .derating = 10,
379 };
380
381 static const struct intel_sa_info adls_sa_info = {
382 .deburst = 16,
383 .deprogbwlimit = 38, /* GB/s */
384 .displayrtids = 256,
385 .derating = 10,
386 };
387
388 static const struct intel_sa_info adlp_sa_info = {
389 .deburst = 16,
390 .deprogbwlimit = 38, /* GB/s */
391 .displayrtids = 256,
392 .derating = 20,
393 };
394
395 static const struct intel_sa_info mtl_sa_info = {
396 .deburst = 32,
397 .deprogbwlimit = 38, /* GB/s */
398 .displayrtids = 256,
399 .derating = 10,
400 };
401
402 static const struct intel_sa_info xe2_hpd_sa_info = {
403 .derating = 30,
404 .deprogbwlimit = 53,
405 /* Other values not used by simplified algorithm */
406 };
407
408 static const struct intel_sa_info xe2_hpd_ecc_sa_info = {
409 .derating = 45,
410 .deprogbwlimit = 53,
411 /* Other values not used by simplified algorithm */
412 };
413
414 static const struct intel_sa_info xe3lpd_sa_info = {
415 .deburst = 32,
416 .deprogbwlimit = 65, /* GB/s */
417 .displayrtids = 256,
418 .derating = 10,
419 };
420
icl_get_bw_info(struct intel_display * display,const struct intel_sa_info * sa)421 static int icl_get_bw_info(struct intel_display *display, const struct intel_sa_info *sa)
422 {
423 struct drm_i915_private *i915 = to_i915(display->drm);
424 struct intel_qgv_info qi = {};
425 bool is_y_tile = true; /* assume y tile may be used */
426 int num_channels = max_t(u8, 1, i915->dram_info.num_channels);
427 int ipqdepth, ipqdepthpch = 16;
428 int dclk_max;
429 int maxdebw;
430 int num_groups = ARRAY_SIZE(display->bw.max);
431 int i, ret;
432
433 ret = icl_get_qgv_points(display, &qi, is_y_tile);
434 if (ret) {
435 drm_dbg_kms(display->drm,
436 "Failed to get memory subsystem information, ignoring bandwidth limits");
437 return ret;
438 }
439
440 dclk_max = icl_sagv_max_dclk(&qi);
441 maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
442 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
443 qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
444
445 for (i = 0; i < num_groups; i++) {
446 struct intel_bw_info *bi = &display->bw.max[i];
447 int clpchgroup;
448 int j;
449
450 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
451 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
452
453 bi->num_qgv_points = qi.num_points;
454 bi->num_psf_gv_points = qi.num_psf_points;
455
456 for (j = 0; j < qi.num_points; j++) {
457 const struct intel_qgv_point *sp = &qi.points[j];
458 int ct, bw;
459
460 /*
461 * Max row cycle time
462 *
463 * FIXME what is the logic behind the
464 * assumed burst length?
465 */
466 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
467 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
468 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
469
470 bi->deratedbw[j] = min(maxdebw,
471 bw * (100 - sa->derating) / 100);
472
473 drm_dbg_kms(display->drm,
474 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
475 i, j, bi->num_planes, bi->deratedbw[j]);
476 }
477 }
478 /*
479 * In case if SAGV is disabled in BIOS, we always get 1
480 * SAGV point, but we can't send PCode commands to restrict it
481 * as it will fail and pointless anyway.
482 */
483 if (qi.num_points == 1)
484 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
485 else
486 display->sagv.status = I915_SAGV_ENABLED;
487
488 return 0;
489 }
490
tgl_get_bw_info(struct intel_display * display,const struct intel_sa_info * sa)491 static int tgl_get_bw_info(struct intel_display *display, const struct intel_sa_info *sa)
492 {
493 struct drm_i915_private *i915 = to_i915(display->drm);
494 struct intel_qgv_info qi = {};
495 const struct dram_info *dram_info = &i915->dram_info;
496 bool is_y_tile = true; /* assume y tile may be used */
497 int num_channels = max_t(u8, 1, dram_info->num_channels);
498 int ipqdepth, ipqdepthpch = 16;
499 int dclk_max;
500 int maxdebw, peakbw;
501 int clperchgroup;
502 int num_groups = ARRAY_SIZE(display->bw.max);
503 int i, ret;
504
505 ret = icl_get_qgv_points(display, &qi, is_y_tile);
506 if (ret) {
507 drm_dbg_kms(display->drm,
508 "Failed to get memory subsystem information, ignoring bandwidth limits");
509 return ret;
510 }
511
512 if (DISPLAY_VER(display) < 14 &&
513 (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
514 num_channels *= 2;
515
516 qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
517
518 if (num_channels < qi.max_numchannels && DISPLAY_VER(display) >= 12)
519 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
520
521 if (DISPLAY_VER(display) >= 12 && num_channels > qi.max_numchannels)
522 drm_warn(display->drm, "Number of channels exceeds max number of channels.");
523 if (qi.max_numchannels != 0)
524 num_channels = min_t(u8, num_channels, qi.max_numchannels);
525
526 dclk_max = icl_sagv_max_dclk(&qi);
527
528 peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
529 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);
530
531 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
532 /*
533 * clperchgroup = 4kpagespermempage * clperchperblock,
534 * clperchperblock = 8 / num_channels * interleave
535 */
536 clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
537
538 for (i = 0; i < num_groups; i++) {
539 struct intel_bw_info *bi = &display->bw.max[i];
540 struct intel_bw_info *bi_next;
541 int clpchgroup;
542 int j;
543
544 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
545
546 if (i < num_groups - 1) {
547 bi_next = &display->bw.max[i + 1];
548
549 if (clpchgroup < clperchgroup)
550 bi_next->num_planes = (ipqdepth - clpchgroup) /
551 clpchgroup + 1;
552 else
553 bi_next->num_planes = 0;
554 }
555
556 bi->num_qgv_points = qi.num_points;
557 bi->num_psf_gv_points = qi.num_psf_points;
558
559 for (j = 0; j < qi.num_points; j++) {
560 const struct intel_qgv_point *sp = &qi.points[j];
561 int ct, bw;
562
563 /*
564 * Max row cycle time
565 *
566 * FIXME what is the logic behind the
567 * assumed burst length?
568 */
569 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
570 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
571 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
572
573 bi->deratedbw[j] = min(maxdebw,
574 bw * (100 - sa->derating) / 100);
575 bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
576 num_channels *
577 qi.channel_width, 8);
578
579 drm_dbg_kms(display->drm,
580 "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
581 i, j, bi->num_planes, bi->deratedbw[j],
582 bi->peakbw[j]);
583 }
584
585 for (j = 0; j < qi.num_psf_points; j++) {
586 const struct intel_psf_gv_point *sp = &qi.psf_points[j];
587
588 bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
589
590 drm_dbg_kms(display->drm,
591 "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
592 i, j, bi->num_planes, bi->psf_bw[j]);
593 }
594 }
595
596 /*
597 * In case if SAGV is disabled in BIOS, we always get 1
598 * SAGV point, but we can't send PCode commands to restrict it
599 * as it will fail and pointless anyway.
600 */
601 if (qi.num_points == 1)
602 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
603 else
604 display->sagv.status = I915_SAGV_ENABLED;
605
606 return 0;
607 }
608
dg2_get_bw_info(struct intel_display * display)609 static void dg2_get_bw_info(struct intel_display *display)
610 {
611 unsigned int deratedbw = display->platform.dg2_g11 ? 38000 : 50000;
612 int num_groups = ARRAY_SIZE(display->bw.max);
613 int i;
614
615 /*
616 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
617 * that doesn't depend on the number of planes enabled. So fill all the
618 * plane group with constant bw information for uniformity with other
619 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
620 * whereas DG2-G11 platforms have 38 GB/s.
621 */
622 for (i = 0; i < num_groups; i++) {
623 struct intel_bw_info *bi = &display->bw.max[i];
624
625 bi->num_planes = 1;
626 /* Need only one dummy QGV point per group */
627 bi->num_qgv_points = 1;
628 bi->deratedbw[0] = deratedbw;
629 }
630
631 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
632 }
633
xe2_hpd_get_bw_info(struct intel_display * display,const struct intel_sa_info * sa)634 static int xe2_hpd_get_bw_info(struct intel_display *display,
635 const struct intel_sa_info *sa)
636 {
637 struct drm_i915_private *i915 = to_i915(display->drm);
638 struct intel_qgv_info qi = {};
639 int num_channels = i915->dram_info.num_channels;
640 int peakbw, maxdebw;
641 int ret, i;
642
643 ret = icl_get_qgv_points(display, &qi, true);
644 if (ret) {
645 drm_dbg_kms(display->drm,
646 "Failed to get memory subsystem information, ignoring bandwidth limits");
647 return ret;
648 }
649
650 peakbw = num_channels * qi.channel_width / 8 * icl_sagv_max_dclk(&qi);
651 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 10);
652
653 for (i = 0; i < qi.num_points; i++) {
654 const struct intel_qgv_point *point = &qi.points[i];
655 int bw = num_channels * (qi.channel_width / 8) * point->dclk;
656
657 display->bw.max[0].deratedbw[i] =
658 min(maxdebw, (100 - sa->derating) * bw / 100);
659 display->bw.max[0].peakbw[i] = bw;
660
661 drm_dbg_kms(display->drm, "QGV %d: deratedbw=%u peakbw: %u\n",
662 i, display->bw.max[0].deratedbw[i],
663 display->bw.max[0].peakbw[i]);
664 }
665
666 /* Bandwidth does not depend on # of planes; set all groups the same */
667 display->bw.max[0].num_planes = 1;
668 display->bw.max[0].num_qgv_points = qi.num_points;
669 for (i = 1; i < ARRAY_SIZE(display->bw.max); i++)
670 memcpy(&display->bw.max[i], &display->bw.max[0],
671 sizeof(display->bw.max[0]));
672
673 /*
674 * Xe2_HPD should always have exactly two QGV points representing
675 * battery and plugged-in operation.
676 */
677 drm_WARN_ON(display->drm, qi.num_points != 2);
678 display->sagv.status = I915_SAGV_ENABLED;
679
680 return 0;
681 }
682
icl_max_bw_index(struct intel_display * display,int num_planes,int qgv_point)683 static unsigned int icl_max_bw_index(struct intel_display *display,
684 int num_planes, int qgv_point)
685 {
686 int i;
687
688 /*
689 * Let's return max bw for 0 planes
690 */
691 num_planes = max(1, num_planes);
692
693 for (i = 0; i < ARRAY_SIZE(display->bw.max); i++) {
694 const struct intel_bw_info *bi =
695 &display->bw.max[i];
696
697 /*
698 * Pcode will not expose all QGV points when
699 * SAGV is forced to off/min/med/max.
700 */
701 if (qgv_point >= bi->num_qgv_points)
702 return UINT_MAX;
703
704 if (num_planes >= bi->num_planes)
705 return i;
706 }
707
708 return UINT_MAX;
709 }
710
tgl_max_bw_index(struct intel_display * display,int num_planes,int qgv_point)711 static unsigned int tgl_max_bw_index(struct intel_display *display,
712 int num_planes, int qgv_point)
713 {
714 int i;
715
716 /*
717 * Let's return max bw for 0 planes
718 */
719 num_planes = max(1, num_planes);
720
721 for (i = ARRAY_SIZE(display->bw.max) - 1; i >= 0; i--) {
722 const struct intel_bw_info *bi =
723 &display->bw.max[i];
724
725 /*
726 * Pcode will not expose all QGV points when
727 * SAGV is forced to off/min/med/max.
728 */
729 if (qgv_point >= bi->num_qgv_points)
730 return UINT_MAX;
731
732 if (num_planes <= bi->num_planes)
733 return i;
734 }
735
736 return 0;
737 }
738
adl_psf_bw(struct intel_display * display,int psf_gv_point)739 static unsigned int adl_psf_bw(struct intel_display *display,
740 int psf_gv_point)
741 {
742 const struct intel_bw_info *bi =
743 &display->bw.max[0];
744
745 return bi->psf_bw[psf_gv_point];
746 }
747
icl_qgv_bw(struct intel_display * display,int num_active_planes,int qgv_point)748 static unsigned int icl_qgv_bw(struct intel_display *display,
749 int num_active_planes, int qgv_point)
750 {
751 unsigned int idx;
752
753 if (DISPLAY_VER(display) >= 12)
754 idx = tgl_max_bw_index(display, num_active_planes, qgv_point);
755 else
756 idx = icl_max_bw_index(display, num_active_planes, qgv_point);
757
758 if (idx >= ARRAY_SIZE(display->bw.max))
759 return 0;
760
761 return display->bw.max[idx].deratedbw[qgv_point];
762 }
763
intel_bw_init_hw(struct intel_display * display)764 void intel_bw_init_hw(struct intel_display *display)
765 {
766 const struct dram_info *dram_info = &to_i915(display->drm)->dram_info;
767
768 if (!HAS_DISPLAY(display))
769 return;
770
771 if (DISPLAY_VER(display) >= 30)
772 tgl_get_bw_info(display, &xe3lpd_sa_info);
773 else if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx &&
774 dram_info->type == INTEL_DRAM_GDDR_ECC)
775 xe2_hpd_get_bw_info(display, &xe2_hpd_ecc_sa_info);
776 else if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx)
777 xe2_hpd_get_bw_info(display, &xe2_hpd_sa_info);
778 else if (DISPLAY_VER(display) >= 14)
779 tgl_get_bw_info(display, &mtl_sa_info);
780 else if (display->platform.dg2)
781 dg2_get_bw_info(display);
782 else if (display->platform.alderlake_p)
783 tgl_get_bw_info(display, &adlp_sa_info);
784 else if (display->platform.alderlake_s)
785 tgl_get_bw_info(display, &adls_sa_info);
786 else if (display->platform.rocketlake)
787 tgl_get_bw_info(display, &rkl_sa_info);
788 else if (DISPLAY_VER(display) == 12)
789 tgl_get_bw_info(display, &tgl_sa_info);
790 else if (DISPLAY_VER(display) == 11)
791 icl_get_bw_info(display, &icl_sa_info);
792 }
793
intel_bw_crtc_num_active_planes(const struct intel_crtc_state * crtc_state)794 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
795 {
796 /*
797 * We assume cursors are small enough
798 * to not not cause bandwidth problems.
799 */
800 return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
801 }
802
intel_bw_crtc_data_rate(const struct intel_crtc_state * crtc_state)803 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
804 {
805 struct intel_display *display = to_intel_display(crtc_state);
806 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
807 unsigned int data_rate = 0;
808 enum plane_id plane_id;
809
810 for_each_plane_id_on_crtc(crtc, plane_id) {
811 /*
812 * We assume cursors are small enough
813 * to not not cause bandwidth problems.
814 */
815 if (plane_id == PLANE_CURSOR)
816 continue;
817
818 data_rate += crtc_state->data_rate[plane_id];
819
820 if (DISPLAY_VER(display) < 11)
821 data_rate += crtc_state->data_rate_y[plane_id];
822 }
823
824 return data_rate;
825 }
826
827 /* "Maximum Pipe Read Bandwidth" */
intel_bw_crtc_min_cdclk(struct intel_display * display,unsigned int data_rate)828 static int intel_bw_crtc_min_cdclk(struct intel_display *display,
829 unsigned int data_rate)
830 {
831 if (DISPLAY_VER(display) < 12)
832 return 0;
833
834 return DIV_ROUND_UP_ULL(mul_u32_u32(data_rate, 10), 512);
835 }
836
intel_bw_num_active_planes(struct intel_display * display,const struct intel_bw_state * bw_state)837 static unsigned int intel_bw_num_active_planes(struct intel_display *display,
838 const struct intel_bw_state *bw_state)
839 {
840 unsigned int num_active_planes = 0;
841 enum pipe pipe;
842
843 for_each_pipe(display, pipe)
844 num_active_planes += bw_state->num_active_planes[pipe];
845
846 return num_active_planes;
847 }
848
intel_bw_data_rate(struct intel_display * display,const struct intel_bw_state * bw_state)849 static unsigned int intel_bw_data_rate(struct intel_display *display,
850 const struct intel_bw_state *bw_state)
851 {
852 struct drm_i915_private *i915 = to_i915(display->drm);
853 unsigned int data_rate = 0;
854 enum pipe pipe;
855
856 for_each_pipe(display, pipe)
857 data_rate += bw_state->data_rate[pipe];
858
859 if (DISPLAY_VER(display) >= 13 && i915_vtd_active(i915))
860 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
861
862 return data_rate;
863 }
864
865 struct intel_bw_state *
intel_atomic_get_old_bw_state(struct intel_atomic_state * state)866 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
867 {
868 struct intel_display *display = to_intel_display(state);
869 struct intel_global_state *bw_state;
870
871 bw_state = intel_atomic_get_old_global_obj_state(state, &display->bw.obj);
872
873 return to_intel_bw_state(bw_state);
874 }
875
876 struct intel_bw_state *
intel_atomic_get_new_bw_state(struct intel_atomic_state * state)877 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
878 {
879 struct intel_display *display = to_intel_display(state);
880 struct intel_global_state *bw_state;
881
882 bw_state = intel_atomic_get_new_global_obj_state(state, &display->bw.obj);
883
884 return to_intel_bw_state(bw_state);
885 }
886
887 struct intel_bw_state *
intel_atomic_get_bw_state(struct intel_atomic_state * state)888 intel_atomic_get_bw_state(struct intel_atomic_state *state)
889 {
890 struct intel_display *display = to_intel_display(state);
891 struct intel_global_state *bw_state;
892
893 bw_state = intel_atomic_get_global_obj_state(state, &display->bw.obj);
894 if (IS_ERR(bw_state))
895 return ERR_CAST(bw_state);
896
897 return to_intel_bw_state(bw_state);
898 }
899
icl_max_bw_qgv_point_mask(struct intel_display * display,int num_active_planes)900 static unsigned int icl_max_bw_qgv_point_mask(struct intel_display *display,
901 int num_active_planes)
902 {
903 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
904 unsigned int max_bw_point = 0;
905 unsigned int max_bw = 0;
906 int i;
907
908 for (i = 0; i < num_qgv_points; i++) {
909 unsigned int max_data_rate =
910 icl_qgv_bw(display, num_active_planes, i);
911
912 /*
913 * We need to know which qgv point gives us
914 * maximum bandwidth in order to disable SAGV
915 * if we find that we exceed SAGV block time
916 * with watermarks. By that moment we already
917 * have those, as it is calculated earlier in
918 * intel_atomic_check,
919 */
920 if (max_data_rate > max_bw) {
921 max_bw_point = BIT(i);
922 max_bw = max_data_rate;
923 }
924 }
925
926 return max_bw_point;
927 }
928
icl_prepare_qgv_points_mask(struct intel_display * display,unsigned int qgv_points,unsigned int psf_points)929 static u16 icl_prepare_qgv_points_mask(struct intel_display *display,
930 unsigned int qgv_points,
931 unsigned int psf_points)
932 {
933 return ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
934 ADLS_PCODE_REQ_PSF_PT(psf_points)) & icl_qgv_points_mask(display);
935 }
936
icl_max_bw_psf_gv_point_mask(struct intel_display * display)937 static unsigned int icl_max_bw_psf_gv_point_mask(struct intel_display *display)
938 {
939 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
940 unsigned int max_bw_point_mask = 0;
941 unsigned int max_bw = 0;
942 int i;
943
944 for (i = 0; i < num_psf_gv_points; i++) {
945 unsigned int max_data_rate = adl_psf_bw(display, i);
946
947 if (max_data_rate > max_bw) {
948 max_bw_point_mask = BIT(i);
949 max_bw = max_data_rate;
950 } else if (max_data_rate == max_bw) {
951 max_bw_point_mask |= BIT(i);
952 }
953 }
954
955 return max_bw_point_mask;
956 }
957
icl_force_disable_sagv(struct intel_display * display,struct intel_bw_state * bw_state)958 static void icl_force_disable_sagv(struct intel_display *display,
959 struct intel_bw_state *bw_state)
960 {
961 unsigned int qgv_points = icl_max_bw_qgv_point_mask(display, 0);
962 unsigned int psf_points = icl_max_bw_psf_gv_point_mask(display);
963
964 bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display,
965 qgv_points,
966 psf_points);
967
968 drm_dbg_kms(display->drm, "Forcing SAGV disable: mask 0x%x\n",
969 bw_state->qgv_points_mask);
970
971 icl_pcode_restrict_qgv_points(display, bw_state->qgv_points_mask);
972 }
973
mtl_find_qgv_points(struct intel_display * display,unsigned int data_rate,unsigned int num_active_planes,struct intel_bw_state * new_bw_state)974 static int mtl_find_qgv_points(struct intel_display *display,
975 unsigned int data_rate,
976 unsigned int num_active_planes,
977 struct intel_bw_state *new_bw_state)
978 {
979 unsigned int best_rate = UINT_MAX;
980 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
981 unsigned int qgv_peak_bw = 0;
982 int i;
983 int ret;
984
985 ret = intel_atomic_lock_global_state(&new_bw_state->base);
986 if (ret)
987 return ret;
988
989 /*
990 * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
991 * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
992 * not enabled. PM Demand code will clamp the value for the register
993 */
994 if (!intel_can_enable_sagv(display, new_bw_state)) {
995 new_bw_state->qgv_point_peakbw = U16_MAX;
996 drm_dbg_kms(display->drm, "No SAGV, use UINT_MAX as peak bw.");
997 return 0;
998 }
999
1000 /*
1001 * Find the best QGV point by comparing the data_rate with max data rate
1002 * offered per plane group
1003 */
1004 for (i = 0; i < num_qgv_points; i++) {
1005 unsigned int bw_index =
1006 tgl_max_bw_index(display, num_active_planes, i);
1007 unsigned int max_data_rate;
1008
1009 if (bw_index >= ARRAY_SIZE(display->bw.max))
1010 continue;
1011
1012 max_data_rate = display->bw.max[bw_index].deratedbw[i];
1013
1014 if (max_data_rate < data_rate)
1015 continue;
1016
1017 if (max_data_rate - data_rate < best_rate) {
1018 best_rate = max_data_rate - data_rate;
1019 qgv_peak_bw = display->bw.max[bw_index].peakbw[i];
1020 }
1021
1022 drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
1023 i, max_data_rate, data_rate, qgv_peak_bw);
1024 }
1025
1026 drm_dbg_kms(display->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
1027 qgv_peak_bw, data_rate);
1028
1029 /*
1030 * The display configuration cannot be supported if no QGV point
1031 * satisfying the required data rate is found
1032 */
1033 if (qgv_peak_bw == 0) {
1034 drm_dbg_kms(display->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
1035 data_rate, num_active_planes);
1036 return -EINVAL;
1037 }
1038
1039 /* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
1040 new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100);
1041
1042 return 0;
1043 }
1044
icl_find_qgv_points(struct intel_display * display,unsigned int data_rate,unsigned int num_active_planes,const struct intel_bw_state * old_bw_state,struct intel_bw_state * new_bw_state)1045 static int icl_find_qgv_points(struct intel_display *display,
1046 unsigned int data_rate,
1047 unsigned int num_active_planes,
1048 const struct intel_bw_state *old_bw_state,
1049 struct intel_bw_state *new_bw_state)
1050 {
1051 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
1052 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
1053 u16 psf_points = 0;
1054 u16 qgv_points = 0;
1055 int i;
1056 int ret;
1057
1058 ret = intel_atomic_lock_global_state(&new_bw_state->base);
1059 if (ret)
1060 return ret;
1061
1062 for (i = 0; i < num_qgv_points; i++) {
1063 unsigned int max_data_rate = icl_qgv_bw(display,
1064 num_active_planes, i);
1065 if (max_data_rate >= data_rate)
1066 qgv_points |= BIT(i);
1067
1068 drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d\n",
1069 i, max_data_rate, data_rate);
1070 }
1071
1072 for (i = 0; i < num_psf_gv_points; i++) {
1073 unsigned int max_data_rate = adl_psf_bw(display, i);
1074
1075 if (max_data_rate >= data_rate)
1076 psf_points |= BIT(i);
1077
1078 drm_dbg_kms(display->drm, "PSF GV point %d: max bw %d"
1079 " required %d\n",
1080 i, max_data_rate, data_rate);
1081 }
1082
1083 /*
1084 * BSpec states that we always should have at least one allowed point
1085 * left, so if we couldn't - simply reject the configuration for obvious
1086 * reasons.
1087 */
1088 if (qgv_points == 0) {
1089 drm_dbg_kms(display->drm, "No QGV points provide sufficient memory"
1090 " bandwidth %d for display configuration(%d active planes).\n",
1091 data_rate, num_active_planes);
1092 return -EINVAL;
1093 }
1094
1095 if (num_psf_gv_points > 0 && psf_points == 0) {
1096 drm_dbg_kms(display->drm, "No PSF GV points provide sufficient memory"
1097 " bandwidth %d for display configuration(%d active planes).\n",
1098 data_rate, num_active_planes);
1099 return -EINVAL;
1100 }
1101
1102 /*
1103 * Leave only single point with highest bandwidth, if
1104 * we can't enable SAGV due to the increased memory latency it may
1105 * cause.
1106 */
1107 if (!intel_can_enable_sagv(display, new_bw_state)) {
1108 qgv_points = icl_max_bw_qgv_point_mask(display, num_active_planes);
1109 drm_dbg_kms(display->drm, "No SAGV, using single QGV point mask 0x%x\n",
1110 qgv_points);
1111 }
1112
1113 /*
1114 * We store the ones which need to be masked as that is what PCode
1115 * actually accepts as a parameter.
1116 */
1117 new_bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display,
1118 qgv_points,
1119 psf_points);
1120 /*
1121 * If the actual mask had changed we need to make sure that
1122 * the commits are serialized(in case this is a nomodeset, nonblocking)
1123 */
1124 if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1125 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1126 if (ret)
1127 return ret;
1128 }
1129
1130 return 0;
1131 }
1132
intel_bw_check_qgv_points(struct intel_display * display,const struct intel_bw_state * old_bw_state,struct intel_bw_state * new_bw_state)1133 static int intel_bw_check_qgv_points(struct intel_display *display,
1134 const struct intel_bw_state *old_bw_state,
1135 struct intel_bw_state *new_bw_state)
1136 {
1137 unsigned int data_rate = intel_bw_data_rate(display, new_bw_state);
1138 unsigned int num_active_planes =
1139 intel_bw_num_active_planes(display, new_bw_state);
1140
1141 data_rate = DIV_ROUND_UP(data_rate, 1000);
1142
1143 if (DISPLAY_VER(display) >= 14)
1144 return mtl_find_qgv_points(display, data_rate, num_active_planes,
1145 new_bw_state);
1146 else
1147 return icl_find_qgv_points(display, data_rate, num_active_planes,
1148 old_bw_state, new_bw_state);
1149 }
1150
intel_dbuf_bw_changed(struct intel_display * display,const struct intel_dbuf_bw * old_dbuf_bw,const struct intel_dbuf_bw * new_dbuf_bw)1151 static bool intel_dbuf_bw_changed(struct intel_display *display,
1152 const struct intel_dbuf_bw *old_dbuf_bw,
1153 const struct intel_dbuf_bw *new_dbuf_bw)
1154 {
1155 enum dbuf_slice slice;
1156
1157 for_each_dbuf_slice(display, slice) {
1158 if (old_dbuf_bw->max_bw[slice] != new_dbuf_bw->max_bw[slice] ||
1159 old_dbuf_bw->active_planes[slice] != new_dbuf_bw->active_planes[slice])
1160 return true;
1161 }
1162
1163 return false;
1164 }
1165
intel_bw_state_changed(struct intel_display * display,const struct intel_bw_state * old_bw_state,const struct intel_bw_state * new_bw_state)1166 static bool intel_bw_state_changed(struct intel_display *display,
1167 const struct intel_bw_state *old_bw_state,
1168 const struct intel_bw_state *new_bw_state)
1169 {
1170 enum pipe pipe;
1171
1172 for_each_pipe(display, pipe) {
1173 const struct intel_dbuf_bw *old_dbuf_bw =
1174 &old_bw_state->dbuf_bw[pipe];
1175 const struct intel_dbuf_bw *new_dbuf_bw =
1176 &new_bw_state->dbuf_bw[pipe];
1177
1178 if (intel_dbuf_bw_changed(display, old_dbuf_bw, new_dbuf_bw))
1179 return true;
1180
1181 if (intel_bw_crtc_min_cdclk(display, old_bw_state->data_rate[pipe]) !=
1182 intel_bw_crtc_min_cdclk(display, new_bw_state->data_rate[pipe]))
1183 return true;
1184 }
1185
1186 return false;
1187 }
1188
skl_plane_calc_dbuf_bw(struct intel_dbuf_bw * dbuf_bw,struct intel_crtc * crtc,enum plane_id plane_id,const struct skl_ddb_entry * ddb,unsigned int data_rate)1189 static void skl_plane_calc_dbuf_bw(struct intel_dbuf_bw *dbuf_bw,
1190 struct intel_crtc *crtc,
1191 enum plane_id plane_id,
1192 const struct skl_ddb_entry *ddb,
1193 unsigned int data_rate)
1194 {
1195 struct intel_display *display = to_intel_display(crtc);
1196 unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(display, ddb);
1197 enum dbuf_slice slice;
1198
1199 /*
1200 * The arbiter can only really guarantee an
1201 * equal share of the total bw to each plane.
1202 */
1203 for_each_dbuf_slice_in_mask(display, slice, dbuf_mask) {
1204 dbuf_bw->max_bw[slice] = max(dbuf_bw->max_bw[slice], data_rate);
1205 dbuf_bw->active_planes[slice] |= BIT(plane_id);
1206 }
1207 }
1208
skl_crtc_calc_dbuf_bw(struct intel_dbuf_bw * dbuf_bw,const struct intel_crtc_state * crtc_state)1209 static void skl_crtc_calc_dbuf_bw(struct intel_dbuf_bw *dbuf_bw,
1210 const struct intel_crtc_state *crtc_state)
1211 {
1212 struct intel_display *display = to_intel_display(crtc_state);
1213 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1214 enum plane_id plane_id;
1215
1216 memset(dbuf_bw, 0, sizeof(*dbuf_bw));
1217
1218 if (!crtc_state->hw.active)
1219 return;
1220
1221 for_each_plane_id_on_crtc(crtc, plane_id) {
1222 /*
1223 * We assume cursors are small enough
1224 * to not cause bandwidth problems.
1225 */
1226 if (plane_id == PLANE_CURSOR)
1227 continue;
1228
1229 skl_plane_calc_dbuf_bw(dbuf_bw, crtc, plane_id,
1230 &crtc_state->wm.skl.plane_ddb[plane_id],
1231 crtc_state->data_rate[plane_id]);
1232
1233 if (DISPLAY_VER(display) < 11)
1234 skl_plane_calc_dbuf_bw(dbuf_bw, crtc, plane_id,
1235 &crtc_state->wm.skl.plane_ddb_y[plane_id],
1236 crtc_state->data_rate[plane_id]);
1237 }
1238 }
1239
1240 /* "Maximum Data Buffer Bandwidth" */
1241 static int
intel_bw_dbuf_min_cdclk(struct intel_display * display,const struct intel_bw_state * bw_state)1242 intel_bw_dbuf_min_cdclk(struct intel_display *display,
1243 const struct intel_bw_state *bw_state)
1244 {
1245 unsigned int total_max_bw = 0;
1246 enum dbuf_slice slice;
1247
1248 for_each_dbuf_slice(display, slice) {
1249 int num_active_planes = 0;
1250 unsigned int max_bw = 0;
1251 enum pipe pipe;
1252
1253 /*
1254 * The arbiter can only really guarantee an
1255 * equal share of the total bw to each plane.
1256 */
1257 for_each_pipe(display, pipe) {
1258 const struct intel_dbuf_bw *dbuf_bw = &bw_state->dbuf_bw[pipe];
1259
1260 max_bw = max(dbuf_bw->max_bw[slice], max_bw);
1261 num_active_planes += hweight8(dbuf_bw->active_planes[slice]);
1262 }
1263 max_bw *= num_active_planes;
1264
1265 total_max_bw = max(total_max_bw, max_bw);
1266 }
1267
1268 return DIV_ROUND_UP(total_max_bw, 64);
1269 }
1270
intel_bw_min_cdclk(struct intel_display * display,const struct intel_bw_state * bw_state)1271 int intel_bw_min_cdclk(struct intel_display *display,
1272 const struct intel_bw_state *bw_state)
1273 {
1274 enum pipe pipe;
1275 int min_cdclk;
1276
1277 min_cdclk = intel_bw_dbuf_min_cdclk(display, bw_state);
1278
1279 for_each_pipe(display, pipe)
1280 min_cdclk = max(min_cdclk,
1281 intel_bw_crtc_min_cdclk(display,
1282 bw_state->data_rate[pipe]));
1283
1284 return min_cdclk;
1285 }
1286
intel_bw_calc_min_cdclk(struct intel_atomic_state * state,bool * need_cdclk_calc)1287 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
1288 bool *need_cdclk_calc)
1289 {
1290 struct intel_display *display = to_intel_display(state);
1291 struct intel_bw_state *new_bw_state = NULL;
1292 const struct intel_bw_state *old_bw_state = NULL;
1293 const struct intel_cdclk_state *cdclk_state;
1294 const struct intel_crtc_state *old_crtc_state;
1295 const struct intel_crtc_state *new_crtc_state;
1296 int old_min_cdclk, new_min_cdclk;
1297 struct intel_crtc *crtc;
1298 int i;
1299
1300 if (DISPLAY_VER(display) < 9)
1301 return 0;
1302
1303 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1304 new_crtc_state, i) {
1305 struct intel_dbuf_bw old_dbuf_bw, new_dbuf_bw;
1306
1307 skl_crtc_calc_dbuf_bw(&old_dbuf_bw, old_crtc_state);
1308 skl_crtc_calc_dbuf_bw(&new_dbuf_bw, new_crtc_state);
1309
1310 if (!intel_dbuf_bw_changed(display, &old_dbuf_bw, &new_dbuf_bw))
1311 continue;
1312
1313 new_bw_state = intel_atomic_get_bw_state(state);
1314 if (IS_ERR(new_bw_state))
1315 return PTR_ERR(new_bw_state);
1316
1317 old_bw_state = intel_atomic_get_old_bw_state(state);
1318
1319 new_bw_state->dbuf_bw[crtc->pipe] = new_dbuf_bw;
1320 }
1321
1322 if (!old_bw_state)
1323 return 0;
1324
1325 if (intel_bw_state_changed(display, old_bw_state, new_bw_state)) {
1326 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
1327 if (ret)
1328 return ret;
1329 }
1330
1331 old_min_cdclk = intel_bw_min_cdclk(display, old_bw_state);
1332 new_min_cdclk = intel_bw_min_cdclk(display, new_bw_state);
1333
1334 /*
1335 * No need to check against the cdclk state if
1336 * the min cdclk doesn't increase.
1337 *
1338 * Ie. we only ever increase the cdclk due to bandwidth
1339 * requirements. This can reduce back and forth
1340 * display blinking due to constant cdclk changes.
1341 */
1342 if (new_min_cdclk <= old_min_cdclk)
1343 return 0;
1344
1345 cdclk_state = intel_atomic_get_cdclk_state(state);
1346 if (IS_ERR(cdclk_state))
1347 return PTR_ERR(cdclk_state);
1348
1349 /*
1350 * No need to recalculate the cdclk state if
1351 * the min cdclk doesn't increase.
1352 *
1353 * Ie. we only ever increase the cdclk due to bandwidth
1354 * requirements. This can reduce back and forth
1355 * display blinking due to constant cdclk changes.
1356 */
1357 if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
1358 return 0;
1359
1360 drm_dbg_kms(display->drm,
1361 "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
1362 new_min_cdclk, cdclk_state->bw_min_cdclk);
1363 *need_cdclk_calc = true;
1364
1365 return 0;
1366 }
1367
intel_bw_check_data_rate(struct intel_atomic_state * state,bool * changed)1368 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
1369 {
1370 struct intel_display *display = to_intel_display(state);
1371 const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1372 struct intel_crtc *crtc;
1373 int i;
1374
1375 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1376 new_crtc_state, i) {
1377 unsigned int old_data_rate =
1378 intel_bw_crtc_data_rate(old_crtc_state);
1379 unsigned int new_data_rate =
1380 intel_bw_crtc_data_rate(new_crtc_state);
1381 unsigned int old_active_planes =
1382 intel_bw_crtc_num_active_planes(old_crtc_state);
1383 unsigned int new_active_planes =
1384 intel_bw_crtc_num_active_planes(new_crtc_state);
1385 struct intel_bw_state *new_bw_state;
1386
1387 /*
1388 * Avoid locking the bw state when
1389 * nothing significant has changed.
1390 */
1391 if (old_data_rate == new_data_rate &&
1392 old_active_planes == new_active_planes)
1393 continue;
1394
1395 new_bw_state = intel_atomic_get_bw_state(state);
1396 if (IS_ERR(new_bw_state))
1397 return PTR_ERR(new_bw_state);
1398
1399 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1400 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1401
1402 *changed = true;
1403
1404 drm_dbg_kms(display->drm,
1405 "[CRTC:%d:%s] data rate %u num active planes %u\n",
1406 crtc->base.base.id, crtc->base.name,
1407 new_bw_state->data_rate[crtc->pipe],
1408 new_bw_state->num_active_planes[crtc->pipe]);
1409 }
1410
1411 return 0;
1412 }
1413
intel_bw_modeset_checks(struct intel_atomic_state * state)1414 static int intel_bw_modeset_checks(struct intel_atomic_state *state)
1415 {
1416 struct intel_display *display = to_intel_display(state);
1417 const struct intel_bw_state *old_bw_state;
1418 struct intel_bw_state *new_bw_state;
1419
1420 if (DISPLAY_VER(display) < 9)
1421 return 0;
1422
1423 new_bw_state = intel_atomic_get_bw_state(state);
1424 if (IS_ERR(new_bw_state))
1425 return PTR_ERR(new_bw_state);
1426
1427 old_bw_state = intel_atomic_get_old_bw_state(state);
1428
1429 new_bw_state->active_pipes =
1430 intel_calc_active_pipes(state, old_bw_state->active_pipes);
1431
1432 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
1433 int ret;
1434
1435 ret = intel_atomic_lock_global_state(&new_bw_state->base);
1436 if (ret)
1437 return ret;
1438 }
1439
1440 return 0;
1441 }
1442
intel_bw_check_sagv_mask(struct intel_atomic_state * state)1443 static int intel_bw_check_sagv_mask(struct intel_atomic_state *state)
1444 {
1445 struct intel_display *display = to_intel_display(state);
1446 const struct intel_crtc_state *old_crtc_state;
1447 const struct intel_crtc_state *new_crtc_state;
1448 const struct intel_bw_state *old_bw_state = NULL;
1449 struct intel_bw_state *new_bw_state = NULL;
1450 struct intel_crtc *crtc;
1451 int ret, i;
1452
1453 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1454 new_crtc_state, i) {
1455 if (intel_crtc_can_enable_sagv(old_crtc_state) ==
1456 intel_crtc_can_enable_sagv(new_crtc_state))
1457 continue;
1458
1459 new_bw_state = intel_atomic_get_bw_state(state);
1460 if (IS_ERR(new_bw_state))
1461 return PTR_ERR(new_bw_state);
1462
1463 old_bw_state = intel_atomic_get_old_bw_state(state);
1464
1465 if (intel_crtc_can_enable_sagv(new_crtc_state))
1466 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
1467 else
1468 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
1469 }
1470
1471 if (!new_bw_state)
1472 return 0;
1473
1474 if (intel_can_enable_sagv(display, new_bw_state) !=
1475 intel_can_enable_sagv(display, old_bw_state)) {
1476 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1477 if (ret)
1478 return ret;
1479 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
1480 ret = intel_atomic_lock_global_state(&new_bw_state->base);
1481 if (ret)
1482 return ret;
1483 }
1484
1485 return 0;
1486 }
1487
intel_bw_atomic_check(struct intel_atomic_state * state,bool any_ms)1488 int intel_bw_atomic_check(struct intel_atomic_state *state, bool any_ms)
1489 {
1490 struct intel_display *display = to_intel_display(state);
1491 bool changed = false;
1492 struct intel_bw_state *new_bw_state;
1493 const struct intel_bw_state *old_bw_state;
1494 int ret;
1495
1496 if (DISPLAY_VER(display) < 9)
1497 return 0;
1498
1499 if (any_ms) {
1500 ret = intel_bw_modeset_checks(state);
1501 if (ret)
1502 return ret;
1503 }
1504
1505 ret = intel_bw_check_sagv_mask(state);
1506 if (ret)
1507 return ret;
1508
1509 /* FIXME earlier gens need some checks too */
1510 if (DISPLAY_VER(display) < 11)
1511 return 0;
1512
1513 ret = intel_bw_check_data_rate(state, &changed);
1514 if (ret)
1515 return ret;
1516
1517 old_bw_state = intel_atomic_get_old_bw_state(state);
1518 new_bw_state = intel_atomic_get_new_bw_state(state);
1519
1520 if (new_bw_state &&
1521 intel_can_enable_sagv(display, old_bw_state) !=
1522 intel_can_enable_sagv(display, new_bw_state))
1523 changed = true;
1524
1525 /*
1526 * If none of our inputs (data rates, number of active
1527 * planes, SAGV yes/no) changed then nothing to do here.
1528 */
1529 if (!changed)
1530 return 0;
1531
1532 ret = intel_bw_check_qgv_points(display, old_bw_state, new_bw_state);
1533 if (ret)
1534 return ret;
1535
1536 return 0;
1537 }
1538
intel_bw_crtc_update(struct intel_bw_state * bw_state,const struct intel_crtc_state * crtc_state)1539 static void intel_bw_crtc_update(struct intel_bw_state *bw_state,
1540 const struct intel_crtc_state *crtc_state)
1541 {
1542 struct intel_display *display = to_intel_display(crtc_state);
1543 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1544
1545 bw_state->data_rate[crtc->pipe] =
1546 intel_bw_crtc_data_rate(crtc_state);
1547 bw_state->num_active_planes[crtc->pipe] =
1548 intel_bw_crtc_num_active_planes(crtc_state);
1549
1550 drm_dbg_kms(display->drm, "pipe %c data rate %u num active planes %u\n",
1551 pipe_name(crtc->pipe),
1552 bw_state->data_rate[crtc->pipe],
1553 bw_state->num_active_planes[crtc->pipe]);
1554 }
1555
intel_bw_update_hw_state(struct intel_display * display)1556 void intel_bw_update_hw_state(struct intel_display *display)
1557 {
1558 struct intel_bw_state *bw_state =
1559 to_intel_bw_state(display->bw.obj.state);
1560 struct intel_crtc *crtc;
1561
1562 if (DISPLAY_VER(display) < 9)
1563 return;
1564
1565 bw_state->active_pipes = 0;
1566 bw_state->pipe_sagv_reject = 0;
1567
1568 for_each_intel_crtc(display->drm, crtc) {
1569 const struct intel_crtc_state *crtc_state =
1570 to_intel_crtc_state(crtc->base.state);
1571 enum pipe pipe = crtc->pipe;
1572
1573 if (crtc_state->hw.active)
1574 bw_state->active_pipes |= BIT(pipe);
1575
1576 if (DISPLAY_VER(display) >= 11)
1577 intel_bw_crtc_update(bw_state, crtc_state);
1578
1579 skl_crtc_calc_dbuf_bw(&bw_state->dbuf_bw[pipe], crtc_state);
1580
1581 /* initially SAGV has been forced off */
1582 bw_state->pipe_sagv_reject |= BIT(pipe);
1583 }
1584 }
1585
intel_bw_crtc_disable_noatomic(struct intel_crtc * crtc)1586 void intel_bw_crtc_disable_noatomic(struct intel_crtc *crtc)
1587 {
1588 struct intel_display *display = to_intel_display(crtc);
1589 struct intel_bw_state *bw_state =
1590 to_intel_bw_state(display->bw.obj.state);
1591 enum pipe pipe = crtc->pipe;
1592
1593 if (DISPLAY_VER(display) < 9)
1594 return;
1595
1596 bw_state->data_rate[pipe] = 0;
1597 bw_state->num_active_planes[pipe] = 0;
1598 memset(&bw_state->dbuf_bw[pipe], 0, sizeof(bw_state->dbuf_bw[pipe]));
1599 }
1600
1601 static struct intel_global_state *
intel_bw_duplicate_state(struct intel_global_obj * obj)1602 intel_bw_duplicate_state(struct intel_global_obj *obj)
1603 {
1604 struct intel_bw_state *state;
1605
1606 state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1607 if (!state)
1608 return NULL;
1609
1610 return &state->base;
1611 }
1612
intel_bw_destroy_state(struct intel_global_obj * obj,struct intel_global_state * state)1613 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1614 struct intel_global_state *state)
1615 {
1616 kfree(state);
1617 }
1618
1619 static const struct intel_global_state_funcs intel_bw_funcs = {
1620 .atomic_duplicate_state = intel_bw_duplicate_state,
1621 .atomic_destroy_state = intel_bw_destroy_state,
1622 };
1623
intel_bw_init(struct intel_display * display)1624 int intel_bw_init(struct intel_display *display)
1625 {
1626 struct intel_bw_state *state;
1627
1628 state = kzalloc(sizeof(*state), GFP_KERNEL);
1629 if (!state)
1630 return -ENOMEM;
1631
1632 intel_atomic_global_obj_init(display, &display->bw.obj,
1633 &state->base, &intel_bw_funcs);
1634
1635 /*
1636 * Limit this only if we have SAGV. And for Display version 14 onwards
1637 * sagv is handled though pmdemand requests
1638 */
1639 if (intel_has_sagv(display) && IS_DISPLAY_VER(display, 11, 13))
1640 icl_force_disable_sagv(display, state);
1641
1642 return 0;
1643 }
1644