1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <drm/drm_atomic_state_helper.h> 7 8 #include "intel_bw.h" 9 #include "intel_display_types.h" 10 #include "intel_sideband.h" 11 #include "intel_atomic.h" 12 #include "intel_pm.h" 13 14 15 /* Parameters for Qclk Geyserville (QGV) */ 16 struct intel_qgv_point { 17 u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd; 18 }; 19 20 struct intel_qgv_info { 21 struct intel_qgv_point points[I915_NUM_QGV_POINTS]; 22 u8 num_points; 23 u8 num_channels; 24 u8 t_bl; 25 enum intel_dram_type dram_type; 26 }; 27 28 static int icl_pcode_read_mem_global_info(struct drm_i915_private *dev_priv, 29 struct intel_qgv_info *qi) 30 { 31 u32 val = 0; 32 int ret; 33 34 ret = sandybridge_pcode_read(dev_priv, 35 ICL_PCODE_MEM_SUBSYSYSTEM_INFO | 36 ICL_PCODE_MEM_SS_READ_GLOBAL_INFO, 37 &val, NULL); 38 if (ret) 39 return ret; 40 41 if (IS_GEN(dev_priv, 12)) { 42 switch (val & 0xf) { 43 case 0: 44 qi->dram_type = INTEL_DRAM_DDR4; 45 break; 46 case 3: 47 qi->dram_type = INTEL_DRAM_LPDDR4; 48 break; 49 case 4: 50 qi->dram_type = INTEL_DRAM_DDR3; 51 break; 52 case 5: 53 qi->dram_type = INTEL_DRAM_LPDDR3; 54 break; 55 default: 56 MISSING_CASE(val & 0xf); 57 break; 58 } 59 } else if (IS_GEN(dev_priv, 11)) { 60 switch (val & 0xf) { 61 case 0: 62 qi->dram_type = INTEL_DRAM_DDR4; 63 break; 64 case 1: 65 qi->dram_type = INTEL_DRAM_DDR3; 66 break; 67 case 2: 68 qi->dram_type = INTEL_DRAM_LPDDR3; 69 break; 70 case 3: 71 qi->dram_type = INTEL_DRAM_LPDDR4; 72 break; 73 default: 74 MISSING_CASE(val & 0xf); 75 break; 76 } 77 } else { 78 MISSING_CASE(INTEL_GEN(dev_priv)); 79 qi->dram_type = INTEL_DRAM_LPDDR3; /* Conservative default */ 80 } 81 82 qi->num_channels = (val & 0xf0) >> 4; 83 qi->num_points = (val & 0xf00) >> 8; 84 85 if (IS_GEN(dev_priv, 12)) 86 qi->t_bl = qi->dram_type == INTEL_DRAM_DDR4 ? 4 : 16; 87 else if (IS_GEN(dev_priv, 11)) 88 qi->t_bl = qi->dram_type == INTEL_DRAM_DDR4 ? 4 : 8; 89 90 return 0; 91 } 92 93 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv, 94 struct intel_qgv_point *sp, 95 int point) 96 { 97 u32 val = 0, val2 = 0; 98 int ret; 99 100 ret = sandybridge_pcode_read(dev_priv, 101 ICL_PCODE_MEM_SUBSYSYSTEM_INFO | 102 ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point), 103 &val, &val2); 104 if (ret) 105 return ret; 106 107 sp->dclk = val & 0xffff; 108 sp->t_rp = (val & 0xff0000) >> 16; 109 sp->t_rcd = (val & 0xff000000) >> 24; 110 111 sp->t_rdpre = val2 & 0xff; 112 sp->t_ras = (val2 & 0xff00) >> 8; 113 114 sp->t_rc = sp->t_rp + sp->t_ras; 115 116 return 0; 117 } 118 119 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv, 120 u32 points_mask) 121 { 122 int ret; 123 124 /* bspec says to keep retrying for at least 1 ms */ 125 ret = skl_pcode_request(dev_priv, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG, 126 points_mask, 127 ICL_PCODE_POINTS_RESTRICTED_MASK, 128 ICL_PCODE_POINTS_RESTRICTED, 129 1); 130 131 if (ret < 0) { 132 drm_err(&dev_priv->drm, "Failed to disable qgv points (%d)\n", ret); 133 return ret; 134 } 135 136 return 0; 137 } 138 139 static int icl_get_qgv_points(struct drm_i915_private *dev_priv, 140 struct intel_qgv_info *qi) 141 { 142 int i, ret; 143 144 ret = icl_pcode_read_mem_global_info(dev_priv, qi); 145 if (ret) 146 return ret; 147 148 if (drm_WARN_ON(&dev_priv->drm, 149 qi->num_points > ARRAY_SIZE(qi->points))) 150 qi->num_points = ARRAY_SIZE(qi->points); 151 152 for (i = 0; i < qi->num_points; i++) { 153 struct intel_qgv_point *sp = &qi->points[i]; 154 155 ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i); 156 if (ret) 157 return ret; 158 159 drm_dbg_kms(&dev_priv->drm, 160 "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n", 161 i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras, 162 sp->t_rcd, sp->t_rc); 163 } 164 165 return 0; 166 } 167 168 static int icl_calc_bw(int dclk, int num, int den) 169 { 170 /* multiples of 16.666MHz (100/6) */ 171 return DIV_ROUND_CLOSEST(num * dclk * 100, den * 6); 172 } 173 174 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi) 175 { 176 u16 dclk = 0; 177 int i; 178 179 for (i = 0; i < qi->num_points; i++) 180 dclk = max(dclk, qi->points[i].dclk); 181 182 return dclk; 183 } 184 185 struct intel_sa_info { 186 u16 displayrtids; 187 u8 deburst, deprogbwlimit; 188 }; 189 190 static const struct intel_sa_info icl_sa_info = { 191 .deburst = 8, 192 .deprogbwlimit = 25, /* GB/s */ 193 .displayrtids = 128, 194 }; 195 196 static const struct intel_sa_info tgl_sa_info = { 197 .deburst = 16, 198 .deprogbwlimit = 34, /* GB/s */ 199 .displayrtids = 256, 200 }; 201 202 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa) 203 { 204 struct intel_qgv_info qi = {}; 205 bool is_y_tile = true; /* assume y tile may be used */ 206 int num_channels; 207 int deinterleave; 208 int ipqdepth, ipqdepthpch; 209 int dclk_max; 210 int maxdebw; 211 int i, ret; 212 213 ret = icl_get_qgv_points(dev_priv, &qi); 214 if (ret) { 215 drm_dbg_kms(&dev_priv->drm, 216 "Failed to get memory subsystem information, ignoring bandwidth limits"); 217 return ret; 218 } 219 num_channels = qi.num_channels; 220 221 deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); 222 dclk_max = icl_sagv_max_dclk(&qi); 223 224 ipqdepthpch = 16; 225 226 maxdebw = min(sa->deprogbwlimit * 1000, 227 icl_calc_bw(dclk_max, 16, 1) * 6 / 10); /* 60% */ 228 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels); 229 230 for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) { 231 struct intel_bw_info *bi = &dev_priv->max_bw[i]; 232 int clpchgroup; 233 int j; 234 235 clpchgroup = (sa->deburst * deinterleave / num_channels) << i; 236 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1; 237 238 bi->num_qgv_points = qi.num_points; 239 240 for (j = 0; j < qi.num_points; j++) { 241 const struct intel_qgv_point *sp = &qi.points[j]; 242 int ct, bw; 243 244 /* 245 * Max row cycle time 246 * 247 * FIXME what is the logic behind the 248 * assumed burst length? 249 */ 250 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd + 251 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre); 252 bw = icl_calc_bw(sp->dclk, clpchgroup * 32 * num_channels, ct); 253 254 bi->deratedbw[j] = min(maxdebw, 255 bw * 9 / 10); /* 90% */ 256 257 drm_dbg_kms(&dev_priv->drm, 258 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n", 259 i, j, bi->num_planes, bi->deratedbw[j]); 260 } 261 262 if (bi->num_planes == 1) 263 break; 264 } 265 266 /* 267 * In case if SAGV is disabled in BIOS, we always get 1 268 * SAGV point, but we can't send PCode commands to restrict it 269 * as it will fail and pointless anyway. 270 */ 271 if (qi.num_points == 1) 272 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; 273 else 274 dev_priv->sagv_status = I915_SAGV_ENABLED; 275 276 return 0; 277 } 278 279 static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, 280 int num_planes, int qgv_point) 281 { 282 int i; 283 284 /* 285 * Let's return max bw for 0 planes 286 */ 287 num_planes = max(1, num_planes); 288 289 for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) { 290 const struct intel_bw_info *bi = 291 &dev_priv->max_bw[i]; 292 293 /* 294 * Pcode will not expose all QGV points when 295 * SAGV is forced to off/min/med/max. 296 */ 297 if (qgv_point >= bi->num_qgv_points) 298 return UINT_MAX; 299 300 if (num_planes >= bi->num_planes) 301 return bi->deratedbw[qgv_point]; 302 } 303 304 return 0; 305 } 306 307 void intel_bw_init_hw(struct drm_i915_private *dev_priv) 308 { 309 if (!HAS_DISPLAY(dev_priv)) 310 return; 311 312 if (IS_GEN(dev_priv, 12)) 313 icl_get_bw_info(dev_priv, &tgl_sa_info); 314 else if (IS_GEN(dev_priv, 11)) 315 icl_get_bw_info(dev_priv, &icl_sa_info); 316 } 317 318 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state) 319 { 320 /* 321 * We assume cursors are small enough 322 * to not not cause bandwidth problems. 323 */ 324 return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR)); 325 } 326 327 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state) 328 { 329 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 330 unsigned int data_rate = 0; 331 enum plane_id plane_id; 332 333 for_each_plane_id_on_crtc(crtc, plane_id) { 334 /* 335 * We assume cursors are small enough 336 * to not not cause bandwidth problems. 337 */ 338 if (plane_id == PLANE_CURSOR) 339 continue; 340 341 data_rate += crtc_state->data_rate[plane_id]; 342 } 343 344 return data_rate; 345 } 346 347 void intel_bw_crtc_update(struct intel_bw_state *bw_state, 348 const struct intel_crtc_state *crtc_state) 349 { 350 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 351 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 352 353 bw_state->data_rate[crtc->pipe] = 354 intel_bw_crtc_data_rate(crtc_state); 355 bw_state->num_active_planes[crtc->pipe] = 356 intel_bw_crtc_num_active_planes(crtc_state); 357 358 drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n", 359 pipe_name(crtc->pipe), 360 bw_state->data_rate[crtc->pipe], 361 bw_state->num_active_planes[crtc->pipe]); 362 } 363 364 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv, 365 const struct intel_bw_state *bw_state) 366 { 367 unsigned int num_active_planes = 0; 368 enum pipe pipe; 369 370 for_each_pipe(dev_priv, pipe) 371 num_active_planes += bw_state->num_active_planes[pipe]; 372 373 return num_active_planes; 374 } 375 376 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv, 377 const struct intel_bw_state *bw_state) 378 { 379 unsigned int data_rate = 0; 380 enum pipe pipe; 381 382 for_each_pipe(dev_priv, pipe) 383 data_rate += bw_state->data_rate[pipe]; 384 385 return data_rate; 386 } 387 388 struct intel_bw_state * 389 intel_atomic_get_old_bw_state(struct intel_atomic_state *state) 390 { 391 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 392 struct intel_global_state *bw_state; 393 394 bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->bw_obj); 395 396 return to_intel_bw_state(bw_state); 397 } 398 399 struct intel_bw_state * 400 intel_atomic_get_new_bw_state(struct intel_atomic_state *state) 401 { 402 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 403 struct intel_global_state *bw_state; 404 405 bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->bw_obj); 406 407 return to_intel_bw_state(bw_state); 408 } 409 410 struct intel_bw_state * 411 intel_atomic_get_bw_state(struct intel_atomic_state *state) 412 { 413 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 414 struct intel_global_state *bw_state; 415 416 bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->bw_obj); 417 if (IS_ERR(bw_state)) 418 return ERR_CAST(bw_state); 419 420 return to_intel_bw_state(bw_state); 421 } 422 423 int intel_bw_atomic_check(struct intel_atomic_state *state) 424 { 425 struct drm_i915_private *dev_priv = to_i915(state->base.dev); 426 struct intel_crtc_state *new_crtc_state, *old_crtc_state; 427 struct intel_bw_state *new_bw_state = NULL; 428 const struct intel_bw_state *old_bw_state = NULL; 429 unsigned int data_rate; 430 unsigned int num_active_planes; 431 struct intel_crtc *crtc; 432 int i, ret; 433 u32 allowed_points = 0; 434 unsigned int max_bw_point = 0, max_bw = 0; 435 unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points; 436 u32 mask = (1 << num_qgv_points) - 1; 437 438 /* FIXME earlier gens need some checks too */ 439 if (INTEL_GEN(dev_priv) < 11) 440 return 0; 441 442 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, 443 new_crtc_state, i) { 444 unsigned int old_data_rate = 445 intel_bw_crtc_data_rate(old_crtc_state); 446 unsigned int new_data_rate = 447 intel_bw_crtc_data_rate(new_crtc_state); 448 unsigned int old_active_planes = 449 intel_bw_crtc_num_active_planes(old_crtc_state); 450 unsigned int new_active_planes = 451 intel_bw_crtc_num_active_planes(new_crtc_state); 452 453 /* 454 * Avoid locking the bw state when 455 * nothing significant has changed. 456 */ 457 if (old_data_rate == new_data_rate && 458 old_active_planes == new_active_planes) 459 continue; 460 461 new_bw_state = intel_atomic_get_bw_state(state); 462 if (IS_ERR(new_bw_state)) 463 return PTR_ERR(new_bw_state); 464 465 new_bw_state->data_rate[crtc->pipe] = new_data_rate; 466 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes; 467 468 drm_dbg_kms(&dev_priv->drm, 469 "pipe %c data rate %u num active planes %u\n", 470 pipe_name(crtc->pipe), 471 new_bw_state->data_rate[crtc->pipe], 472 new_bw_state->num_active_planes[crtc->pipe]); 473 } 474 475 if (!new_bw_state) 476 return 0; 477 478 ret = intel_atomic_lock_global_state(&new_bw_state->base); 479 if (ret) 480 return ret; 481 482 data_rate = intel_bw_data_rate(dev_priv, new_bw_state); 483 data_rate = DIV_ROUND_UP(data_rate, 1000); 484 485 num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state); 486 487 for (i = 0; i < num_qgv_points; i++) { 488 unsigned int max_data_rate; 489 490 max_data_rate = icl_max_bw(dev_priv, num_active_planes, i); 491 /* 492 * We need to know which qgv point gives us 493 * maximum bandwidth in order to disable SAGV 494 * if we find that we exceed SAGV block time 495 * with watermarks. By that moment we already 496 * have those, as it is calculated earlier in 497 * intel_atomic_check, 498 */ 499 if (max_data_rate > max_bw) { 500 max_bw_point = i; 501 max_bw = max_data_rate; 502 } 503 if (max_data_rate >= data_rate) 504 allowed_points |= BIT(i); 505 drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n", 506 i, max_data_rate, data_rate); 507 } 508 509 /* 510 * BSpec states that we always should have at least one allowed point 511 * left, so if we couldn't - simply reject the configuration for obvious 512 * reasons. 513 */ 514 if (allowed_points == 0) { 515 drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory" 516 " bandwidth %d for display configuration(%d active planes).\n", 517 data_rate, num_active_planes); 518 return -EINVAL; 519 } 520 521 /* 522 * Leave only single point with highest bandwidth, if 523 * we can't enable SAGV due to the increased memory latency it may 524 * cause. 525 */ 526 if (!intel_can_enable_sagv(dev_priv, new_bw_state)) { 527 allowed_points = BIT(max_bw_point); 528 drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n", 529 max_bw_point); 530 } 531 /* 532 * We store the ones which need to be masked as that is what PCode 533 * actually accepts as a parameter. 534 */ 535 new_bw_state->qgv_points_mask = ~allowed_points & mask; 536 537 old_bw_state = intel_atomic_get_old_bw_state(state); 538 /* 539 * If the actual mask had changed we need to make sure that 540 * the commits are serialized(in case this is a nomodeset, nonblocking) 541 */ 542 if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) { 543 ret = intel_atomic_serialize_global_state(&new_bw_state->base); 544 if (ret) 545 return ret; 546 } 547 548 return 0; 549 } 550 551 static struct intel_global_state * 552 intel_bw_duplicate_state(struct intel_global_obj *obj) 553 { 554 struct intel_bw_state *state; 555 556 state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL); 557 if (!state) 558 return NULL; 559 560 return &state->base; 561 } 562 563 static void intel_bw_destroy_state(struct intel_global_obj *obj, 564 struct intel_global_state *state) 565 { 566 kfree(state); 567 } 568 569 static const struct intel_global_state_funcs intel_bw_funcs = { 570 .atomic_duplicate_state = intel_bw_duplicate_state, 571 .atomic_destroy_state = intel_bw_destroy_state, 572 }; 573 574 int intel_bw_init(struct drm_i915_private *dev_priv) 575 { 576 struct intel_bw_state *state; 577 578 state = kzalloc(sizeof(*state), GFP_KERNEL); 579 if (!state) 580 return -ENOMEM; 581 582 intel_atomic_global_obj_init(dev_priv, &dev_priv->bw_obj, 583 &state->base, &intel_bw_funcs); 584 585 return 0; 586 } 587